mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-09 14:02:56 +09:00
test(e2e): prepull distroless images via staged static sleep
Distroless images (cert-manager and friends) ship no shell or /bin/sleep, so the previous `command: ["sleep", "infinity"]` failed with exec ENOENT. Stage busybox:musl (statically linked) into a shared emptyDir from an initContainer; every prepull container then execs /shared/sleep regardless of what the image itself ships. The glibc busybox tag is unusable here — it needs /lib64/ld-linux-x86-64.so.2 which distroless images lack. Re-adds cert-manager to the prepull set in the install bats so its 4 images are cached on every node before the HelmRelease installs. Verified locally in a single-node kind v1.32: all 4 cert-manager-* images plus busybox pulled and held Ready under this DaemonSet. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
This commit is contained in:
1 parent
5a4ac9c202
commit
a46bd60c0a
2 files changed
+34
-20
No files matched your search
@@ -24,16 +24,18 @@
|
||||
# skipped. Writing to a file makes `set -e` trip on a render failure
|
||||
# without using `var=$(helm template ...)` captures, which `set -x`
|
||||
# would expand into the trace and balloon CI logs.
|
||||
local kubeovn_yaml linstor_yaml
|
||||
local kubeovn_yaml linstor_yaml certmanager_yaml
|
||||
kubeovn_yaml=$(mktemp)
|
||||
linstor_yaml=$(mktemp)
|
||||
certmanager_yaml=$(mktemp)
|
||||
helm template packages/system/kubeovn > "$kubeovn_yaml"
|
||||
helm template packages/system/linstor > "$linstor_yaml"
|
||||
cat "$kubeovn_yaml" "$linstor_yaml" | yq -N '
|
||||
helm template packages/system/cert-manager > "$certmanager_yaml"
|
||||
cat "$kubeovn_yaml" "$linstor_yaml" "$certmanager_yaml" | yq -N '
|
||||
(..|select(has("containers"))|.containers[]|.image),
|
||||
(..|select(has("initContainers"))|.initContainers[]|.image)
|
||||
' | hack/e2e-prepull-images.sh
|
||||
rm -f "$kubeovn_yaml" "$linstor_yaml"
|
||||
rm -f "$kubeovn_yaml" "$linstor_yaml" "$certmanager_yaml"
|
||||
}
|
||||
|
||||
@test "Install Cozystack" {
|
||||
|
||||
+29
-17
@@ -4,16 +4,15 @@
|
||||
# Reads image references from stdin, one per line. Empty lines and lines
|
||||
# starting with '#' are ignored.
|
||||
#
|
||||
# Some workloads (OVN raft, LINSTOR) are sensitive to peer-readiness at
|
||||
# startup: if nodes pull the image at different speeds, one replica boots
|
||||
# before its peers are reachable, exhausts its raft connection retries, and
|
||||
# the HelmRelease install times out. Pre-pulling eliminates the pull stagger
|
||||
# so all replicas start within milliseconds of each other.
|
||||
# Some workloads (OVN raft, LINSTOR, cert-manager) are sensitive to slow or
|
||||
# staggered pulls at install time: a HelmRelease can time out while one node
|
||||
# is still pulling. Pre-pulling eliminates the stagger.
|
||||
#
|
||||
# Implementation: a DaemonSet with one regular container per image (parallel
|
||||
# pulls — total time = max of any one image rather than sum). kubectl rollout
|
||||
# status blocks until all pods are Ready (= all containers running = all
|
||||
# images cached on every node), then we delete the DaemonSet.
|
||||
# Distroless images (cert-manager, etc.) ship no shell or `sleep`, so we
|
||||
# can't just `command: ["sleep", "infinity"]`. Trick: an initContainer copies
|
||||
# busybox (a static binary) into a shared emptyDir, then every prepull
|
||||
# container execs /shared/sleep — the kernel execve's the binary from the
|
||||
# volume, the container's filesystem doesn't need to provide it.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -38,8 +37,11 @@ for image in "${images[@]}"; do
|
||||
containers+="
|
||||
- name: pull-${i}
|
||||
image: ${image}
|
||||
command: [\"sleep\", \"infinity\"]
|
||||
command: [\"/shared/sleep\", \"infinity\"]
|
||||
imagePullPolicy: IfNotPresent
|
||||
volumeMounts:
|
||||
- name: bin
|
||||
mountPath: /shared
|
||||
resources:
|
||||
requests:
|
||||
cpu: 1m
|
||||
@@ -65,17 +67,27 @@ spec:
|
||||
app: e2e-image-prepuller
|
||||
spec:
|
||||
# hostNetwork bypasses the CNI: this script runs BEFORE Cozystack
|
||||
# installs kube-ovn, so the cluster has no working CNI yet and a normal
|
||||
# pod would stay ContainerCreating with NetworkPluginNotReady, never
|
||||
# reaching image-pull. With hostNetwork the pod sandbox is created on
|
||||
# the host's namespace and the kubelet pulls images right away.
|
||||
# Same pattern the cozystack-operator pod uses for the same reason.
|
||||
# installs kube-ovn, so a normal pod would stay ContainerCreating with
|
||||
# NetworkPluginNotReady.
|
||||
hostNetwork: true
|
||||
# No need for an SA token — this pod just sleeps while images pull.
|
||||
automountServiceAccountToken: false
|
||||
tolerations:
|
||||
# Reach every node including control-plane and nodes still coming up.
|
||||
- operator: Exists
|
||||
initContainers:
|
||||
# Stage a static \`sleep\` into the shared volume so distroless images
|
||||
# (no shell, no /bin/sleep) can still keep their container alive.
|
||||
# musl tag = statically linked; the glibc default needs an ELF
|
||||
# interpreter that distroless images don't ship, so execve fails with
|
||||
# "no such file or directory" even though the binary is right there.
|
||||
- name: stage-sleep
|
||||
image: busybox:musl
|
||||
command: ["cp", "/bin/busybox", "/shared/sleep"]
|
||||
volumeMounts:
|
||||
- name: bin
|
||||
mountPath: /shared
|
||||
volumes:
|
||||
- name: bin
|
||||
emptyDir: {}
|
||||
containers:${containers}
|
||||
EOF
|
||||
|
||||
|
||||
Reference in new issue
Block a user