#!/usr/bin/env bats # ----------------------------------------------------------------------------- # Cozystack E2E provisioning — CONTAINER lane # # The counterpart to hack/e2e-prepare-cluster.bats. That file boots three QEMU # guests; this one takes over after hack/e2e-container-up.sh has started three # Talos containers on the runner's docker daemon and attached this sandbox to # their network. Everything from bootstrap onward is deliberately the same shape # as the QEMU lane so the two stay comparable, and both hand off an identical # cluster to hack/e2e-install-cozystack.bats. # # Why the split: containers cannot nest inside the sandbox the way QEMU guests # do, so the nodes must be siblings on the host daemon for hack/e2e-compose.yaml's # /dev and /run/udev binds to resolve against the real host devtmpfs. Only the # host-side half needs a docker socket; this half needs only the network. # # Addressing, CPU and memory match the QEMU lane exactly, so the MetalLB # 192.168.123.200-250 pool and hack/e2e-post-install-prep.sh are unchanged. # The one deliberate difference is the API endpoint: container mode has no ARP # VIP, so .11 stands in for the QEMU lane's 192.168.123.10. # ----------------------------------------------------------------------------- HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)" E2E_CONTAINER_UP_LIB=true . "$HACK_DIR/e2e-container-up.sh" @test "Talos API is reachable on all three container nodes" { # e2e-container-up.sh has already started them; this is the handoff check, and # it is a wait rather than a probe because container start is asynchronous. if ! timeout 180 sh -ec 'until nc -nz 192.168.123.11 50000 && nc -nz 192.168.123.12 50000 && nc -nz 192.168.123.13 50000; do sleep 1; done'; then echo "Talos API never came up on 192.168.123.11-13:50000." >&2 echo "The sandbox reaches the nodes over the compose network it was attached to;" >&2 echo "if this times out, check that hack/e2e-container-up.sh ran and that the" >&2 echo "sandbox is on the cozy-e2e_talos network." >&2 exit 1 fi } @test "Host kernel modules are usable from inside the nodes" { # THE trap of this lane. machine.kernel.modules is a silent no-op in container # mode -- kernel_module_spec.go returns early on ModeContainer with no error # and no event -- so a lane that carried the QEMU patch over would look like it # worked and fail much later as a CNI or storage error. e2e-container-up.sh # loads these on the host; this asserts the nodes actually see them, which is # the property that matters and is not implied by the host check alone. for mod in openvswitch zfs; do if ! talosctl read /proc/modules -n 192.168.123.11 -e 192.168.123.11 | grep -q "^${mod} "; then echo "Kernel module '${mod}' is not visible from srv1." >&2 echo "Container nodes share the host kernel and Talos cannot load modules in" >&2 echo "container mode, so this must be loaded on the host before they start." >&2 exit 1 fi done } @test "/dev/kvm is present inside the nodes" { # KubeVirt's device plugin needs it, and its presence is what makes the tenant # workers run under real KVM rather than TCG emulation. if ! talosctl ls /dev/kvm -n 192.168.123.11 -e 192.168.123.11 >/dev/null 2>&1; then echo "/dev/kvm is missing inside srv1 — KubeVirt cannot advertise devices.kubevirt.io/kvm." >&2 echo "hack/e2e-compose.yaml binds the host /dev; check the host exposes /dev/kvm." >&2 exit 1 fi } @test "Bootstrap Talos cluster" { # Retry for up to 120s: talosctl bootstrap refuses with "time is not in sync # yet" until NTP converges on a freshly started node. This is the same # allowance the QEMU lane makes, and it is a genuine infrastructure wait, not # a retry around product logic. # # If it never converges, the usual cause is that the nodes cannot reach the # internet at all rather than slow NTP -- most often because docker sets # `-P FORWARD DROP` and the compose bridge's traffic is being dropped. if ! timeout 120 sh -ec 'until talosctl bootstrap -n 192.168.123.11 -e 192.168.123.11; do sleep 2; done'; then echo "talosctl bootstrap never succeeded." >&2 echo "If the error was 'time is not in sync yet', the nodes likely have no egress:" >&2 echo "check 'iptables -S FORWARD' on the host for a DROP policy over the compose bridge." >&2 talosctl dmesg -n 192.168.123.11 -e 192.168.123.11 || true exit 1 fi } @test "Wait until etcd is healthy" { # Same 5m budget and the same reason as the QEMU lane: a 3-node etcd converges # through Talos's serialized learner promotion, so the third member is only # promoted once the second has caught up. if ! timeout 300 sh -ec 'until talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.11 >/dev/null 2>&1; do sleep 1; done'; then talosctl dmesg -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.11 || true exit 1 fi timeout 60 sh -ec 'while talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.11 2>&1 | grep -q "rpc error"; do sleep 1; done' } @test "Retrieve kubeconfig and register all three nodes" { rm -f kubeconfig # No ARP VIP in container mode, so the endpoint is a node IP rather than the # QEMU lane's 192.168.123.10. talosctl kubeconfig kubeconfig -e 192.168.123.11 -n 192.168.123.11 if ! timeout 120 sh -ec 'until [ $(kubectl get node --no-headers | wc -l) -eq 3 ]; do sleep 1; done'; then echo "Fewer than three nodes registered:" >&2 kubectl get nodes -o wide >&2 || true exit 1 fi } @test "Node allocatable matches the container limits" { # Container mode exposes host-wide capacity through /proc, so without the # systemReserved values generated by e2e-container-up.sh every one of these # nodes advertises the runner's full 32 CPU / ~125 GiB. That lets the # scheduler pack far beyond each 8 CPU / 24 GiB cgroup and turns later OOMs # into apparent workload flakes. Check the scheduler-visible result, not just # the machine-config input. local node cpu cpu_m memory memory_kib local node_cpu_limit="${COZY_E2E_NODE_CPUS:-8}" local node_memory_limit_mib="${COZY_E2E_NODE_MEMORY_MIB:-24576}" local max_cpu_m=$(( node_cpu_limit * 1000 )) local min_cpu_m=$(( max_cpu_m - 1000 )) local max_memory_kib=$(( node_memory_limit_mib * 1024 )) local min_memory_kib=$(( max_memory_kib - 1024 * 1024 )) for node in srv1 srv2 srv3; do cpu=$(kubectl get node "$node" -o jsonpath='{.status.allocatable.cpu}') case "$cpu" in *m) cpu_m=${cpu%m} ;; *[!0-9]* | '') echo "node $node reported an unreadable allocatable CPU quantity: '$cpu'" >&2 return 1 ;; *) cpu_m=$(( cpu * 1000 )) ;; esac memory=$(kubectl get node "$node" -o jsonpath='{.status.allocatable.memory}') if ! memory_kib=$(kubernetes_memory_to_kib "$memory"); then echo "node $node reported an unreadable allocatable memory quantity: '$memory'" >&2 return 1 fi if [ "$cpu_m" -lt "$min_cpu_m" ] || [ "$cpu_m" -gt "$max_cpu_m" ]; then echo "node $node allocatable CPU $cpu is outside ${min_cpu_m}-${max_cpu_m}m" >&2 return 1 fi if [ "$memory_kib" -lt "$min_memory_kib" ] || [ "$memory_kib" -gt "$max_memory_kib" ]; then echo "node $node allocatable memory $memory is outside ${min_memory_kib}-${max_memory_kib}Ki" >&2 return 1 fi echo "$node allocatable: ${cpu}/${memory}" done }