Files
cozystack/hack/cilium-ipam-migration.bats
T
Myasnikov Daniil a2d0e4cb50 fix(cilium): pass the CiliumNode list to jq on stdin in the IPAM hook
The hook fed the whole `kubectl get ciliumnodes -o json` to jq as one
argument. Cilium copies every Node label onto its CiliumNode, 22 to 30
KiB per node on Talos or with NFD and GPU labels, so the list passes the
128 KiB Linux allows a single argument within six nodes. jq then fails
to start with "Argument list too long", the hook Job fails before it
moves any node, and the cilium HelmRelease loops on UpgradeFailed with
the HelmReleases behind it stuck in DependencyNotReady.

The hook renders on every upgrade of cilium in cluster-pool mode, so a
cluster that already finished the migration hits the same failure on its
next values change.

Assisted-by: LLM
Signed-off-by: Myasnikov Daniil <myasnikovdaniil2001@gmail.com>
2026-11-04 06:57:03 +00:00

591 lines
22 KiB
Bash

#!/usr/bin/env bats
# Behavioural unit test for packages/system/cilium/files/ipam-migration.sh, the
# script the cilium chart embeds into its post-upgrade hook Job when ipam.mode
# is cluster-pool.
#
# The script is EXECUTED against a fake kubectl that keeps CiliumNode, Node,
# agent pod and kube-ovn Subnet state in files and models the two controllers
# the script waits on. cilium-operator allocates a /29 from the pool to a
# CiliumNode only once its spec.ipam.podCIDRs is empty, and the agent DaemonSet
# replaces a deleted agent pod with a Ready one whose router and Ingress IPs
# come from the node's current allocation. Either can be told not to act, which
# is how the timeout paths are reached.
#
# hack/cozytest.sh knows @test and bash only, with no setup() or teardown() and
# no run/$status/$output, so each test calls init_state itself and every
# negative assertion is an explicit `if ...; then ...; false; fi`.
load test_helper
SCRIPT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)/packages/system/cilium/files/ipam-migration.sh"
# init_state builds an isolated workspace with the fake kubectl first on PATH,
# the environment the hook Job sets and a kube-ovn default subnet that already
# excludes the pool and has nothing allocated inside it.
init_state() {
STATE="$(mktemp -d)"
mkdir -p "$STATE/bin" "$STATE/ciliumnodes" "$STATE/nodes" "$STATE/pods"
export STATE PATH="$STATE/bin:$PATH" KLOG="$STATE/kubectl.log"
export NAMESPACE=cozy-cilium POOL=10.244.248.0/21 MASK=29
export ALLOCATION_TIMEOUT=3 AGENT_TIMEOUT=3 POLL_INTERVAL=1
export OPERATOR=allocate AGENT=restart
: > "$KLOG"
echo 0 > "$STATE/next-block"
echo '{"items": []}' > "$STATE/ips.json"
set_subnet '["10.244.0.1","10.244.248.0..10.244.255.255"]' "10.244.0.2-10.244.0.30,10.244.3.7"
cat > "$STATE/bin/kubectl" <<'KEOF'
#!/bin/bash
set -euo pipefail
echo "$*" >> "$KLOG"
args=()
field= patch=
while [ $# -gt 0 ]; do
case $1 in
-n|-l|-o) shift ;;
--field-selector) field=$2; shift ;;
-p) patch=$2; shift ;;
-*) ;;
*) args+=("$1") ;;
esac
shift
done
set -- "${args[@]}"
verb=$1 kind=$2
allocate() {
local file=$1 n
if [ "$OPERATOR" != allocate ] || jq -e '.spec.ipam.podCIDRs' "$file" >/dev/null; then
return 0
fi
n=$(cat "$STATE/next-block")
echo $((n + 1)) > "$STATE/next-block"
jq --arg c "10.244.248.$((n * 8))/29" '.spec.ipam.podCIDRs = [$c]' "$file" > "$file.tmp"
mv "$file.tmp" "$file"
}
case "$verb $kind" in
"get ciliumnodes")
for f in "$STATE"/ciliumnodes/*.json; do allocate "$f"; done
jq -s '{items: .}' "$STATE"/ciliumnodes/*.json
;;
"get ciliumnode")
name=$3
if [ "${FAIL_GET_CILIUMNODE:-}" = "$name" ]; then
echo "error: the server is currently unable to handle the request" >&2
exit 1
fi
allocate "$STATE/ciliumnodes/$name.json"
cat "$STATE/ciliumnodes/$name.json"
# RACE_CIDR models a writer that gets in between this read and the patch.
if [ -n "${RACE_CIDR:-}" ] && [ ! -f "$STATE/raced" ]; then
touch "$STATE/raced"
jq --arg c "$RACE_CIDR" '.spec.ipam.podCIDRs = [$c]' "$STATE/ciliumnodes/$name.json" > "$STATE/race.tmp"
mv "$STATE/race.tmp" "$STATE/ciliumnodes/$name.json"
fi
;;
"patch ciliumnode")
name=$3
file="$STATE/ciliumnodes/$name.json"
expected=$(jq -c '.[] | select(.op == "test") | .value' <<<"$patch")
actual=$(jq -c '.spec.ipam.podCIDRs' "$file")
if [ "$expected" != "$actual" ]; then
echo "The request is invalid: the server rejected our request due to an error in our request" >&2
exit 1
fi
jq 'del(.spec.ipam.podCIDRs)' "$file" > "$file.tmp"
mv "$file.tmp" "$file"
;;
"get customresourcedefinition")
if [ -f "$STATE/subnet.json" ]; then
echo "customresourcedefinition.apiextensions.k8s.io/subnets.kubeovn.io"
fi
;;
"get subnets.kubeovn.io")
if [ -z "${3:-}" ]; then
jq '{items: [.]}' "$STATE/subnet.json"
exit 0
fi
# A read by name models kube-ovn publishing the status after the patch.
# OVN_LAG reads go by before status.v4availableIPrange stops offering the
# end of the /16, and OVN_STUCK means it never does.
n=$(( $(cat "$STATE/status-reads" 2>/dev/null || echo 0) + 1 ))
echo "$n" > "$STATE/status-reads"
if [ -z "${OVN_STUCK:-}" ] && [ "$n" -gt "${OVN_LAG:-0}" ]; then
jq '.status.v4availableIPrange = "10.244.0.31-10.244.247.255"' "$STATE/subnet.json" > "$STATE/subnet.tmp"
mv "$STATE/subnet.tmp" "$STATE/subnet.json"
fi
cat "$STATE/subnet.json"
;;
"get ips.kubeovn.io")
cat "$STATE/ips.json"
;;
"patch subnets.kubeovn.io")
# RACE_SUBNET models kube-ovn rewriting excludeIps between the script's read
# and its patch, once or on every attempt.
if [ -n "${RACE_SUBNET:-}" ] && { [ "$RACE_SUBNET" = always ] || [ ! -f "$STATE/subnet-raced" ]; }; then
touch "$STATE/subnet-raced"
n=$(( $(cat "$STATE/subnet-races" 2>/dev/null || echo 0) + 1 ))
echo "$n" > "$STATE/subnet-races"
jq --arg e "10.244.1.$n" '.spec.excludeIps += [$e]' "$STATE/subnet.json" > "$STATE/subnet.tmp"
mv "$STATE/subnet.tmp" "$STATE/subnet.json"
fi
expected=$(jq -c '.[] | select(.op == "test") | .value' <<<"$patch")
actual=$(jq -c '.spec.excludeIps' "$STATE/subnet.json")
if [ "$expected" != "$actual" ]; then
echo "The request is invalid: the server rejected our request due to an error in our request" >&2
exit 1
fi
jq --argjson v "$(jq -c '.[] | select(.op == "replace") | .value' <<<"$patch")" '.spec.excludeIps = $v' \
"$STATE/subnet.json" > "$STATE/subnet.tmp"
mv "$STATE/subnet.tmp" "$STATE/subnet.json"
;;
"get node")
name=$3
if [ -f "$STATE/nodes/$name.json" ]; then
cat "$STATE/nodes/$name.json"
fi
;;
"get pods")
node=${field#spec.nodeName=}
jq -s --arg n "$node" '{items: [.[] | select(.spec.nodeName == $n)]}' "$STATE"/pods/*.json 2>/dev/null || echo '{"items": []}'
;;
"delete pod")
name=$3
node=$(jq -r '.spec.nodeName' "$STATE/pods/$name.json")
rm "$STATE/pods/$name.json"
if [ "$AGENT" = restart ]; then
jq -n --arg p "$name-r" --arg n "$node" \
'{metadata: {name: $p}, spec: {nodeName: $n}, status: {conditions: [{type: "Ready", status: "True"}]}}' \
> "$STATE/pods/$name-r.json"
file="$STATE/ciliumnodes/$node.json"
block=$(jq -r '.spec.ipam.podCIDRs[0] | split("/")[0]' "$file")
jq --arg r "${block%.*}.$(( ${block##*.} + 2 ))" --arg i "${block%.*}.$(( ${block##*.} + 3 ))" \
'.spec.addresses = [{type: "InternalIP", ip: "192.0.2.1"}, {type: "CiliumInternalIP", ip: $r}] | .spec.ingress.ipv4 = $i' \
"$file" > "$file.tmp"
mv "$file.tmp" "$file"
fi
;;
*)
echo "fake kubectl: unexpected call: $verb $kind" >&2
exit 1
;;
esac
KEOF
chmod +x "$STATE/bin/kubectl"
}
# set_subnet <excludeIps JSON> <status.v4usingIPrange> [status.v4availableIPrange]
# By default status.v4availableIPrange stops short of the pool once excludeIps
# lists it, the way kube-ovn reports it.
set_subnet() {
available=10.244.0.31-10.244.255.255
if printf '%s' "$1" | jq -e 'index("10.244.248.0..10.244.255.255")' >/dev/null; then
available=10.244.0.31-10.244.247.255
fi
available=${3:-$available}
jq -n --argjson e "$1" --arg u "$2" --arg a "$available" \
'{metadata: {name: "ovn-default"}, spec: {default: true, cidrBlock: "10.244.0.0/16", excludeIps: $e}, status: {v4usingIPrange: $u, v4availableIPrange: $a}}' \
> "$STATE/subnet.json"
}
# add_ip <IP object name> <subnet> <v4 address>
add_ip() {
jq --arg n "$1" --arg s "$2" --arg i "$3" '.items += [{metadata: {name: $n}, spec: {subnet: $s, v4IpAddress: $i}}]' \
"$STATE/ips.json" > "$STATE/ips.tmp"
mv "$STATE/ips.tmp" "$STATE/ips.json"
}
# add_node <name> <podCIDR or ""> <router IP> <ingress IP> [Ready status]
# Writes a CiliumNode, its Node and one Ready agent pod.
add_node() {
local name=$1 cidr=$2 router=$3 ingress=$4 ready=${5:-True}
jq -n --arg n "$name" --arg c "$cidr" --arg r "$router" --arg i "$ingress" '
{metadata: {name: $n},
spec: ({addresses: [{type: "InternalIP", ip: "192.0.2.1"}, {type: "CiliumInternalIP", ip: $r}],
ingress: {ipv4: $i}, ipam: {}}
| if $c == "" then . else .ipam.podCIDRs = [$c] end)}' > "$STATE/ciliumnodes/$name.json"
jq -n --arg n "$name" --arg s "$ready" \
'{metadata: {name: $n}, status: {conditions: [{type: "Ready", status: $s}]}}' > "$STATE/nodes/$name.json"
jq -n --arg p "cilium-$name" --arg n "$name" \
'{metadata: {name: $p}, spec: {nodeName: $n}, status: {conditions: [{type: "Ready", status: "True"}]}}' \
> "$STATE/pods/cilium-$name.json"
}
pod_cidrs_of() {
jq -c '.spec.ipam.podCIDRs' "$STATE/ciliumnodes/$1.json"
}
@test "stale nodes are moved onto the pool and their agents restarted one at a time" {
init_state
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
add_node n2 10.244.4.0/24 10.244.4.251 10.244.4.240
"$SCRIPT"
[ "$(pod_cidrs_of n1)" = '["10.244.248.0/29"]' ]
[ "$(pod_cidrs_of n2)" = '["10.244.248.8/29"]' ]
[ "$(jq -r '.spec.ingress.ipv4' "$STATE/ciliumnodes/n1.json")" = 10.244.248.3 ]
[ -f "$STATE/pods/cilium-n1-r.json" ]
[ -f "$STATE/pods/cilium-n2-r.json" ]
# n1 is cleared, allocated and restarted before n2 is touched at all.
first_n2=$(grep -n "ciliumnode n2" "$KLOG" | head -1 | cut -d: -f1)
n1_restart=$(grep -n "delete pod cilium-n1" "$KLOG" | cut -d: -f1)
[ "$n1_restart" -lt "$first_n2" ]
rm -rf "$STATE"
}
@test "an agent that came up before the operator allocated its CIDR is restarted" {
init_state
# An empty field the operator fills, and an agent still holding addresses
# from the old /24.
add_node n1 "" 10.244.2.223 10.244.2.135
"$SCRIPT"
[ "$(pod_cidrs_of n1)" = '["10.244.248.0/29"]' ]
grep -q "^-n cozy-cilium delete pod cilium-n1" "$KLOG"
rm -rf "$STATE"
}
@test "a node whose agent already took its addresses from the pool is left alone" {
init_state
add_node n1 10.244.248.24/29 10.244.248.26 10.244.248.27
"$SCRIPT"
if grep -q -e "^patch" -e "delete pod" "$KLOG"; then
echo "BUG: a node already on the pool, or the subnet that already excludes it, was touched"
false
fi
rm -rf "$STATE"
}
@test "an agent whose Ingress IP alone is still outside its allocation is restarted" {
init_state
add_node n1 10.244.248.24/29 10.244.248.26 10.244.2.135
"$SCRIPT"
grep -q "delete pod cilium-n1" "$KLOG"
rm -rf "$STATE"
}
@test "the pool is excluded from the kube-ovn default subnet before any node moves" {
init_state
set_subnet '["10.244.0.1"]' "10.244.0.2-10.244.0.30"
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
"$SCRIPT"
[ "$(jq -c '.spec.excludeIps' "$STATE/subnet.json")" = '["10.244.0.1","10.244.248.0..10.244.255.255","10.244.2.135"]' ]
exclude=$(grep -n "^patch subnets.kubeovn.io" "$KLOG" | cut -d: -f1)
clear=$(grep -n "^patch ciliumnode n1" "$KLOG" | cut -d: -f1)
[ "$exclude" -lt "$clear" ]
rm -rf "$STATE"
}
@test "the Ingress IPs the agents leave behind go into the same patch as the pool, the router IPs do not" {
init_state
set_subnet '["10.244.0.1"]' "10.244.0.2-10.244.0.30"
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
add_node n2 10.244.4.0/24 10.244.4.251 10.244.4.240
"$SCRIPT"
[ "$(jq -c '.spec.excludeIps' "$STATE/subnet.json")" = '["10.244.0.1","10.244.248.0..10.244.255.255","10.244.2.135","10.244.4.240"]' ]
[ "$(grep -c "^patch subnets.kubeovn.io" "$KLOG")" -eq 1 ]
rm -rf "$STATE"
}
@test "an Ingress IP that already lies in the pool is covered by the pool range alone" {
init_state
set_subnet '["10.244.0.1"]' "10.244.0.2-10.244.0.30"
add_node n1 10.244.248.24/29 10.244.248.26 10.244.248.27
"$SCRIPT"
[ "$(jq -c '.spec.excludeIps' "$STATE/subnet.json")" = '["10.244.0.1","10.244.248.0..10.244.255.255"]' ]
rm -rf "$STATE"
}
@test "an Ingress IP the subnet already excludes is not added again" {
init_state
set_subnet '["10.244.0.1","10.244.2.130..10.244.2.140"]' "10.244.0.2-10.244.0.30"
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
"$SCRIPT"
[ "$(jq -c '.spec.excludeIps' "$STATE/subnet.json")" = '["10.244.0.1","10.244.2.130..10.244.2.140","10.244.248.0..10.244.255.255"]' ]
rm -rf "$STATE"
}
@test "a subnet rewritten between the read and the patch is read again and patched on top of the new list" {
init_state
export RACE_SUBNET=once
set_subnet '["10.244.0.1"]' "10.244.0.2-10.244.0.30"
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
"$SCRIPT"
[ "$(jq -c '.spec.excludeIps' "$STATE/subnet.json")" = '["10.244.0.1","10.244.1.1","10.244.248.0..10.244.255.255","10.244.2.135"]' ]
[ "$(grep -c "^patch subnets.kubeovn.io" "$KLOG")" -eq 2 ]
[ "$(pod_cidrs_of n1)" = '["10.244.248.0/29"]' ]
rm -rf "$STATE"
}
@test "a subnet that keeps changing under the patch stops the run before any node moves" {
init_state
export RACE_SUBNET=always
set_subnet '["10.244.0.1"]' "10.244.0.2-10.244.0.30"
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
rc=0
"$SCRIPT" > "$STATE/out" 2>&1 || rc=$?
[ "$rc" -ne 0 ]
grep -q "kept changing under the patch" "$STATE/out"
if grep -q "^patch ciliumnode" "$KLOG"; then
echo "BUG: a node was moved although the pool was never excluded"
false
fi
rm -rf "$STATE"
}
@test "a pool excluded by an earlier attempt that still has pods on it stops the run" {
init_state
# kube-ovn leaves excluded addresses out of status.v4usingIPrange, so only
# the IP object shows the pod still sitting in the pool.
add_ip db-0.tenant-a ovn-default 10.244.250.7
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
rc=0
"$SCRIPT" > "$STATE/out" 2>&1 || rc=$?
[ "$rc" -ne 0 ]
grep -q "still has db-0.tenant-a=10.244.250.7 in use inside 10.244.248.0/21" "$STATE/out"
if grep -q "^patch ciliumnode" "$KLOG"; then
echo "BUG: a node was moved onto a pool a pod still holds an address in"
false
fi
rm -rf "$STATE"
}
@test "an IP object of another subnet with an address in the pool range does not stop the run" {
init_state
add_ip vm-0.tenant-b vpc-b-subnet 10.244.250.7
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
"$SCRIPT"
[ "$(pod_cidrs_of n1)" = '["10.244.248.0/29"]' ]
rm -rf "$STATE"
}
@test "the IP objects are read only after kube-ovn stops offering the pool" {
init_state
export OVN_LAG=2
set_subnet '["10.244.0.1"]' "10.244.0.2-10.244.0.30"
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
"$SCRIPT"
[ "$(cat "$STATE/status-reads")" -eq 3 ]
reads=$(grep -n "^get subnets.kubeovn.io ovn-default" "$KLOG" | tail -1 | cut -d: -f1)
ips=$(grep -n "^get ips.kubeovn.io" "$KLOG" | cut -d: -f1)
[ "$reads" -lt "$ips" ]
[ "$(pod_cidrs_of n1)" = '["10.244.248.0/29"]' ]
rm -rf "$STATE"
}
@test "a subnet whose status never drops the pool stops the run before any node moves" {
init_state
export OVN_STUCK=1
set_subnet '["10.244.0.1"]' "10.244.0.2-10.244.0.30"
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
rc=0
"$SCRIPT" > "$STATE/out" 2>&1 || rc=$?
[ "$rc" -ne 0 ]
grep -q "kube-ovn has not applied the exclusion" "$STATE/out"
if grep -q "^patch ciliumnode" "$KLOG"; then
echo "BUG: a node was moved before kube-ovn applied the exclusion"
false
fi
rm -rf "$STATE"
}
@test "a wider exclusion that already covers the pool is left as it is" {
init_state
set_subnet '["10.244.0.1","10.244.240.0..10.244.255.255"]' "10.244.0.2-10.244.0.30" "10.244.0.31-10.244.239.255"
add_node n1 10.244.248.24/29 10.244.248.26 10.244.248.27
"$SCRIPT"
if grep -q "^patch subnets.kubeovn.io" "$KLOG"; then
echo "BUG: an exclusion that already covers the pool was patched again"
false
fi
rm -rf "$STATE"
}
@test "addresses kube-ovn already handed out from the pool stop the run before any node moves" {
init_state
set_subnet '["10.244.0.1"]' "10.244.0.2-10.244.0.30,10.244.249.4-10.244.249.6"
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
rc=0
"$SCRIPT" > "$STATE/out" 2>&1 || rc=$?
[ "$rc" -ne 0 ]
grep -q "has already handed out 10.244.249.4-10.244.249.6 from 10.244.248.0/21" "$STATE/out"
# The exclusion still lands, so kube-ovn stops handing out more from the pool.
[ "$(jq -c '.spec.excludeIps' "$STATE/subnet.json")" = '["10.244.0.1","10.244.248.0..10.244.255.255","10.244.2.135"]' ]
if grep -q "^patch ciliumnode" "$KLOG"; then
echo "BUG: a node was moved onto a pool kube-ovn still hands addresses from"
false
fi
rm -rf "$STATE"
}
@test "a pool outside the default subnet stops the run before anything changes" {
init_state
export POOL=100.65.0.0/16
set_subnet '["10.244.0.1"]' "10.244.0.2-10.244.0.30"
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
rc=0
"$SCRIPT" > "$STATE/out" 2>&1 || rc=$?
[ "$rc" -ne 0 ]
grep -q "100.65.0.0/16 lies outside subnet ovn-default (10.244.0.0/16)" "$STATE/out"
if grep -q -e "^patch subnets.kubeovn.io" -e "^patch ciliumnode" "$KLOG"; then
echo "BUG: something was changed for a pool outside kube-ovn's subnet"
false
fi
rm -rf "$STATE"
}
@test "more CiliumNodes than the pool has blocks stops the run before anything changes" {
init_state
export POOL=10.244.248.0/29
set_subnet '["10.244.0.1"]' "10.244.0.2-10.244.0.30"
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
add_node n2 10.244.4.0/24 10.244.4.251 10.244.4.240
rc=0
"$SCRIPT" > "$STATE/out" 2>&1 || rc=$?
[ "$rc" -ne 0 ]
grep -q "there are 2 CiliumNodes but 10.244.248.0/29 holds only 1 /29 blocks" "$STATE/out"
if grep -q -e "^patch" "$KLOG"; then
echo "BUG: something was changed although the pool cannot hold every node"
false
fi
rm -rf "$STATE"
}
@test "a pool with exactly one block per node is enough" {
init_state
export POOL=10.244.248.0/28
set_subnet '["10.244.0.1"]' "10.244.0.2-10.244.0.30"
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
add_node n2 10.244.4.0/24 10.244.4.251 10.244.4.240
"$SCRIPT"
[ "$(pod_cidrs_of n1)" = '["10.244.248.0/29"]' ]
[ "$(pod_cidrs_of n2)" = '["10.244.248.8/29"]' ]
rm -rf "$STATE"
}
@test "a usage range that crosses the start of the pool stops the run" {
init_state
# kube-ovn's ascending pointer reaches the block from below, so the first
# range to touch it starts outside.
set_subnet '["10.244.0.1"]' "10.244.0.2-10.244.0.30,10.244.247.250-10.244.248.3"
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
rc=0
"$SCRIPT" > "$STATE/out" 2>&1 || rc=$?
[ "$rc" -ne 0 ]
grep -q "has already handed out 10.244.247.250-10.244.248.3 from 10.244.248.0/21" "$STATE/out"
if grep -q "^patch ciliumnode" "$KLOG"; then
echo "BUG: a node was moved onto a pool kube-ovn still hands addresses from"
false
fi
rm -rf "$STATE"
}
@test "a pod on a node's current Ingress IP is named in the log and does not stop the run" {
init_state
add_ip web-0.tenant-c ovn-default 10.244.2.135
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
"$SCRIPT" > "$STATE/out" 2>&1
grep -q "web-0.tenant-c holds 10.244.2.135, the Ingress IP of n1 before the move" "$STATE/out"
[ "$(pod_cidrs_of n1)" = '["10.244.248.0/29"]' ]
rm -rf "$STATE"
}
@test "a CiliumNode list larger than one argument may hold still moves every node and names the pod on an old Ingress IP" {
init_state
add_ip web-0.tenant-c ovn-default 10.244.1.135
add_ip web-1.tenant-c ovn-default 10.244.4.135
add_ip db-0.tenant-c ovn-default 10.244.9.9
# Cilium copies every Node label onto the CiliumNode, which takes six real
# ones past the 128 KiB Linux allows a single argument. One annotation stands
# in for the labels and keeps the list over that limit even printed compact.
for i in 1 2 3 4 5 6; do
add_node "n$i" "10.244.$i.0/24" "10.244.$i.223" "10.244.$i.135"
jq '.metadata.annotations.padding = ("x" * 24576)' "$STATE/ciliumnodes/n$i.json" > "$STATE/node.tmp"
mv "$STATE/node.tmp" "$STATE/ciliumnodes/n$i.json"
done
[ "$(kubectl get ciliumnodes -o json | jq -c . | wc -c)" -gt 131072 ]
"$SCRIPT" > "$STATE/out" 2>&1
# Each pod is named with the node whose Ingress IP it holds, and a pod on an
# address that is no node's Ingress IP is not named at all.
grep -q "web-0.tenant-c holds 10.244.1.135, the Ingress IP of n1 before the move" "$STATE/out"
grep -q "web-1.tenant-c holds 10.244.4.135, the Ingress IP of n4 before the move" "$STATE/out"
if grep -q "db-0.tenant-c" "$STATE/out"; then
cat "$STATE/out"
false
fi
[ "$(pod_cidrs_of n6)" = '["10.244.248.40/29"]' ]
rm -rf "$STATE"
}
@test "a cluster without kube-ovn skips the exclusion and still moves its nodes" {
init_state
rm "$STATE/subnet.json"
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
"$SCRIPT"
[ "$(pod_cidrs_of n1)" = '["10.244.248.0/29"]' ]
if grep -q -e "^get subnets.kubeovn.io" -e "^patch subnets.kubeovn.io" "$KLOG"; then
echo "BUG: subnets were queried on a cluster that has no kube-ovn"
false
fi
rm -rf "$STATE"
}
@test "the run fails when the operator never allocates, and still handles the other nodes" {
init_state
export OPERATOR=none
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
add_node n2 10.244.248.24/29 10.244.248.26 10.244.4.240
rc=0
"$SCRIPT" > "$STATE/out" 2>&1 || rc=$?
[ "$rc" -ne 0 ]
grep -q "n1: no CIDR from 10.244.248.0/21" "$STATE/out"
grep -q "delete pod cilium-n2" "$KLOG"
rm -rf "$STATE"
}
@test "the run fails when a restarted agent does not come back" {
init_state
export AGENT=stuck
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
rc=0
"$SCRIPT" > "$STATE/out" 2>&1 || rc=$?
[ "$rc" -ne 0 ]
grep -q "n1: agent not ready on 10.244.248.0/29" "$STATE/out"
rm -rf "$STATE"
}
@test "the agent of a node that is not Ready is not restarted" {
init_state
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135 False
"$SCRIPT"
[ "$(pod_cidrs_of n1)" = '["10.244.248.0/29"]' ]
if grep -q "delete pod" "$KLOG"; then
echo "BUG: the agent of a NotReady node was deleted"
false
fi
rm -rf "$STATE"
}
@test "a failed CiliumNode read stops the run before anything is cleared" {
init_state
export FAIL_GET_CILIUMNODE=n1
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
rc=0
"$SCRIPT" > "$STATE/out" 2>&1 || rc=$?
[ "$rc" -ne 0 ]
if grep -q "^patch ciliumnode" "$KLOG"; then
echo "BUG: a node was patched after its read failed"
false
fi
rm -rf "$STATE"
}
@test "a CIDR written between the read and the clear makes the patch fail instead of being removed" {
init_state
export OPERATOR=none RACE_CIDR=10.244.9.0/24
add_node n1 10.244.2.0/24 10.244.2.223 10.244.2.135
rc=0
"$SCRIPT" > "$STATE/out" 2>&1 || rc=$?
[ "$rc" -ne 0 ]
[ "$(pod_cidrs_of n1)" = '["10.244.9.0/24"]' ]
rm -rf "$STATE"
}