From 593c10e0e4dd94db3e501cce6bf28fd652db732d Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Tue, 3 Nov 2026 15:32:59 +0300 Subject: [PATCH] test(e2e): add a GPUClass suite on a fake extended resource CI has no GPU device plugin. Check class budgets at application writes, class expansion at VMI admission and usage in the launcher Workload without claiming a booted guest or a synced VMI. Populate the blank disk before choosing a compatible GPU node: a local WFFC volume bound elsewhere leaves the launcher unschedulable. Validated offline with 133 Bats tests and 137 assertion/script/selector mutations, including refusal paths. Live controller and scheduler checks remain unproven. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- hack/e2e-chainsaw/README.md | 4 +- hack/e2e-chainsaw/gpuclass/chainsaw-test.yaml | 444 ++++++++++++++++++ hack/e2e-chainsaw/gpuclass/disk-node.jq | 40 ++ hack/e2e-chainsaw/gpuclass/wait-release.sh | 39 ++ hack/gpuclass-e2e_test.bats | 170 +++++++ hack/select-e2e.sh | 16 +- hack/select-e2e_test.bats | 44 +- hack/select-install.sh | 3 + hack/select-install_test.bats | 8 + 9 files changed, 748 insertions(+), 20 deletions(-) create mode 100644 hack/e2e-chainsaw/gpuclass/chainsaw-test.yaml create mode 100644 hack/e2e-chainsaw/gpuclass/disk-node.jq create mode 100644 hack/e2e-chainsaw/gpuclass/wait-release.sh create mode 100644 hack/gpuclass-e2e_test.bats diff --git a/hack/e2e-chainsaw/README.md b/hack/e2e-chainsaw/README.md index 98400bc55..39714e890 100644 --- a/hack/e2e-chainsaw/README.md +++ b/hack/e2e-chainsaw/README.md @@ -26,7 +26,7 @@ A suite is a directory directly under `hack/e2e-chainsaw/` in or below which Cha A parked `chainsaw-test.yaml.disabled` is none of these: the directory is then not a suite, is selected by nothing and cannot be allowlisted. -Suites: `postgres`, `bucket`, `mariadb`, `mongodb`, `redis`, `valkey`, `qdrant`, `clickhouse`, `kafka`, `kafka-metadata`, `rabbitmq`, `etcd`, `openbao`, `harbor`, `foundationdb`, `external-dns`, `kuberture`, `vminstance`, `gateway`, `opensearch`, `monitoring`, `kubernetes-latest`, `kubernetes-previous`, `securitygroup`, `seaweedfs`, `computeplane`. `kubernetes-proxmox` is parked as `chainsaw-test.yaml.disabled` and runs only in `.github/workflows/e2e-proxmox.yaml`, which needs a Proxmox hypervisor. +Suites: `postgres`, `bucket`, `mariadb`, `mongodb`, `redis`, `valkey`, `qdrant`, `clickhouse`, `kafka`, `kafka-metadata`, `rabbitmq`, `etcd`, `openbao`, `harbor`, `foundationdb`, `external-dns`, `kuberture`, `vminstance`, `gateway`, `opensearch`, `monitoring`, `kubernetes-latest`, `kubernetes-previous`, `securitygroup`, `seaweedfs`, `computeplane`, `gpuclass`. `kubernetes-proxmox` is parked as `chainsaw-test.yaml.disabled` and runs only in `.github/workflows/e2e-proxmox.yaml`, which needs a Proxmox hypervisor. ## What Chainsaw buys over the BATS suite @@ -82,3 +82,5 @@ chainsaw test postgres/ ## CI integration The `e2e` job in `.github/workflows/pull-requests.yaml` runs `make -C packages/core/testing test-chainsaw` (which execs `hack/e2e-chainsaw-run.sh` inside the e2e sandbox) and uploads `chainsaw-report.xml`. The runner calls `chainsaw test` once per suite directory, fails a suite whose pods outlive its cleanup by more than the default `timeouts.cleanup` (`LEAKED-PODS suite=` in the log; see `docs/agents/e2e-testing.md` ยง8 and `_lib/leak-allowlist.txt` for known, filed leaks), and merges the per-suite reports into that one file. A local `chainsaw test` as shown above runs the same tests without the leak check. The sandbox image (`packages/core/testing/images/e2e-sandbox/Dockerfile`) ships the `chainsaw` binary alongside `kubectl`/`helm`/`mc`/`jq`/`nc`, built from source with a patch that keeps an assert polling through transient API errors such as an etcd leader change; a released `chainsaw` run locally fails the assert on the first such error instead. + +The `gpuclass` suite advertises a fake extended resource on a node compatible with its already populated disk. It checks class status, declared VM and worker-pool budgets, vGPU entitlement refusal, VMI GPU entries, launcher tolerations and scheduling, and GPU units in the launcher pod's existing `Workload`. CI has no GPU device plugin: the suite does not prove device allocation, VMI sync, guest boot or guest drivers. diff --git a/hack/e2e-chainsaw/gpuclass/chainsaw-test.yaml b/hack/e2e-chainsaw/gpuclass/chainsaw-test.yaml new file mode 100644 index 000000000..d65f67dc5 --- /dev/null +++ b/hack/e2e-chainsaw/gpuclass/chainsaw-test.yaml @@ -0,0 +1,444 @@ +# CI has no GPU device plugin. Fake capacity proves admission, scheduling and +# launcher-pod accounting, not VMI sync, a booted guest or device allocation. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: gpuclass +spec: + catch: + - events: {} + - events: + namespace: tenant-test-gpu + - describe: + apiVersion: cozystack.io/v1alpha1 + kind: GPUClass + name: e2e-fake + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + namespace: tenant-test-gpu + - describe: + apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + namespace: tenant-test-gpu + - describe: + apiVersion: v1 + kind: PersistentVolumeClaim + namespace: tenant-test-gpu + - describe: + apiVersion: kubevirt.io/v1 + kind: VirtualMachineInstance + namespace: tenant-test-gpu + - describe: + apiVersion: v1 + kind: Pod + namespace: tenant-test-gpu + - podLogs: + namespace: cozy-system + selector: app=cozystack-controller + tail: 200 + steps: + - name: gpu-paths-are-on + try: + - assert: + timeout: 2m + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: cozystack-controller + namespace: cozy-system + spec: + template: + spec: + (containers[?name == 'cozystack-controller']): + - (contains(args, '--enable-gpu')): true + - assert: + timeout: 2m + resource: + apiVersion: kubevirt.io/v1 + kind: KubeVirt + metadata: + name: kubevirt + namespace: cozy-kubevirt + spec: + configuration: + (permittedHostDevices.pciHostDevices[?resourceName == 'nvidia.com/E2E_FAKE_GPU']): + - externalResourceProvider: true + + - name: tenant-has-class-and-total-budgets + try: + - apply: + resource: + apiVersion: apps.cozystack.io/v1alpha1 + kind: Tenant + metadata: + name: gpu + spec: + resourceQuotas: + cpu: "4" + memory: 8Gi + storage: 10Gi + e2e-fake.gpuclass.cozystack.io/devices: "2" + gpuclass.cozystack.io/devices: "2" + - assert: + timeout: 2m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: tenant-gpu + # Above tenant-rd's 15m install timeout, with room for Flux to start it. + - script: + timeout: 18m + content: | + set -eu + timeout 1020 bash ./wait-release.sh tenant-test tenant-gpu + - assert: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: tenant-test-gpu + labels: + namespace.cozystack.io/vgpu: "false" + + - name: populate-disk-before-selecting-the-gpu-node + try: + - apply: + resource: + apiVersion: apps.cozystack.io/v1alpha1 + kind: VMDisk + metadata: + name: gpu + namespace: tenant-test-gpu + spec: + source: {} + storage: 1Gi + storageClass: ($values.storageClass) + bindImmediately: true + - assert: + timeout: 2m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: vm-disk-gpu + namespace: tenant-test-gpu + # vm-disk-rd uses the API server's 10m install timeout. + - script: + timeout: 12m + content: | + set -eu + timeout 660 bash ./wait-release.sh tenant-test-gpu vm-disk-gpu + - assert: + timeout: 2m + resource: + apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: vm-disk-gpu + namespace: tenant-test-gpu + annotations: + cdi.kubevirt.io/storage.bind.immediate.requested: "true" + status: + phase: Succeeded + - assert: + resource: + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: vm-disk-gpu + namespace: tenant-test-gpu + spec: + (volumeName != null): true + status: + phase: Bound + + - name: advertise-capacity-only-where-the-disk-can-run + try: + - script: + content: | + set -eu + context=$(kubectl --context="" config current-context) + pv=$(kubectl --context "$context" --namespace tenant-test-gpu get pvc vm-disk-gpu --output=jsonpath='{.spec.volumeName}') + [ -n "$pv" ] + volumes=$(kubectl --context "$context" get pv "$pv" --output=json | jq '[.]') + node=$(kubectl --context "$context" get nodes --output=json | + jq --exit-status --raw-output --argjson volumes "$volumes" --from-file ./disk-node.jq) + kubectl --context "$context" get node "$node" --output=json | + jq --exit-status '.status.capacity["nvidia.com/E2E_FAKE_GPU"] == null and .metadata.labels["e2e.cozystack.io/fake-gpu"] == null' + kubectl --context "$context" label node "$node" e2e.cozystack.io/fake-gpu=true + kubectl --context "$context" patch node "$node" --subresource=status --type=json \ + --patch '[{"op":"add","path":"/status/capacity/nvidia.com~1E2E_FAKE_GPU","value":"2"}]' + - assert: + timeout: 2m + resource: + apiVersion: v1 + kind: Node + metadata: + labels: + e2e.cozystack.io/fake-gpu: "true" + status: + allocatable: + nvidia.com/E2E_FAKE_GPU: "2" + # Nodes are not owned by Chainsaw; the label survives a failed status patch. + cleanup: + - script: + content: | + set -eu + context=$(kubectl --context="" config current-context) + nodes=$(kubectl --context "$context" get nodes --selector=e2e.cozystack.io/fake-gpu=true --output=jsonpath='{.items[*].metadata.name}') + for node in $nodes; do + if [ -n "$(kubectl --context "$context" get node "$node" --output=jsonpath='{.status.capacity.nvidia\.com/E2E_FAKE_GPU}')" ]; then + kubectl --context "$context" patch node "$node" --subresource=status --type=json \ + --patch '[{"op":"remove","path":"/status/capacity/nvidia.com~1E2E_FAKE_GPU"}]' + fi + kubectl --context "$context" label node "$node" e2e.cozystack.io/fake-gpu- + done + + - name: class-status-reflects-fake-capacity + try: + - apply: + resource: + apiVersion: cozystack.io/v1alpha1 + kind: GPUClass + metadata: + name: e2e-fake + spec: + displayName: E2E fake GPU + vendor: nvidia + model: FAKE + mode: passthrough + maxCount: 2 + tolerations: + - key: e2e.cozystack.io/gpu + operator: Equal + value: fake + effect: NoSchedule + source: + devicePlugin: + resourceName: nvidia.com/E2E_FAKE_GPU + - apply: + resource: + apiVersion: cozystack.io/v1alpha1 + kind: GPUClass + metadata: + name: e2e-vgpu + spec: + vendor: nvidia + model: FAKE + mode: vgpu + profile: FAKE-1C + source: + devicePlugin: + resourceName: nvidia.com/E2E_FAKE_VGPU + - assert: + timeout: 2m + resource: + apiVersion: cozystack.io/v1alpha1 + kind: GPUClass + metadata: + name: e2e-fake + status: + free: 2 + maxFreePerNode: 2 + (conditions[?type == 'ResourceNameNotPermitted']): + - status: "False" + (conditions[?type == 'ResourceNameConflict']): + - status: "False" + + - name: stopped-vm-reserves-the-class-count + try: + - apply: + resource: + apiVersion: apps.cozystack.io/v1alpha1 + kind: VMInstance + metadata: + name: gpu + namespace: tenant-test-gpu + spec: + runStrategy: Halted + instanceType: u1.medium + disks: + - name: gpu + gpus: + - className: e2e-fake + count: 2 + - assert: + timeout: 2m + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: vm-instance-gpu + namespace: tenant-test-gpu + spec: + values: + runStrategy: Halted + gpus: + - className: e2e-fake + count: 2 + - script: + timeout: 12m + content: | + set -eu + timeout 660 bash ./wait-release.sh tenant-test-gpu vm-instance-gpu + - script: + content: | + set -eu + context=$(kubectl --context="" config current-context) + for scenario in class total workers; do + case "$scenario" in + class) kind=VMInstance; class=e2e-fake; extra='runStrategy: Halted'; need=1; key=e2e-fake.gpuclass.cozystack.io/devices ;; + total) kind=VMInstance; class=e2e-vgpu; extra='runStrategy: Halted'; need=1; key=gpuclass.cozystack.io/devices ;; + workers) kind=KubernetesNodes; class=e2e-fake; extra='cluster: unused + maxReplicas: 3'; need=3; key=e2e-fake.gpuclass.cozystack.io/devices ;; + esac + if out=$(kubectl --context "$context" --namespace tenant-test-gpu create --dry-run=server --filename - 2>&1 <&2 + false + fi + printf '%s\n' "$out" + printf '%s\n' "$out" | grep --fixed-strings "GPU budget \"$key\" of tenant pool tenant-test-gpu is 2 and 2 is already reserved; this application needs $need" + done + + - name: unentitled-vgpu-is-refused-at-vmi-admission + try: + - script: + content: | + set -eu + context=$(kubectl --context="" config current-context) + if out=$(kubectl --context "$context" --namespace tenant-test-gpu create --dry-run=server --filename - 2>&1 <<'EOF' + apiVersion: kubevirt.io/v1 + kind: VirtualMachineInstance + metadata: + name: unentitled + labels: + cozystack.io/gpu-reference: "true" + annotations: + cozystack.io/gpus: '[{"className":"e2e-vgpu","count":1}]' + spec: + domain: + resources: + requests: + memory: 1Gi + devices: {} + EOF + ); then + echo "FAIL: unentitled vGPU VMI was accepted: $out" >&2 + false + fi + printf '%s\n' "$out" + printf '%s\n' "$out" | grep --fixed-strings 'GPUClass e2e-vgpu is vGPU and the tenant is not entitled to vGPU' + + - name: class-entry-expands-to-kubevirt-gpus + try: + - patch: + resource: + apiVersion: apps.cozystack.io/v1alpha1 + kind: VMInstance + metadata: + name: gpu + namespace: tenant-test-gpu + spec: + runStrategy: Always + - assert: + timeout: 5m + resource: + apiVersion: kubevirt.io/v1 + kind: VirtualMachineInstance + metadata: + name: vm-instance-gpu + namespace: tenant-test-gpu + labels: + cozystack.io/gpu-reference: "true" + spec: + domain: + devices: + gpus: + - name: gpu-0-0 + deviceName: nvidia.com/E2E_FAKE_GPU + - name: gpu-0-1 + deviceName: nvidia.com/E2E_FAKE_GPU + (length(gpus)): 2 + + - name: launcher-schedules-with-class-tolerations + try: + - assert: + timeout: 5m + resource: + apiVersion: v1 + kind: Pod + metadata: + namespace: tenant-test-gpu + labels: + kubevirt.io: virt-launcher + vm.kubevirt.io/name: vm-instance-gpu + cozystack.io/gpu-reference: "true" + (annotations."kubevirt.io/ephemeral-provisioning" == null): true + spec: + (nodeName != null): true + tolerations: + - key: e2e.cozystack.io/gpu + operator: Equal + value: fake + effect: NoSchedule + (containers[?name == 'compute']): + - resources: + requests: + nvidia.com/E2E_FAKE_GPU: "2" + - script: + content: | + set -eu + context=$(kubectl --context="" config current-context) + node=$(kubectl --context "$context" get nodes --selector=e2e.cozystack.io/fake-gpu=true --output=jsonpath='{.items[*].metadata.name}') + kubectl --context "$context" --namespace tenant-test-gpu get pods --selector=kubevirt.io=virt-launcher,vm.kubevirt.io/name=vm-instance-gpu --output=json | + jq --exit-status --arg node "$node" ' + [.items[] | select(.metadata.annotations["kubevirt.io/ephemeral-provisioning"] == null)] | + length == 1 and .[0].spec.nodeName == $node' + + - name: launcher-workload-reports-the-gpu-units + try: + - assert: + timeout: 2m + resource: + apiVersion: cozystack.io/v1alpha1 + kind: Workload + metadata: + namespace: tenant-test-gpu + labels: + workloads.cozystack.io/monitor: vm-instance-gpu + (ownerReferences[?kind == 'Pod']): + - apiVersion: v1 + status: + kind: virtual-machine + resources: + nvidia.com/E2E_FAKE_GPU: "2" + - script: + content: | + set -eu + context=$(kubectl --context="" config current-context) + pod=$(kubectl --context "$context" --namespace tenant-test-gpu get pods --selector=kubevirt.io=virt-launcher,vm.kubevirt.io/name=vm-instance-gpu --output=json | + jq --exit-status --raw-output '[.items[] | select(.metadata.annotations["kubevirt.io/ephemeral-provisioning"] == null)] | if length == 1 then .[0].metadata.uid else error("expected one launcher") end') + kubectl --context "$context" --namespace tenant-test-gpu get workloads --output=json | + jq --exit-status --arg uid "$pod" ' + [.items[] | select(.status.resources["nvidia.com/E2E_FAKE_GPU"] != null)] | + length == 1 and .[0].status.resources["nvidia.com/E2E_FAKE_GPU"] == "2" and + any(.[0].metadata.ownerReferences[]?; .kind == "Pod" and .uid == $uid)' + - script: + timeout: 12m + content: | + set -eu + timeout 660 bash ./wait-release.sh tenant-test-gpu vm-instance-gpu diff --git a/hack/e2e-chainsaw/gpuclass/disk-node.jq b/hack/e2e-chainsaw/gpuclass/disk-node.jq new file mode 100644 index 000000000..7d16a6be4 --- /dev/null +++ b/hack/e2e-chainsaw/gpuclass/disk-node.jq @@ -0,0 +1,40 @@ +def int64: + if type != "string" or (test("^[+-]?[0-9]+$") | not) then null + else + startswith("-") as $negative | + sub("^[+-]"; "") | sub("^0+"; "") | + (if . == "" then "0" else . end) as $digits | + (if $negative then "9223372036854775808" else "9223372036854775807" end) as $limit | + if [($digits | length), $digits] > [19, $limit] then null + else {negative: ($negative and $digits != "0"), digits: $digits} end + end; +def compareIntegers($left; $right): + ($left | int64) as $a | ($right | int64) as $b | + if $a == null or $b == null then null + elif $a.negative != $b.negative then (if $a.negative then -1 else 1 end) + elif $a.digits == $b.digits then 0 + else + (if [($a.digits | length), $a.digits] > [($b.digits | length), $b.digits] then 1 else -1 end) * + (if $a.negative then -1 else 1 end) + end; +def requirement($values): + . as $r | $values[$r.key] as $v | + if .operator == "In" then $v != null and (.values | index($v)) != null + elif .operator == "NotIn" then (.values | index($v)) == null + elif .operator == "Exists" then $v != null + elif .operator == "DoesNotExist" then $v == null + elif .operator == "Gt" then compareIntegers($v; .values[0]) as $n | $n != null and $n > 0 + elif .operator == "Lt" then compareIntegers($v; .values[0]) as $n | $n != null and $n < 0 + else error("unsupported PV node affinity operator: " + .operator) + end; +def permits($node): + .spec.nodeAffinity.required as $required | + $required == null or any($required.nodeSelectorTerms[]?; + ((.matchExpressions // []) + (.matchFields // []) | length) > 0 and + all(.matchExpressions[]?; requirement($node.metadata.labels // {})) and + all(.matchFields[]?; requirement({"metadata.name": $node.metadata.name}))); +[.items[] | + select(.spec.unschedulable != true) | + select(any(.status.conditions[]?; .type == "Ready" and .status == "True")) | + . as $node | select(all($volumes[]; permits($node))) | + .metadata.name] | sort | first // error("no ready node is compatible with every GPU VM disk") diff --git a/hack/e2e-chainsaw/gpuclass/wait-release.sh b/hack/e2e-chainsaw/gpuclass/wait-release.sh new file mode 100644 index 000000000..d1bad529d --- /dev/null +++ b/hack/e2e-chainsaw/gpuclass/wait-release.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -euo pipefail + +# shellcheck source=hack/e2e-chainsaw/_lib/remediation-guard.sh +source "$(dirname -- "${BASH_SOURCE[0]}")/../_lib/remediation-guard.sh" + +context=${COZY_E2E_CONTEXT:-$(kubectl --context="" config current-context)} +exec 3< <(exec kubectl --context "$context" --namespace "$1" get helmreleases \ + --field-selector "metadata.name=$2" --watch --output='jsonpath={@}{"\n"}') +watcher=$! +trap 'kill "$watcher" 2>/dev/null || :; wait "$watcher" 2>/dev/null || :' EXIT + +while IFS= read -r event <&3; do + statuses=$(printf '%s\n' "$event" | jq --raw-output '(.items // [.])[] | .status.history[]?.status // empty') + if helmrelease_has_remediation_cycle "$statuses"; then + printf 'HelmRelease %s/%s has failed history: %s\n' "$1" "$2" "$statuses" >&2 + exit 1 + fi + verdict=$(printf '%s\n' "$event" | jq --raw-output ' + (.items // [.])[] | + if .spec.install.strategy.name != "RetryOnFailure" then + error("Application HelmRelease must use RetryOnFailure") + elif any(.status.conditions[]?; (.type == "Stalled" and .status == "True") or + (.type == "Ready" and .status == "False" and (.reason | endswith("Failed")))) then + error("HelmRelease failed: " + (.status | tojson)) + elif any(.status.conditions[]?; .type == "Ready" and .status == "True") then + if (.status.history | type) != "array" or (.status.history | length) == 0 or + any(.status.history[]; (.status | type) != "string" or .status == "") then + error("Ready HelmRelease has no readable action history") + elif .status.observedGeneration == .metadata.generation then "ready" + else empty end + else empty end') + if [ "$verdict" = ready ]; then + exit 0 + fi +done + +echo "HelmRelease $1/$2 watch ended before observed Ready" >&2 +exit 1 diff --git a/hack/gpuclass-e2e_test.bats b/hack/gpuclass-e2e_test.bats new file mode 100644 index 000000000..8bee822d6 --- /dev/null +++ b/hack/gpuclass-e2e_test.bats @@ -0,0 +1,170 @@ +#!/usr/bin/env bats + +load test_helper + +@test "GPU release watch requires readable history and the application retry strategy" { + tmp=$(mktemp --directory) + printf '%s\n' '#!/bin/bash' 'if [ "$GPU_HR_ERROR" = true ]; then exit 1; fi' 'cat "$GPU_HR_EVENT"' > "$tmp/kubectl" + chmod +x "$tmp/kubectl" + good='{"metadata":{"generation":1},"spec":{"install":{"strategy":{"name":"RetryOnFailure"}}},"status":{"observedGeneration":1,"conditions":[{"type":"Ready","status":"True"}],"history":[{"status":"deployed"}]}}' + for state in ready superseded pending initial-list stalled failed stale history-failed history-uninstalled missing-history empty-history object-history missing-status null-status empty-status missing-strategy wrong-strategy read-error malformed; do + event="$good" + read_error=false + case "$state" in + superseded) event=$(printf '%s\n' "$good" | jq --compact-output '.status.history += [{status: "superseded"}]') ;; + pending) event=$(printf '%s\n' "$good" | jq --compact-output '.status.conditions = [{type: "Ready", status: "False", reason: "Progressing"}] | .status.history = []') ;; + initial-list) event='{"items":[]}' ;; + stale) event=$(printf '%s\n' "$good" | jq --compact-output '.status.observedGeneration = 0') ;; + history-*) event=$(printf '%s\n' "$good" | jq --compact-output --arg state "${state#history-}" '.status.history = [{status: $state}]') ;; + stalled) event=$(printf '%s\n' "$good" | jq --compact-output '.status.conditions = [{type: "Stalled", status: "True", message: "broken chart"}]') ;; + failed) event=$(printf '%s\n' "$good" | jq --compact-output '.status.conditions = [{type: "Ready", status: "False", reason: "InstallFailed", message: "broken chart"}]') ;; + missing-history) event=$(printf '%s\n' "$good" | jq --compact-output 'del(.status.history)') ;; + empty-history) event=$(printf '%s\n' "$good" | jq --compact-output '.status.history = []') ;; + object-history) event=$(printf '%s\n' "$good" | jq --compact-output '.status.history = {snapshot: {status: "deployed"}}') ;; + missing-status) event=$(printf '%s\n' "$good" | jq --compact-output '.status.history = [{}]') ;; + null-status) event=$(printf '%s\n' "$good" | jq --compact-output '.status.history = [{status: null}]') ;; + empty-status) event=$(printf '%s\n' "$good" | jq --compact-output '.status.history = [{status: ""}]') ;; + missing-strategy) event=$(printf '%s\n' "$good" | jq --compact-output 'del(.spec.install.strategy)') ;; + wrong-strategy) event=$(printf '%s\n' "$good" | jq --compact-output '.spec.install.strategy.name = "RemediateOnFailure"') ;; + read-error) read_error=true ;; + malformed) event='{' ;; + esac + printf '%s\n' "$event" > "$tmp/event" + case "$state" in + pending|initial-list|stalled|failed) printf '%s\n' "$good" >> "$tmp/event" ;; + esac + if PATH="$tmp:$PATH" GPU_HR_EVENT="$tmp/event" GPU_HR_ERROR="$read_error" COZY_E2E_CONTEXT=offline KUBECONFIG=/dev/null bash hack/e2e-chainsaw/gpuclass/wait-release.sh test release; then + case "$state" in ready|superseded|pending|initial-list) ;; *) echo "FAIL: $state was accepted"; false ;; esac + else + case "$state" in ready|superseded|pending|initial-list) echo "FAIL: $state was refused"; false ;; esac + fi + done + unlink "$tmp/kubectl" + unlink "$tmp/event" + rmdir "$tmp" +} + +@test "GPU disk placement intersects every PV affinity term with ready nodes" { + nodes='{"items":[{"metadata":{"name":"node-a","labels":{"topology.example.com/zone":"a"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"node-b","labels":{"topology.example.com/zone":"b"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}' + volumes='[{"spec":{"nodeAffinity":{"required":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"topology.example.com/zone","operator":"In","values":["b"]}]}]}}}}]' + output=$(printf '%s\n' "$nodes" | jq --exit-status --raw-output --argjson volumes "$volumes" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq) + [ "$output" = node-b ] + incompatible=$(printf '%s\n' "$volumes" | jq '. + [{spec: {nodeAffinity: {required: {nodeSelectorTerms: [{matchFields: [{key: "metadata.name", operator: "In", values: ["node-a"]}]}]}}}}]') + if printf '%s\n' "$nodes" | jq --exit-status --raw-output --argjson volumes "$incompatible" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq; then + echo "FAIL: disks on incompatible nodes were accepted" >&2 + false + fi +} + +@test "GPU disk placement refuses unschedulable or unready nodes" { + [ -s hack/e2e-chainsaw/gpuclass/disk-node.jq ] + for state in cordoned unready; do + case "$state" in + cordoned) spec='{"unschedulable":true}'; ready=True ;; + unready) spec='{}'; ready=False ;; + esac + node=$(jq --null-input --argjson spec "$spec" --arg ready "$ready" '{items: [{metadata: {name: "node-a"}, spec: $spec, status: {conditions: [{type: "Ready", status: $ready}]}}]}') + if printf '%s\n' "$node" | jq --exit-status --raw-output --argjson volumes '[{}]' --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq; then + echo "FAIL: an unusable GPU node was selected" >&2 + false + fi + done +} + +@test "GPU disk placement evaluates PV selector operators and OR terms" { + node='{"items":[{"metadata":{"name":"node-a","labels":{"size":"4"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}' + for operator in In NotIn Exists DoesNotExist Gt Lt; do + case "$operator" in + In) yes=4; no=5; key=size ;; + NotIn) yes=5; no=4; key=size ;; + Exists) yes=4; no=4; key=size ;; + DoesNotExist) yes=4; no=4; key=missing ;; + Gt) yes=3; no=4; key=size ;; + Lt) yes=5; no=4; key=size ;; + esac + volumes=$(jq --null-input --arg op "$operator" --arg key "$key" --arg value "$yes" '[{spec: {nodeAffinity: {required: {nodeSelectorTerms: [{matchExpressions: [{key: $key, operator: $op, values: [$value]}]}]}}}}]') + [ "$(printf '%s\n' "$node" | jq --exit-status --raw-output --argjson volumes "$volumes" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq)" = node-a ] + case "$operator" in + Exists) bad=$(printf '%s\n' "$volumes" | jq '.[0].spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].key = "missing"') ;; + DoesNotExist) bad=$(printf '%s\n' "$volumes" | jq '.[0].spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].key = "size"') ;; + *) bad=$(printf '%s\n' "$volumes" | jq --arg value "$no" '.[0].spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].values = [$value]') ;; + esac + if printf '%s\n' "$node" | jq --exit-status --raw-output --argjson volumes "$bad" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq; then + echo "FAIL: $operator mismatch was accepted" >&2 + false + fi + either=$(jq --null-input --argjson yes "$volumes" --argjson no "$bad" '$yes | .[0].spec.nodeAffinity.required.nodeSelectorTerms += $no[0].spec.nodeAffinity.required.nodeSelectorTerms') + [ "$(printf '%s\n' "$node" | jq --exit-status --raw-output --argjson volumes "$either" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq)" = node-a ] + done +} + +@test "GPU disk placement requires every expression and field within a term" { + nodes='{"items":[{"metadata":{"name":"node-a","labels":{"zone":"a"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"node-b","labels":{"zone":"b","gpu":"true"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}' + volumes='[{"spec":{"nodeAffinity":{"required":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"zone","operator":"In","values":["a","b"]},{"key":"gpu","operator":"Exists"}]}]}}}}]' + [ "$(printf '%s\n' "$nodes" | jq --exit-status --raw-output --argjson volumes "$volumes" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq)" = node-b ] + fields=$(printf '%s\n' "$volumes" | jq '.[0].spec.nodeAffinity.required.nodeSelectorTerms[0] = {matchFields: [{key: "metadata.name", operator: "In", values: ["node-a"]}, {key: "metadata.name", operator: "NotIn", values: ["node-a"]}]}') + if printf '%s\n' "$nodes" | jq --exit-status --raw-output --argjson volumes "$fields" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq; then + echo "FAIL: conflicting PV matchFields were accepted" >&2 + false + fi +} + +@test "GPU disk placement skips nonnumeric labels in numeric comparisons" { + nodes='{"items":[{"metadata":{"name":"node-a","labels":{"size":"large"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"node-b","labels":{"size":"4"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}' + volumes='[{"spec":{"nodeAffinity":{"required":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"size","operator":"Gt","values":["3"]}]}]}}}}]' + [ "$(printf '%s\n' "$nodes" | jq --exit-status --raw-output --argjson volumes "$volumes" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq)" = node-b ] +} + +@test "GPU disk numeric affinity uses exact signed int64 comparisons" { + while IFS='|' read -r operator threshold rejected accepted; do + nodes=$(jq --null-input --arg rejected "$rejected" --arg accepted "$accepted" '[[$rejected, "node-a"], [$accepted, "node-b"]] | map({metadata: {name: .[1], labels: {size: .[0]}}, status: {conditions: [{type: "Ready", status: "True"}]}}) | {items: .} | if $rejected == "" then del(.items[0].metadata.labels.size) else . end') + volumes=$(jq --null-input --arg op "$operator" --arg threshold "$threshold" '[{spec: {nodeAffinity: {required: {nodeSelectorTerms: [{matchExpressions: [{key: "size", operator: $op, values: [$threshold]}]}]}}}}]') + [ "$(printf '%s\n' "$nodes" | jq --exit-status --raw-output --argjson volumes "$volumes" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq)" = node-b ] + done <<'CASES' +Gt|3|4.5|4 +Lt|3|2.5|2 +Gt|3|4e1|4 +Gt|3|0x10|4 +Gt|3|4_0|4 +Gt|3|9223372036854775808|4 +Lt|0|-9223372036854775809|-1 +Gt|9007199254740992|9007199254740992|9007199254740993 +Gt|9223372036854775806|9223372036854775806|9223372036854775807 +Lt|9223372036854775807|9223372036854775807|9223372036854775806 +Gt|0003|0003|0004 +Gt|-2|-3|-1 +Lt|-2|-1|-3 +Gt|0|-0|1 +Gt|-9223372036854775808|-9223372036854775808|-9223372036854775807 +Lt|-9223372036854775807|-9223372036854775807|-9223372036854775808 +Gt|+3|+3|+4 +Gt|3||4 +Lt|0|-0|-1 +Gt|3|0003|0004 +Gt|0003|3|4 +Gt|3|+3|+4 +CASES +} + +@test "GPU suite guards VM install and upgrade retry histories" { + suite="$PWD/hack/e2e-chainsaw/gpuclass" + tmp=$(mktemp --directory) + printf '%s\n' '#!/bin/bash' 'case " $* " in *" --namespace tenant-test-gpu get helmreleases --field-selector metadata.name=vm-instance-gpu --watch "*) cat "$GPU_HR_EVENT" ;; *) exit 1 ;; esac' > "$tmp/kubectl" + chmod +x "$tmp/kubectl" + for step in stopped-vm-reserves-the-class-count launcher-workload-reports-the-gpu-units; do + count=$(STEP="$step" yq '[.spec.steps[] | select(.name == strenv(STEP)) | .try[].script.content // "" | select(contains("wait-release.sh"))] | length' "$suite/chainsaw-test.yaml") + [ "$count" = 1 ] + content=$(STEP="$step" yq --unwrapScalar '.spec.steps[] | select(.name == strenv(STEP)) | .try[].script.content // "" | select(contains("wait-release.sh"))' "$suite/chainsaw-test.yaml") + for state in ready failed; do + jq --null-input --compact-output --arg state "$state" '{metadata: {generation: 1}, spec: {install: {strategy: {name: "RetryOnFailure"}}}, status: {observedGeneration: 1, conditions: [{type: "Ready", status: "True"}], history: [{status: (if $state == "failed" then "failed" else "deployed" end)}]}}' > "$tmp/event" + if (cd "$suite" && PATH="$tmp:$PATH" GPU_HR_EVENT="$tmp/event" COZY_E2E_CONTEXT=offline KUBECONFIG=/dev/null sh -c "$content"); then + [ "$state" = ready ] + else + [ "$state" = failed ] + fi + done + done + unlink "$tmp/kubectl" + unlink "$tmp/event" + rmdir "$tmp" +} diff --git a/hack/select-e2e.sh b/hack/select-e2e.sh index fcb7651e3..b98ad57eb 100755 --- a/hack/select-e2e.sh +++ b/hack/select-e2e.sh @@ -245,14 +245,10 @@ escalate_to_full_suite() { # this list read as longer than the set of real exceptions. src_to_suites() { case "$1" in - vm-instance-application) echo vminstance ;; - # The vminstance suite creates a VMDisk and asserts the DataVolume behind it - # (hack/e2e-chainsaw/vminstance/vmdisk.yaml, vmdisk-vmi.yaml), so it is the - # suite that covers this source and everything reaching it, kubevirt-cdi - # included. Without the entry the source maps to a `vm-disk` suite that does - # not exist, is dropped by intersect_suites(), and a CDI change reaches - # nothing runnable through it. - vm-disk-application) echo vminstance ;; + vm-instance-application) echo "vminstance gpuclass" ;; + # Both VM suites populate VMDisk resources, so they cover CDI changes too. + vm-disk-application) echo "vminstance gpuclass" ;; + gpu-operator) echo gpuclass ;; # The VMInstance and VMDisk backup round-trips are the only Tests that run a # Velero backup and restore; without this the source maps to a `velero` # suite that does not exist and a Velero change runs neither. @@ -266,8 +262,8 @@ src_to_suites() { # apps/kubernetes-nodes also belongs to computeplane-application, which has # a suite; without this entry an edit there selects computeplane alone, # whose pools go through the module allowlist, and drops the suites that - # create KubernetesNodes natively. - kubernetes-nodes-application) echo "kubernetes-latest kubernetes-previous computeplane" ;; + # create KubernetesNodes natively. gpuclass checks its GPU budget admission. + kubernetes-nodes-application) echo "kubernetes-latest kubernetes-previous computeplane gpuclass" ;; securitygroup-controller) echo securitygroup ;; # The kafka app source covers both the app suite and the metadata-backup # roundtrip, so an edit to packages/apps/kafka (or kafka-operator, which diff --git a/hack/select-e2e_test.bats b/hack/select-e2e_test.bats index 6ce198830..56cb73eaa 100644 --- a/hack/select-e2e_test.bats +++ b/hack/select-e2e_test.bats @@ -127,18 +127,45 @@ assert_full_suite() { # kubevirt-cdi reaches vm-disk-application and nothing else runnable, so # until vm-disk-application mapped to a suite every CDI change ran all of # them. The vminstance suite creates a VMDisk and asserts the DataVolume - # behind it, so it is the suite that covers CDI. + # behind it, so it is the suite that covers CDI; gpuclass attaches a + # VMDisk to its VMs too. tmp=$(mktemp -d) cp -r packages/core/platform/sources "$tmp/sources" echo "packages/system/kubevirt-cdi/values.yaml" > "$tmp/diff" output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") - [ "$output" = "vminstance" ] + [ "$output" = "gpuclass vminstance" ] closure=$(hack/select-install.sh "$output" "$tmp/sources") printf '%s\n' "$closure" | tr ' ' '\n' | grep -Fxq cozystack.vm-disk-application printf '%s\n' "$closure" | tr ' ' '\n' | grep -Fxq cozystack.kubevirt-cdi rm -rf "$tmp" } +@test "a GPU change selects gpuclass, a change downstream of gpu-operator does not" { + # gpu-operator and the VM applications map to gpuclass by name. hami + # depends on gpu-operator, so the walk must not carry it the other way. + tmp=$(mktemp -d) + cp -r packages/core/platform/sources "$tmp/sources" + for path in packages/system/gpu-operator/values.yaml packages/apps/kubernetes-nodes/values.yaml packages/apps/vm-instance/values.yaml; do + echo "$path" > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") + if ! printf '%s\n' "$output" | tr ' ' '\n' | grep -Fxq gpuclass; then + echo "$path does not select gpuclass: $output" >&2 + return 1 + fi + done + echo "packages/apps/vm-instance/values.yaml" > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") + [ "$output" = "gpuclass vminstance" ] + echo "packages/system/hami/values.yaml" > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") + [ -n "$output" ] + if printf '%s\n' "$output" | tr ' ' '\n' | grep -Fxq gpuclass; then + echo "a hami change selects gpuclass: $output" >&2 + return 1 + fi + rm -rf "$tmp" +} + @test "a CNI change selects every suite the graph can reach" { # Named for what it measures rather than for the full suite it once claimed. # `packages/system/cilium` is owned by cozystack.networking and resolves @@ -1085,30 +1112,29 @@ assert_full_suite() { rm -rf "$tmp" } -@test "a kubernetes-nodes pool chart change selects computeplane and both kubernetes suites" { +@test "a kubernetes-nodes pool chart change selects native, module and GPU budget suites" { # apps/kubernetes-nodes belongs to kubernetes-nodes-application and to # computeplane-application. The kubernetes suites create KubernetesNodes # natively and computeplane renders pools through its module, so a pool - # chart change has to reach all three, not only the source that happens - # to have a suite of its own name. + # chart change must also reach the GPU budget admission checks. tmp=$(mktemp -d) cp -r packages/core/platform/sources "$tmp/sources" echo "packages/apps/kubernetes-nodes/values.yaml" > "$tmp/diff" output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") assert_selection "a pool chart change must exercise native and module pools" \ - "$output" "computeplane kubernetes-latest kubernetes-previous" + "$output" "computeplane gpuclass kubernetes-latest kubernetes-previous" rm -rf "$tmp" } -@test "a kubernetes-nodes-rd change selects computeplane and both kubernetes suites" { +@test "a kubernetes-nodes-rd change selects native, module and GPU budget suites" { # kubernetes-nodes-rd ships only in kubernetes-nodes-application, which - # maps to the same three suites as the pool chart it registers. + # maps to the same suites as the pool chart it registers. tmp=$(mktemp -d) cp -r packages/core/platform/sources "$tmp/sources" echo "packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml" > "$tmp/diff" output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") assert_selection "a kubernetes-nodes-rd change must exercise native and module pools" \ - "$output" "computeplane kubernetes-latest kubernetes-previous" + "$output" "computeplane gpuclass kubernetes-latest kubernetes-previous" rm -rf "$tmp" } diff --git a/hack/select-install.sh b/hack/select-install.sh index 15acf8aca..aa3962396 100755 --- a/hack/select-install.sh +++ b/hack/select-install.sh @@ -127,6 +127,9 @@ suite_to_source() { vminstance) echo "cozystack.vm-instance-application cozystack.vm-disk-application" return ;; + gpuclass) + echo "cozystack.gpu-operator cozystack.vm-instance-application cozystack.vm-disk-application cozystack.kubernetes-nodes-application" + return ;; securitygroup) echo cozystack.securitygroup-controller ; return ;; kafka-metadata) echo cozystack.kafka-application ; return ;; esac diff --git a/hack/select-install_test.bats b/hack/select-install_test.bats index 462abb60a..235d69f04 100644 --- a/hack/select-install_test.bats +++ b/hack/select-install_test.bats @@ -106,6 +106,14 @@ assert_output_graph_error() { printf '%s\n' "$output" | tr ' ' '\n' | grep -Fxq cozystack.kubevirt-cdi } +@test "gpuclass installs gpu-operator and its application APIs" { + output=$(hack/select-install.sh "gpuclass") + assert_contains_package "$output" cozystack.kubernetes-nodes-application + assert_contains_package "$output" cozystack.gpu-operator + assert_contains_package "$output" cozystack.vm-instance-application + assert_contains_package "$output" cozystack.vm-disk-application +} + @test "multiple suites union their closures" { output=$(hack/select-install.sh "postgres kafka") assert_contains_package "$output" cozystack.postgres-application