diff --git a/hack/e2e-chainsaw/README.md b/hack/e2e-chainsaw/README.md
index 98400bc55..39714e890 100644
--- a/hack/e2e-chainsaw/README.md
+++ b/hack/e2e-chainsaw/README.md
@@ -26,7 +26,7 @@ A suite is a directory directly under `hack/e2e-chainsaw/` in or below which Cha
A parked `chainsaw-test.yaml.disabled` is none of these: the directory is then not a suite, is selected by nothing and cannot be allowlisted.
-Suites: `postgres`, `bucket`, `mariadb`, `mongodb`, `redis`, `valkey`, `qdrant`, `clickhouse`, `kafka`, `kafka-metadata`, `rabbitmq`, `etcd`, `openbao`, `harbor`, `foundationdb`, `external-dns`, `kuberture`, `vminstance`, `gateway`, `opensearch`, `monitoring`, `kubernetes-latest`, `kubernetes-previous`, `securitygroup`, `seaweedfs`, `computeplane`. `kubernetes-proxmox` is parked as `chainsaw-test.yaml.disabled` and runs only in `.github/workflows/e2e-proxmox.yaml`, which needs a Proxmox hypervisor.
+Suites: `postgres`, `bucket`, `mariadb`, `mongodb`, `redis`, `valkey`, `qdrant`, `clickhouse`, `kafka`, `kafka-metadata`, `rabbitmq`, `etcd`, `openbao`, `harbor`, `foundationdb`, `external-dns`, `kuberture`, `vminstance`, `gateway`, `opensearch`, `monitoring`, `kubernetes-latest`, `kubernetes-previous`, `securitygroup`, `seaweedfs`, `computeplane`, `gpuclass`. `kubernetes-proxmox` is parked as `chainsaw-test.yaml.disabled` and runs only in `.github/workflows/e2e-proxmox.yaml`, which needs a Proxmox hypervisor.
## What Chainsaw buys over the BATS suite
@@ -82,3 +82,5 @@ chainsaw test postgres/
## CI integration
The `e2e` job in `.github/workflows/pull-requests.yaml` runs `make -C packages/core/testing test-chainsaw` (which execs `hack/e2e-chainsaw-run.sh` inside the e2e sandbox) and uploads `chainsaw-report.xml`. The runner calls `chainsaw test` once per suite directory, fails a suite whose pods outlive its cleanup by more than the default `timeouts.cleanup` (`LEAKED-PODS suite=
` in the log; see `docs/agents/e2e-testing.md` ยง8 and `_lib/leak-allowlist.txt` for known, filed leaks), and merges the per-suite reports into that one file. A local `chainsaw test` as shown above runs the same tests without the leak check. The sandbox image (`packages/core/testing/images/e2e-sandbox/Dockerfile`) ships the `chainsaw` binary alongside `kubectl`/`helm`/`mc`/`jq`/`nc`, built from source with a patch that keeps an assert polling through transient API errors such as an etcd leader change; a released `chainsaw` run locally fails the assert on the first such error instead.
+
+The `gpuclass` suite advertises a fake extended resource on a node compatible with its already populated disk. It checks class status, declared VM and worker-pool budgets, vGPU entitlement refusal, VMI GPU entries, launcher tolerations and scheduling, and GPU units in the launcher pod's existing `Workload`. CI has no GPU device plugin: the suite does not prove device allocation, VMI sync, guest boot or guest drivers.
diff --git a/hack/e2e-chainsaw/gpuclass/chainsaw-test.yaml b/hack/e2e-chainsaw/gpuclass/chainsaw-test.yaml
new file mode 100644
index 000000000..d65f67dc5
--- /dev/null
+++ b/hack/e2e-chainsaw/gpuclass/chainsaw-test.yaml
@@ -0,0 +1,444 @@
+# CI has no GPU device plugin. Fake capacity proves admission, scheduling and
+# launcher-pod accounting, not VMI sync, a booted guest or device allocation.
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+ name: gpuclass
+spec:
+ catch:
+ - events: {}
+ - events:
+ namespace: tenant-test-gpu
+ - describe:
+ apiVersion: cozystack.io/v1alpha1
+ kind: GPUClass
+ name: e2e-fake
+ - describe:
+ apiVersion: helm.toolkit.fluxcd.io/v2
+ kind: HelmRelease
+ namespace: tenant-test-gpu
+ - describe:
+ apiVersion: cdi.kubevirt.io/v1beta1
+ kind: DataVolume
+ namespace: tenant-test-gpu
+ - describe:
+ apiVersion: v1
+ kind: PersistentVolumeClaim
+ namespace: tenant-test-gpu
+ - describe:
+ apiVersion: kubevirt.io/v1
+ kind: VirtualMachineInstance
+ namespace: tenant-test-gpu
+ - describe:
+ apiVersion: v1
+ kind: Pod
+ namespace: tenant-test-gpu
+ - podLogs:
+ namespace: cozy-system
+ selector: app=cozystack-controller
+ tail: 200
+ steps:
+ - name: gpu-paths-are-on
+ try:
+ - assert:
+ timeout: 2m
+ resource:
+ apiVersion: apps/v1
+ kind: Deployment
+ metadata:
+ name: cozystack-controller
+ namespace: cozy-system
+ spec:
+ template:
+ spec:
+ (containers[?name == 'cozystack-controller']):
+ - (contains(args, '--enable-gpu')): true
+ - assert:
+ timeout: 2m
+ resource:
+ apiVersion: kubevirt.io/v1
+ kind: KubeVirt
+ metadata:
+ name: kubevirt
+ namespace: cozy-kubevirt
+ spec:
+ configuration:
+ (permittedHostDevices.pciHostDevices[?resourceName == 'nvidia.com/E2E_FAKE_GPU']):
+ - externalResourceProvider: true
+
+ - name: tenant-has-class-and-total-budgets
+ try:
+ - apply:
+ resource:
+ apiVersion: apps.cozystack.io/v1alpha1
+ kind: Tenant
+ metadata:
+ name: gpu
+ spec:
+ resourceQuotas:
+ cpu: "4"
+ memory: 8Gi
+ storage: 10Gi
+ e2e-fake.gpuclass.cozystack.io/devices: "2"
+ gpuclass.cozystack.io/devices: "2"
+ - assert:
+ timeout: 2m
+ resource:
+ apiVersion: helm.toolkit.fluxcd.io/v2
+ kind: HelmRelease
+ metadata:
+ name: tenant-gpu
+ # Above tenant-rd's 15m install timeout, with room for Flux to start it.
+ - script:
+ timeout: 18m
+ content: |
+ set -eu
+ timeout 1020 bash ./wait-release.sh tenant-test tenant-gpu
+ - assert:
+ resource:
+ apiVersion: v1
+ kind: Namespace
+ metadata:
+ name: tenant-test-gpu
+ labels:
+ namespace.cozystack.io/vgpu: "false"
+
+ - name: populate-disk-before-selecting-the-gpu-node
+ try:
+ - apply:
+ resource:
+ apiVersion: apps.cozystack.io/v1alpha1
+ kind: VMDisk
+ metadata:
+ name: gpu
+ namespace: tenant-test-gpu
+ spec:
+ source: {}
+ storage: 1Gi
+ storageClass: ($values.storageClass)
+ bindImmediately: true
+ - assert:
+ timeout: 2m
+ resource:
+ apiVersion: helm.toolkit.fluxcd.io/v2
+ kind: HelmRelease
+ metadata:
+ name: vm-disk-gpu
+ namespace: tenant-test-gpu
+ # vm-disk-rd uses the API server's 10m install timeout.
+ - script:
+ timeout: 12m
+ content: |
+ set -eu
+ timeout 660 bash ./wait-release.sh tenant-test-gpu vm-disk-gpu
+ - assert:
+ timeout: 2m
+ resource:
+ apiVersion: cdi.kubevirt.io/v1beta1
+ kind: DataVolume
+ metadata:
+ name: vm-disk-gpu
+ namespace: tenant-test-gpu
+ annotations:
+ cdi.kubevirt.io/storage.bind.immediate.requested: "true"
+ status:
+ phase: Succeeded
+ - assert:
+ resource:
+ apiVersion: v1
+ kind: PersistentVolumeClaim
+ metadata:
+ name: vm-disk-gpu
+ namespace: tenant-test-gpu
+ spec:
+ (volumeName != null): true
+ status:
+ phase: Bound
+
+ - name: advertise-capacity-only-where-the-disk-can-run
+ try:
+ - script:
+ content: |
+ set -eu
+ context=$(kubectl --context="" config current-context)
+ pv=$(kubectl --context "$context" --namespace tenant-test-gpu get pvc vm-disk-gpu --output=jsonpath='{.spec.volumeName}')
+ [ -n "$pv" ]
+ volumes=$(kubectl --context "$context" get pv "$pv" --output=json | jq '[.]')
+ node=$(kubectl --context "$context" get nodes --output=json |
+ jq --exit-status --raw-output --argjson volumes "$volumes" --from-file ./disk-node.jq)
+ kubectl --context "$context" get node "$node" --output=json |
+ jq --exit-status '.status.capacity["nvidia.com/E2E_FAKE_GPU"] == null and .metadata.labels["e2e.cozystack.io/fake-gpu"] == null'
+ kubectl --context "$context" label node "$node" e2e.cozystack.io/fake-gpu=true
+ kubectl --context "$context" patch node "$node" --subresource=status --type=json \
+ --patch '[{"op":"add","path":"/status/capacity/nvidia.com~1E2E_FAKE_GPU","value":"2"}]'
+ - assert:
+ timeout: 2m
+ resource:
+ apiVersion: v1
+ kind: Node
+ metadata:
+ labels:
+ e2e.cozystack.io/fake-gpu: "true"
+ status:
+ allocatable:
+ nvidia.com/E2E_FAKE_GPU: "2"
+ # Nodes are not owned by Chainsaw; the label survives a failed status patch.
+ cleanup:
+ - script:
+ content: |
+ set -eu
+ context=$(kubectl --context="" config current-context)
+ nodes=$(kubectl --context "$context" get nodes --selector=e2e.cozystack.io/fake-gpu=true --output=jsonpath='{.items[*].metadata.name}')
+ for node in $nodes; do
+ if [ -n "$(kubectl --context "$context" get node "$node" --output=jsonpath='{.status.capacity.nvidia\.com/E2E_FAKE_GPU}')" ]; then
+ kubectl --context "$context" patch node "$node" --subresource=status --type=json \
+ --patch '[{"op":"remove","path":"/status/capacity/nvidia.com~1E2E_FAKE_GPU"}]'
+ fi
+ kubectl --context "$context" label node "$node" e2e.cozystack.io/fake-gpu-
+ done
+
+ - name: class-status-reflects-fake-capacity
+ try:
+ - apply:
+ resource:
+ apiVersion: cozystack.io/v1alpha1
+ kind: GPUClass
+ metadata:
+ name: e2e-fake
+ spec:
+ displayName: E2E fake GPU
+ vendor: nvidia
+ model: FAKE
+ mode: passthrough
+ maxCount: 2
+ tolerations:
+ - key: e2e.cozystack.io/gpu
+ operator: Equal
+ value: fake
+ effect: NoSchedule
+ source:
+ devicePlugin:
+ resourceName: nvidia.com/E2E_FAKE_GPU
+ - apply:
+ resource:
+ apiVersion: cozystack.io/v1alpha1
+ kind: GPUClass
+ metadata:
+ name: e2e-vgpu
+ spec:
+ vendor: nvidia
+ model: FAKE
+ mode: vgpu
+ profile: FAKE-1C
+ source:
+ devicePlugin:
+ resourceName: nvidia.com/E2E_FAKE_VGPU
+ - assert:
+ timeout: 2m
+ resource:
+ apiVersion: cozystack.io/v1alpha1
+ kind: GPUClass
+ metadata:
+ name: e2e-fake
+ status:
+ free: 2
+ maxFreePerNode: 2
+ (conditions[?type == 'ResourceNameNotPermitted']):
+ - status: "False"
+ (conditions[?type == 'ResourceNameConflict']):
+ - status: "False"
+
+ - name: stopped-vm-reserves-the-class-count
+ try:
+ - apply:
+ resource:
+ apiVersion: apps.cozystack.io/v1alpha1
+ kind: VMInstance
+ metadata:
+ name: gpu
+ namespace: tenant-test-gpu
+ spec:
+ runStrategy: Halted
+ instanceType: u1.medium
+ disks:
+ - name: gpu
+ gpus:
+ - className: e2e-fake
+ count: 2
+ - assert:
+ timeout: 2m
+ resource:
+ apiVersion: helm.toolkit.fluxcd.io/v2
+ kind: HelmRelease
+ metadata:
+ name: vm-instance-gpu
+ namespace: tenant-test-gpu
+ spec:
+ values:
+ runStrategy: Halted
+ gpus:
+ - className: e2e-fake
+ count: 2
+ - script:
+ timeout: 12m
+ content: |
+ set -eu
+ timeout 660 bash ./wait-release.sh tenant-test-gpu vm-instance-gpu
+ - script:
+ content: |
+ set -eu
+ context=$(kubectl --context="" config current-context)
+ for scenario in class total workers; do
+ case "$scenario" in
+ class) kind=VMInstance; class=e2e-fake; extra='runStrategy: Halted'; need=1; key=e2e-fake.gpuclass.cozystack.io/devices ;;
+ total) kind=VMInstance; class=e2e-vgpu; extra='runStrategy: Halted'; need=1; key=gpuclass.cozystack.io/devices ;;
+ workers) kind=KubernetesNodes; class=e2e-fake; extra='cluster: unused
+ maxReplicas: 3'; need=3; key=e2e-fake.gpuclass.cozystack.io/devices ;;
+ esac
+ if out=$(kubectl --context "$context" --namespace tenant-test-gpu create --dry-run=server --filename - 2>&1 <&2
+ false
+ fi
+ printf '%s\n' "$out"
+ printf '%s\n' "$out" | grep --fixed-strings "GPU budget \"$key\" of tenant pool tenant-test-gpu is 2 and 2 is already reserved; this application needs $need"
+ done
+
+ - name: unentitled-vgpu-is-refused-at-vmi-admission
+ try:
+ - script:
+ content: |
+ set -eu
+ context=$(kubectl --context="" config current-context)
+ if out=$(kubectl --context "$context" --namespace tenant-test-gpu create --dry-run=server --filename - 2>&1 <<'EOF'
+ apiVersion: kubevirt.io/v1
+ kind: VirtualMachineInstance
+ metadata:
+ name: unentitled
+ labels:
+ cozystack.io/gpu-reference: "true"
+ annotations:
+ cozystack.io/gpus: '[{"className":"e2e-vgpu","count":1}]'
+ spec:
+ domain:
+ resources:
+ requests:
+ memory: 1Gi
+ devices: {}
+ EOF
+ ); then
+ echo "FAIL: unentitled vGPU VMI was accepted: $out" >&2
+ false
+ fi
+ printf '%s\n' "$out"
+ printf '%s\n' "$out" | grep --fixed-strings 'GPUClass e2e-vgpu is vGPU and the tenant is not entitled to vGPU'
+
+ - name: class-entry-expands-to-kubevirt-gpus
+ try:
+ - patch:
+ resource:
+ apiVersion: apps.cozystack.io/v1alpha1
+ kind: VMInstance
+ metadata:
+ name: gpu
+ namespace: tenant-test-gpu
+ spec:
+ runStrategy: Always
+ - assert:
+ timeout: 5m
+ resource:
+ apiVersion: kubevirt.io/v1
+ kind: VirtualMachineInstance
+ metadata:
+ name: vm-instance-gpu
+ namespace: tenant-test-gpu
+ labels:
+ cozystack.io/gpu-reference: "true"
+ spec:
+ domain:
+ devices:
+ gpus:
+ - name: gpu-0-0
+ deviceName: nvidia.com/E2E_FAKE_GPU
+ - name: gpu-0-1
+ deviceName: nvidia.com/E2E_FAKE_GPU
+ (length(gpus)): 2
+
+ - name: launcher-schedules-with-class-tolerations
+ try:
+ - assert:
+ timeout: 5m
+ resource:
+ apiVersion: v1
+ kind: Pod
+ metadata:
+ namespace: tenant-test-gpu
+ labels:
+ kubevirt.io: virt-launcher
+ vm.kubevirt.io/name: vm-instance-gpu
+ cozystack.io/gpu-reference: "true"
+ (annotations."kubevirt.io/ephemeral-provisioning" == null): true
+ spec:
+ (nodeName != null): true
+ tolerations:
+ - key: e2e.cozystack.io/gpu
+ operator: Equal
+ value: fake
+ effect: NoSchedule
+ (containers[?name == 'compute']):
+ - resources:
+ requests:
+ nvidia.com/E2E_FAKE_GPU: "2"
+ - script:
+ content: |
+ set -eu
+ context=$(kubectl --context="" config current-context)
+ node=$(kubectl --context "$context" get nodes --selector=e2e.cozystack.io/fake-gpu=true --output=jsonpath='{.items[*].metadata.name}')
+ kubectl --context "$context" --namespace tenant-test-gpu get pods --selector=kubevirt.io=virt-launcher,vm.kubevirt.io/name=vm-instance-gpu --output=json |
+ jq --exit-status --arg node "$node" '
+ [.items[] | select(.metadata.annotations["kubevirt.io/ephemeral-provisioning"] == null)] |
+ length == 1 and .[0].spec.nodeName == $node'
+
+ - name: launcher-workload-reports-the-gpu-units
+ try:
+ - assert:
+ timeout: 2m
+ resource:
+ apiVersion: cozystack.io/v1alpha1
+ kind: Workload
+ metadata:
+ namespace: tenant-test-gpu
+ labels:
+ workloads.cozystack.io/monitor: vm-instance-gpu
+ (ownerReferences[?kind == 'Pod']):
+ - apiVersion: v1
+ status:
+ kind: virtual-machine
+ resources:
+ nvidia.com/E2E_FAKE_GPU: "2"
+ - script:
+ content: |
+ set -eu
+ context=$(kubectl --context="" config current-context)
+ pod=$(kubectl --context "$context" --namespace tenant-test-gpu get pods --selector=kubevirt.io=virt-launcher,vm.kubevirt.io/name=vm-instance-gpu --output=json |
+ jq --exit-status --raw-output '[.items[] | select(.metadata.annotations["kubevirt.io/ephemeral-provisioning"] == null)] | if length == 1 then .[0].metadata.uid else error("expected one launcher") end')
+ kubectl --context "$context" --namespace tenant-test-gpu get workloads --output=json |
+ jq --exit-status --arg uid "$pod" '
+ [.items[] | select(.status.resources["nvidia.com/E2E_FAKE_GPU"] != null)] |
+ length == 1 and .[0].status.resources["nvidia.com/E2E_FAKE_GPU"] == "2" and
+ any(.[0].metadata.ownerReferences[]?; .kind == "Pod" and .uid == $uid)'
+ - script:
+ timeout: 12m
+ content: |
+ set -eu
+ timeout 660 bash ./wait-release.sh tenant-test-gpu vm-instance-gpu
diff --git a/hack/e2e-chainsaw/gpuclass/disk-node.jq b/hack/e2e-chainsaw/gpuclass/disk-node.jq
new file mode 100644
index 000000000..7d16a6be4
--- /dev/null
+++ b/hack/e2e-chainsaw/gpuclass/disk-node.jq
@@ -0,0 +1,40 @@
+def int64:
+ if type != "string" or (test("^[+-]?[0-9]+$") | not) then null
+ else
+ startswith("-") as $negative |
+ sub("^[+-]"; "") | sub("^0+"; "") |
+ (if . == "" then "0" else . end) as $digits |
+ (if $negative then "9223372036854775808" else "9223372036854775807" end) as $limit |
+ if [($digits | length), $digits] > [19, $limit] then null
+ else {negative: ($negative and $digits != "0"), digits: $digits} end
+ end;
+def compareIntegers($left; $right):
+ ($left | int64) as $a | ($right | int64) as $b |
+ if $a == null or $b == null then null
+ elif $a.negative != $b.negative then (if $a.negative then -1 else 1 end)
+ elif $a.digits == $b.digits then 0
+ else
+ (if [($a.digits | length), $a.digits] > [($b.digits | length), $b.digits] then 1 else -1 end) *
+ (if $a.negative then -1 else 1 end)
+ end;
+def requirement($values):
+ . as $r | $values[$r.key] as $v |
+ if .operator == "In" then $v != null and (.values | index($v)) != null
+ elif .operator == "NotIn" then (.values | index($v)) == null
+ elif .operator == "Exists" then $v != null
+ elif .operator == "DoesNotExist" then $v == null
+ elif .operator == "Gt" then compareIntegers($v; .values[0]) as $n | $n != null and $n > 0
+ elif .operator == "Lt" then compareIntegers($v; .values[0]) as $n | $n != null and $n < 0
+ else error("unsupported PV node affinity operator: " + .operator)
+ end;
+def permits($node):
+ .spec.nodeAffinity.required as $required |
+ $required == null or any($required.nodeSelectorTerms[]?;
+ ((.matchExpressions // []) + (.matchFields // []) | length) > 0 and
+ all(.matchExpressions[]?; requirement($node.metadata.labels // {})) and
+ all(.matchFields[]?; requirement({"metadata.name": $node.metadata.name})));
+[.items[] |
+ select(.spec.unschedulable != true) |
+ select(any(.status.conditions[]?; .type == "Ready" and .status == "True")) |
+ . as $node | select(all($volumes[]; permits($node))) |
+ .metadata.name] | sort | first // error("no ready node is compatible with every GPU VM disk")
diff --git a/hack/e2e-chainsaw/gpuclass/wait-release.sh b/hack/e2e-chainsaw/gpuclass/wait-release.sh
new file mode 100644
index 000000000..d1bad529d
--- /dev/null
+++ b/hack/e2e-chainsaw/gpuclass/wait-release.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+set -euo pipefail
+
+# shellcheck source=hack/e2e-chainsaw/_lib/remediation-guard.sh
+source "$(dirname -- "${BASH_SOURCE[0]}")/../_lib/remediation-guard.sh"
+
+context=${COZY_E2E_CONTEXT:-$(kubectl --context="" config current-context)}
+exec 3< <(exec kubectl --context "$context" --namespace "$1" get helmreleases \
+ --field-selector "metadata.name=$2" --watch --output='jsonpath={@}{"\n"}')
+watcher=$!
+trap 'kill "$watcher" 2>/dev/null || :; wait "$watcher" 2>/dev/null || :' EXIT
+
+while IFS= read -r event <&3; do
+ statuses=$(printf '%s\n' "$event" | jq --raw-output '(.items // [.])[] | .status.history[]?.status // empty')
+ if helmrelease_has_remediation_cycle "$statuses"; then
+ printf 'HelmRelease %s/%s has failed history: %s\n' "$1" "$2" "$statuses" >&2
+ exit 1
+ fi
+ verdict=$(printf '%s\n' "$event" | jq --raw-output '
+ (.items // [.])[] |
+ if .spec.install.strategy.name != "RetryOnFailure" then
+ error("Application HelmRelease must use RetryOnFailure")
+ elif any(.status.conditions[]?; (.type == "Stalled" and .status == "True") or
+ (.type == "Ready" and .status == "False" and (.reason | endswith("Failed")))) then
+ error("HelmRelease failed: " + (.status | tojson))
+ elif any(.status.conditions[]?; .type == "Ready" and .status == "True") then
+ if (.status.history | type) != "array" or (.status.history | length) == 0 or
+ any(.status.history[]; (.status | type) != "string" or .status == "") then
+ error("Ready HelmRelease has no readable action history")
+ elif .status.observedGeneration == .metadata.generation then "ready"
+ else empty end
+ else empty end')
+ if [ "$verdict" = ready ]; then
+ exit 0
+ fi
+done
+
+echo "HelmRelease $1/$2 watch ended before observed Ready" >&2
+exit 1
diff --git a/hack/gpuclass-e2e_test.bats b/hack/gpuclass-e2e_test.bats
new file mode 100644
index 000000000..8bee822d6
--- /dev/null
+++ b/hack/gpuclass-e2e_test.bats
@@ -0,0 +1,170 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+@test "GPU release watch requires readable history and the application retry strategy" {
+ tmp=$(mktemp --directory)
+ printf '%s\n' '#!/bin/bash' 'if [ "$GPU_HR_ERROR" = true ]; then exit 1; fi' 'cat "$GPU_HR_EVENT"' > "$tmp/kubectl"
+ chmod +x "$tmp/kubectl"
+ good='{"metadata":{"generation":1},"spec":{"install":{"strategy":{"name":"RetryOnFailure"}}},"status":{"observedGeneration":1,"conditions":[{"type":"Ready","status":"True"}],"history":[{"status":"deployed"}]}}'
+ for state in ready superseded pending initial-list stalled failed stale history-failed history-uninstalled missing-history empty-history object-history missing-status null-status empty-status missing-strategy wrong-strategy read-error malformed; do
+ event="$good"
+ read_error=false
+ case "$state" in
+ superseded) event=$(printf '%s\n' "$good" | jq --compact-output '.status.history += [{status: "superseded"}]') ;;
+ pending) event=$(printf '%s\n' "$good" | jq --compact-output '.status.conditions = [{type: "Ready", status: "False", reason: "Progressing"}] | .status.history = []') ;;
+ initial-list) event='{"items":[]}' ;;
+ stale) event=$(printf '%s\n' "$good" | jq --compact-output '.status.observedGeneration = 0') ;;
+ history-*) event=$(printf '%s\n' "$good" | jq --compact-output --arg state "${state#history-}" '.status.history = [{status: $state}]') ;;
+ stalled) event=$(printf '%s\n' "$good" | jq --compact-output '.status.conditions = [{type: "Stalled", status: "True", message: "broken chart"}]') ;;
+ failed) event=$(printf '%s\n' "$good" | jq --compact-output '.status.conditions = [{type: "Ready", status: "False", reason: "InstallFailed", message: "broken chart"}]') ;;
+ missing-history) event=$(printf '%s\n' "$good" | jq --compact-output 'del(.status.history)') ;;
+ empty-history) event=$(printf '%s\n' "$good" | jq --compact-output '.status.history = []') ;;
+ object-history) event=$(printf '%s\n' "$good" | jq --compact-output '.status.history = {snapshot: {status: "deployed"}}') ;;
+ missing-status) event=$(printf '%s\n' "$good" | jq --compact-output '.status.history = [{}]') ;;
+ null-status) event=$(printf '%s\n' "$good" | jq --compact-output '.status.history = [{status: null}]') ;;
+ empty-status) event=$(printf '%s\n' "$good" | jq --compact-output '.status.history = [{status: ""}]') ;;
+ missing-strategy) event=$(printf '%s\n' "$good" | jq --compact-output 'del(.spec.install.strategy)') ;;
+ wrong-strategy) event=$(printf '%s\n' "$good" | jq --compact-output '.spec.install.strategy.name = "RemediateOnFailure"') ;;
+ read-error) read_error=true ;;
+ malformed) event='{' ;;
+ esac
+ printf '%s\n' "$event" > "$tmp/event"
+ case "$state" in
+ pending|initial-list|stalled|failed) printf '%s\n' "$good" >> "$tmp/event" ;;
+ esac
+ if PATH="$tmp:$PATH" GPU_HR_EVENT="$tmp/event" GPU_HR_ERROR="$read_error" COZY_E2E_CONTEXT=offline KUBECONFIG=/dev/null bash hack/e2e-chainsaw/gpuclass/wait-release.sh test release; then
+ case "$state" in ready|superseded|pending|initial-list) ;; *) echo "FAIL: $state was accepted"; false ;; esac
+ else
+ case "$state" in ready|superseded|pending|initial-list) echo "FAIL: $state was refused"; false ;; esac
+ fi
+ done
+ unlink "$tmp/kubectl"
+ unlink "$tmp/event"
+ rmdir "$tmp"
+}
+
+@test "GPU disk placement intersects every PV affinity term with ready nodes" {
+ nodes='{"items":[{"metadata":{"name":"node-a","labels":{"topology.example.com/zone":"a"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"node-b","labels":{"topology.example.com/zone":"b"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}'
+ volumes='[{"spec":{"nodeAffinity":{"required":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"topology.example.com/zone","operator":"In","values":["b"]}]}]}}}}]'
+ output=$(printf '%s\n' "$nodes" | jq --exit-status --raw-output --argjson volumes "$volumes" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq)
+ [ "$output" = node-b ]
+ incompatible=$(printf '%s\n' "$volumes" | jq '. + [{spec: {nodeAffinity: {required: {nodeSelectorTerms: [{matchFields: [{key: "metadata.name", operator: "In", values: ["node-a"]}]}]}}}}]')
+ if printf '%s\n' "$nodes" | jq --exit-status --raw-output --argjson volumes "$incompatible" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq; then
+ echo "FAIL: disks on incompatible nodes were accepted" >&2
+ false
+ fi
+}
+
+@test "GPU disk placement refuses unschedulable or unready nodes" {
+ [ -s hack/e2e-chainsaw/gpuclass/disk-node.jq ]
+ for state in cordoned unready; do
+ case "$state" in
+ cordoned) spec='{"unschedulable":true}'; ready=True ;;
+ unready) spec='{}'; ready=False ;;
+ esac
+ node=$(jq --null-input --argjson spec "$spec" --arg ready "$ready" '{items: [{metadata: {name: "node-a"}, spec: $spec, status: {conditions: [{type: "Ready", status: $ready}]}}]}')
+ if printf '%s\n' "$node" | jq --exit-status --raw-output --argjson volumes '[{}]' --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq; then
+ echo "FAIL: an unusable GPU node was selected" >&2
+ false
+ fi
+ done
+}
+
+@test "GPU disk placement evaluates PV selector operators and OR terms" {
+ node='{"items":[{"metadata":{"name":"node-a","labels":{"size":"4"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}'
+ for operator in In NotIn Exists DoesNotExist Gt Lt; do
+ case "$operator" in
+ In) yes=4; no=5; key=size ;;
+ NotIn) yes=5; no=4; key=size ;;
+ Exists) yes=4; no=4; key=size ;;
+ DoesNotExist) yes=4; no=4; key=missing ;;
+ Gt) yes=3; no=4; key=size ;;
+ Lt) yes=5; no=4; key=size ;;
+ esac
+ volumes=$(jq --null-input --arg op "$operator" --arg key "$key" --arg value "$yes" '[{spec: {nodeAffinity: {required: {nodeSelectorTerms: [{matchExpressions: [{key: $key, operator: $op, values: [$value]}]}]}}}}]')
+ [ "$(printf '%s\n' "$node" | jq --exit-status --raw-output --argjson volumes "$volumes" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq)" = node-a ]
+ case "$operator" in
+ Exists) bad=$(printf '%s\n' "$volumes" | jq '.[0].spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].key = "missing"') ;;
+ DoesNotExist) bad=$(printf '%s\n' "$volumes" | jq '.[0].spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].key = "size"') ;;
+ *) bad=$(printf '%s\n' "$volumes" | jq --arg value "$no" '.[0].spec.nodeAffinity.required.nodeSelectorTerms[0].matchExpressions[0].values = [$value]') ;;
+ esac
+ if printf '%s\n' "$node" | jq --exit-status --raw-output --argjson volumes "$bad" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq; then
+ echo "FAIL: $operator mismatch was accepted" >&2
+ false
+ fi
+ either=$(jq --null-input --argjson yes "$volumes" --argjson no "$bad" '$yes | .[0].spec.nodeAffinity.required.nodeSelectorTerms += $no[0].spec.nodeAffinity.required.nodeSelectorTerms')
+ [ "$(printf '%s\n' "$node" | jq --exit-status --raw-output --argjson volumes "$either" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq)" = node-a ]
+ done
+}
+
+@test "GPU disk placement requires every expression and field within a term" {
+ nodes='{"items":[{"metadata":{"name":"node-a","labels":{"zone":"a"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"node-b","labels":{"zone":"b","gpu":"true"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}'
+ volumes='[{"spec":{"nodeAffinity":{"required":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"zone","operator":"In","values":["a","b"]},{"key":"gpu","operator":"Exists"}]}]}}}}]'
+ [ "$(printf '%s\n' "$nodes" | jq --exit-status --raw-output --argjson volumes "$volumes" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq)" = node-b ]
+ fields=$(printf '%s\n' "$volumes" | jq '.[0].spec.nodeAffinity.required.nodeSelectorTerms[0] = {matchFields: [{key: "metadata.name", operator: "In", values: ["node-a"]}, {key: "metadata.name", operator: "NotIn", values: ["node-a"]}]}')
+ if printf '%s\n' "$nodes" | jq --exit-status --raw-output --argjson volumes "$fields" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq; then
+ echo "FAIL: conflicting PV matchFields were accepted" >&2
+ false
+ fi
+}
+
+@test "GPU disk placement skips nonnumeric labels in numeric comparisons" {
+ nodes='{"items":[{"metadata":{"name":"node-a","labels":{"size":"large"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"node-b","labels":{"size":"4"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}'
+ volumes='[{"spec":{"nodeAffinity":{"required":{"nodeSelectorTerms":[{"matchExpressions":[{"key":"size","operator":"Gt","values":["3"]}]}]}}}}]'
+ [ "$(printf '%s\n' "$nodes" | jq --exit-status --raw-output --argjson volumes "$volumes" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq)" = node-b ]
+}
+
+@test "GPU disk numeric affinity uses exact signed int64 comparisons" {
+ while IFS='|' read -r operator threshold rejected accepted; do
+ nodes=$(jq --null-input --arg rejected "$rejected" --arg accepted "$accepted" '[[$rejected, "node-a"], [$accepted, "node-b"]] | map({metadata: {name: .[1], labels: {size: .[0]}}, status: {conditions: [{type: "Ready", status: "True"}]}}) | {items: .} | if $rejected == "" then del(.items[0].metadata.labels.size) else . end')
+ volumes=$(jq --null-input --arg op "$operator" --arg threshold "$threshold" '[{spec: {nodeAffinity: {required: {nodeSelectorTerms: [{matchExpressions: [{key: "size", operator: $op, values: [$threshold]}]}]}}}}]')
+ [ "$(printf '%s\n' "$nodes" | jq --exit-status --raw-output --argjson volumes "$volumes" --from-file hack/e2e-chainsaw/gpuclass/disk-node.jq)" = node-b ]
+ done <<'CASES'
+Gt|3|4.5|4
+Lt|3|2.5|2
+Gt|3|4e1|4
+Gt|3|0x10|4
+Gt|3|4_0|4
+Gt|3|9223372036854775808|4
+Lt|0|-9223372036854775809|-1
+Gt|9007199254740992|9007199254740992|9007199254740993
+Gt|9223372036854775806|9223372036854775806|9223372036854775807
+Lt|9223372036854775807|9223372036854775807|9223372036854775806
+Gt|0003|0003|0004
+Gt|-2|-3|-1
+Lt|-2|-1|-3
+Gt|0|-0|1
+Gt|-9223372036854775808|-9223372036854775808|-9223372036854775807
+Lt|-9223372036854775807|-9223372036854775807|-9223372036854775808
+Gt|+3|+3|+4
+Gt|3||4
+Lt|0|-0|-1
+Gt|3|0003|0004
+Gt|0003|3|4
+Gt|3|+3|+4
+CASES
+}
+
+@test "GPU suite guards VM install and upgrade retry histories" {
+ suite="$PWD/hack/e2e-chainsaw/gpuclass"
+ tmp=$(mktemp --directory)
+ printf '%s\n' '#!/bin/bash' 'case " $* " in *" --namespace tenant-test-gpu get helmreleases --field-selector metadata.name=vm-instance-gpu --watch "*) cat "$GPU_HR_EVENT" ;; *) exit 1 ;; esac' > "$tmp/kubectl"
+ chmod +x "$tmp/kubectl"
+ for step in stopped-vm-reserves-the-class-count launcher-workload-reports-the-gpu-units; do
+ count=$(STEP="$step" yq '[.spec.steps[] | select(.name == strenv(STEP)) | .try[].script.content // "" | select(contains("wait-release.sh"))] | length' "$suite/chainsaw-test.yaml")
+ [ "$count" = 1 ]
+ content=$(STEP="$step" yq --unwrapScalar '.spec.steps[] | select(.name == strenv(STEP)) | .try[].script.content // "" | select(contains("wait-release.sh"))' "$suite/chainsaw-test.yaml")
+ for state in ready failed; do
+ jq --null-input --compact-output --arg state "$state" '{metadata: {generation: 1}, spec: {install: {strategy: {name: "RetryOnFailure"}}}, status: {observedGeneration: 1, conditions: [{type: "Ready", status: "True"}], history: [{status: (if $state == "failed" then "failed" else "deployed" end)}]}}' > "$tmp/event"
+ if (cd "$suite" && PATH="$tmp:$PATH" GPU_HR_EVENT="$tmp/event" COZY_E2E_CONTEXT=offline KUBECONFIG=/dev/null sh -c "$content"); then
+ [ "$state" = ready ]
+ else
+ [ "$state" = failed ]
+ fi
+ done
+ done
+ unlink "$tmp/kubectl"
+ unlink "$tmp/event"
+ rmdir "$tmp"
+}
diff --git a/hack/select-e2e.sh b/hack/select-e2e.sh
index fcb7651e3..b98ad57eb 100755
--- a/hack/select-e2e.sh
+++ b/hack/select-e2e.sh
@@ -245,14 +245,10 @@ escalate_to_full_suite() {
# this list read as longer than the set of real exceptions.
src_to_suites() {
case "$1" in
- vm-instance-application) echo vminstance ;;
- # The vminstance suite creates a VMDisk and asserts the DataVolume behind it
- # (hack/e2e-chainsaw/vminstance/vmdisk.yaml, vmdisk-vmi.yaml), so it is the
- # suite that covers this source and everything reaching it, kubevirt-cdi
- # included. Without the entry the source maps to a `vm-disk` suite that does
- # not exist, is dropped by intersect_suites(), and a CDI change reaches
- # nothing runnable through it.
- vm-disk-application) echo vminstance ;;
+ vm-instance-application) echo "vminstance gpuclass" ;;
+ # Both VM suites populate VMDisk resources, so they cover CDI changes too.
+ vm-disk-application) echo "vminstance gpuclass" ;;
+ gpu-operator) echo gpuclass ;;
# The VMInstance and VMDisk backup round-trips are the only Tests that run a
# Velero backup and restore; without this the source maps to a `velero`
# suite that does not exist and a Velero change runs neither.
@@ -266,8 +262,8 @@ src_to_suites() {
# apps/kubernetes-nodes also belongs to computeplane-application, which has
# a suite; without this entry an edit there selects computeplane alone,
# whose pools go through the module allowlist, and drops the suites that
- # create KubernetesNodes natively.
- kubernetes-nodes-application) echo "kubernetes-latest kubernetes-previous computeplane" ;;
+ # create KubernetesNodes natively. gpuclass checks its GPU budget admission.
+ kubernetes-nodes-application) echo "kubernetes-latest kubernetes-previous computeplane gpuclass" ;;
securitygroup-controller) echo securitygroup ;;
# The kafka app source covers both the app suite and the metadata-backup
# roundtrip, so an edit to packages/apps/kafka (or kafka-operator, which
diff --git a/hack/select-e2e_test.bats b/hack/select-e2e_test.bats
index 6ce198830..56cb73eaa 100644
--- a/hack/select-e2e_test.bats
+++ b/hack/select-e2e_test.bats
@@ -127,18 +127,45 @@ assert_full_suite() {
# kubevirt-cdi reaches vm-disk-application and nothing else runnable, so
# until vm-disk-application mapped to a suite every CDI change ran all of
# them. The vminstance suite creates a VMDisk and asserts the DataVolume
- # behind it, so it is the suite that covers CDI.
+ # behind it, so it is the suite that covers CDI; gpuclass attaches a
+ # VMDisk to its VMs too.
tmp=$(mktemp -d)
cp -r packages/core/platform/sources "$tmp/sources"
echo "packages/system/kubevirt-cdi/values.yaml" > "$tmp/diff"
output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources")
- [ "$output" = "vminstance" ]
+ [ "$output" = "gpuclass vminstance" ]
closure=$(hack/select-install.sh "$output" "$tmp/sources")
printf '%s\n' "$closure" | tr ' ' '\n' | grep -Fxq cozystack.vm-disk-application
printf '%s\n' "$closure" | tr ' ' '\n' | grep -Fxq cozystack.kubevirt-cdi
rm -rf "$tmp"
}
+@test "a GPU change selects gpuclass, a change downstream of gpu-operator does not" {
+ # gpu-operator and the VM applications map to gpuclass by name. hami
+ # depends on gpu-operator, so the walk must not carry it the other way.
+ tmp=$(mktemp -d)
+ cp -r packages/core/platform/sources "$tmp/sources"
+ for path in packages/system/gpu-operator/values.yaml packages/apps/kubernetes-nodes/values.yaml packages/apps/vm-instance/values.yaml; do
+ echo "$path" > "$tmp/diff"
+ output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources")
+ if ! printf '%s\n' "$output" | tr ' ' '\n' | grep -Fxq gpuclass; then
+ echo "$path does not select gpuclass: $output" >&2
+ return 1
+ fi
+ done
+ echo "packages/apps/vm-instance/values.yaml" > "$tmp/diff"
+ output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources")
+ [ "$output" = "gpuclass vminstance" ]
+ echo "packages/system/hami/values.yaml" > "$tmp/diff"
+ output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources")
+ [ -n "$output" ]
+ if printf '%s\n' "$output" | tr ' ' '\n' | grep -Fxq gpuclass; then
+ echo "a hami change selects gpuclass: $output" >&2
+ return 1
+ fi
+ rm -rf "$tmp"
+}
+
@test "a CNI change selects every suite the graph can reach" {
# Named for what it measures rather than for the full suite it once claimed.
# `packages/system/cilium` is owned by cozystack.networking and resolves
@@ -1085,30 +1112,29 @@ assert_full_suite() {
rm -rf "$tmp"
}
-@test "a kubernetes-nodes pool chart change selects computeplane and both kubernetes suites" {
+@test "a kubernetes-nodes pool chart change selects native, module and GPU budget suites" {
# apps/kubernetes-nodes belongs to kubernetes-nodes-application and to
# computeplane-application. The kubernetes suites create KubernetesNodes
# natively and computeplane renders pools through its module, so a pool
- # chart change has to reach all three, not only the source that happens
- # to have a suite of its own name.
+ # chart change must also reach the GPU budget admission checks.
tmp=$(mktemp -d)
cp -r packages/core/platform/sources "$tmp/sources"
echo "packages/apps/kubernetes-nodes/values.yaml" > "$tmp/diff"
output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources")
assert_selection "a pool chart change must exercise native and module pools" \
- "$output" "computeplane kubernetes-latest kubernetes-previous"
+ "$output" "computeplane gpuclass kubernetes-latest kubernetes-previous"
rm -rf "$tmp"
}
-@test "a kubernetes-nodes-rd change selects computeplane and both kubernetes suites" {
+@test "a kubernetes-nodes-rd change selects native, module and GPU budget suites" {
# kubernetes-nodes-rd ships only in kubernetes-nodes-application, which
- # maps to the same three suites as the pool chart it registers.
+ # maps to the same suites as the pool chart it registers.
tmp=$(mktemp -d)
cp -r packages/core/platform/sources "$tmp/sources"
echo "packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml" > "$tmp/diff"
output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources")
assert_selection "a kubernetes-nodes-rd change must exercise native and module pools" \
- "$output" "computeplane kubernetes-latest kubernetes-previous"
+ "$output" "computeplane gpuclass kubernetes-latest kubernetes-previous"
rm -rf "$tmp"
}
diff --git a/hack/select-install.sh b/hack/select-install.sh
index 15acf8aca..aa3962396 100755
--- a/hack/select-install.sh
+++ b/hack/select-install.sh
@@ -127,6 +127,9 @@ suite_to_source() {
vminstance)
echo "cozystack.vm-instance-application cozystack.vm-disk-application"
return ;;
+ gpuclass)
+ echo "cozystack.gpu-operator cozystack.vm-instance-application cozystack.vm-disk-application cozystack.kubernetes-nodes-application"
+ return ;;
securitygroup) echo cozystack.securitygroup-controller ; return ;;
kafka-metadata) echo cozystack.kafka-application ; return ;;
esac
diff --git a/hack/select-install_test.bats b/hack/select-install_test.bats
index 462abb60a..235d69f04 100644
--- a/hack/select-install_test.bats
+++ b/hack/select-install_test.bats
@@ -106,6 +106,14 @@ assert_output_graph_error() {
printf '%s\n' "$output" | tr ' ' '\n' | grep -Fxq cozystack.kubevirt-cdi
}
+@test "gpuclass installs gpu-operator and its application APIs" {
+ output=$(hack/select-install.sh "gpuclass")
+ assert_contains_package "$output" cozystack.kubernetes-nodes-application
+ assert_contains_package "$output" cozystack.gpu-operator
+ assert_contains_package "$output" cozystack.vm-instance-application
+ assert_contains_package "$output" cozystack.vm-disk-application
+}
+
@test "multiple suites union their closures" {
output=$(hack/select-install.sh "postgres kafka")
assert_contains_package "$output" cozystack.postgres-application