Files
cozystack/internal/gpu/controller/usage_test.go
Aleksei Sviridkin 4225be12cc feat(gpu): bind, move and release GPU claims in the leader
The binder is the only writer of a claim's placement, so budget and
capacity checks that admission already ran cannot be raced past by two
replicas. Every event collapses into one pass over all GPU objects,
which serializes the decisions without per-object locks, and claims
are read past the cache so a unit booked in the last pass is never
booked again from a stale view.

A VM starts through its launcher pod, which the pod webhook holds at a
scheduling gate. The pass creates the VMI's template claims, gathers
the whole set onto one node that every class of the set tolerates and
has room on, then pins the pod there and removes the gate in one
update, since the API server lets a pod's node affinity change only
while the pod is gated. A set that cannot be placed keeps its pod
gated with a condition on its claims and a Warning event on the VMI,
once per change of reason, so the VM waits for capacity instead of
being refused into KubeVirt's backoff. The set goes only to a node the
pod's own node selector and required affinity accept and, when the VMI
has a volume bound to a node's PersistentVolume, that volume reaches.
Gated pods go before moves and new orders.

A pin cannot be undone, so a pinned pod its node refuses for lack of
resources is deleted and the node is skipped for that set for ten
minutes, or until its allocatable, taints or readiness change. The
template claims of the set go with its failed VMI, so the controller
also keeps the rejection in memory for the VM's next VMI. Only a VMI
whose pod is already pinned is deleted to free a claim whose node
cannot keep it; a VMI whose pod is gated has its set gathered instead.

The full pass also moves held sets off nodes that cannot keep them,
rebinds Lost sets before new orders, and binds new orders onto the
fullest node with room. Finalizers and usage records run with the
feature disabled too, so turning GPU off never hangs a namespace or
class deletion. Each usage record is owned by its claim and sits under
no WorkloadMonitor, so no tenant label can add or drop one, and
garbage collection removes it with the claim. Without a monitor label
the Workload controller would take such a record for an orphan, so it
now leaves a record owned by a GPUClaim to that owner.

The claims one VM names move, go Lost and are rebound as a set. Every
pass restores that invariant before it places anything, gathering a
set left partly Lost, by a write that failed half way or by a VMI that
ended next to held members, onto one node or sending all of it to
Lost, so each later step only ever changes whole sets.

A Lost claim already counts against its tenant's budgets, so the pod
of its still running VMI is not counted again under a raw GPU key; for
node capacity it is, since the claim itself holds no units. A pod is a
VMI's launcher only when the VMI lists it in status.activePods,
because whoever creates a pod sets its owner reference. The KubeVirt
watches start once KubeVirt serves its kinds, whose CRDs a fresh
install adds after this controller starts.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
2026-10-26 11:22:36 +03:00

228 lines
8.4 KiB
Go

/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controller
import (
"context"
"strings"
"testing"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/validation"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
gpuv1alpha1 "github.com/cozystack/cozystack/api/gpu/v1alpha1"
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
cozycontroller "github.com/cozystack/cozystack/internal/controller"
"github.com/cozystack/cozystack/internal/gpu"
)
func (e *env) record(claim string) *cozyv1alpha1.Workload {
e.t.Helper()
w := &cozyv1alpha1.Workload{}
if err := e.c.Get(context.Background(), key(ns, "gpu-"+claim), w); err != nil {
return nil
}
return w
}
func (e *env) recordExists(claim string) bool { return e.record(claim) != nil }
func TestUsageRecordExistsExactlyWhileBoundOrAttached(t *testing.T) {
e := newEnv(t,
class("l40s", rnL40S),
node("n1", map[string]int64{rnL40S: 2}),
claim("bound", "l40s", 2, bound("n1")),
claim("attached", "l40s", 1, bound("n1"), attached("vm1")),
vmiObj("vm1", "Running", "n1", ""),
claim("pending", "l40s", 4),
claim("lost", "l40s", 1, lost("gone")),
)
e.pass()
for name, want := range map[string]bool{"bound": true, "attached": true, "pending": false, "lost": false} {
if got := e.recordExists(name); got != want {
t.Errorf("record for %s exists = %v, want %v", name, got, want)
}
}
rec := e.record("bound")
if rec.Status.Kind != gpu.WorkloadKind || len(rec.Labels) != 0 {
t.Fatalf("record %+v", rec)
}
if q := rec.Status.Resources["gpu.cozystack.io/l40s"]; q.Cmp(resource.MustParse("2")) != 0 {
t.Fatalf("resources %v, want gpu.cozystack.io/l40s: 2", rec.Status.Resources)
}
monitors := &cozyv1alpha1.WorkloadMonitorList{}
if err := e.c.List(t.Context(), monitors); err != nil || len(monitors.Items) != 0 {
t.Fatalf("GPU usage records need no WorkloadMonitor, found %d (%v)", len(monitors.Items), err)
}
}
// The record's owner is its claim, so garbage collection removes it with
// the claim even when this controller is down.
func TestUsageRecordIsOwnedByItsClaim(t *testing.T) {
d := claim("a", "l40s", 1, bound("n1"))
e := newEnv(t, class("l40s", rnL40S), node("n1", map[string]int64{rnL40S: 2}), d)
e.pass()
refs := e.record("a").OwnerReferences
if len(refs) != 1 || refs[0].Kind != "GPUClaim" || refs[0].APIVersion != gpuv1alpha1.GroupVersion.String() ||
refs[0].UID != d.UID || refs[0].Name != "a" {
t.Fatalf("owner references %+v do not name the claim", refs)
}
e.delete(d)
e.pass()
if e.recordExists("a") {
t.Fatal("record kept after its claim was released")
}
}
func TestUsageRecordIgnoresTenantLabelsOnTheClaim(t *testing.T) {
d := claim("a", "l40s", 1, bound("n1"))
d.Labels = map[string]string{"workloads.cozystack.io/monitor": "other", "team": "x"}
e := newEnv(t, class("l40s", rnL40S), node("n1", map[string]int64{rnL40S: 2}), d)
e.pass()
if rec := e.record("a"); rec == nil || len(rec.Labels) != 0 {
t.Fatalf("record labels %v", rec)
}
}
func TestUsageRecordGoesWhenTheClaimIsLost(t *testing.T) {
n1 := node("n1", map[string]int64{rnL40S: 1})
e := newEnv(t,
class("l40s", rnL40S),
n1,
claim("a", "l40s", 1, bound("n1")),
)
e.pass()
if !e.recordExists("a") {
t.Fatal("a Bound claim has no usage record")
}
e.delete(n1)
e.pass()
expectPlacement(t, e.claim("a"), gpuv1alpha1.GPUClaimLost, "n1")
if e.recordExists("a") {
t.Fatal("a Lost claim keeps its usage record")
}
}
// A GPU record has no monitor label, which the Workload controller would
// otherwise read as an orphan; it leaves a record owned by a GPUClaim to
// its owner.
func TestUsageRecordSurvivesWorkloadReconciles(t *testing.T) {
e := newEnv(t, class("l40s", rnL40S), node("n1", map[string]int64{rnL40S: 2}), claim("a", "l40s", 1, bound("n1")))
e.pass()
workloads := &cozycontroller.WorkloadReconciler{Client: e.c, Scheme: e.c.Scheme()}
if _, err := workloads.Reconcile(context.Background(), ctrl.Request{NamespacedName: key(ns, "gpu-a")}); err != nil {
t.Fatalf("Workload reconcile: %v", err)
}
rec := e.record("a")
if rec == nil || rec.Status.Kind != gpu.WorkloadKind {
t.Fatalf("record after the reconcile: %+v", rec)
}
}
// Template claim names reach 253 characters for long VMI names, and such a
// claim still needs a usage record.
func TestUsageRecordOfAClaimWithAMaximalNameFitsAndIsStable(t *testing.T) {
// The cut lands right after the dot, which must not end a name part.
long := strings.Repeat("w", 237) + "." + strings.Repeat("y", 15)
if len(long) != 253 {
t.Fatalf("fixture is %d characters", len(long))
}
d := claim(long, "l40s", 1, bound("n1"))
e := newEnv(t, class("l40s", rnL40S), node("n1", map[string]int64{rnL40S: 2}), d)
e.pass()
name := recordName(d)
if errs := validation.IsDNS1123Subdomain(name); len(errs) > 0 {
t.Fatalf("record name %q: %v", name, errs)
}
if other := claim(long[:252]+"z", "l40s", 1); recordName(other) == name {
t.Fatal("two claims differing only past the cut share a record")
}
rec := &cozyv1alpha1.Workload{}
if err := e.c.Get(t.Context(), key(ns, name), rec); err != nil {
t.Fatalf("record: %v", err)
}
e.pass()
records := &cozyv1alpha1.WorkloadList{}
_ = e.c.List(t.Context(), records)
if len(records.Items) != 1 {
t.Fatalf("%d records after a second pass, want 1", len(records.Items))
}
}
// A tenant names its own claims, so it can name one after the hashed
// record of another; neither that nor two long names sharing the cut may
// make two held claims share one record.
func TestUsageRecordNamesOfLongAndCraftedClaimsDoNotCollide(t *testing.T) {
long := strings.Repeat("a", 253)
crafted := strings.TrimPrefix(recordName(claim(long, "l40s", 1)), "gpu-")
crafted = strings.TrimPrefix(crafted, "gpuh-")
if len(crafted) > 253 {
t.Fatalf("fixture too long: %d", len(crafted))
}
twin := strings.Repeat("a", 238) + strings.Repeat("b", 15)
names := map[string]string{}
for _, n := range []string{long, crafted, twin} {
r := recordName(claim(n, "l40s", 1))
if other, ok := names[r]; ok {
t.Fatalf("claims %q and %q share record %q", other, n, r)
}
names[r] = n
if errs := validation.IsDNS1123Subdomain(r); len(errs) > 0 {
t.Fatalf("record %q: %v", r, errs)
}
}
if !strings.HasPrefix(recordName(claim(long, "l40s", 1)), "gpuh-") {
t.Fatal("a hashed record name must carry its own prefix")
}
if h := strings.TrimPrefix(recordName(claim(long, "l40s", 1)), "gpuh-"); len(h) < 33 || len(strings.Split(h, "-")[len(strings.Split(h, "-"))-1]) != 32 {
t.Fatalf("hashed record %q must end in a 128-bit hash", h)
}
}
// A record a stale cache shows as missing already exists; the create's
// AlreadyExists is not a failure of the pass.
func TestAUsageRecordCreateOnAStaleCacheIsNotAnError(t *testing.T) {
e := newEnv(t, class("l40s", rnL40S), node("n1", map[string]int64{rnL40S: 2}), claim("a", "l40s", 1, bound("n1")))
e.r.Client = interceptor.NewClient(e.c.(client.WithWatch), interceptor.Funcs{
Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
if _, ok := obj.(*cozyv1alpha1.Workload); ok {
return apierrors.NewAlreadyExists(cozyv1alpha1.GroupVersion.WithResource("workloads").GroupResource(), obj.GetName())
}
return c.Create(ctx, obj, opts...)
},
})
if _, err := e.r.Reconcile(t.Context(), ctrl.Request{NamespacedName: passKey}); err != nil {
t.Fatalf("pass: %v", err)
}
}
// Records of CPU, memory and storage share the namespace and the kind.
func TestOnlyRecordsOwnedByAClaimArePruned(t *testing.T) {
other := &cozyv1alpha1.Workload{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "vm-instance-vm1",
Labels: map[string]string{"workloads.cozystack.io/monitor": "vm1"}}}
e := newEnv(t, class("l40s", rnL40S), node("n1", map[string]int64{rnL40S: 1}), other)
e.pass()
if e.gone(other) {
t.Fatal("a usage record no GPUClaim owns was deleted")
}
}