diff --git a/internal/gpu/config.go b/internal/gpu/config.go index 2b200361c..4beb50b21 100644 --- a/internal/gpu/config.go +++ b/internal/gpu/config.go @@ -28,7 +28,9 @@ type Config struct { Enabled bool // ResourceNames is the hoster's gpu.resourceNames list. The effective - // set adds class and agent-reported names (model.ResourceNames). + // set (model.ResourceNames) adds the classes' names, the names node + // agents report and the NVIDIA (10DE) and mdev names of the KubeVirt + // permittedHostDevices table. ResourceNames []string // RawResourceNames is the hoster's gpu.rawResourceNames; empty means diff --git a/internal/gpu/controller/kubevirt.go b/internal/gpu/controller/kubevirt.go index 9240ea74a..6312022ab 100644 --- a/internal/gpu/controller/kubevirt.go +++ b/internal/gpu/controller/kubevirt.go @@ -17,8 +17,6 @@ limitations under the License. package controller import ( - "strings" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -34,12 +32,6 @@ var ( kubeVirtGVK = schema.GroupVersionKind{Group: "kubevirt.io", Version: "v1", Kind: "KubeVirt"} ) -// The KubeVirt CR the platform renders permittedHostDevices into. -const ( - kubeVirtNamespace = "cozy-kubevirt" - kubeVirtName = "kubevirt" -) - // vmi is the part of a VirtualMachineInstance the controller reads. type vmi struct { Name, Namespace string @@ -137,66 +129,3 @@ func vmFrom(u *unstructured.Unstructured) *vm { } return out } - -type permittedEntry struct { - resourceName, vendorSelector string - mediated bool -} - -// permittedEntries reads the KubeVirt CR's PCI and mediated -// permittedHostDevices lists, where GPUs are declared, and false when the -// table is unset. -func permittedEntries(kv *unstructured.Unstructured) ([]permittedEntry, bool) { - table, ok, _ := unstructured.NestedMap(kv.Object, "spec", "configuration", "permittedHostDevices") - if !ok || table == nil { - return nil, false - } - var out []permittedEntry - for _, list := range []string{"pciHostDevices", "mediatedDevices"} { - entries, _, _ := unstructured.NestedSlice(table, list) - for _, e := range entries { - m, ok := e.(map[string]any) - if !ok { - continue - } - name, ok := m["resourceName"].(string) - if !ok { - continue - } - selector, _ := m["pciVendorSelector"].(string) - out = append(out, permittedEntry{resourceName: name, vendorSelector: selector, mediated: list == "mediatedDevices"}) - } - } - return out, true -} - -// permittedResourceNames returns the resource names of the KubeVirt CR's -// permittedHostDevices table, or nil when the table is unset: KubeVirt -// checks GPU entries against it only when it is non-nil -// (renderresources.go validatePermittedHostDevices in v1.9.0). -func permittedResourceNames(kv *unstructured.Unstructured) sets.Set[string] { - entries, ok := permittedEntries(kv) - if !ok { - return nil - } - out := sets.New[string]() - for _, e := range entries { - out.Insert(e.resourceName) - } - return out -} - -// permittedGPUNames returns the table's GPU names: its NVIDIA (vendor -// 10DE) PCI entries and its mediated entries. They join the effective GPU -// resource-name set besides the hoster's list; a NIC in the table, such as -// nvidia.com/hostdev of a ConnectX VF, is not a GPU. -func permittedGPUNames(kv *unstructured.Unstructured) []string { - entries, _ := permittedEntries(kv) - var out []string - for _, e := range entries { - if e.mediated || strings.HasPrefix(strings.ToUpper(e.vendorSelector), "10DE:") { - out = append(out, e.resourceName) - } - } - return out -} diff --git a/internal/gpu/controller/state.go b/internal/gpu/controller/state.go index 087863ecd..8abbff3db 100644 --- a/internal/gpu/controller/state.go +++ b/internal/gpu/controller/state.go @@ -18,7 +18,6 @@ package controller import ( "context" - "encoding/json" "fmt" "slices" "sort" @@ -28,7 +27,6 @@ import ( helmv2 "github.com/fluxcd/helm-controller/api/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" @@ -196,9 +194,9 @@ func (r *Reconciler) load(ctx context.Context) (*world, error) { } var tableGPUs []string for i := range kvs.Items { - if kvs.Items[i].GetNamespace() == kubeVirtNamespace && kvs.Items[i].GetName() == kubeVirtName { - w.permitted = permittedResourceNames(&kvs.Items[i]) - tableGPUs = permittedGPUNames(&kvs.Items[i]) + if kvs.Items[i].GetNamespace() == model.KubeVirtNamespace && kvs.Items[i].GetName() == model.KubeVirtName { + w.permitted = model.PermittedResourceNames(&kvs.Items[i]) + tableGPUs = model.PermittedGPUNames(&kvs.Items[i]) } } @@ -296,18 +294,12 @@ func (r *Reconciler) tenantQuotas(ctx context.Context) (map[string]corev1.Resour if hr.Spec.Values == nil { continue } - var values struct { - ResourceQuotas map[string]resource.Quantity `json:"resourceQuotas"` - } ns := tenantNamespace(hr.Namespace, strings.TrimPrefix(hr.Name, tenantNamespacePrefix)) - if err := json.Unmarshal(hr.Spec.Values.Raw, &values); err != nil { - bad[ns] = fmt.Errorf("decode values of tenant %s/%s: %w", hr.Namespace, hr.Name, err) + rl, err := model.DeclaredQuotas(hr.Spec.Values.Raw) + if err != nil { + bad[ns] = fmt.Errorf("tenant %s/%s: %w", hr.Namespace, hr.Name, err) continue } - rl := corev1.ResourceList{} - for k, v := range values.ResourceQuotas { - rl[corev1.ResourceName(k)] = v - } out[ns] = rl } return out, bad, nil @@ -455,24 +447,18 @@ func (w *world) buildSnapshot(quotas map[string]corev1.ResourceList, pods []core // A Lost claim covers its VMI's pod for budgets, which already count // the claim, but not for node capacity: it holds no units, so the pod // still running on its node holds them itself. - coveredForBudget := map[types.NamespacedName]map[corev1.ResourceName]int64{} + coveredForBudget := model.CoveredForBudget(w.claims, s.Classes) coveredForCapacity := map[types.NamespacedName]map[corev1.ResourceName]int64{} - add := func(m map[types.NamespacedName]map[corev1.ResourceName]int64, k types.NamespacedName, rn corev1.ResourceName, n int64) { - if m[k] == nil { - m[k] = map[corev1.ResourceName]int64{} - } - m[k][rn] += n - } for _, d := range w.claims { rn := w.resourceName(d.Spec.ClassName) - if d.Status.AttachedTo == nil || rn == "" { + if d.Status.AttachedTo == nil || rn == "" || !holds(d) { continue } k := key(d.Namespace, d.Status.AttachedTo.VMI) - add(coveredForBudget, k, rn, int64(d.Spec.Count)) - if holds(d) { - add(coveredForCapacity, k, rn, int64(d.Spec.Count)) + if coveredForCapacity[k] == nil { + coveredForCapacity[k] = map[corev1.ResourceName]int64{} } + coveredForCapacity[k][rn] += int64(d.Spec.Count) } for i := range pods { p := &pods[i] diff --git a/internal/gpu/model/kubevirt.go b/internal/gpu/model/kubevirt.go new file mode 100644 index 000000000..2b7e6c62b --- /dev/null +++ b/internal/gpu/model/kubevirt.go @@ -0,0 +1,93 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/sets" +) + +// The KubeVirt CR the platform renders permittedHostDevices into. +const ( + KubeVirtNamespace = "cozy-kubevirt" + KubeVirtName = "kubevirt" +) + +type permittedEntry struct { + resourceName, vendorSelector string + mediated bool +} + +// permittedEntries reads the KubeVirt CR's PCI and mediated +// permittedHostDevices lists, where GPUs are declared, and false when the +// table is unset. +func permittedEntries(kv *unstructured.Unstructured) ([]permittedEntry, bool) { + table, ok, _ := unstructured.NestedMap(kv.Object, "spec", "configuration", "permittedHostDevices") + if !ok || table == nil { + return nil, false + } + var out []permittedEntry + for _, list := range []string{"pciHostDevices", "mediatedDevices"} { + entries, _, _ := unstructured.NestedSlice(table, list) + for _, e := range entries { + m, ok := e.(map[string]any) + if !ok { + continue + } + name, ok := m["resourceName"].(string) + if !ok { + continue + } + selector, _ := m["pciVendorSelector"].(string) + out = append(out, permittedEntry{resourceName: name, vendorSelector: selector, mediated: list == "mediatedDevices"}) + } + } + return out, true +} + +// PermittedResourceNames returns the resource names of the KubeVirt CR's +// permittedHostDevices table, or nil when the table is unset: KubeVirt +// checks GPU entries against it only when it is non-nil +// (renderresources.go validatePermittedHostDevices in v1.9.0). +func PermittedResourceNames(kv *unstructured.Unstructured) sets.Set[string] { + entries, ok := permittedEntries(kv) + if !ok { + return nil + } + out := sets.New[string]() + for _, e := range entries { + out.Insert(e.resourceName) + } + return out +} + +// PermittedGPUNames returns the table's GPU names: its NVIDIA (vendor +// 10DE) PCI entries and its mediated entries. They join the effective GPU +// resource-name set besides the hoster's list; a NIC in the table, such as +// nvidia.com/hostdev of a ConnectX VF, is not a GPU. +func PermittedGPUNames(kv *unstructured.Unstructured) []string { + entries, _ := permittedEntries(kv) + var out []string + for _, e := range entries { + if e.mediated || strings.HasPrefix(strings.ToUpper(e.vendorSelector), "10DE:") { + out = append(out, e.resourceName) + } + } + return out +} diff --git a/internal/gpu/model/kubevirt_test.go b/internal/gpu/model/kubevirt_test.go new file mode 100644 index 000000000..c8acd757d --- /dev/null +++ b/internal/gpu/model/kubevirt_test.go @@ -0,0 +1,52 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "slices" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/sets" +) + +func TestPermittedNames(t *testing.T) { + kv := &unstructured.Unstructured{Object: map[string]any{}} + if PermittedResourceNames(kv) != nil || PermittedGPUNames(kv) != nil { + t.Fatal("an unset table permits everything and names no GPU") + } + _ = unstructured.SetNestedField(kv.Object, map[string]any{ + "pciHostDevices": []any{ + map[string]any{"pciVendorSelector": "10de:26B9", "resourceName": "nvidia.com/L40S"}, + map[string]any{"pciVendorSelector": "15B3:101E", "resourceName": "nvidia.com/hostdev"}, + }, + "mediatedDevices": []any{ + map[string]any{"mdevNameSelector": "GRID A100-40C", "resourceName": "nvidia.com/GRID_A100-40C"}, + }, + "usbHostDevices": []any{map[string]any{"resourceName": "kubevirt.io/usb"}}, + }, "spec", "configuration", "permittedHostDevices") + + want := sets.New("nvidia.com/L40S", "nvidia.com/hostdev", "nvidia.com/GRID_A100-40C") + if got := PermittedResourceNames(kv); !got.Equal(want) { + t.Fatalf("permitted = %v, want %v", sets.List(got), sets.List(want)) + } + got := PermittedGPUNames(kv) + slices.Sort(got) + if !slices.Equal(got, []string{"nvidia.com/GRID_A100-40C", "nvidia.com/L40S"}) { + t.Fatalf("GPU names = %v: the 10DE PCI and mdev entries, not the NIC", got) + } +} diff --git a/internal/gpu/model/launcher.go b/internal/gpu/model/launcher.go index eebabcd94..b7bc147ec 100644 --- a/internal/gpu/model/launcher.go +++ b/internal/gpu/model/launcher.go @@ -21,6 +21,8 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + + gpuv1alpha1 "github.com/cozystack/cozystack/api/gpu/v1alpha1" ) // ActivePods is a VMI's status.activePods: the UIDs of the pods KubeVirt @@ -56,3 +58,24 @@ func LauncherVMI(p *corev1.Pod, activePods func(types.NamespacedName) (sets.Set[ } return types.NamespacedName{}, false, nil } + +// CoveredForBudget maps a VMI to the units, per resource name, that its +// attached claims account for, so its launcher pod does not count them +// against a raw budget key a second time. Every attached claim counts, a +// Lost one included, since it still counts against budgets itself; node +// capacity needs another rule, where a Lost claim holds nothing. +func CoveredForBudget(claims []*gpuv1alpha1.GPUClaim, classes map[string]Class) map[types.NamespacedName]map[corev1.ResourceName]int64 { + out := map[types.NamespacedName]map[corev1.ResourceName]int64{} + for _, d := range claims { + rn := classes[d.Spec.ClassName].ResourceName + if d.Status.AttachedTo == nil || rn == "" { + continue + } + k := types.NamespacedName{Namespace: d.Namespace, Name: d.Status.AttachedTo.VMI} + if out[k] == nil { + out[k] = map[corev1.ResourceName]int64{} + } + out[k][rn] += int64(d.Spec.Count) + } + return out +} diff --git a/internal/gpu/model/launcher_test.go b/internal/gpu/model/launcher_test.go index 1bcdf0dae..3271637f9 100644 --- a/internal/gpu/model/launcher_test.go +++ b/internal/gpu/model/launcher_test.go @@ -17,6 +17,7 @@ limitations under the License. package model import ( + "reflect" "testing" corev1 "k8s.io/api/core/v1" @@ -24,6 +25,8 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + + gpuv1alpha1 "github.com/cozystack/cozystack/api/gpu/v1alpha1" ) func TestLauncherVMI(t *testing.T) { @@ -65,3 +68,26 @@ func TestActivePods(t *testing.T) { t.Fatalf("ActivePods = %v", got) } } + +func TestCoveredForBudget(t *testing.T) { + claim := func(name, class string, count int32, phase gpuv1alpha1.GPUClaimPhase, vmi string) *gpuv1alpha1.GPUClaim { + d := &gpuv1alpha1.GPUClaim{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-a", Name: name}, + Spec: gpuv1alpha1.GPUClaimSpec{ClassName: class, Count: count}, Status: gpuv1alpha1.GPUClaimStatus{Phase: phase}} + if vmi != "" { + d.Status.AttachedTo = &gpuv1alpha1.GPUClaimAttachment{VMI: vmi} + } + return d + } + classes := map[string]Class{"h100": {Name: "h100", ResourceName: "nvidia.com/H100"}, "dra": {Name: "dra"}} + got := CoveredForBudget([]*gpuv1alpha1.GPUClaim{ + claim("a", "h100", 2, gpuv1alpha1.GPUClaimAttached, "vm"), + claim("b", "h100", 1, gpuv1alpha1.GPUClaimLost, "vm"), + claim("c", "h100", 4, gpuv1alpha1.GPUClaimBound, ""), + claim("d", "dra", 1, gpuv1alpha1.GPUClaimAttached, "vm"), + claim("e", "gone", 1, gpuv1alpha1.GPUClaimAttached, "vm"), + }, classes) + want := map[types.NamespacedName]map[corev1.ResourceName]int64{{Namespace: "tenant-a", Name: "vm"}: {"nvidia.com/H100": 3}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v: attached claims of a device-plugin class, a Lost one included", got, want) + } +} diff --git a/internal/gpu/model/quota.go b/internal/gpu/model/quota.go index e9ea841cb..238131170 100644 --- a/internal/gpu/model/quota.go +++ b/internal/gpu/model/quota.go @@ -17,9 +17,12 @@ limitations under the License. package model import ( + "encoding/json" + "fmt" "strings" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "github.com/cozystack/cozystack/internal/gpu" ) @@ -87,3 +90,24 @@ func RawQuotaLimit(quota corev1.ResourceList, name corev1.ResourceName) (int64, } return limit, found } + +// DeclaredQuotas reads resourceQuotas from a tenant HelmRelease's values, +// every key as written: GPU keys are never rendered into a namespace +// ResourceQuota, so the release is where budgets are declared. A value +// that is not a quantity fails the read rather than reading as no budget. +func DeclaredQuotas(values []byte) (corev1.ResourceList, error) { + out := corev1.ResourceList{} + if len(values) == 0 { + return out, nil + } + var v struct { + ResourceQuotas map[string]resource.Quantity `json:"resourceQuotas"` + } + if err := json.Unmarshal(values, &v); err != nil { + return nil, fmt.Errorf("decode tenant resourceQuotas: %w", err) + } + for k, q := range v.ResourceQuotas { + out[corev1.ResourceName(k)] = q + } + return out, nil +} diff --git a/internal/gpu/model/quota_test.go b/internal/gpu/model/quota_test.go index 65c88eed9..79215f023 100644 --- a/internal/gpu/model/quota_test.go +++ b/internal/gpu/model/quota_test.go @@ -114,3 +114,20 @@ func TestRawQuotaLimit(t *testing.T) { }) } } + +func TestDeclaredQuotas(t *testing.T) { + got, err := DeclaredQuotas([]byte(`{"host":"x","resourceQuotas":{"gpu.cozystack.io/total":"4","cpu":"2"}}`)) + if err != nil { + t.Fatal(err) + } + want := corev1.ResourceList{"gpu.cozystack.io/total": resource.MustParse("4"), "cpu": resource.MustParse("2")} + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + if got, err := DeclaredQuotas(nil); err != nil || len(got) != 0 { + t.Fatalf("no values: %v, %v", got, err) + } + if _, err := DeclaredQuotas([]byte(`{"resourceQuotas":{"gpu.cozystack.io/total":"four"}}`)); err == nil { + t.Fatal("a value that is not a quantity must fail the read, not read as no budget") + } +}