refactor(gpu): move the KubeVirt table, quota and coverage readers to the model

The admission webhooks and the option sources need the same readings
the controller makes: the permittedHostDevices table KubeVirt checks
GPU entries against and the table's GPU names, a tenant's declared
resourceQuotas, and the units a VMI's attached claims cover for
budgets. They move into the model unchanged, and the controller reads
them from there, so each has one copy.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
Aleksei Sviridkin committed 2026-10-26 11:53:07 +03:00
1 parent ebca80fe38
commit ca0c8944b2
9 files changed
+249 -97

No files matched your search

+3 -1
View File
@@ -28,7 +28,9 @@ type Config struct {
Enabled bool
// ResourceNames is the hoster's gpu.resourceNames list. The effective
// set adds class and agent-reported names (model.ResourceNames).
// set (model.ResourceNames) adds the classes' names, the names node
// agents report and the NVIDIA (10DE) and mdev names of the KubeVirt
// permittedHostDevices table.
ResourceNames []string
// RawResourceNames is the hoster's gpu.rawResourceNames; empty means
-71
View File
@@ -17,8 +17,6 @@ limitations under the License.
package controller
import (
"strings"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
@@ -34,12 +32,6 @@ var (
kubeVirtGVK = schema.GroupVersionKind{Group: "kubevirt.io", Version: "v1", Kind: "KubeVirt"}
)
// The KubeVirt CR the platform renders permittedHostDevices into.
const (
kubeVirtNamespace = "cozy-kubevirt"
kubeVirtName = "kubevirt"
)
// vmi is the part of a VirtualMachineInstance the controller reads.
type vmi struct {
Name, Namespace string
@@ -137,66 +129,3 @@ func vmFrom(u *unstructured.Unstructured) *vm {
}
return out
}
type permittedEntry struct {
resourceName, vendorSelector string
mediated bool
}
// permittedEntries reads the KubeVirt CR's PCI and mediated
// permittedHostDevices lists, where GPUs are declared, and false when the
// table is unset.
func permittedEntries(kv *unstructured.Unstructured) ([]permittedEntry, bool) {
table, ok, _ := unstructured.NestedMap(kv.Object, "spec", "configuration", "permittedHostDevices")
if !ok || table == nil {
return nil, false
}
var out []permittedEntry
for _, list := range []string{"pciHostDevices", "mediatedDevices"} {
entries, _, _ := unstructured.NestedSlice(table, list)
for _, e := range entries {
m, ok := e.(map[string]any)
if !ok {
continue
}
name, ok := m["resourceName"].(string)
if !ok {
continue
}
selector, _ := m["pciVendorSelector"].(string)
out = append(out, permittedEntry{resourceName: name, vendorSelector: selector, mediated: list == "mediatedDevices"})
}
}
return out, true
}
// permittedResourceNames returns the resource names of the KubeVirt CR's
// permittedHostDevices table, or nil when the table is unset: KubeVirt
// checks GPU entries against it only when it is non-nil
// (renderresources.go validatePermittedHostDevices in v1.9.0).
func permittedResourceNames(kv *unstructured.Unstructured) sets.Set[string] {
entries, ok := permittedEntries(kv)
if !ok {
return nil
}
out := sets.New[string]()
for _, e := range entries {
out.Insert(e.resourceName)
}
return out
}
// permittedGPUNames returns the table's GPU names: its NVIDIA (vendor
// 10DE) PCI entries and its mediated entries. They join the effective GPU
// resource-name set besides the hoster's list; a NIC in the table, such as
// nvidia.com/hostdev of a ConnectX VF, is not a GPU.
func permittedGPUNames(kv *unstructured.Unstructured) []string {
entries, _ := permittedEntries(kv)
var out []string
for _, e := range entries {
if e.mediated || strings.HasPrefix(strings.ToUpper(e.vendorSelector), "10DE:") {
out = append(out, e.resourceName)
}
}
return out
}
+11 -25
View File
@@ -18,7 +18,6 @@ package controller
import (
"context"
"encoding/json"
"fmt"
"slices"
"sort"
@@ -28,7 +27,6 @@ import (
helmv2 "github.com/fluxcd/helm-controller/api/v2"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
@@ -196,9 +194,9 @@ func (r *Reconciler) load(ctx context.Context) (*world, error) {
}
var tableGPUs []string
for i := range kvs.Items {
if kvs.Items[i].GetNamespace() == kubeVirtNamespace && kvs.Items[i].GetName() == kubeVirtName {
w.permitted = permittedResourceNames(&kvs.Items[i])
tableGPUs = permittedGPUNames(&kvs.Items[i])
if kvs.Items[i].GetNamespace() == model.KubeVirtNamespace && kvs.Items[i].GetName() == model.KubeVirtName {
w.permitted = model.PermittedResourceNames(&kvs.Items[i])
tableGPUs = model.PermittedGPUNames(&kvs.Items[i])
}
}
@@ -296,18 +294,12 @@ func (r *Reconciler) tenantQuotas(ctx context.Context) (map[string]corev1.Resour
if hr.Spec.Values == nil {
continue
}
var values struct {
ResourceQuotas map[string]resource.Quantity `json:"resourceQuotas"`
}
ns := tenantNamespace(hr.Namespace, strings.TrimPrefix(hr.Name, tenantNamespacePrefix))
if err := json.Unmarshal(hr.Spec.Values.Raw, &values); err != nil {
bad[ns] = fmt.Errorf("decode values of tenant %s/%s: %w", hr.Namespace, hr.Name, err)
rl, err := model.DeclaredQuotas(hr.Spec.Values.Raw)
if err != nil {
bad[ns] = fmt.Errorf("tenant %s/%s: %w", hr.Namespace, hr.Name, err)
continue
}
rl := corev1.ResourceList{}
for k, v := range values.ResourceQuotas {
rl[corev1.ResourceName(k)] = v
}
out[ns] = rl
}
return out, bad, nil
@@ -455,24 +447,18 @@ func (w *world) buildSnapshot(quotas map[string]corev1.ResourceList, pods []core
// A Lost claim covers its VMI's pod for budgets, which already count
// the claim, but not for node capacity: it holds no units, so the pod
// still running on its node holds them itself.
coveredForBudget := map[types.NamespacedName]map[corev1.ResourceName]int64{}
coveredForBudget := model.CoveredForBudget(w.claims, s.Classes)
coveredForCapacity := map[types.NamespacedName]map[corev1.ResourceName]int64{}
add := func(m map[types.NamespacedName]map[corev1.ResourceName]int64, k types.NamespacedName, rn corev1.ResourceName, n int64) {
if m[k] == nil {
m[k] = map[corev1.ResourceName]int64{}
}
m[k][rn] += n
}
for _, d := range w.claims {
rn := w.resourceName(d.Spec.ClassName)
if d.Status.AttachedTo == nil || rn == "" {
if d.Status.AttachedTo == nil || rn == "" || !holds(d) {
continue
}
k := key(d.Namespace, d.Status.AttachedTo.VMI)
add(coveredForBudget, k, rn, int64(d.Spec.Count))
if holds(d) {
add(coveredForCapacity, k, rn, int64(d.Spec.Count))
if coveredForCapacity[k] == nil {
coveredForCapacity[k] = map[corev1.ResourceName]int64{}
}
coveredForCapacity[k][rn] += int64(d.Spec.Count)
}
for i := range pods {
p := &pods[i]
+93
View File
@@ -0,0 +1,93 @@
/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package model
import (
"strings"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/sets"
)
// The KubeVirt CR the platform renders permittedHostDevices into.
const (
KubeVirtNamespace = "cozy-kubevirt"
KubeVirtName = "kubevirt"
)
type permittedEntry struct {
resourceName, vendorSelector string
mediated bool
}
// permittedEntries reads the KubeVirt CR's PCI and mediated
// permittedHostDevices lists, where GPUs are declared, and false when the
// table is unset.
func permittedEntries(kv *unstructured.Unstructured) ([]permittedEntry, bool) {
table, ok, _ := unstructured.NestedMap(kv.Object, "spec", "configuration", "permittedHostDevices")
if !ok || table == nil {
return nil, false
}
var out []permittedEntry
for _, list := range []string{"pciHostDevices", "mediatedDevices"} {
entries, _, _ := unstructured.NestedSlice(table, list)
for _, e := range entries {
m, ok := e.(map[string]any)
if !ok {
continue
}
name, ok := m["resourceName"].(string)
if !ok {
continue
}
selector, _ := m["pciVendorSelector"].(string)
out = append(out, permittedEntry{resourceName: name, vendorSelector: selector, mediated: list == "mediatedDevices"})
}
}
return out, true
}
// PermittedResourceNames returns the resource names of the KubeVirt CR's
// permittedHostDevices table, or nil when the table is unset: KubeVirt
// checks GPU entries against it only when it is non-nil
// (renderresources.go validatePermittedHostDevices in v1.9.0).
func PermittedResourceNames(kv *unstructured.Unstructured) sets.Set[string] {
entries, ok := permittedEntries(kv)
if !ok {
return nil
}
out := sets.New[string]()
for _, e := range entries {
out.Insert(e.resourceName)
}
return out
}
// PermittedGPUNames returns the table's GPU names: its NVIDIA (vendor
// 10DE) PCI entries and its mediated entries. They join the effective GPU
// resource-name set besides the hoster's list; a NIC in the table, such as
// nvidia.com/hostdev of a ConnectX VF, is not a GPU.
func PermittedGPUNames(kv *unstructured.Unstructured) []string {
entries, _ := permittedEntries(kv)
var out []string
for _, e := range entries {
if e.mediated || strings.HasPrefix(strings.ToUpper(e.vendorSelector), "10DE:") {
out = append(out, e.resourceName)
}
}
return out
}
+52
View File
@@ -0,0 +1,52 @@
/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package model
import (
"slices"
"testing"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/sets"
)
func TestPermittedNames(t *testing.T) {
kv := &unstructured.Unstructured{Object: map[string]any{}}
if PermittedResourceNames(kv) != nil || PermittedGPUNames(kv) != nil {
t.Fatal("an unset table permits everything and names no GPU")
}
_ = unstructured.SetNestedField(kv.Object, map[string]any{
"pciHostDevices": []any{
map[string]any{"pciVendorSelector": "10de:26B9", "resourceName": "nvidia.com/L40S"},
map[string]any{"pciVendorSelector": "15B3:101E", "resourceName": "nvidia.com/hostdev"},
},
"mediatedDevices": []any{
map[string]any{"mdevNameSelector": "GRID A100-40C", "resourceName": "nvidia.com/GRID_A100-40C"},
},
"usbHostDevices": []any{map[string]any{"resourceName": "kubevirt.io/usb"}},
}, "spec", "configuration", "permittedHostDevices")
want := sets.New("nvidia.com/L40S", "nvidia.com/hostdev", "nvidia.com/GRID_A100-40C")
if got := PermittedResourceNames(kv); !got.Equal(want) {
t.Fatalf("permitted = %v, want %v", sets.List(got), sets.List(want))
}
got := PermittedGPUNames(kv)
slices.Sort(got)
if !slices.Equal(got, []string{"nvidia.com/GRID_A100-40C", "nvidia.com/L40S"}) {
t.Fatalf("GPU names = %v: the 10DE PCI and mdev entries, not the NIC", got)
}
}
+23
View File
@@ -21,6 +21,8 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
gpuv1alpha1 "github.com/cozystack/cozystack/api/gpu/v1alpha1"
)
// ActivePods is a VMI's status.activePods: the UIDs of the pods KubeVirt
@@ -56,3 +58,24 @@ func LauncherVMI(p *corev1.Pod, activePods func(types.NamespacedName) (sets.Set[
}
return types.NamespacedName{}, false, nil
}
// CoveredForBudget maps a VMI to the units, per resource name, that its
// attached claims account for, so its launcher pod does not count them
// against a raw budget key a second time. Every attached claim counts, a
// Lost one included, since it still counts against budgets itself; node
// capacity needs another rule, where a Lost claim holds nothing.
func CoveredForBudget(claims []*gpuv1alpha1.GPUClaim, classes map[string]Class) map[types.NamespacedName]map[corev1.ResourceName]int64 {
out := map[types.NamespacedName]map[corev1.ResourceName]int64{}
for _, d := range claims {
rn := classes[d.Spec.ClassName].ResourceName
if d.Status.AttachedTo == nil || rn == "" {
continue
}
k := types.NamespacedName{Namespace: d.Namespace, Name: d.Status.AttachedTo.VMI}
if out[k] == nil {
out[k] = map[corev1.ResourceName]int64{}
}
out[k][rn] += int64(d.Spec.Count)
}
return out
}
+26
View File
@@ -17,6 +17,7 @@ limitations under the License.
package model
import (
"reflect"
"testing"
corev1 "k8s.io/api/core/v1"
@@ -24,6 +25,8 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
gpuv1alpha1 "github.com/cozystack/cozystack/api/gpu/v1alpha1"
)
func TestLauncherVMI(t *testing.T) {
@@ -65,3 +68,26 @@ func TestActivePods(t *testing.T) {
t.Fatalf("ActivePods = %v", got)
}
}
func TestCoveredForBudget(t *testing.T) {
claim := func(name, class string, count int32, phase gpuv1alpha1.GPUClaimPhase, vmi string) *gpuv1alpha1.GPUClaim {
d := &gpuv1alpha1.GPUClaim{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-a", Name: name},
Spec: gpuv1alpha1.GPUClaimSpec{ClassName: class, Count: count}, Status: gpuv1alpha1.GPUClaimStatus{Phase: phase}}
if vmi != "" {
d.Status.AttachedTo = &gpuv1alpha1.GPUClaimAttachment{VMI: vmi}
}
return d
}
classes := map[string]Class{"h100": {Name: "h100", ResourceName: "nvidia.com/H100"}, "dra": {Name: "dra"}}
got := CoveredForBudget([]*gpuv1alpha1.GPUClaim{
claim("a", "h100", 2, gpuv1alpha1.GPUClaimAttached, "vm"),
claim("b", "h100", 1, gpuv1alpha1.GPUClaimLost, "vm"),
claim("c", "h100", 4, gpuv1alpha1.GPUClaimBound, ""),
claim("d", "dra", 1, gpuv1alpha1.GPUClaimAttached, "vm"),
claim("e", "gone", 1, gpuv1alpha1.GPUClaimAttached, "vm"),
}, classes)
want := map[types.NamespacedName]map[corev1.ResourceName]int64{{Namespace: "tenant-a", Name: "vm"}: {"nvidia.com/H100": 3}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("got %v, want %v: attached claims of a device-plugin class, a Lost one included", got, want)
}
}
+24
View File
@@ -17,9 +17,12 @@ limitations under the License.
package model
import (
"encoding/json"
"fmt"
"strings"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"github.com/cozystack/cozystack/internal/gpu"
)
@@ -87,3 +90,24 @@ func RawQuotaLimit(quota corev1.ResourceList, name corev1.ResourceName) (int64,
}
return limit, found
}
// DeclaredQuotas reads resourceQuotas from a tenant HelmRelease's values,
// every key as written: GPU keys are never rendered into a namespace
// ResourceQuota, so the release is where budgets are declared. A value
// that is not a quantity fails the read rather than reading as no budget.
func DeclaredQuotas(values []byte) (corev1.ResourceList, error) {
out := corev1.ResourceList{}
if len(values) == 0 {
return out, nil
}
var v struct {
ResourceQuotas map[string]resource.Quantity `json:"resourceQuotas"`
}
if err := json.Unmarshal(values, &v); err != nil {
return nil, fmt.Errorf("decode tenant resourceQuotas: %w", err)
}
for k, q := range v.ResourceQuotas {
out[corev1.ResourceName(k)] = q
}
return out, nil
}
+17
View File
@@ -114,3 +114,20 @@ func TestRawQuotaLimit(t *testing.T) {
})
}
}
func TestDeclaredQuotas(t *testing.T) {
got, err := DeclaredQuotas([]byte(`{"host":"x","resourceQuotas":{"gpu.cozystack.io/total":"4","cpu":"2"}}`))
if err != nil {
t.Fatal(err)
}
want := corev1.ResourceList{"gpu.cozystack.io/total": resource.MustParse("4"), "cpu": resource.MustParse("2")}
if !reflect.DeepEqual(got, want) {
t.Fatalf("got %v, want %v", got, want)
}
if got, err := DeclaredQuotas(nil); err != nil || len(got) != 0 {
t.Fatalf("no values: %v, %v", got, err)
}
if _, err := DeclaredQuotas([]byte(`{"resourceQuotas":{"gpu.cozystack.io/total":"four"}}`)); err == nil {
t.Fatal("a value that is not a quantity must fail the read, not read as no budget")
}
}