mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-03 10:37:21 +09:00
refactor(gpu): move the KubeVirt table, quota and coverage readers to the model
The admission webhooks and the option sources need the same readings the controller makes: the permittedHostDevices table KubeVirt checks GPU entries against and the table's GPU names, a tenant's declared resourceQuotas, and the units a VMI's attached claims cover for budgets. They move into the model unchanged, and the controller reads them from there, so each has one copy. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
9 files changed
+249
-97
No files matched your search
@@ -28,7 +28,9 @@ type Config struct {
|
||||
Enabled bool
|
||||
|
||||
// ResourceNames is the hoster's gpu.resourceNames list. The effective
|
||||
// set adds class and agent-reported names (model.ResourceNames).
|
||||
// set (model.ResourceNames) adds the classes' names, the names node
|
||||
// agents report and the NVIDIA (10DE) and mdev names of the KubeVirt
|
||||
// permittedHostDevices table.
|
||||
ResourceNames []string
|
||||
|
||||
// RawResourceNames is the hoster's gpu.rawResourceNames; empty means
|
||||
|
||||
@@ -17,8 +17,6 @@ limitations under the License.
|
||||
package controller
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
@@ -34,12 +32,6 @@ var (
|
||||
kubeVirtGVK = schema.GroupVersionKind{Group: "kubevirt.io", Version: "v1", Kind: "KubeVirt"}
|
||||
)
|
||||
|
||||
// The KubeVirt CR the platform renders permittedHostDevices into.
|
||||
const (
|
||||
kubeVirtNamespace = "cozy-kubevirt"
|
||||
kubeVirtName = "kubevirt"
|
||||
)
|
||||
|
||||
// vmi is the part of a VirtualMachineInstance the controller reads.
|
||||
type vmi struct {
|
||||
Name, Namespace string
|
||||
@@ -137,66 +129,3 @@ func vmFrom(u *unstructured.Unstructured) *vm {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type permittedEntry struct {
|
||||
resourceName, vendorSelector string
|
||||
mediated bool
|
||||
}
|
||||
|
||||
// permittedEntries reads the KubeVirt CR's PCI and mediated
|
||||
// permittedHostDevices lists, where GPUs are declared, and false when the
|
||||
// table is unset.
|
||||
func permittedEntries(kv *unstructured.Unstructured) ([]permittedEntry, bool) {
|
||||
table, ok, _ := unstructured.NestedMap(kv.Object, "spec", "configuration", "permittedHostDevices")
|
||||
if !ok || table == nil {
|
||||
return nil, false
|
||||
}
|
||||
var out []permittedEntry
|
||||
for _, list := range []string{"pciHostDevices", "mediatedDevices"} {
|
||||
entries, _, _ := unstructured.NestedSlice(table, list)
|
||||
for _, e := range entries {
|
||||
m, ok := e.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name, ok := m["resourceName"].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
selector, _ := m["pciVendorSelector"].(string)
|
||||
out = append(out, permittedEntry{resourceName: name, vendorSelector: selector, mediated: list == "mediatedDevices"})
|
||||
}
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// permittedResourceNames returns the resource names of the KubeVirt CR's
|
||||
// permittedHostDevices table, or nil when the table is unset: KubeVirt
|
||||
// checks GPU entries against it only when it is non-nil
|
||||
// (renderresources.go validatePermittedHostDevices in v1.9.0).
|
||||
func permittedResourceNames(kv *unstructured.Unstructured) sets.Set[string] {
|
||||
entries, ok := permittedEntries(kv)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
out := sets.New[string]()
|
||||
for _, e := range entries {
|
||||
out.Insert(e.resourceName)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// permittedGPUNames returns the table's GPU names: its NVIDIA (vendor
|
||||
// 10DE) PCI entries and its mediated entries. They join the effective GPU
|
||||
// resource-name set besides the hoster's list; a NIC in the table, such as
|
||||
// nvidia.com/hostdev of a ConnectX VF, is not a GPU.
|
||||
func permittedGPUNames(kv *unstructured.Unstructured) []string {
|
||||
entries, _ := permittedEntries(kv)
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if e.mediated || strings.HasPrefix(strings.ToUpper(e.vendorSelector), "10DE:") {
|
||||
out = append(out, e.resourceName)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -18,7 +18,6 @@ package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -28,7 +27,6 @@ import (
|
||||
helmv2 "github.com/fluxcd/helm-controller/api/v2"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
@@ -196,9 +194,9 @@ func (r *Reconciler) load(ctx context.Context) (*world, error) {
|
||||
}
|
||||
var tableGPUs []string
|
||||
for i := range kvs.Items {
|
||||
if kvs.Items[i].GetNamespace() == kubeVirtNamespace && kvs.Items[i].GetName() == kubeVirtName {
|
||||
w.permitted = permittedResourceNames(&kvs.Items[i])
|
||||
tableGPUs = permittedGPUNames(&kvs.Items[i])
|
||||
if kvs.Items[i].GetNamespace() == model.KubeVirtNamespace && kvs.Items[i].GetName() == model.KubeVirtName {
|
||||
w.permitted = model.PermittedResourceNames(&kvs.Items[i])
|
||||
tableGPUs = model.PermittedGPUNames(&kvs.Items[i])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -296,18 +294,12 @@ func (r *Reconciler) tenantQuotas(ctx context.Context) (map[string]corev1.Resour
|
||||
if hr.Spec.Values == nil {
|
||||
continue
|
||||
}
|
||||
var values struct {
|
||||
ResourceQuotas map[string]resource.Quantity `json:"resourceQuotas"`
|
||||
}
|
||||
ns := tenantNamespace(hr.Namespace, strings.TrimPrefix(hr.Name, tenantNamespacePrefix))
|
||||
if err := json.Unmarshal(hr.Spec.Values.Raw, &values); err != nil {
|
||||
bad[ns] = fmt.Errorf("decode values of tenant %s/%s: %w", hr.Namespace, hr.Name, err)
|
||||
rl, err := model.DeclaredQuotas(hr.Spec.Values.Raw)
|
||||
if err != nil {
|
||||
bad[ns] = fmt.Errorf("tenant %s/%s: %w", hr.Namespace, hr.Name, err)
|
||||
continue
|
||||
}
|
||||
rl := corev1.ResourceList{}
|
||||
for k, v := range values.ResourceQuotas {
|
||||
rl[corev1.ResourceName(k)] = v
|
||||
}
|
||||
out[ns] = rl
|
||||
}
|
||||
return out, bad, nil
|
||||
@@ -455,24 +447,18 @@ func (w *world) buildSnapshot(quotas map[string]corev1.ResourceList, pods []core
|
||||
// A Lost claim covers its VMI's pod for budgets, which already count
|
||||
// the claim, but not for node capacity: it holds no units, so the pod
|
||||
// still running on its node holds them itself.
|
||||
coveredForBudget := map[types.NamespacedName]map[corev1.ResourceName]int64{}
|
||||
coveredForBudget := model.CoveredForBudget(w.claims, s.Classes)
|
||||
coveredForCapacity := map[types.NamespacedName]map[corev1.ResourceName]int64{}
|
||||
add := func(m map[types.NamespacedName]map[corev1.ResourceName]int64, k types.NamespacedName, rn corev1.ResourceName, n int64) {
|
||||
if m[k] == nil {
|
||||
m[k] = map[corev1.ResourceName]int64{}
|
||||
}
|
||||
m[k][rn] += n
|
||||
}
|
||||
for _, d := range w.claims {
|
||||
rn := w.resourceName(d.Spec.ClassName)
|
||||
if d.Status.AttachedTo == nil || rn == "" {
|
||||
if d.Status.AttachedTo == nil || rn == "" || !holds(d) {
|
||||
continue
|
||||
}
|
||||
k := key(d.Namespace, d.Status.AttachedTo.VMI)
|
||||
add(coveredForBudget, k, rn, int64(d.Spec.Count))
|
||||
if holds(d) {
|
||||
add(coveredForCapacity, k, rn, int64(d.Spec.Count))
|
||||
if coveredForCapacity[k] == nil {
|
||||
coveredForCapacity[k] = map[corev1.ResourceName]int64{}
|
||||
}
|
||||
coveredForCapacity[k][rn] += int64(d.Spec.Count)
|
||||
}
|
||||
for i := range pods {
|
||||
p := &pods[i]
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
)
|
||||
|
||||
// The KubeVirt CR the platform renders permittedHostDevices into.
|
||||
const (
|
||||
KubeVirtNamespace = "cozy-kubevirt"
|
||||
KubeVirtName = "kubevirt"
|
||||
)
|
||||
|
||||
type permittedEntry struct {
|
||||
resourceName, vendorSelector string
|
||||
mediated bool
|
||||
}
|
||||
|
||||
// permittedEntries reads the KubeVirt CR's PCI and mediated
|
||||
// permittedHostDevices lists, where GPUs are declared, and false when the
|
||||
// table is unset.
|
||||
func permittedEntries(kv *unstructured.Unstructured) ([]permittedEntry, bool) {
|
||||
table, ok, _ := unstructured.NestedMap(kv.Object, "spec", "configuration", "permittedHostDevices")
|
||||
if !ok || table == nil {
|
||||
return nil, false
|
||||
}
|
||||
var out []permittedEntry
|
||||
for _, list := range []string{"pciHostDevices", "mediatedDevices"} {
|
||||
entries, _, _ := unstructured.NestedSlice(table, list)
|
||||
for _, e := range entries {
|
||||
m, ok := e.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name, ok := m["resourceName"].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
selector, _ := m["pciVendorSelector"].(string)
|
||||
out = append(out, permittedEntry{resourceName: name, vendorSelector: selector, mediated: list == "mediatedDevices"})
|
||||
}
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
// PermittedResourceNames returns the resource names of the KubeVirt CR's
|
||||
// permittedHostDevices table, or nil when the table is unset: KubeVirt
|
||||
// checks GPU entries against it only when it is non-nil
|
||||
// (renderresources.go validatePermittedHostDevices in v1.9.0).
|
||||
func PermittedResourceNames(kv *unstructured.Unstructured) sets.Set[string] {
|
||||
entries, ok := permittedEntries(kv)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
out := sets.New[string]()
|
||||
for _, e := range entries {
|
||||
out.Insert(e.resourceName)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PermittedGPUNames returns the table's GPU names: its NVIDIA (vendor
|
||||
// 10DE) PCI entries and its mediated entries. They join the effective GPU
|
||||
// resource-name set besides the hoster's list; a NIC in the table, such as
|
||||
// nvidia.com/hostdev of a ConnectX VF, is not a GPU.
|
||||
func PermittedGPUNames(kv *unstructured.Unstructured) []string {
|
||||
entries, _ := permittedEntries(kv)
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if e.mediated || strings.HasPrefix(strings.ToUpper(e.vendorSelector), "10DE:") {
|
||||
out = append(out, e.resourceName)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
)
|
||||
|
||||
func TestPermittedNames(t *testing.T) {
|
||||
kv := &unstructured.Unstructured{Object: map[string]any{}}
|
||||
if PermittedResourceNames(kv) != nil || PermittedGPUNames(kv) != nil {
|
||||
t.Fatal("an unset table permits everything and names no GPU")
|
||||
}
|
||||
_ = unstructured.SetNestedField(kv.Object, map[string]any{
|
||||
"pciHostDevices": []any{
|
||||
map[string]any{"pciVendorSelector": "10de:26B9", "resourceName": "nvidia.com/L40S"},
|
||||
map[string]any{"pciVendorSelector": "15B3:101E", "resourceName": "nvidia.com/hostdev"},
|
||||
},
|
||||
"mediatedDevices": []any{
|
||||
map[string]any{"mdevNameSelector": "GRID A100-40C", "resourceName": "nvidia.com/GRID_A100-40C"},
|
||||
},
|
||||
"usbHostDevices": []any{map[string]any{"resourceName": "kubevirt.io/usb"}},
|
||||
}, "spec", "configuration", "permittedHostDevices")
|
||||
|
||||
want := sets.New("nvidia.com/L40S", "nvidia.com/hostdev", "nvidia.com/GRID_A100-40C")
|
||||
if got := PermittedResourceNames(kv); !got.Equal(want) {
|
||||
t.Fatalf("permitted = %v, want %v", sets.List(got), sets.List(want))
|
||||
}
|
||||
got := PermittedGPUNames(kv)
|
||||
slices.Sort(got)
|
||||
if !slices.Equal(got, []string{"nvidia.com/GRID_A100-40C", "nvidia.com/L40S"}) {
|
||||
t.Fatalf("GPU names = %v: the 10DE PCI and mdev entries, not the NIC", got)
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,8 @@ import (
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
|
||||
gpuv1alpha1 "github.com/cozystack/cozystack/api/gpu/v1alpha1"
|
||||
)
|
||||
|
||||
// ActivePods is a VMI's status.activePods: the UIDs of the pods KubeVirt
|
||||
@@ -56,3 +58,24 @@ func LauncherVMI(p *corev1.Pod, activePods func(types.NamespacedName) (sets.Set[
|
||||
}
|
||||
return types.NamespacedName{}, false, nil
|
||||
}
|
||||
|
||||
// CoveredForBudget maps a VMI to the units, per resource name, that its
|
||||
// attached claims account for, so its launcher pod does not count them
|
||||
// against a raw budget key a second time. Every attached claim counts, a
|
||||
// Lost one included, since it still counts against budgets itself; node
|
||||
// capacity needs another rule, where a Lost claim holds nothing.
|
||||
func CoveredForBudget(claims []*gpuv1alpha1.GPUClaim, classes map[string]Class) map[types.NamespacedName]map[corev1.ResourceName]int64 {
|
||||
out := map[types.NamespacedName]map[corev1.ResourceName]int64{}
|
||||
for _, d := range claims {
|
||||
rn := classes[d.Spec.ClassName].ResourceName
|
||||
if d.Status.AttachedTo == nil || rn == "" {
|
||||
continue
|
||||
}
|
||||
k := types.NamespacedName{Namespace: d.Namespace, Name: d.Status.AttachedTo.VMI}
|
||||
if out[k] == nil {
|
||||
out[k] = map[corev1.ResourceName]int64{}
|
||||
}
|
||||
out[k][rn] += int64(d.Spec.Count)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
package model
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
@@ -24,6 +25,8 @@ import (
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
|
||||
gpuv1alpha1 "github.com/cozystack/cozystack/api/gpu/v1alpha1"
|
||||
)
|
||||
|
||||
func TestLauncherVMI(t *testing.T) {
|
||||
@@ -65,3 +68,26 @@ func TestActivePods(t *testing.T) {
|
||||
t.Fatalf("ActivePods = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoveredForBudget(t *testing.T) {
|
||||
claim := func(name, class string, count int32, phase gpuv1alpha1.GPUClaimPhase, vmi string) *gpuv1alpha1.GPUClaim {
|
||||
d := &gpuv1alpha1.GPUClaim{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-a", Name: name},
|
||||
Spec: gpuv1alpha1.GPUClaimSpec{ClassName: class, Count: count}, Status: gpuv1alpha1.GPUClaimStatus{Phase: phase}}
|
||||
if vmi != "" {
|
||||
d.Status.AttachedTo = &gpuv1alpha1.GPUClaimAttachment{VMI: vmi}
|
||||
}
|
||||
return d
|
||||
}
|
||||
classes := map[string]Class{"h100": {Name: "h100", ResourceName: "nvidia.com/H100"}, "dra": {Name: "dra"}}
|
||||
got := CoveredForBudget([]*gpuv1alpha1.GPUClaim{
|
||||
claim("a", "h100", 2, gpuv1alpha1.GPUClaimAttached, "vm"),
|
||||
claim("b", "h100", 1, gpuv1alpha1.GPUClaimLost, "vm"),
|
||||
claim("c", "h100", 4, gpuv1alpha1.GPUClaimBound, ""),
|
||||
claim("d", "dra", 1, gpuv1alpha1.GPUClaimAttached, "vm"),
|
||||
claim("e", "gone", 1, gpuv1alpha1.GPUClaimAttached, "vm"),
|
||||
}, classes)
|
||||
want := map[types.NamespacedName]map[corev1.ResourceName]int64{{Namespace: "tenant-a", Name: "vm"}: {"nvidia.com/H100": 3}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v, want %v: attached claims of a device-plugin class, a Lost one included", got, want)
|
||||
}
|
||||
}
|
||||
@@ -17,9 +17,12 @@ limitations under the License.
|
||||
package model
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
|
||||
"github.com/cozystack/cozystack/internal/gpu"
|
||||
)
|
||||
@@ -87,3 +90,24 @@ func RawQuotaLimit(quota corev1.ResourceList, name corev1.ResourceName) (int64,
|
||||
}
|
||||
return limit, found
|
||||
}
|
||||
|
||||
// DeclaredQuotas reads resourceQuotas from a tenant HelmRelease's values,
|
||||
// every key as written: GPU keys are never rendered into a namespace
|
||||
// ResourceQuota, so the release is where budgets are declared. A value
|
||||
// that is not a quantity fails the read rather than reading as no budget.
|
||||
func DeclaredQuotas(values []byte) (corev1.ResourceList, error) {
|
||||
out := corev1.ResourceList{}
|
||||
if len(values) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
var v struct {
|
||||
ResourceQuotas map[string]resource.Quantity `json:"resourceQuotas"`
|
||||
}
|
||||
if err := json.Unmarshal(values, &v); err != nil {
|
||||
return nil, fmt.Errorf("decode tenant resourceQuotas: %w", err)
|
||||
}
|
||||
for k, q := range v.ResourceQuotas {
|
||||
out[corev1.ResourceName(k)] = q
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -114,3 +114,20 @@ func TestRawQuotaLimit(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeclaredQuotas(t *testing.T) {
|
||||
got, err := DeclaredQuotas([]byte(`{"host":"x","resourceQuotas":{"gpu.cozystack.io/total":"4","cpu":"2"}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := corev1.ResourceList{"gpu.cozystack.io/total": resource.MustParse("4"), "cpu": resource.MustParse("2")}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("got %v, want %v", got, want)
|
||||
}
|
||||
if got, err := DeclaredQuotas(nil); err != nil || len(got) != 0 {
|
||||
t.Fatalf("no values: %v, %v", got, err)
|
||||
}
|
||||
if _, err := DeclaredQuotas([]byte(`{"resourceQuotas":{"gpu.cozystack.io/total":"four"}}`)); err == nil {
|
||||
t.Fatal("a value that is not a quantity must fail the read, not read as no budget")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user