mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-07 04:23:50 +09:00
feat(gpu): read class entries and the GPU resource-name set
Malformed class annotations must fail rather than silently boot a VM without its requested GPUs. Admission also needs to distinguish GPUs from unrelated host devices when refusing additional requests beside class entries. No single resource-name source identifies every GPU: a pod-only resource may appear only in the hoster's list, while KubeVirt's permitted-device table can also contain NICs. Sharing these interpretations prevents the consumers from accepting different annotations or treating a NIC as a GPU. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
5 files changed
+201
No files matched your search
@@ -16,6 +16,10 @@ limitations under the License.
|
||||
|
||||
package gpu
|
||||
|
||||
// MaxVMIUnits bounds tenant-controlled per-unit admission expansion. It is
|
||||
// an implementation safety limit, not a class capacity or tenant budget.
|
||||
const MaxVMIUnits = 1024
|
||||
|
||||
const (
|
||||
// LabelGPUReference, valued LabelValueTrue, marks a VMI template with
|
||||
// class entries; KubeVirt copies it to the VMI and its virt-launcher
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apimachinery/pkg/util/validation"
|
||||
)
|
||||
|
||||
// ClassEntry is one entry of the cozystack.io/gpus annotation: count GPUs
|
||||
// of a class.
|
||||
type ClassEntry struct {
|
||||
ClassName string `json:"className"`
|
||||
Count int32 `json:"count"`
|
||||
}
|
||||
|
||||
// ParseClassEntries reads the cozystack.io/gpus annotation. Count defaults
|
||||
// to 1. An empty list, an unknown field or an entry without a valid class
|
||||
// name is an error, so a mistyped annotation never reads as no GPUs.
|
||||
func ParseClassEntries(raw string) ([]ClassEntry, error) {
|
||||
dec := json.NewDecoder(bytes.NewReader([]byte(raw)))
|
||||
dec.DisallowUnknownFields()
|
||||
var entries []struct {
|
||||
ClassName string `json:"className"`
|
||||
Count *int32 `json:"count"`
|
||||
}
|
||||
if err := dec.Decode(&entries); err != nil {
|
||||
return nil, fmt.Errorf("decode: %w", err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return nil, errors.New("no entries")
|
||||
}
|
||||
out := make([]ClassEntry, 0, len(entries))
|
||||
for i, e := range entries {
|
||||
if errs := validation.IsDNS1123Label(e.ClassName); len(errs) > 0 {
|
||||
return nil, fmt.Errorf("entry %d: className %q: %s", i, e.ClassName, strings.Join(errs, "; "))
|
||||
}
|
||||
count := int32(1)
|
||||
if e.Count != nil {
|
||||
count = *e.Count
|
||||
}
|
||||
if count < 1 {
|
||||
return nil, fmt.Errorf("entry %d: count %d is below 1", i, count)
|
||||
}
|
||||
out = append(out, ClassEntry{ClassName: e.ClassName, Count: count})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// GPUResourceNames is the set of GPU resource names: the hoster's list,
|
||||
// the classes' resource names and the GPU names of the KubeVirt
|
||||
// permittedHostDevices table. None of them replaces another.
|
||||
func GPUResourceNames(hoster []string, classes []Class, table []string) sets.Set[string] {
|
||||
out := sets.New[string]()
|
||||
for _, n := range hoster {
|
||||
out.Insert(n)
|
||||
}
|
||||
for _, c := range classes {
|
||||
out.Insert(string(c.ResourceName))
|
||||
}
|
||||
for _, n := range table {
|
||||
out.Insert(n)
|
||||
}
|
||||
out.Delete("")
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
)
|
||||
|
||||
func TestParseClassEntries(t *testing.T) {
|
||||
tests := []struct {
|
||||
name, raw string
|
||||
want []ClassEntry
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "two entries", raw: `[{"className":"h100","count":2},{"className":"l40s-24c"}]`, want: []ClassEntry{{ClassName: "h100", Count: 2}, {ClassName: "l40s-24c", Count: 1}}},
|
||||
{name: "empty annotation", raw: "", wantErr: true},
|
||||
{name: "empty list", raw: "[]", wantErr: true},
|
||||
{name: "not JSON", raw: "h100", wantErr: true},
|
||||
{name: "no class name", raw: `[{"count":1}]`, wantErr: true},
|
||||
{name: "class name not a DNS label", raw: `[{"className":"H100"}]`, wantErr: true},
|
||||
{name: "zero count", raw: `[{"className":"h100","count":0}]`, wantErr: true},
|
||||
{name: "negative count", raw: `[{"className":"h100","count":-1}]`, wantErr: true},
|
||||
{name: "unknown field", raw: `[{"className":"h100","name":"nvidia.com/x"}]`, wantErr: true},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := ParseClassEntries(tc.raw)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Fatalf("err = %v, wantErr %v", err, tc.wantErr)
|
||||
}
|
||||
if !tc.wantErr && !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("ParseClassEntries = %+v, want %+v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPUResourceNames(t *testing.T) {
|
||||
got := GPUResourceNames([]string{"amd.com/gpu", ""}, []Class{{ResourceName: h100Name}, {}}, []string{"nvidia.com/L40S"})
|
||||
if want := []string{"amd.com/gpu", "nvidia.com/GH100_H100", "nvidia.com/L40S"}; !reflect.DeepEqual(sets.List(got), want) {
|
||||
t.Errorf("GPUResourceNames = %v, want %v", sets.List(got), want)
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,8 @@ limitations under the License.
|
||||
package model
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
)
|
||||
@@ -50,3 +52,29 @@ func PermittedResourceNames(kv *unstructured.Unstructured) sets.Set[string] {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PermittedGPUNames returns the table's GPU names: its NVIDIA (vendor
|
||||
// 10DE) PCI entries and its mediated entries. A NIC in the table, such as
|
||||
// nvidia.com/hostdev of a ConnectX VF, is not a GPU.
|
||||
func PermittedGPUNames(kv *unstructured.Unstructured) []string {
|
||||
table, ok, _ := unstructured.NestedMap(kv.Object, "spec", "configuration", "permittedHostDevices")
|
||||
if !ok || table == nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, list := range []string{"pciHostDevices", "mediatedDevices"} {
|
||||
entries, _, _ := unstructured.NestedSlice(table, list)
|
||||
for _, e := range entries {
|
||||
m, ok := e.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name, _ := m["resourceName"].(string)
|
||||
selector, _ := m["pciVendorSelector"].(string)
|
||||
if name != "" && (list == "mediatedDevices" || strings.HasPrefix(strings.ToUpper(selector), "10DE:")) {
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
package model
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
@@ -48,3 +49,25 @@ func TestPermittedResourceNames(t *testing.T) {
|
||||
t.Fatalf("permitted = %v, want %v", sets.List(got), sets.List(want))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPermittedGPUNames(t *testing.T) {
|
||||
kv := &unstructured.Unstructured{Object: map[string]any{}}
|
||||
if got := PermittedGPUNames(kv); got != nil {
|
||||
t.Fatalf("an unset table names no GPU, got %v", got)
|
||||
}
|
||||
_ = unstructured.SetNestedField(kv.Object, map[string]any{
|
||||
"pciHostDevices": []any{
|
||||
map[string]any{"pciVendorSelector": "10de:26B9", "resourceName": "nvidia.com/L40S"},
|
||||
map[string]any{"pciVendorSelector": "10DE:2335", "resourceName": "nvidia.com/H200"},
|
||||
map[string]any{"pciVendorSelector": "15B3:101E", "resourceName": "nvidia.com/hostdev"},
|
||||
},
|
||||
"mediatedDevices": []any{
|
||||
map[string]any{"mdevNameSelector": "GRID A100-40C", "resourceName": "nvidia.com/GRID_A100-40C"},
|
||||
},
|
||||
}, "spec", "configuration", "permittedHostDevices")
|
||||
got := PermittedGPUNames(kv)
|
||||
slices.Sort(got)
|
||||
if want := []string{"nvidia.com/GRID_A100-40C", "nvidia.com/H200", "nvidia.com/L40S"}; !slices.Equal(got, want) {
|
||||
t.Fatalf("GPU names = %v, want %v: the 10DE PCI and mdev entries, not the NIC", got, want)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user