Files
cozystack/internal/gpu/model/release.go
Aleksei Sviridkin 2aaf7f6872 feat(gpu): count the KubeVirt table's GPU names as GPU resource names
The NVIDIA and mediated entries of the KubeVirt permittedHostDevices
table are GPU resource names besides the hoster's gpu.resourceNames
list, whether the hoster sets one or not. Without that, a
GPU name only the table carries, on a node whose agent reports nothing
and with no class of its own, would not be a GPU resource name: under
forbidden the raw-name policy would let a VM or a pod request it
directly, and the VMI webhook would take it for a non-GPU host device.

The binder adds those names to the effective set it publishes in the
settings ConfigMap, which the policy and cozystack-api read, and the VMI
webhook adds them when it looks for a VMI's own GPU entries. A PCI entry
of another vendor, such as a ConnectX VF, stays out. Whether a class's
resource name is in the table at all is still checked against every
PCI and mediated entry.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
2026-10-21 13:31:33 +03:00

223 lines
7.7 KiB
Go

/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package model
import (
"encoding/json"
"fmt"
"strings"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
gpuv1alpha1 "github.com/cozystack/cozystack/api/gpu/v1alpha1"
)
// The KubeVirt CR the platform renders permittedHostDevices into.
const (
KubeVirtNamespace = "cozy-kubevirt"
KubeVirtName = "kubevirt"
)
// TenantReleaseRef maps the namespace a tenant owns to the name and
// namespace of that tenant's HelmRelease, the inverse of the tenant
// namespace naming in cozystack-api (REST.computeTenantNamespace):
//
// "tenant-root" -> ("tenant-root", "tenant-root") // root is its own parent
// "tenant-foo" -> ("tenant-foo", "tenant-root") // foo lives in root
// "tenant-foo-bar" -> ("tenant-bar", "tenant-foo") // bar lives in foo
// "tenant-foo-bar-baz" -> ("tenant-baz", "tenant-foo-bar")
func TenantReleaseRef(ownedNamespace, prefix string) (name, namespace string, ok bool) {
if ownedNamespace == rootTenantNamespace {
return prefix + "root", rootTenantNamespace, true
}
if !strings.HasPrefix(ownedNamespace, prefix) {
return "", "", false
}
segments := strings.Split(ownedNamespace, "-")
last := segments[len(segments)-1]
if len(segments) == 2 {
return prefix + last, rootTenantNamespace, true
}
return prefix + last, strings.Join(segments[:len(segments)-1], "-"), true
}
// TenantNamespaceOf is the namespace the tenant of a HelmRelease owns.
func TenantNamespaceOf(releaseNamespace, releaseName, prefix string) string {
app := strings.TrimPrefix(releaseName, prefix)
if releaseNamespace == rootTenantNamespace {
return prefix + app
}
return releaseNamespace + "-" + app
}
// DeclaredQuotas reads resourceQuotas from a tenant HelmRelease's values,
// every key as written: GPU keys are never rendered into a namespace
// ResourceQuota, so the release is where budgets are declared. A value that
// is not a quantity fails the read rather than reading as no budget.
func DeclaredQuotas(values []byte) (corev1.ResourceList, error) {
out := corev1.ResourceList{}
if len(values) == 0 {
return out, nil
}
var v struct {
ResourceQuotas map[string]resource.Quantity `json:"resourceQuotas"`
}
if err := json.Unmarshal(values, &v); err != nil {
return nil, fmt.Errorf("decode tenant resourceQuotas: %w", err)
}
for k, q := range v.ResourceQuotas {
out[corev1.ResourceName(k)] = q
}
return out, nil
}
// PermittedResourceNames returns the resource names of the KubeVirt CR's
// permittedHostDevices table, or nil when the table is unset and KubeVirt
// checks nothing (renderresources.go validatePermittedHostDevices in
// v1.9.0). It reads the PCI and mediated lists, where GPUs are declared.
func PermittedResourceNames(kv *unstructured.Unstructured) sets.Set[string] {
entries, ok := permittedEntries(kv)
if !ok {
return nil
}
out := sets.New[string]()
for _, e := range entries {
out.Insert(e.resourceName)
}
return out
}
// PermittedGPUNames returns, sorted, the table's GPU names: its NVIDIA
// (vendor 10DE) PCI entries and its mediated entries. They join the
// effective GPU resource-name set besides the hoster's gpu.resourceNames
// list, not in place of it; a NIC in the table, such as nvidia.com/hostdev
// of a ConnectX VF, is not a GPU.
func PermittedGPUNames(kv *unstructured.Unstructured) []string {
entries, ok := permittedEntries(kv)
if !ok {
return nil
}
out := sets.New[string]()
for _, e := range entries {
if e.mediated || strings.HasPrefix(strings.ToUpper(e.vendorSelector), "10DE:") {
out.Insert(e.resourceName)
}
}
return sets.List(out)
}
type permittedEntry struct {
resourceName, vendorSelector string
mediated bool
}
func permittedEntries(kv *unstructured.Unstructured) ([]permittedEntry, bool) {
table, ok, _ := unstructured.NestedMap(kv.Object, "spec", "configuration", "permittedHostDevices")
if !ok || table == nil {
return nil, false
}
var out []permittedEntry
for _, list := range []string{"pciHostDevices", "mediatedDevices"} {
entries, _, _ := unstructured.NestedSlice(table, list)
for _, e := range entries {
m, ok := e.(map[string]any)
if !ok {
continue
}
name, ok := m["resourceName"].(string)
if !ok {
continue
}
selector, _ := m["pciVendorSelector"].(string)
out = append(out, permittedEntry{resourceName: name, vendorSelector: selector, mediated: list == "mediatedDevices"})
}
}
return out, true
}
// Coverage maps a VMI to the units, per resource name, that its attached
// devices account for, so its virt-launcher pod does not count them twice.
type Coverage map[types.NamespacedName]map[corev1.ResourceName]int64
// CoveredForBudget counts every attached device, in any phase: a Lost
// device still counts against budgets itself. Do not use it for node
// capacity, where that device holds nothing and its pod holds the units.
func CoveredForBudget(devices []*gpuv1alpha1.GPUDevice, classes map[string]Class) Coverage {
return covered(devices, classes, func(*gpuv1alpha1.GPUDevice) bool { return true })
}
// CoveredForCapacity counts only attached devices that hold units on a
// node. Do not use it for budgets, where it would count a Lost device's
// units twice.
func CoveredForCapacity(devices []*gpuv1alpha1.GPUDevice, classes map[string]Class) Coverage {
return covered(devices, classes, func(d *gpuv1alpha1.GPUDevice) bool { return HoldsGPUs(d.Status.Phase) })
}
func covered(devices []*gpuv1alpha1.GPUDevice, classes map[string]Class, keep func(*gpuv1alpha1.GPUDevice) bool) Coverage {
out := Coverage{}
for _, d := range devices {
rn := classes[d.Spec.ClassName].ResourceName
if d.Status.AttachedTo == nil || rn == "" || !keep(d) {
continue
}
k := types.NamespacedName{Namespace: d.Namespace, Name: d.Status.AttachedTo.VMI}
if out[k] == nil {
out[k] = map[corev1.ResourceName]int64{}
}
out[k][rn] += int64(d.Spec.Count)
}
return out
}
// ActivePods is a VMI's status.activePods: the UIDs of the pods KubeVirt
// runs it in.
func ActivePods(vmi *unstructured.Unstructured) sets.Set[types.UID] {
out := sets.New[types.UID]()
active, _, _ := unstructured.NestedMap(vmi.Object, "status", "activePods")
for uid := range active {
out.Insert(types.UID(uid))
}
return out
}
// LauncherVMI returns the VMI a pod is the launcher of: the KubeVirt VMI
// its owner reference names, and only when that VMI lists the pod in
// status.activePods. Whoever creates a pod sets its owner references and
// labels, while no tenant role can write a VMI's status. activePods returns
// the named VMI's active pods, nil when there is no such VMI.
func LauncherVMI(p *corev1.Pod, activePods func(types.NamespacedName) (sets.Set[types.UID], error)) (types.NamespacedName, bool, error) {
for _, ref := range p.OwnerReferences {
if ref.Kind != "VirtualMachineInstance" || ref.APIVersion != "kubevirt.io/v1" {
continue
}
k := types.NamespacedName{Namespace: p.Namespace, Name: ref.Name}
active, err := activePods(k)
if err != nil {
return types.NamespacedName{}, false, err
}
if !active.Has(p.UID) {
return types.NamespacedName{}, false, nil
}
return k, true, nil
}
return types.NamespacedName{}, false, nil
}