feat(api): add gpu.cozystack.io GPUClass, GPUDevice and GPUNode

GPUs become a catalog of hoster offers, tenant reservations against
them and a per-node inventory, so a tenant can hold a GPU while its VM
is off and an order that does not fit is refused at create time.

The CRDs enforce what the apiserver alone can: a class name is a DNS
label other than total because it becomes part of a quota key, the
source is a union with dra reserved for a later member, a device's
spec is immutable and orders at least one GPU, and bound devices are
listable per node through a field selector on status.nodeName. The
CRDs are kept on chart uninstall, since they ship as plain templates
and deleting GPUDevice would drop every hold and usage record.

maxCount carries no schema default: it defaults to 1 for vgpu classes
and to no limit for passthrough, which a structural default cannot
express, so readers resolve it through the shared model.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
Aleksei Sviridkin committed 2026-10-20 15:29:05 +03:00
1 parent d1d4e92acb
commit 0d58dedfe9
10 files changed
+2060 -1

No files matched your search

+166
View File
@@ -0,0 +1,166 @@
/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha1
import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GPUMode is how a class's GPUs reach a VM.
// +kubebuilder:validation:Enum=passthrough;vgpu
type GPUMode string
const (
GPUModePassthrough GPUMode = "passthrough"
GPUModeVGPU GPUMode = "vgpu"
)
// OnCordonPolicy decides what happens to an unattached device on a cordoned node.
// +kubebuilder:validation:Enum=Move;Stay
type OnCordonPolicy string
const (
// OnCordonMove moves an unattached device to another node of its class
// when one has room, keeping its hold.
OnCordonMove OnCordonPolicy = "Move"
// OnCordonStay keeps the device on its node until the cordon is lifted.
OnCordonStay OnCordonPolicy = "Stay"
)
// DevicePluginSource serves a class through a Kubernetes device plugin.
type DevicePluginSource struct {
// ResourceName is the extended resource the device plugin advertises for
// this class's GPUs. Two classes may share it only when their node
// selectors do not overlap.
// +required
// +kubebuilder:validation:MinLength=3
// +kubebuilder:validation:MaxLength=317
// +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$`
ResourceName string `json:"resourceName"`
}
// GPUClassSource is the backend that serves a class. Exactly one member is
// set. The name "dra" is reserved for a DRA-backed member.
// +kubebuilder:validation:XValidation:rule="[has(self.devicePlugin)].filter(x, x).size() == 1",message="exactly one source must be set; devicePlugin is the only source supported, dra is reserved"
type GPUClassSource struct {
// +optional
DevicePlugin *DevicePluginSource `json:"devicePlugin,omitempty"`
}
// GPUClassSpec defines one GPU offer of the hoster's catalog.
// +kubebuilder:validation:XValidation:rule="self.mode == 'vgpu' ? has(self.profile) : !has(self.profile)",message="profile is required for mode vgpu and not allowed for mode passthrough"
type GPUClassSpec struct {
// DisplayName is the human-readable name of the offer.
// +optional
// +kubebuilder:validation:MaxLength=256
DisplayName string `json:"displayName,omitempty"`
// +required
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=63
Vendor string `json:"vendor"`
// +required
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=63
Model string `json:"model"`
// +required
Mode GPUMode `json:"mode"`
// Profile is the vGPU profile, such as L40S-24C. Set only for mode vgpu.
// +optional
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=63
Profile string `json:"profile,omitempty"`
// Memory is the GPU memory of one unit of the class.
// +optional
Memory *resource.Quantity `json:"memory,omitempty"`
// MaxCount is how many GPUs of this class one VM may take. Unset means 1
// for mode vgpu and, for mode passthrough, no limit beyond the GPUs of
// one node; read it through the shared model rather than the raw field.
// +optional
// +kubebuilder:validation:Minimum=1
MaxCount *int32 `json:"maxCount,omitempty"`
// OnCordon decides whether an unattached device leaves a cordoned node.
// +optional
// +kubebuilder:default=Move
OnCordon OnCordonPolicy `json:"onCordon,omitempty"`
// NodeSelector selects the nodes that serve this class.
// +optional
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
// +required
Source GPUClassSource `json:"source"`
}
// GPUClassStatus is readable by every authenticated user, so it carries no
// node names and no counts beyond free capacity.
type GPUClassStatus struct {
// Free is the number of free units of the class on eligible nodes.
// Absent until the controller has computed it.
// +optional
Free *int32 `json:"free,omitempty"`
// MaxFreePerNode is the largest number of free units on one node.
// +optional
MaxFreePerNode *int32 `json:"maxFreePerNode,omitempty"`
// +optional
// +listType=map
// +listMapKey=type
Conditions []metav1.Condition `json:"conditions,omitempty"`
}
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:metadata:annotations="helm.sh/resource-policy=keep"
// +kubebuilder:resource:scope=Cluster
// +kubebuilder:printcolumn:name="Mode",type="string",JSONPath=".spec.mode"
// +kubebuilder:printcolumn:name="Model",type="string",JSONPath=".spec.model"
// +kubebuilder:printcolumn:name="Profile",type="string",JSONPath=".spec.profile"
// +kubebuilder:printcolumn:name="Free",type="integer",JSONPath=".status.free"
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
// +kubebuilder:validation:XValidation:rule="size(self.metadata.name) <= 63 && self.metadata.name.matches('^[a-z0-9]([-a-z0-9]*[a-z0-9])?$')",message="GPUClass name must be a DNS label, since it becomes part of a quota key"
// +kubebuilder:validation:XValidation:rule="self.metadata.name != 'total'",message="GPUClass name total is reserved for the gpu.cozystack.io/total quota key"
// GPUClass is one GPU offer of the hoster's catalog.
type GPUClass struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec GPUClassSpec `json:"spec"`
Status GPUClassStatus `json:"status,omitempty"`
}
// +kubebuilder:object:root=true
// GPUClassList contains a list of GPUClass.
type GPUClassList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []GPUClass `json:"items"`
}
func init() {
SchemeBuilder.Register(&GPUClass{}, &GPUClassList{})
}
+163
View File
@@ -0,0 +1,163 @@
/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// GPUDevicePhase decides what a device holds and costs; the shared model
// package carries the table.
// +kubebuilder:validation:Enum=Pending;Bound;Attached;Lost
type GPUDevicePhase string
const (
GPUDevicePending GPUDevicePhase = "Pending"
GPUDeviceBound GPUDevicePhase = "Bound"
GPUDeviceAttached GPUDevicePhase = "Attached"
GPUDeviceLost GPUDevicePhase = "Lost"
)
// BindResult is the binder's answer to a BindRequest.
// +kubebuilder:validation:Enum=Succeeded;Refused
type BindResult string
const (
BindSucceeded BindResult = "Succeeded"
BindRefused BindResult = "Refused"
)
// GPUDeviceSpec is immutable: a change is a new order.
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec is immutable; order a new GPUDevice instead"
type GPUDeviceSpec struct {
// ClassName is the GPUClass the device is ordered against.
// +required
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=63
// +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`
ClassName string `json:"className"`
// Count is the number of GPUs of the class, always on one node.
// +required
// +kubebuilder:validation:Minimum=1
Count int32 `json:"count"`
}
// GPUDeviceAttachment names the VMI a device is attached to.
type GPUDeviceAttachment struct {
// VMI is the name of the VirtualMachineInstance in the device's namespace.
// It is recorded at admission, before the VMI exists.
// +required
// +kubebuilder:validation:MinLength=1
VMI string `json:"vmi"`
}
// BindRequest asks the binder to place a set of devices onto one node. The
// admission webhook writes ID, VMI and Devices; the binder writes Result and
// Message. internal/gpu/doc.go carries the full contract.
type BindRequest struct {
// ID is shared by every device of one request and is unique per request.
// +required
// +kubebuilder:validation:MinLength=1
ID string `json:"id"`
// VMI is the VirtualMachineInstance the set is gathered for.
// +required
// +kubebuilder:validation:MinLength=1
VMI string `json:"vmi"`
// Devices names every device of the set, in the device's namespace,
// this one included.
// +required
// +kubebuilder:validation:MinItems=1
// +listType=set
Devices []string `json:"devices"`
// RequestedAt lets the controller expire a request whose VMI never appears.
// +required
RequestedAt metav1.Time `json:"requestedAt"`
// +optional
Result BindResult `json:"result,omitempty"`
// Message explains a refusal and names each device and its node.
// +optional
Message string `json:"message,omitempty"`
}
// GPUDeviceStatus is written by the controller and the admission webhook;
// tenants cannot write it.
type GPUDeviceStatus struct {
// +optional
Phase GPUDevicePhase `json:"phase,omitempty"`
// NodeName is the node the device holds its units on.
// +optional
NodeName string `json:"nodeName,omitempty"`
// +optional
AttachedTo *GPUDeviceAttachment `json:"attachedTo,omitempty"`
// BoundAt is when the device was last bound to a node.
// +optional
BoundAt *metav1.Time `json:"boundAt,omitempty"`
// +optional
BindRequest *BindRequest `json:"bindRequest,omitempty"`
// +optional
// +listType=map
// +listMapKey=type
Conditions []metav1.Condition `json:"conditions,omitempty"`
}
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// keep: an uninstall or a failed reinstall of the controller chart must not
// delete every hold and the usage history along with the CRD.
// +kubebuilder:metadata:annotations="helm.sh/resource-policy=keep"
// +kubebuilder:resource:scope=Namespaced
// +kubebuilder:selectablefield:JSONPath=`.status.nodeName`
// +kubebuilder:printcolumn:name="Class",type="string",JSONPath=".spec.className"
// +kubebuilder:printcolumn:name="Count",type="integer",JSONPath=".spec.count"
// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase"
// +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.nodeName"
// +kubebuilder:printcolumn:name="VMI",type="string",JSONPath=".status.attachedTo.vmi"
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
// GPUDevice is a tenant's GPU, or a set of GPUs of one class used together
// on one node.
type GPUDevice struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec GPUDeviceSpec `json:"spec"`
Status GPUDeviceStatus `json:"status,omitempty"`
}
// +kubebuilder:object:root=true
// GPUDeviceList contains a list of GPUDevice.
type GPUDeviceList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []GPUDevice `json:"items"`
}
func init() {
SchemeBuilder.Register(&GPUDevice{}, &GPUDeviceList{})
}
+124
View File
@@ -0,0 +1,124 @@
/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// PCIFunction is the role of a PCI function in GPU virtualization.
// +kubebuilder:validation:Enum=pf;vf;whole
type PCIFunction string
const (
// PCIFunctionPF is an SR-IOV physical function that carries VFs.
PCIFunctionPF PCIFunction = "pf"
// PCIFunctionVF is an SR-IOV virtual function, one vGPU.
PCIFunctionVF PCIFunction = "vf"
// PCIFunctionWhole is a GPU served whole, without VFs.
PCIFunctionWhole PCIFunction = "whole"
)
// GPUPodReference names the pod that uses a device.
type GPUPodReference struct {
// +required
Namespace string `json:"namespace"`
// +required
Pod string `json:"pod"`
}
// GPUNodeDevice is one PCI function of a GPU.
type GPUNodeDevice struct {
// +required
// +kubebuilder:validation:Pattern=`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$`
PCIAddress string `json:"pciAddress"`
// +optional
Model string `json:"model,omitempty"`
// +required
Function PCIFunction `json:"function"`
// VFs is the number of VFs a physical function carries.
// +optional
VFs int32 `json:"vfs,omitempty"`
// Profile is the vGPU profile written to a VF.
// +optional
Profile string `json:"profile,omitempty"`
// ResourceName is the device-plugin resource that advertises this
// function, as kubelet's pod-resources API reports it.
// +optional
ResourceName string `json:"resourceName,omitempty"`
// Healthy is the device plugin's health of the function. Absent for a
// function no device plugin advertises.
// +optional
Healthy *bool `json:"healthy,omitempty"`
// +optional
UsedBy *GPUPodReference `json:"usedBy,omitempty"`
}
// GPUNodeStatus is the GPU inventory of one node, written by the node agent.
type GPUNodeStatus struct {
// +optional
// +listType=map
// +listMapKey=pciAddress
Devices []GPUNodeDevice `json:"devices,omitempty"`
// ResourceNames are the device-plugin resource names on this node whose
// devices are GPU PCI functions. They join the cluster's GPU
// resource-name set.
// +optional
// +listType=set
ResourceNames []string `json:"resourceNames,omitempty"`
// +optional
// +listType=map
// +listMapKey=type
Conditions []metav1.Condition `json:"conditions,omitempty"`
}
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:metadata:annotations="helm.sh/resource-policy=keep"
// +kubebuilder:resource:scope=Cluster
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
// GPUNode is the GPU inventory of the node of the same name. It is owned by
// that Node and goes away with it.
type GPUNode struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Status GPUNodeStatus `json:"status,omitempty"`
}
// +kubebuilder:object:root=true
// GPUNodeList contains a list of GPUNode.
type GPUNodeList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []GPUNode `json:"items"`
}
func init() {
SchemeBuilder.Register(&GPUNode{}, &GPUNodeList{})
}
+38
View File
@@ -0,0 +1,38 @@
/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package v1alpha1 contains API Schema definitions for the
// gpu.cozystack.io v1alpha1 API group: the GPU catalog (GPUClass), tenant
// GPU reservations (GPUDevice) and the per-node GPU inventory (GPUNode).
// +kubebuilder:object:generate=true
// +groupName=gpu.cozystack.io
package v1alpha1
import (
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/scheme"
)
var (
// GroupVersion is group version used to register these objects.
GroupVersion = schema.GroupVersion{Group: "gpu.cozystack.io", Version: "v1alpha1"}
// SchemeBuilder is used to add go types to the GroupVersionKind scheme.
SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
// AddToScheme adds the types in this group-version to the given scheme.
AddToScheme = SchemeBuilder.AddToScheme
)
+463
View File
@@ -0,0 +1,463 @@
//go:build !ignore_autogenerated
/*
Copyright 2025 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by controller-gen. DO NOT EDIT.
package v1alpha1
import (
"k8s.io/apimachinery/pkg/apis/meta/v1"
runtime "k8s.io/apimachinery/pkg/runtime"
)
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BindRequest) DeepCopyInto(out *BindRequest) {
*out = *in
if in.Devices != nil {
in, out := &in.Devices, &out.Devices
*out = make([]string, len(*in))
copy(*out, *in)
}
in.RequestedAt.DeepCopyInto(&out.RequestedAt)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BindRequest.
func (in *BindRequest) DeepCopy() *BindRequest {
if in == nil {
return nil
}
out := new(BindRequest)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DevicePluginSource) DeepCopyInto(out *DevicePluginSource) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevicePluginSource.
func (in *DevicePluginSource) DeepCopy() *DevicePluginSource {
if in == nil {
return nil
}
out := new(DevicePluginSource)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUClass) DeepCopyInto(out *GPUClass) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUClass.
func (in *GPUClass) DeepCopy() *GPUClass {
if in == nil {
return nil
}
out := new(GPUClass)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *GPUClass) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUClassList) DeepCopyInto(out *GPUClassList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]GPUClass, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUClassList.
func (in *GPUClassList) DeepCopy() *GPUClassList {
if in == nil {
return nil
}
out := new(GPUClassList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *GPUClassList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUClassSource) DeepCopyInto(out *GPUClassSource) {
*out = *in
if in.DevicePlugin != nil {
in, out := &in.DevicePlugin, &out.DevicePlugin
*out = new(DevicePluginSource)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUClassSource.
func (in *GPUClassSource) DeepCopy() *GPUClassSource {
if in == nil {
return nil
}
out := new(GPUClassSource)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUClassSpec) DeepCopyInto(out *GPUClassSpec) {
*out = *in
if in.Memory != nil {
in, out := &in.Memory, &out.Memory
x := (*in).DeepCopy()
*out = &x
}
if in.MaxCount != nil {
in, out := &in.MaxCount, &out.MaxCount
*out = new(int32)
**out = **in
}
if in.NodeSelector != nil {
in, out := &in.NodeSelector, &out.NodeSelector
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
in.Source.DeepCopyInto(&out.Source)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUClassSpec.
func (in *GPUClassSpec) DeepCopy() *GPUClassSpec {
if in == nil {
return nil
}
out := new(GPUClassSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUClassStatus) DeepCopyInto(out *GPUClassStatus) {
*out = *in
if in.Free != nil {
in, out := &in.Free, &out.Free
*out = new(int32)
**out = **in
}
if in.MaxFreePerNode != nil {
in, out := &in.MaxFreePerNode, &out.MaxFreePerNode
*out = new(int32)
**out = **in
}
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]v1.Condition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUClassStatus.
func (in *GPUClassStatus) DeepCopy() *GPUClassStatus {
if in == nil {
return nil
}
out := new(GPUClassStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUDevice) DeepCopyInto(out *GPUDevice) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
out.Spec = in.Spec
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUDevice.
func (in *GPUDevice) DeepCopy() *GPUDevice {
if in == nil {
return nil
}
out := new(GPUDevice)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *GPUDevice) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUDeviceAttachment) DeepCopyInto(out *GPUDeviceAttachment) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUDeviceAttachment.
func (in *GPUDeviceAttachment) DeepCopy() *GPUDeviceAttachment {
if in == nil {
return nil
}
out := new(GPUDeviceAttachment)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUDeviceList) DeepCopyInto(out *GPUDeviceList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]GPUDevice, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUDeviceList.
func (in *GPUDeviceList) DeepCopy() *GPUDeviceList {
if in == nil {
return nil
}
out := new(GPUDeviceList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *GPUDeviceList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUDeviceSpec) DeepCopyInto(out *GPUDeviceSpec) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUDeviceSpec.
func (in *GPUDeviceSpec) DeepCopy() *GPUDeviceSpec {
if in == nil {
return nil
}
out := new(GPUDeviceSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUDeviceStatus) DeepCopyInto(out *GPUDeviceStatus) {
*out = *in
if in.AttachedTo != nil {
in, out := &in.AttachedTo, &out.AttachedTo
*out = new(GPUDeviceAttachment)
**out = **in
}
if in.BoundAt != nil {
in, out := &in.BoundAt, &out.BoundAt
*out = (*in).DeepCopy()
}
if in.BindRequest != nil {
in, out := &in.BindRequest, &out.BindRequest
*out = new(BindRequest)
(*in).DeepCopyInto(*out)
}
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]v1.Condition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUDeviceStatus.
func (in *GPUDeviceStatus) DeepCopy() *GPUDeviceStatus {
if in == nil {
return nil
}
out := new(GPUDeviceStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUNode) DeepCopyInto(out *GPUNode) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUNode.
func (in *GPUNode) DeepCopy() *GPUNode {
if in == nil {
return nil
}
out := new(GPUNode)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *GPUNode) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUNodeDevice) DeepCopyInto(out *GPUNodeDevice) {
*out = *in
if in.Healthy != nil {
in, out := &in.Healthy, &out.Healthy
*out = new(bool)
**out = **in
}
if in.UsedBy != nil {
in, out := &in.UsedBy, &out.UsedBy
*out = new(GPUPodReference)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUNodeDevice.
func (in *GPUNodeDevice) DeepCopy() *GPUNodeDevice {
if in == nil {
return nil
}
out := new(GPUNodeDevice)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUNodeList) DeepCopyInto(out *GPUNodeList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]GPUNode, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUNodeList.
func (in *GPUNodeList) DeepCopy() *GPUNodeList {
if in == nil {
return nil
}
out := new(GPUNodeList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *GPUNodeList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUNodeStatus) DeepCopyInto(out *GPUNodeStatus) {
*out = *in
if in.Devices != nil {
in, out := &in.Devices, &out.Devices
*out = make([]GPUNodeDevice, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.ResourceNames != nil {
in, out := &in.ResourceNames, &out.ResourceNames
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]v1.Condition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUNodeStatus.
func (in *GPUNodeStatus) DeepCopy() *GPUNodeStatus {
if in == nil {
return nil
}
out := new(GPUNodeStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *GPUPodReference) DeepCopyInto(out *GPUPodReference) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUPodReference.
func (in *GPUPodReference) DeepCopy() *GPUPodReference {
if in == nil {
return nil
}
out := new(GPUPodReference)
in.DeepCopyInto(out)
return out
}
+1 -1
View File
@@ -92,7 +92,7 @@ kube::codegen::gen_client \
"${SCRIPT_ROOT}/pkg/apis"
$CONTROLLER_GEN object:headerFile="hack/boilerplate.go.txt" paths="./api/..."
$CONTROLLER_GEN rbac:roleName=manager-role crd paths="./api/v1alpha1/..." paths="./api/backups/..." paths="./api/gateway/..." paths="./api/internalapi/..." paths="./api/migration/..." output:crd:artifacts:config=${TMPDIR}
$CONTROLLER_GEN rbac:roleName=manager-role crd paths="./api/v1alpha1/..." paths="./api/backups/..." paths="./api/gateway/..." paths="./api/gpu/..." paths="./api/internalapi/..." paths="./api/migration/..." output:crd:artifacts:config=${TMPDIR}
mv ${TMPDIR}/cozystack.io_packages.yaml ${OPERATOR_CRDDIR}/cozystack.io_packages.yaml
mv ${TMPDIR}/cozystack.io_packagesources.yaml ${OPERATOR_CRDDIR}/cozystack.io_packagesources.yaml
+461
View File
@@ -0,0 +1,461 @@
/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package apitest checks the shipped gpu.cozystack.io CRDs the way the
// apiserver applies them, so a marker that silently failed to generate
// shows up as a test failure rather than as an admitted bad object.
package apitest
import (
"context"
"os"
"path/filepath"
"testing"
apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextvalidation "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation"
"k8s.io/apiextensions-apiserver/pkg/apiserver/schema"
schemacel "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel"
"k8s.io/apiextensions-apiserver/pkg/apiserver/schema/defaulting"
"k8s.io/apiextensions-apiserver/pkg/apiserver/schema/listtype"
"k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning"
apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation"
utiljson "k8s.io/apimachinery/pkg/util/json"
"k8s.io/apimachinery/pkg/util/validation/field"
celconfig "k8s.io/apiserver/pkg/apis/cel"
"sigs.k8s.io/yaml"
)
const definitionsDir = "../../../packages/system/cozystack-controller/definitions"
var crdFiles = map[string]string{
"GPUClass": "gpu.cozystack.io_gpuclasses.yaml",
"GPUDevice": "gpu.cozystack.io_gpudevices.yaml",
"GPUNode": "gpu.cozystack.io_gpunodes.yaml",
}
func readCRD(t *testing.T, kind string) *apiextensionsv1.CustomResourceDefinition {
t.Helper()
raw, err := os.ReadFile(filepath.Join(definitionsDir, crdFiles[kind]))
if err != nil {
t.Fatalf("read %s CRD: %v", kind, err)
}
var crd apiextensionsv1.CustomResourceDefinition
if err := yaml.Unmarshal(raw, &crd); err != nil {
t.Fatalf("unmarshal %s CRD: %v", kind, err)
}
return &crd
}
// v1alpha1 fails the test when the version is missing, so a renamed
// version cannot turn every assertion below into a no-op.
func v1alpha1(t *testing.T, crd *apiextensionsv1.CustomResourceDefinition) *apiextensionsv1.CustomResourceDefinitionVersion {
t.Helper()
for i := range crd.Spec.Versions {
if v := &crd.Spec.Versions[i]; v.Name == "v1alpha1" && v.Schema != nil && v.Schema.OpenAPIV3Schema != nil {
return v
}
}
t.Fatalf("%s: no v1alpha1 version with a schema", crd.Name)
return nil
}
// admission runs the steps the apiserver applies to a custom resource
// write, in its order: prune, default, structural schema, list types, CEL
// rules.
type admission struct {
structural *schema.Structural
schema apiservervalidation.SchemaValidator
cel *schemacel.Validator
}
func newAdmission(t *testing.T, kind string) *admission {
t.Helper()
props := v1alpha1(t, readCRD(t, kind)).Schema.OpenAPIV3Schema
var internal apiextensions.JSONSchemaProps
if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(props, &internal, nil); err != nil {
t.Fatalf("convert schema: %v", err)
}
structural, err := schema.NewStructural(&internal)
if err != nil {
t.Fatalf("structural schema: %v", err)
}
validator, _, err := apiservervalidation.NewSchemaValidator(&internal)
if err != nil {
t.Fatalf("schema validator: %v", err)
}
return &admission{
structural: structural,
schema: validator,
cel: schemacel.NewValidator(structural, true, celconfig.PerCallLimit),
}
}
func decode(t *testing.T, doc string) map[string]any {
t.Helper()
if doc == "" {
return nil
}
js, err := yaml.YAMLToJSON([]byte(doc))
if err != nil {
t.Fatalf("yaml: %v", err)
}
var obj map[string]any
// utiljson decodes integers as int64, as the apiserver does.
if err := utiljson.Unmarshal(js, &obj); err != nil {
t.Fatalf("json: %v", err)
}
return obj
}
// admit returns the object as stored and the errors the apiserver would
// report. old is empty for a create.
func (a *admission) admit(t *testing.T, doc, old string) (map[string]any, field.ErrorList) {
t.Helper()
obj, oldObj := decode(t, doc), decode(t, old)
for _, o := range []map[string]any{obj, oldObj} {
if o != nil {
pruning.Prune(o, a.structural, true)
defaulting.Default(o, a.structural)
}
}
errs := apiservervalidation.ValidateCustomResource(nil, obj, a.schema)
errs = append(errs, listtype.ValidateListSetsAndMaps(nil, a.structural, obj)...)
var oldArg any
if oldObj != nil {
oldArg = oldObj
}
celErrs, _ := a.cel.Validate(context.TODO(), nil, a.structural, obj, oldArg, celconfig.RuntimeCELCostBudget)
return obj, append(errs, celErrs...)
}
func TestCRDsPassInstallTimeValidation(t *testing.T) {
for kind := range crdFiles {
t.Run(kind, func(t *testing.T) {
crd := readCRD(t, kind)
crd.Status.StoredVersions = []string{"v1alpha1"}
var internal apiextensions.CustomResourceDefinition
if err := apiextensionsv1.Convert_v1_CustomResourceDefinition_To_apiextensions_CustomResourceDefinition(crd, &internal, nil); err != nil {
t.Fatalf("convert CRD: %v", err)
}
for _, e := range apiextvalidation.ValidateCustomResourceDefinition(context.TODO(), &internal) {
t.Errorf("apiserver would reject this CRD: %v", e)
}
})
}
}
// The CRDs render as ordinary chart templates, so without the keep policy
// a controller uninstall deletes every GPUDevice, its hold and its usage
// record with the CRD.
func TestCRDsAreKeptOnChartUninstall(t *testing.T) {
for kind := range crdFiles {
if got := readCRD(t, kind).Annotations["helm.sh/resource-policy"]; got != "keep" {
t.Errorf("%s: helm.sh/resource-policy = %q, want keep", kind, got)
}
}
}
// Tenants can write a GPUDevice but not its status; everything the binder
// decides lives there.
func TestCRDsHaveStatusSubresource(t *testing.T) {
for kind := range crdFiles {
v := v1alpha1(t, readCRD(t, kind))
if v.Subresources == nil || v.Subresources.Status == nil {
t.Errorf("%s: no status subresource", kind)
}
}
}
func TestGPUDeviceNodeNameIsSelectable(t *testing.T) {
for _, f := range v1alpha1(t, readCRD(t, "GPUDevice")).SelectableFields {
if f.JSONPath == ".status.nodeName" {
return
}
}
t.Error("GPUDevice does not declare .status.nodeName in selectableFields")
}
func TestScope(t *testing.T) {
want := map[string]apiextensionsv1.ResourceScope{
"GPUClass": apiextensionsv1.ClusterScoped,
"GPUDevice": apiextensionsv1.NamespaceScoped,
"GPUNode": apiextensionsv1.ClusterScoped,
}
for kind, scope := range want {
if got := readCRD(t, kind).Spec.Scope; got != scope {
t.Errorf("%s: scope %s, want %s", kind, got, scope)
}
}
}
func gpuClass(name, spec string) string {
return "apiVersion: gpu.cozystack.io/v1alpha1\nkind: GPUClass\nmetadata:\n name: " + name + "\nspec:\n" + spec
}
const passthroughSpec = ` vendor: nvidia
model: H100
mode: passthrough
source:
devicePlugin:
resourceName: nvidia.com/GH100_H100
`
const vgpuSpec = ` vendor: nvidia
model: L40S
mode: vgpu
profile: L40S-24C
maxCount: 2
memory: 24Gi
nodeSelector:
gpu.cozystack.io/10de-26b9: L40S-24C
source:
devicePlugin:
resourceName: nvidia.com/NVIDIA_L40S-24C
`
func TestGPUClassCreate(t *testing.T) {
a := newAdmission(t, "GPUClass")
tests := []struct {
name string
obj string
wantReject bool
}{
{name: "passthrough class", obj: gpuClass("h100", passthroughSpec)},
{name: "vgpu class with profile", obj: gpuClass("l40s-24c", vgpuSpec)},
{name: "name total is reserved", obj: gpuClass("total", passthroughSpec), wantReject: true},
{name: "name with a dot is not a DNS label", obj: gpuClass("h100.whole", passthroughSpec), wantReject: true},
{name: "upper-case name", obj: gpuClass("H100", passthroughSpec), wantReject: true},
{name: "name longer than 63", obj: gpuClass("a234567890123456789012345678901234567890123456789012345678901234", passthroughSpec), wantReject: true},
{name: "name of exactly 63", obj: gpuClass("a23456789012345678901234567890123456789012345678901234567890123", passthroughSpec)},
{name: "vgpu without profile", obj: gpuClass("x", ` vendor: nvidia
model: L40S
mode: vgpu
source:
devicePlugin:
resourceName: nvidia.com/NVIDIA_L40S-24C
`), wantReject: true},
{name: "passthrough with profile", obj: gpuClass("x", passthroughSpec+" profile: H100-80C\n"), wantReject: true},
{name: "unknown mode", obj: gpuClass("x", ` vendor: nvidia
model: H100
mode: mig
source:
devicePlugin:
resourceName: nvidia.com/GH100_H100
`), wantReject: true},
{name: "maxCount 0", obj: gpuClass("x", passthroughSpec+" maxCount: 0\n"), wantReject: true},
{name: "maxCount 1", obj: gpuClass("x", passthroughSpec+" maxCount: 1\n")},
{name: "onCordon Stay", obj: gpuClass("x", passthroughSpec+" onCordon: Stay\n")},
{name: "unknown onCordon", obj: gpuClass("x", passthroughSpec+" onCordon: Drain\n"), wantReject: true},
{name: "no source", obj: gpuClass("x", " vendor: nvidia\n model: H100\n mode: passthrough\n"), wantReject: true},
{name: "empty source", obj: gpuClass("x", " vendor: nvidia\n model: H100\n mode: passthrough\n source: {}\n"), wantReject: true},
{
// dra is not in the schema, so pruning drops it and the
// union rule sees no member.
name: "reserved dra source",
obj: gpuClass("x", ` vendor: nvidia
model: H100
mode: passthrough
source:
dra:
deviceClassName: gpu.nvidia.com
`),
wantReject: true,
},
{name: "devicePlugin without resourceName", obj: gpuClass("x", " vendor: nvidia\n model: H100\n mode: passthrough\n source:\n devicePlugin: {}\n"), wantReject: true},
{name: "resourceName without a domain", obj: gpuClass("x", " vendor: nvidia\n model: H100\n mode: passthrough\n source:\n devicePlugin:\n resourceName: gpu\n"), wantReject: true},
{name: "no vendor", obj: gpuClass("x", " model: H100\n mode: passthrough\n source:\n devicePlugin:\n resourceName: nvidia.com/GH100_H100\n"), wantReject: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, errs := a.admit(t, tc.obj, "")
if got := len(errs) > 0; got != tc.wantReject {
t.Errorf("rejected = %v, want %v; errors: %v", got, tc.wantReject, errs)
}
})
}
}
func TestGPUClassOnCordonDefaultsToMove(t *testing.T) {
obj, errs := newAdmission(t, "GPUClass").admit(t, gpuClass("h100", passthroughSpec), "")
if len(errs) > 0 {
t.Fatalf("unexpected errors: %v", errs)
}
if got := obj["spec"].(map[string]any)["onCordon"]; got != "Move" {
t.Errorf("onCordon = %v, want Move", got)
}
}
// maxCount has no schema default: its default depends on mode, which a
// structural default cannot express.
func TestGPUClassMaxCountHasNoSchemaDefault(t *testing.T) {
obj, errs := newAdmission(t, "GPUClass").admit(t, gpuClass("l40s", ` vendor: nvidia
model: L40S
mode: vgpu
profile: L40S-24C
source:
devicePlugin:
resourceName: nvidia.com/NVIDIA_L40S-24C
`), "")
if len(errs) > 0 {
t.Fatalf("unexpected errors: %v", errs)
}
if v, ok := obj["spec"].(map[string]any)["maxCount"]; ok {
t.Errorf("maxCount defaulted to %v by the schema", v)
}
}
// The CRD carries no transition rule on GPUClass: onCordon, resourceName
// and nodeSelector stay changeable, and freezing the rest while the
// finalizer is present needs the finalizer, which CRD CEL cannot read.
func TestGPUClassUpdateIsNotFrozenByTheCRD(t *testing.T) {
a := newAdmission(t, "GPUClass")
old := gpuClass("l40s-24c", vgpuSpec)
tests := []struct{ name, obj string }{
{"onCordon", gpuClass("l40s-24c", vgpuSpec+" onCordon: Stay\n")},
{"resourceName", gpuClass("l40s-24c", ` vendor: nvidia
model: L40S
mode: vgpu
profile: L40S-24C
maxCount: 2
memory: 24Gi
nodeSelector:
gpu.cozystack.io/10de-26b9: L40S-24C
source:
devicePlugin:
resourceName: nvidia.com/L40S-24C
`)},
{"nodeSelector", gpuClass("l40s-24c", ` vendor: nvidia
model: L40S
mode: vgpu
profile: L40S-24C
maxCount: 2
memory: 24Gi
nodeSelector:
pool: b
source:
devicePlugin:
resourceName: nvidia.com/NVIDIA_L40S-24C
`)},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if _, errs := a.admit(t, tc.obj, old); len(errs) > 0 {
t.Errorf("update refused: %v", errs)
}
})
}
}
func gpuDevice(spec string) string {
return "apiVersion: gpu.cozystack.io/v1alpha1\nkind: GPUDevice\nmetadata:\n name: train-0\n namespace: tenant-acme\nspec:\n" + spec
}
func TestGPUDeviceCreate(t *testing.T) {
a := newAdmission(t, "GPUDevice")
tests := []struct {
name string
spec string
wantReject bool
}{
{name: "count 1", spec: " className: l40s-24c\n count: 1\n"},
{name: "count 4", spec: " className: h100\n count: 4\n"},
{name: "count 0", spec: " className: h100\n count: 0\n", wantReject: true},
{name: "negative count", spec: " className: h100\n count: -1\n", wantReject: true},
{name: "no count", spec: " className: h100\n", wantReject: true},
{name: "no className", spec: " count: 1\n", wantReject: true},
{name: "className not a DNS label", spec: " className: H100.whole\n count: 1\n", wantReject: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, errs := a.admit(t, gpuDevice(tc.spec), "")
if got := len(errs) > 0; got != tc.wantReject {
t.Errorf("rejected = %v, want %v; errors: %v", got, tc.wantReject, errs)
}
})
}
}
func TestGPUDeviceSpecIsImmutable(t *testing.T) {
a := newAdmission(t, "GPUDevice")
old := gpuDevice(" className: l40s-24c\n count: 1\n")
tests := []struct {
name string
obj string
wantReject bool
}{
{name: "count changed", obj: gpuDevice(" className: l40s-24c\n count: 2\n"), wantReject: true},
{name: "className changed", obj: gpuDevice(" className: h100\n count: 1\n"), wantReject: true},
{name: "spec unchanged, status written", obj: gpuDevice(" className: l40s-24c\n count: 1\nstatus:\n phase: Bound\n nodeName: gpu-node-3\n")},
{name: "labels changed", obj: "apiVersion: gpu.cozystack.io/v1alpha1\nkind: GPUDevice\nmetadata:\n name: train-0\n namespace: tenant-acme\n labels:\n pool: inference\nspec:\n className: l40s-24c\n count: 1\n"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, errs := a.admit(t, tc.obj, old)
if got := len(errs) > 0; got != tc.wantReject {
t.Errorf("rejected = %v, want %v; errors: %v", got, tc.wantReject, errs)
}
})
}
}
func TestGPUDeviceStatusPhaseEnum(t *testing.T) {
a := newAdmission(t, "GPUDevice")
for phase, wantReject := range map[string]bool{"Pending": false, "Bound": false, "Attached": false, "Lost": false, "Running": true} {
t.Run(phase, func(t *testing.T) {
_, errs := a.admit(t, gpuDevice(" className: h100\n count: 1\nstatus:\n phase: "+phase+"\n"), "")
if got := len(errs) > 0; got != wantReject {
t.Errorf("rejected = %v, want %v; errors: %v", got, wantReject, errs)
}
})
}
}
func TestGPUNodeInventory(t *testing.T) {
a := newAdmission(t, "GPUNode")
node := func(devices string) string {
return "apiVersion: gpu.cozystack.io/v1alpha1\nkind: GPUNode\nmetadata:\n name: gpu-node-3\nstatus:\n resourceNames: [nvidia.com/NVIDIA_L40S-24C]\n devices:\n" + devices
}
tests := []struct {
name string
devices string
wantReject bool
}{
{name: "pf and vf", devices: ` - pciAddress: "0000:41:00.0"
model: L40S
function: pf
vfs: 16
- pciAddress: "0000:41:00.4"
function: vf
profile: L40S-24C
resourceName: nvidia.com/NVIDIA_L40S-24C
healthy: true
usedBy: {namespace: tenant-acme, pod: virt-launcher-vm-instance-trainer-x7k2p}
`},
{name: "unknown function", devices: " - pciAddress: \"0000:41:00.0\"\n function: mig\n", wantReject: true},
{name: "malformed PCI address", devices: " - pciAddress: \"41:00.0\"\n function: whole\n", wantReject: true},
{name: "duplicate PCI address", devices: " - pciAddress: \"0000:41:00.0\"\n function: whole\n - pciAddress: \"0000:41:00.0\"\n function: whole\n", wantReject: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, errs := a.admit(t, node(tc.devices), "")
if got := len(errs) > 0; got != tc.wantReject {
t.Errorf("rejected = %v, want %v; errors: %v", got, tc.wantReject, errs)
}
})
}
}
@@ -0,0 +1,236 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.16.4
helm.sh/resource-policy: keep
name: gpuclasses.gpu.cozystack.io
spec:
group: gpu.cozystack.io
names:
kind: GPUClass
listKind: GPUClassList
plural: gpuclasses
singular: gpuclass
scope: Cluster
versions:
- additionalPrinterColumns:
- jsonPath: .spec.mode
name: Mode
type: string
- jsonPath: .spec.model
name: Model
type: string
- jsonPath: .spec.profile
name: Profile
type: string
- jsonPath: .status.free
name: Free
type: integer
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1alpha1
schema:
openAPIV3Schema:
description: GPUClass is one GPU offer of the hoster's catalog.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: GPUClassSpec defines one GPU offer of the hoster's catalog.
properties:
displayName:
description: DisplayName is the human-readable name of the offer.
maxLength: 256
type: string
maxCount:
description: |-
MaxCount is how many GPUs of this class one VM may take. Unset means 1
for mode vgpu and, for mode passthrough, no limit beyond the GPUs of
one node; read it through the shared model rather than the raw field.
format: int32
minimum: 1
type: integer
memory:
anyOf:
- type: integer
- type: string
description: Memory is the GPU memory of one unit of the class.
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
mode:
description: GPUMode is how a class's GPUs reach a VM.
enum:
- passthrough
- vgpu
type: string
model:
maxLength: 63
minLength: 1
type: string
nodeSelector:
additionalProperties:
type: string
description: NodeSelector selects the nodes that serve this class.
type: object
onCordon:
default: Move
description: OnCordon decides whether an unattached device leaves
a cordoned node.
enum:
- Move
- Stay
type: string
profile:
description: Profile is the vGPU profile, such as L40S-24C. Set only
for mode vgpu.
maxLength: 63
minLength: 1
type: string
source:
description: |-
GPUClassSource is the backend that serves a class. Exactly one member is
set. The name "dra" is reserved for a DRA-backed member.
properties:
devicePlugin:
description: DevicePluginSource serves a class through a Kubernetes
device plugin.
properties:
resourceName:
description: |-
ResourceName is the extended resource the device plugin advertises for
this class's GPUs. Two classes may share it only when their node
selectors do not overlap.
maxLength: 317
minLength: 3
pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$
type: string
required:
- resourceName
type: object
type: object
x-kubernetes-validations:
- message: exactly one source must be set; devicePlugin is the only
source supported, dra is reserved
rule: '[has(self.devicePlugin)].filter(x, x).size() == 1'
vendor:
maxLength: 63
minLength: 1
type: string
required:
- mode
- model
- source
- vendor
type: object
x-kubernetes-validations:
- message: profile is required for mode vgpu and not allowed for mode
passthrough
rule: 'self.mode == ''vgpu'' ? has(self.profile) : !has(self.profile)'
status:
description: |-
GPUClassStatus is readable by every authenticated user, so it carries no
node names and no counts beyond free capacity.
properties:
conditions:
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
free:
description: |-
Free is the number of free units of the class on eligible nodes.
Absent until the controller has computed it.
format: int32
type: integer
maxFreePerNode:
description: MaxFreePerNode is the largest number of free units on
one node.
format: int32
type: integer
type: object
required:
- spec
type: object
x-kubernetes-validations:
- message: GPUClass name must be a DNS label, since it becomes part of a quota
key
rule: size(self.metadata.name) <= 63 && self.metadata.name.matches('^[a-z0-9]([-a-z0-9]*[a-z0-9])?$')
- message: GPUClass name total is reserved for the gpu.cozystack.io/total
quota key
rule: self.metadata.name != 'total'
served: true
storage: true
subresources:
status: {}
@@ -0,0 +1,232 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.16.4
helm.sh/resource-policy: keep
name: gpudevices.gpu.cozystack.io
spec:
group: gpu.cozystack.io
names:
kind: GPUDevice
listKind: GPUDeviceList
plural: gpudevices
singular: gpudevice
scope: Namespaced
versions:
- additionalPrinterColumns:
- jsonPath: .spec.className
name: Class
type: string
- jsonPath: .spec.count
name: Count
type: integer
- jsonPath: .status.phase
name: Phase
type: string
- jsonPath: .status.nodeName
name: Node
type: string
- jsonPath: .status.attachedTo.vmi
name: VMI
type: string
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1alpha1
schema:
openAPIV3Schema:
description: |-
GPUDevice is a tenant's GPU, or a set of GPUs of one class used together
on one node.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
spec:
description: 'GPUDeviceSpec is immutable: a change is a new order.'
properties:
className:
description: ClassName is the GPUClass the device is ordered against.
maxLength: 63
minLength: 1
pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
type: string
count:
description: Count is the number of GPUs of the class, always on one
node.
format: int32
minimum: 1
type: integer
required:
- className
- count
type: object
x-kubernetes-validations:
- message: spec is immutable; order a new GPUDevice instead
rule: self == oldSelf
status:
description: |-
GPUDeviceStatus is written by the controller and the admission webhook;
tenants cannot write it.
properties:
attachedTo:
description: GPUDeviceAttachment names the VMI a device is attached
to.
properties:
vmi:
description: |-
VMI is the name of the VirtualMachineInstance in the device's namespace.
It is recorded at admission, before the VMI exists.
minLength: 1
type: string
required:
- vmi
type: object
bindRequest:
description: |-
BindRequest asks the binder to place a set of devices onto one node. The
admission webhook writes ID, VMI and Devices; the binder writes Result and
Message. internal/gpu/doc.go carries the full contract.
properties:
devices:
description: |-
Devices names every device of the set, in the device's namespace,
this one included.
items:
type: string
minItems: 1
type: array
x-kubernetes-list-type: set
id:
description: ID is shared by every device of one request and is
unique per request.
minLength: 1
type: string
message:
description: Message explains a refusal and names each device
and its node.
type: string
requestedAt:
description: RequestedAt lets the controller expire a request
whose VMI never appears.
format: date-time
type: string
result:
description: BindResult is the binder's answer to a BindRequest.
enum:
- Succeeded
- Refused
type: string
vmi:
description: VMI is the VirtualMachineInstance the set is gathered
for.
minLength: 1
type: string
required:
- devices
- id
- requestedAt
- vmi
type: object
boundAt:
description: BoundAt is when the device was last bound to a node.
format: date-time
type: string
conditions:
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
nodeName:
description: NodeName is the node the device holds its units on.
type: string
phase:
description: |-
GPUDevicePhase decides what a device holds and costs; the shared model
package carries the table.
enum:
- Pending
- Bound
- Attached
- Lost
type: string
type: object
required:
- spec
type: object
selectableFields:
- jsonPath: .status.nodeName
served: true
storage: true
subresources:
status: {}
@@ -0,0 +1,176 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.16.4
helm.sh/resource-policy: keep
name: gpunodes.gpu.cozystack.io
spec:
group: gpu.cozystack.io
names:
kind: GPUNode
listKind: GPUNodeList
plural: gpunodes
singular: gpunode
scope: Cluster
versions:
- additionalPrinterColumns:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1alpha1
schema:
openAPIV3Schema:
description: |-
GPUNode is the GPU inventory of the node of the same name. It is owned by
that Node and goes away with it.
properties:
apiVersion:
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
status:
description: GPUNodeStatus is the GPU inventory of one node, written by
the node agent.
properties:
conditions:
items:
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
lastTransitionTime is the last time the condition transitioned from one status to another.
This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: |-
message is a human readable message indicating details about the transition.
This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: |-
observedGeneration represents the .metadata.generation that the condition was set based upon.
For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: |-
reason contains a programmatic identifier indicating the reason for the condition's last transition.
Producers of specific condition types may define expected values and meanings for this field,
and whether the values are considered a guaranteed API.
The value should be a CamelCase string.
This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase.
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
devices:
items:
description: GPUNodeDevice is one PCI function of a GPU.
properties:
function:
description: PCIFunction is the role of a PCI function in GPU
virtualization.
enum:
- pf
- vf
- whole
type: string
healthy:
description: |-
Healthy is the device plugin's health of the function. Absent for a
function no device plugin advertises.
type: boolean
model:
type: string
pciAddress:
pattern: ^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$
type: string
profile:
description: Profile is the vGPU profile written to a VF.
type: string
resourceName:
description: |-
ResourceName is the device-plugin resource that advertises this
function, as kubelet's pod-resources API reports it.
type: string
usedBy:
description: GPUPodReference names the pod that uses a device.
properties:
namespace:
type: string
pod:
type: string
required:
- namespace
- pod
type: object
vfs:
description: VFs is the number of VFs a physical function carries.
format: int32
type: integer
required:
- function
- pciAddress
type: object
type: array
x-kubernetes-list-map-keys:
- pciAddress
x-kubernetes-list-type: map
resourceNames:
description: |-
ResourceNames are the device-plugin resource names on this node whose
devices are GPU PCI functions. They join the cluster's GPU
resource-name set.
items:
type: string
type: array
x-kubernetes-list-type: set
type: object
type: object
served: true
storage: true
subresources:
status: {}