diff --git a/api/gpu/v1alpha1/gpuclass_types.go b/api/gpu/v1alpha1/gpuclass_types.go new file mode 100644 index 000000000..91bdd51fc --- /dev/null +++ b/api/gpu/v1alpha1/gpuclass_types.go @@ -0,0 +1,166 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// GPUMode is how a class's GPUs reach a VM. +// +kubebuilder:validation:Enum=passthrough;vgpu +type GPUMode string + +const ( + GPUModePassthrough GPUMode = "passthrough" + GPUModeVGPU GPUMode = "vgpu" +) + +// OnCordonPolicy decides what happens to an unattached device on a cordoned node. +// +kubebuilder:validation:Enum=Move;Stay +type OnCordonPolicy string + +const ( + // OnCordonMove moves an unattached device to another node of its class + // when one has room, keeping its hold. + OnCordonMove OnCordonPolicy = "Move" + // OnCordonStay keeps the device on its node until the cordon is lifted. + OnCordonStay OnCordonPolicy = "Stay" +) + +// DevicePluginSource serves a class through a Kubernetes device plugin. +type DevicePluginSource struct { + // ResourceName is the extended resource the device plugin advertises for + // this class's GPUs. Two classes may share it only when their node + // selectors do not overlap. + // +required + // +kubebuilder:validation:MinLength=3 + // +kubebuilder:validation:MaxLength=317 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$` + ResourceName string `json:"resourceName"` +} + +// GPUClassSource is the backend that serves a class. Exactly one member is +// set. The name "dra" is reserved for a DRA-backed member. +// +kubebuilder:validation:XValidation:rule="[has(self.devicePlugin)].filter(x, x).size() == 1",message="exactly one source must be set; devicePlugin is the only source supported, dra is reserved" +type GPUClassSource struct { + // +optional + DevicePlugin *DevicePluginSource `json:"devicePlugin,omitempty"` +} + +// GPUClassSpec defines one GPU offer of the hoster's catalog. +// +kubebuilder:validation:XValidation:rule="self.mode == 'vgpu' ? has(self.profile) : !has(self.profile)",message="profile is required for mode vgpu and not allowed for mode passthrough" +type GPUClassSpec struct { + // DisplayName is the human-readable name of the offer. + // +optional + // +kubebuilder:validation:MaxLength=256 + DisplayName string `json:"displayName,omitempty"` + + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Vendor string `json:"vendor"` + + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Model string `json:"model"` + + // +required + Mode GPUMode `json:"mode"` + + // Profile is the vGPU profile, such as L40S-24C. Set only for mode vgpu. + // +optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + Profile string `json:"profile,omitempty"` + + // Memory is the GPU memory of one unit of the class. + // +optional + Memory *resource.Quantity `json:"memory,omitempty"` + + // MaxCount is how many GPUs of this class one VM may take. Unset means 1 + // for mode vgpu and, for mode passthrough, no limit beyond the GPUs of + // one node; read it through the shared model rather than the raw field. + // +optional + // +kubebuilder:validation:Minimum=1 + MaxCount *int32 `json:"maxCount,omitempty"` + + // OnCordon decides whether an unattached device leaves a cordoned node. + // +optional + // +kubebuilder:default=Move + OnCordon OnCordonPolicy `json:"onCordon,omitempty"` + + // NodeSelector selects the nodes that serve this class. + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // +required + Source GPUClassSource `json:"source"` +} + +// GPUClassStatus is readable by every authenticated user, so it carries no +// node names and no counts beyond free capacity. +type GPUClassStatus struct { + // Free is the number of free units of the class on eligible nodes. + // Absent until the controller has computed it. + // +optional + Free *int32 `json:"free,omitempty"` + + // MaxFreePerNode is the largest number of free units on one node. + // +optional + MaxFreePerNode *int32 `json:"maxFreePerNode,omitempty"` + + // +optional + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:metadata:annotations="helm.sh/resource-policy=keep" +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:printcolumn:name="Mode",type="string",JSONPath=".spec.mode" +// +kubebuilder:printcolumn:name="Model",type="string",JSONPath=".spec.model" +// +kubebuilder:printcolumn:name="Profile",type="string",JSONPath=".spec.profile" +// +kubebuilder:printcolumn:name="Free",type="integer",JSONPath=".status.free" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:validation:XValidation:rule="size(self.metadata.name) <= 63 && self.metadata.name.matches('^[a-z0-9]([-a-z0-9]*[a-z0-9])?$')",message="GPUClass name must be a DNS label, since it becomes part of a quota key" +// +kubebuilder:validation:XValidation:rule="self.metadata.name != 'total'",message="GPUClass name total is reserved for the gpu.cozystack.io/total quota key" + +// GPUClass is one GPU offer of the hoster's catalog. +type GPUClass struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec GPUClassSpec `json:"spec"` + Status GPUClassStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// GPUClassList contains a list of GPUClass. +type GPUClassList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GPUClass `json:"items"` +} + +func init() { + SchemeBuilder.Register(&GPUClass{}, &GPUClassList{}) +} diff --git a/api/gpu/v1alpha1/gpudevice_types.go b/api/gpu/v1alpha1/gpudevice_types.go new file mode 100644 index 000000000..9849e519f --- /dev/null +++ b/api/gpu/v1alpha1/gpudevice_types.go @@ -0,0 +1,163 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// GPUDevicePhase decides what a device holds and costs; the shared model +// package carries the table. +// +kubebuilder:validation:Enum=Pending;Bound;Attached;Lost +type GPUDevicePhase string + +const ( + GPUDevicePending GPUDevicePhase = "Pending" + GPUDeviceBound GPUDevicePhase = "Bound" + GPUDeviceAttached GPUDevicePhase = "Attached" + GPUDeviceLost GPUDevicePhase = "Lost" +) + +// BindResult is the binder's answer to a BindRequest. +// +kubebuilder:validation:Enum=Succeeded;Refused +type BindResult string + +const ( + BindSucceeded BindResult = "Succeeded" + BindRefused BindResult = "Refused" +) + +// GPUDeviceSpec is immutable: a change is a new order. +// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec is immutable; order a new GPUDevice instead" +type GPUDeviceSpec struct { + // ClassName is the GPUClass the device is ordered against. + // +required + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` + ClassName string `json:"className"` + + // Count is the number of GPUs of the class, always on one node. + // +required + // +kubebuilder:validation:Minimum=1 + Count int32 `json:"count"` +} + +// GPUDeviceAttachment names the VMI a device is attached to. +type GPUDeviceAttachment struct { + // VMI is the name of the VirtualMachineInstance in the device's namespace. + // It is recorded at admission, before the VMI exists. + // +required + // +kubebuilder:validation:MinLength=1 + VMI string `json:"vmi"` +} + +// BindRequest asks the binder to place a set of devices onto one node. The +// admission webhook writes ID, VMI and Devices; the binder writes Result and +// Message. internal/gpu/doc.go carries the full contract. +type BindRequest struct { + // ID is shared by every device of one request and is unique per request. + // +required + // +kubebuilder:validation:MinLength=1 + ID string `json:"id"` + + // VMI is the VirtualMachineInstance the set is gathered for. + // +required + // +kubebuilder:validation:MinLength=1 + VMI string `json:"vmi"` + + // Devices names every device of the set, in the device's namespace, + // this one included. + // +required + // +kubebuilder:validation:MinItems=1 + // +listType=set + Devices []string `json:"devices"` + + // RequestedAt lets the controller expire a request whose VMI never appears. + // +required + RequestedAt metav1.Time `json:"requestedAt"` + + // +optional + Result BindResult `json:"result,omitempty"` + + // Message explains a refusal and names each device and its node. + // +optional + Message string `json:"message,omitempty"` +} + +// GPUDeviceStatus is written by the controller and the admission webhook; +// tenants cannot write it. +type GPUDeviceStatus struct { + // +optional + Phase GPUDevicePhase `json:"phase,omitempty"` + + // NodeName is the node the device holds its units on. + // +optional + NodeName string `json:"nodeName,omitempty"` + + // +optional + AttachedTo *GPUDeviceAttachment `json:"attachedTo,omitempty"` + + // BoundAt is when the device was last bound to a node. + // +optional + BoundAt *metav1.Time `json:"boundAt,omitempty"` + + // +optional + BindRequest *BindRequest `json:"bindRequest,omitempty"` + + // +optional + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// keep: an uninstall or a failed reinstall of the controller chart must not +// delete every hold and the usage history along with the CRD. +// +kubebuilder:metadata:annotations="helm.sh/resource-policy=keep" +// +kubebuilder:resource:scope=Namespaced +// +kubebuilder:selectablefield:JSONPath=`.status.nodeName` +// +kubebuilder:printcolumn:name="Class",type="string",JSONPath=".spec.className" +// +kubebuilder:printcolumn:name="Count",type="integer",JSONPath=".spec.count" +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase" +// +kubebuilder:printcolumn:name="Node",type="string",JSONPath=".status.nodeName" +// +kubebuilder:printcolumn:name="VMI",type="string",JSONPath=".status.attachedTo.vmi" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// GPUDevice is a tenant's GPU, or a set of GPUs of one class used together +// on one node. +type GPUDevice struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec GPUDeviceSpec `json:"spec"` + Status GPUDeviceStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// GPUDeviceList contains a list of GPUDevice. +type GPUDeviceList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GPUDevice `json:"items"` +} + +func init() { + SchemeBuilder.Register(&GPUDevice{}, &GPUDeviceList{}) +} diff --git a/api/gpu/v1alpha1/gpunode_types.go b/api/gpu/v1alpha1/gpunode_types.go new file mode 100644 index 000000000..5ae839efe --- /dev/null +++ b/api/gpu/v1alpha1/gpunode_types.go @@ -0,0 +1,124 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// PCIFunction is the role of a PCI function in GPU virtualization. +// +kubebuilder:validation:Enum=pf;vf;whole +type PCIFunction string + +const ( + // PCIFunctionPF is an SR-IOV physical function that carries VFs. + PCIFunctionPF PCIFunction = "pf" + // PCIFunctionVF is an SR-IOV virtual function, one vGPU. + PCIFunctionVF PCIFunction = "vf" + // PCIFunctionWhole is a GPU served whole, without VFs. + PCIFunctionWhole PCIFunction = "whole" +) + +// GPUPodReference names the pod that uses a device. +type GPUPodReference struct { + // +required + Namespace string `json:"namespace"` + // +required + Pod string `json:"pod"` +} + +// GPUNodeDevice is one PCI function of a GPU. +type GPUNodeDevice struct { + // +required + // +kubebuilder:validation:Pattern=`^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$` + PCIAddress string `json:"pciAddress"` + + // +optional + Model string `json:"model,omitempty"` + + // +required + Function PCIFunction `json:"function"` + + // VFs is the number of VFs a physical function carries. + // +optional + VFs int32 `json:"vfs,omitempty"` + + // Profile is the vGPU profile written to a VF. + // +optional + Profile string `json:"profile,omitempty"` + + // ResourceName is the device-plugin resource that advertises this + // function, as kubelet's pod-resources API reports it. + // +optional + ResourceName string `json:"resourceName,omitempty"` + + // Healthy is the device plugin's health of the function. Absent for a + // function no device plugin advertises. + // +optional + Healthy *bool `json:"healthy,omitempty"` + + // +optional + UsedBy *GPUPodReference `json:"usedBy,omitempty"` +} + +// GPUNodeStatus is the GPU inventory of one node, written by the node agent. +type GPUNodeStatus struct { + // +optional + // +listType=map + // +listMapKey=pciAddress + Devices []GPUNodeDevice `json:"devices,omitempty"` + + // ResourceNames are the device-plugin resource names on this node whose + // devices are GPU PCI functions. They join the cluster's GPU + // resource-name set. + // +optional + // +listType=set + ResourceNames []string `json:"resourceNames,omitempty"` + + // +optional + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:metadata:annotations="helm.sh/resource-policy=keep" +// +kubebuilder:resource:scope=Cluster +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// GPUNode is the GPU inventory of the node of the same name. It is owned by +// that Node and goes away with it. +type GPUNode struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Status GPUNodeStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// GPUNodeList contains a list of GPUNode. +type GPUNodeList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GPUNode `json:"items"` +} + +func init() { + SchemeBuilder.Register(&GPUNode{}, &GPUNodeList{}) +} diff --git a/api/gpu/v1alpha1/groupversion_info.go b/api/gpu/v1alpha1/groupversion_info.go new file mode 100644 index 000000000..f0f803384 --- /dev/null +++ b/api/gpu/v1alpha1/groupversion_info.go @@ -0,0 +1,38 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the +// gpu.cozystack.io v1alpha1 API group: the GPU catalog (GPUClass), tenant +// GPU reservations (GPUDevice) and the per-node GPU inventory (GPUNode). +// +kubebuilder:object:generate=true +// +groupName=gpu.cozystack.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "gpu.cozystack.io", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/gpu/v1alpha1/zz_generated.deepcopy.go b/api/gpu/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000..d16a3c678 --- /dev/null +++ b/api/gpu/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,463 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2025 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BindRequest) DeepCopyInto(out *BindRequest) { + *out = *in + if in.Devices != nil { + in, out := &in.Devices, &out.Devices + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.RequestedAt.DeepCopyInto(&out.RequestedAt) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BindRequest. +func (in *BindRequest) DeepCopy() *BindRequest { + if in == nil { + return nil + } + out := new(BindRequest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DevicePluginSource) DeepCopyInto(out *DevicePluginSource) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevicePluginSource. +func (in *DevicePluginSource) DeepCopy() *DevicePluginSource { + if in == nil { + return nil + } + out := new(DevicePluginSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUClass) DeepCopyInto(out *GPUClass) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUClass. +func (in *GPUClass) DeepCopy() *GPUClass { + if in == nil { + return nil + } + out := new(GPUClass) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GPUClass) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUClassList) DeepCopyInto(out *GPUClassList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GPUClass, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUClassList. +func (in *GPUClassList) DeepCopy() *GPUClassList { + if in == nil { + return nil + } + out := new(GPUClassList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GPUClassList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUClassSource) DeepCopyInto(out *GPUClassSource) { + *out = *in + if in.DevicePlugin != nil { + in, out := &in.DevicePlugin, &out.DevicePlugin + *out = new(DevicePluginSource) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUClassSource. +func (in *GPUClassSource) DeepCopy() *GPUClassSource { + if in == nil { + return nil + } + out := new(GPUClassSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUClassSpec) DeepCopyInto(out *GPUClassSpec) { + *out = *in + if in.Memory != nil { + in, out := &in.Memory, &out.Memory + x := (*in).DeepCopy() + *out = &x + } + if in.MaxCount != nil { + in, out := &in.MaxCount, &out.MaxCount + *out = new(int32) + **out = **in + } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + in.Source.DeepCopyInto(&out.Source) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUClassSpec. +func (in *GPUClassSpec) DeepCopy() *GPUClassSpec { + if in == nil { + return nil + } + out := new(GPUClassSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUClassStatus) DeepCopyInto(out *GPUClassStatus) { + *out = *in + if in.Free != nil { + in, out := &in.Free, &out.Free + *out = new(int32) + **out = **in + } + if in.MaxFreePerNode != nil { + in, out := &in.MaxFreePerNode, &out.MaxFreePerNode + *out = new(int32) + **out = **in + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUClassStatus. +func (in *GPUClassStatus) DeepCopy() *GPUClassStatus { + if in == nil { + return nil + } + out := new(GPUClassStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUDevice) DeepCopyInto(out *GPUDevice) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUDevice. +func (in *GPUDevice) DeepCopy() *GPUDevice { + if in == nil { + return nil + } + out := new(GPUDevice) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GPUDevice) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUDeviceAttachment) DeepCopyInto(out *GPUDeviceAttachment) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUDeviceAttachment. +func (in *GPUDeviceAttachment) DeepCopy() *GPUDeviceAttachment { + if in == nil { + return nil + } + out := new(GPUDeviceAttachment) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUDeviceList) DeepCopyInto(out *GPUDeviceList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GPUDevice, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUDeviceList. +func (in *GPUDeviceList) DeepCopy() *GPUDeviceList { + if in == nil { + return nil + } + out := new(GPUDeviceList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GPUDeviceList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUDeviceSpec) DeepCopyInto(out *GPUDeviceSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUDeviceSpec. +func (in *GPUDeviceSpec) DeepCopy() *GPUDeviceSpec { + if in == nil { + return nil + } + out := new(GPUDeviceSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUDeviceStatus) DeepCopyInto(out *GPUDeviceStatus) { + *out = *in + if in.AttachedTo != nil { + in, out := &in.AttachedTo, &out.AttachedTo + *out = new(GPUDeviceAttachment) + **out = **in + } + if in.BoundAt != nil { + in, out := &in.BoundAt, &out.BoundAt + *out = (*in).DeepCopy() + } + if in.BindRequest != nil { + in, out := &in.BindRequest, &out.BindRequest + *out = new(BindRequest) + (*in).DeepCopyInto(*out) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUDeviceStatus. +func (in *GPUDeviceStatus) DeepCopy() *GPUDeviceStatus { + if in == nil { + return nil + } + out := new(GPUDeviceStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUNode) DeepCopyInto(out *GPUNode) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUNode. +func (in *GPUNode) DeepCopy() *GPUNode { + if in == nil { + return nil + } + out := new(GPUNode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GPUNode) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUNodeDevice) DeepCopyInto(out *GPUNodeDevice) { + *out = *in + if in.Healthy != nil { + in, out := &in.Healthy, &out.Healthy + *out = new(bool) + **out = **in + } + if in.UsedBy != nil { + in, out := &in.UsedBy, &out.UsedBy + *out = new(GPUPodReference) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUNodeDevice. +func (in *GPUNodeDevice) DeepCopy() *GPUNodeDevice { + if in == nil { + return nil + } + out := new(GPUNodeDevice) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUNodeList) DeepCopyInto(out *GPUNodeList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GPUNode, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUNodeList. +func (in *GPUNodeList) DeepCopy() *GPUNodeList { + if in == nil { + return nil + } + out := new(GPUNodeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GPUNodeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUNodeStatus) DeepCopyInto(out *GPUNodeStatus) { + *out = *in + if in.Devices != nil { + in, out := &in.Devices, &out.Devices + *out = make([]GPUNodeDevice, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ResourceNames != nil { + in, out := &in.ResourceNames, &out.ResourceNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUNodeStatus. +func (in *GPUNodeStatus) DeepCopy() *GPUNodeStatus { + if in == nil { + return nil + } + out := new(GPUNodeStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GPUPodReference) DeepCopyInto(out *GPUPodReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUPodReference. +func (in *GPUPodReference) DeepCopy() *GPUPodReference { + if in == nil { + return nil + } + out := new(GPUPodReference) + in.DeepCopyInto(out) + return out +} diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 95fcee632..e8d0bac90 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -92,7 +92,7 @@ kube::codegen::gen_client \ "${SCRIPT_ROOT}/pkg/apis" $CONTROLLER_GEN object:headerFile="hack/boilerplate.go.txt" paths="./api/..." -$CONTROLLER_GEN rbac:roleName=manager-role crd paths="./api/v1alpha1/..." paths="./api/backups/..." paths="./api/gateway/..." paths="./api/internalapi/..." paths="./api/migration/..." output:crd:artifacts:config=${TMPDIR} +$CONTROLLER_GEN rbac:roleName=manager-role crd paths="./api/v1alpha1/..." paths="./api/backups/..." paths="./api/gateway/..." paths="./api/gpu/..." paths="./api/internalapi/..." paths="./api/migration/..." output:crd:artifacts:config=${TMPDIR} mv ${TMPDIR}/cozystack.io_packages.yaml ${OPERATOR_CRDDIR}/cozystack.io_packages.yaml mv ${TMPDIR}/cozystack.io_packagesources.yaml ${OPERATOR_CRDDIR}/cozystack.io_packagesources.yaml diff --git a/internal/gpu/apitest/crdcel_test.go b/internal/gpu/apitest/crdcel_test.go new file mode 100644 index 000000000..beced9f26 --- /dev/null +++ b/internal/gpu/apitest/crdcel_test.go @@ -0,0 +1,461 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package apitest checks the shipped gpu.cozystack.io CRDs the way the +// apiserver applies them, so a marker that silently failed to generate +// shows up as a test failure rather than as an admitted bad object. +package apitest + +import ( + "context" + "os" + "path/filepath" + "testing" + + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextvalidation "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + schemacel "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/defaulting" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/listtype" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning" + apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" + utiljson "k8s.io/apimachinery/pkg/util/json" + "k8s.io/apimachinery/pkg/util/validation/field" + celconfig "k8s.io/apiserver/pkg/apis/cel" + "sigs.k8s.io/yaml" +) + +const definitionsDir = "../../../packages/system/cozystack-controller/definitions" + +var crdFiles = map[string]string{ + "GPUClass": "gpu.cozystack.io_gpuclasses.yaml", + "GPUDevice": "gpu.cozystack.io_gpudevices.yaml", + "GPUNode": "gpu.cozystack.io_gpunodes.yaml", +} + +func readCRD(t *testing.T, kind string) *apiextensionsv1.CustomResourceDefinition { + t.Helper() + raw, err := os.ReadFile(filepath.Join(definitionsDir, crdFiles[kind])) + if err != nil { + t.Fatalf("read %s CRD: %v", kind, err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(raw, &crd); err != nil { + t.Fatalf("unmarshal %s CRD: %v", kind, err) + } + return &crd +} + +// v1alpha1 fails the test when the version is missing, so a renamed +// version cannot turn every assertion below into a no-op. +func v1alpha1(t *testing.T, crd *apiextensionsv1.CustomResourceDefinition) *apiextensionsv1.CustomResourceDefinitionVersion { + t.Helper() + for i := range crd.Spec.Versions { + if v := &crd.Spec.Versions[i]; v.Name == "v1alpha1" && v.Schema != nil && v.Schema.OpenAPIV3Schema != nil { + return v + } + } + t.Fatalf("%s: no v1alpha1 version with a schema", crd.Name) + return nil +} + +// admission runs the steps the apiserver applies to a custom resource +// write, in its order: prune, default, structural schema, list types, CEL +// rules. +type admission struct { + structural *schema.Structural + schema apiservervalidation.SchemaValidator + cel *schemacel.Validator +} + +func newAdmission(t *testing.T, kind string) *admission { + t.Helper() + props := v1alpha1(t, readCRD(t, kind)).Schema.OpenAPIV3Schema + var internal apiextensions.JSONSchemaProps + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(props, &internal, nil); err != nil { + t.Fatalf("convert schema: %v", err) + } + structural, err := schema.NewStructural(&internal) + if err != nil { + t.Fatalf("structural schema: %v", err) + } + validator, _, err := apiservervalidation.NewSchemaValidator(&internal) + if err != nil { + t.Fatalf("schema validator: %v", err) + } + return &admission{ + structural: structural, + schema: validator, + cel: schemacel.NewValidator(structural, true, celconfig.PerCallLimit), + } +} + +func decode(t *testing.T, doc string) map[string]any { + t.Helper() + if doc == "" { + return nil + } + js, err := yaml.YAMLToJSON([]byte(doc)) + if err != nil { + t.Fatalf("yaml: %v", err) + } + var obj map[string]any + // utiljson decodes integers as int64, as the apiserver does. + if err := utiljson.Unmarshal(js, &obj); err != nil { + t.Fatalf("json: %v", err) + } + return obj +} + +// admit returns the object as stored and the errors the apiserver would +// report. old is empty for a create. +func (a *admission) admit(t *testing.T, doc, old string) (map[string]any, field.ErrorList) { + t.Helper() + obj, oldObj := decode(t, doc), decode(t, old) + for _, o := range []map[string]any{obj, oldObj} { + if o != nil { + pruning.Prune(o, a.structural, true) + defaulting.Default(o, a.structural) + } + } + errs := apiservervalidation.ValidateCustomResource(nil, obj, a.schema) + errs = append(errs, listtype.ValidateListSetsAndMaps(nil, a.structural, obj)...) + var oldArg any + if oldObj != nil { + oldArg = oldObj + } + celErrs, _ := a.cel.Validate(context.TODO(), nil, a.structural, obj, oldArg, celconfig.RuntimeCELCostBudget) + return obj, append(errs, celErrs...) +} + +func TestCRDsPassInstallTimeValidation(t *testing.T) { + for kind := range crdFiles { + t.Run(kind, func(t *testing.T) { + crd := readCRD(t, kind) + crd.Status.StoredVersions = []string{"v1alpha1"} + var internal apiextensions.CustomResourceDefinition + if err := apiextensionsv1.Convert_v1_CustomResourceDefinition_To_apiextensions_CustomResourceDefinition(crd, &internal, nil); err != nil { + t.Fatalf("convert CRD: %v", err) + } + for _, e := range apiextvalidation.ValidateCustomResourceDefinition(context.TODO(), &internal) { + t.Errorf("apiserver would reject this CRD: %v", e) + } + }) + } +} + +// The CRDs render as ordinary chart templates, so without the keep policy +// a controller uninstall deletes every GPUDevice, its hold and its usage +// record with the CRD. +func TestCRDsAreKeptOnChartUninstall(t *testing.T) { + for kind := range crdFiles { + if got := readCRD(t, kind).Annotations["helm.sh/resource-policy"]; got != "keep" { + t.Errorf("%s: helm.sh/resource-policy = %q, want keep", kind, got) + } + } +} + +// Tenants can write a GPUDevice but not its status; everything the binder +// decides lives there. +func TestCRDsHaveStatusSubresource(t *testing.T) { + for kind := range crdFiles { + v := v1alpha1(t, readCRD(t, kind)) + if v.Subresources == nil || v.Subresources.Status == nil { + t.Errorf("%s: no status subresource", kind) + } + } +} + +func TestGPUDeviceNodeNameIsSelectable(t *testing.T) { + for _, f := range v1alpha1(t, readCRD(t, "GPUDevice")).SelectableFields { + if f.JSONPath == ".status.nodeName" { + return + } + } + t.Error("GPUDevice does not declare .status.nodeName in selectableFields") +} + +func TestScope(t *testing.T) { + want := map[string]apiextensionsv1.ResourceScope{ + "GPUClass": apiextensionsv1.ClusterScoped, + "GPUDevice": apiextensionsv1.NamespaceScoped, + "GPUNode": apiextensionsv1.ClusterScoped, + } + for kind, scope := range want { + if got := readCRD(t, kind).Spec.Scope; got != scope { + t.Errorf("%s: scope %s, want %s", kind, got, scope) + } + } +} + +func gpuClass(name, spec string) string { + return "apiVersion: gpu.cozystack.io/v1alpha1\nkind: GPUClass\nmetadata:\n name: " + name + "\nspec:\n" + spec +} + +const passthroughSpec = ` vendor: nvidia + model: H100 + mode: passthrough + source: + devicePlugin: + resourceName: nvidia.com/GH100_H100 +` + +const vgpuSpec = ` vendor: nvidia + model: L40S + mode: vgpu + profile: L40S-24C + maxCount: 2 + memory: 24Gi + nodeSelector: + gpu.cozystack.io/10de-26b9: L40S-24C + source: + devicePlugin: + resourceName: nvidia.com/NVIDIA_L40S-24C +` + +func TestGPUClassCreate(t *testing.T) { + a := newAdmission(t, "GPUClass") + tests := []struct { + name string + obj string + wantReject bool + }{ + {name: "passthrough class", obj: gpuClass("h100", passthroughSpec)}, + {name: "vgpu class with profile", obj: gpuClass("l40s-24c", vgpuSpec)}, + {name: "name total is reserved", obj: gpuClass("total", passthroughSpec), wantReject: true}, + {name: "name with a dot is not a DNS label", obj: gpuClass("h100.whole", passthroughSpec), wantReject: true}, + {name: "upper-case name", obj: gpuClass("H100", passthroughSpec), wantReject: true}, + {name: "name longer than 63", obj: gpuClass("a234567890123456789012345678901234567890123456789012345678901234", passthroughSpec), wantReject: true}, + {name: "name of exactly 63", obj: gpuClass("a23456789012345678901234567890123456789012345678901234567890123", passthroughSpec)}, + {name: "vgpu without profile", obj: gpuClass("x", ` vendor: nvidia + model: L40S + mode: vgpu + source: + devicePlugin: + resourceName: nvidia.com/NVIDIA_L40S-24C +`), wantReject: true}, + {name: "passthrough with profile", obj: gpuClass("x", passthroughSpec+" profile: H100-80C\n"), wantReject: true}, + {name: "unknown mode", obj: gpuClass("x", ` vendor: nvidia + model: H100 + mode: mig + source: + devicePlugin: + resourceName: nvidia.com/GH100_H100 +`), wantReject: true}, + {name: "maxCount 0", obj: gpuClass("x", passthroughSpec+" maxCount: 0\n"), wantReject: true}, + {name: "maxCount 1", obj: gpuClass("x", passthroughSpec+" maxCount: 1\n")}, + {name: "onCordon Stay", obj: gpuClass("x", passthroughSpec+" onCordon: Stay\n")}, + {name: "unknown onCordon", obj: gpuClass("x", passthroughSpec+" onCordon: Drain\n"), wantReject: true}, + {name: "no source", obj: gpuClass("x", " vendor: nvidia\n model: H100\n mode: passthrough\n"), wantReject: true}, + {name: "empty source", obj: gpuClass("x", " vendor: nvidia\n model: H100\n mode: passthrough\n source: {}\n"), wantReject: true}, + { + // dra is not in the schema, so pruning drops it and the + // union rule sees no member. + name: "reserved dra source", + obj: gpuClass("x", ` vendor: nvidia + model: H100 + mode: passthrough + source: + dra: + deviceClassName: gpu.nvidia.com +`), + wantReject: true, + }, + {name: "devicePlugin without resourceName", obj: gpuClass("x", " vendor: nvidia\n model: H100\n mode: passthrough\n source:\n devicePlugin: {}\n"), wantReject: true}, + {name: "resourceName without a domain", obj: gpuClass("x", " vendor: nvidia\n model: H100\n mode: passthrough\n source:\n devicePlugin:\n resourceName: gpu\n"), wantReject: true}, + {name: "no vendor", obj: gpuClass("x", " model: H100\n mode: passthrough\n source:\n devicePlugin:\n resourceName: nvidia.com/GH100_H100\n"), wantReject: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, errs := a.admit(t, tc.obj, "") + if got := len(errs) > 0; got != tc.wantReject { + t.Errorf("rejected = %v, want %v; errors: %v", got, tc.wantReject, errs) + } + }) + } +} + +func TestGPUClassOnCordonDefaultsToMove(t *testing.T) { + obj, errs := newAdmission(t, "GPUClass").admit(t, gpuClass("h100", passthroughSpec), "") + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if got := obj["spec"].(map[string]any)["onCordon"]; got != "Move" { + t.Errorf("onCordon = %v, want Move", got) + } +} + +// maxCount has no schema default: its default depends on mode, which a +// structural default cannot express. +func TestGPUClassMaxCountHasNoSchemaDefault(t *testing.T) { + obj, errs := newAdmission(t, "GPUClass").admit(t, gpuClass("l40s", ` vendor: nvidia + model: L40S + mode: vgpu + profile: L40S-24C + source: + devicePlugin: + resourceName: nvidia.com/NVIDIA_L40S-24C +`), "") + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if v, ok := obj["spec"].(map[string]any)["maxCount"]; ok { + t.Errorf("maxCount defaulted to %v by the schema", v) + } +} + +// The CRD carries no transition rule on GPUClass: onCordon, resourceName +// and nodeSelector stay changeable, and freezing the rest while the +// finalizer is present needs the finalizer, which CRD CEL cannot read. +func TestGPUClassUpdateIsNotFrozenByTheCRD(t *testing.T) { + a := newAdmission(t, "GPUClass") + old := gpuClass("l40s-24c", vgpuSpec) + tests := []struct{ name, obj string }{ + {"onCordon", gpuClass("l40s-24c", vgpuSpec+" onCordon: Stay\n")}, + {"resourceName", gpuClass("l40s-24c", ` vendor: nvidia + model: L40S + mode: vgpu + profile: L40S-24C + maxCount: 2 + memory: 24Gi + nodeSelector: + gpu.cozystack.io/10de-26b9: L40S-24C + source: + devicePlugin: + resourceName: nvidia.com/L40S-24C +`)}, + {"nodeSelector", gpuClass("l40s-24c", ` vendor: nvidia + model: L40S + mode: vgpu + profile: L40S-24C + maxCount: 2 + memory: 24Gi + nodeSelector: + pool: b + source: + devicePlugin: + resourceName: nvidia.com/NVIDIA_L40S-24C +`)}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if _, errs := a.admit(t, tc.obj, old); len(errs) > 0 { + t.Errorf("update refused: %v", errs) + } + }) + } +} + +func gpuDevice(spec string) string { + return "apiVersion: gpu.cozystack.io/v1alpha1\nkind: GPUDevice\nmetadata:\n name: train-0\n namespace: tenant-acme\nspec:\n" + spec +} + +func TestGPUDeviceCreate(t *testing.T) { + a := newAdmission(t, "GPUDevice") + tests := []struct { + name string + spec string + wantReject bool + }{ + {name: "count 1", spec: " className: l40s-24c\n count: 1\n"}, + {name: "count 4", spec: " className: h100\n count: 4\n"}, + {name: "count 0", spec: " className: h100\n count: 0\n", wantReject: true}, + {name: "negative count", spec: " className: h100\n count: -1\n", wantReject: true}, + {name: "no count", spec: " className: h100\n", wantReject: true}, + {name: "no className", spec: " count: 1\n", wantReject: true}, + {name: "className not a DNS label", spec: " className: H100.whole\n count: 1\n", wantReject: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, errs := a.admit(t, gpuDevice(tc.spec), "") + if got := len(errs) > 0; got != tc.wantReject { + t.Errorf("rejected = %v, want %v; errors: %v", got, tc.wantReject, errs) + } + }) + } +} + +func TestGPUDeviceSpecIsImmutable(t *testing.T) { + a := newAdmission(t, "GPUDevice") + old := gpuDevice(" className: l40s-24c\n count: 1\n") + tests := []struct { + name string + obj string + wantReject bool + }{ + {name: "count changed", obj: gpuDevice(" className: l40s-24c\n count: 2\n"), wantReject: true}, + {name: "className changed", obj: gpuDevice(" className: h100\n count: 1\n"), wantReject: true}, + {name: "spec unchanged, status written", obj: gpuDevice(" className: l40s-24c\n count: 1\nstatus:\n phase: Bound\n nodeName: gpu-node-3\n")}, + {name: "labels changed", obj: "apiVersion: gpu.cozystack.io/v1alpha1\nkind: GPUDevice\nmetadata:\n name: train-0\n namespace: tenant-acme\n labels:\n pool: inference\nspec:\n className: l40s-24c\n count: 1\n"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, errs := a.admit(t, tc.obj, old) + if got := len(errs) > 0; got != tc.wantReject { + t.Errorf("rejected = %v, want %v; errors: %v", got, tc.wantReject, errs) + } + }) + } +} + +func TestGPUDeviceStatusPhaseEnum(t *testing.T) { + a := newAdmission(t, "GPUDevice") + for phase, wantReject := range map[string]bool{"Pending": false, "Bound": false, "Attached": false, "Lost": false, "Running": true} { + t.Run(phase, func(t *testing.T) { + _, errs := a.admit(t, gpuDevice(" className: h100\n count: 1\nstatus:\n phase: "+phase+"\n"), "") + if got := len(errs) > 0; got != wantReject { + t.Errorf("rejected = %v, want %v; errors: %v", got, wantReject, errs) + } + }) + } +} + +func TestGPUNodeInventory(t *testing.T) { + a := newAdmission(t, "GPUNode") + node := func(devices string) string { + return "apiVersion: gpu.cozystack.io/v1alpha1\nkind: GPUNode\nmetadata:\n name: gpu-node-3\nstatus:\n resourceNames: [nvidia.com/NVIDIA_L40S-24C]\n devices:\n" + devices + } + tests := []struct { + name string + devices string + wantReject bool + }{ + {name: "pf and vf", devices: ` - pciAddress: "0000:41:00.0" + model: L40S + function: pf + vfs: 16 + - pciAddress: "0000:41:00.4" + function: vf + profile: L40S-24C + resourceName: nvidia.com/NVIDIA_L40S-24C + healthy: true + usedBy: {namespace: tenant-acme, pod: virt-launcher-vm-instance-trainer-x7k2p} +`}, + {name: "unknown function", devices: " - pciAddress: \"0000:41:00.0\"\n function: mig\n", wantReject: true}, + {name: "malformed PCI address", devices: " - pciAddress: \"41:00.0\"\n function: whole\n", wantReject: true}, + {name: "duplicate PCI address", devices: " - pciAddress: \"0000:41:00.0\"\n function: whole\n - pciAddress: \"0000:41:00.0\"\n function: whole\n", wantReject: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, errs := a.admit(t, node(tc.devices), "") + if got := len(errs) > 0; got != tc.wantReject { + t.Errorf("rejected = %v, want %v; errors: %v", got, tc.wantReject, errs) + } + }) + } +} diff --git a/packages/system/cozystack-controller/definitions/gpu.cozystack.io_gpuclasses.yaml b/packages/system/cozystack-controller/definitions/gpu.cozystack.io_gpuclasses.yaml new file mode 100644 index 000000000..c927b081a --- /dev/null +++ b/packages/system/cozystack-controller/definitions/gpu.cozystack.io_gpuclasses.yaml @@ -0,0 +1,236 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + helm.sh/resource-policy: keep + name: gpuclasses.gpu.cozystack.io +spec: + group: gpu.cozystack.io + names: + kind: GPUClass + listKind: GPUClassList + plural: gpuclasses + singular: gpuclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.mode + name: Mode + type: string + - jsonPath: .spec.model + name: Model + type: string + - jsonPath: .spec.profile + name: Profile + type: string + - jsonPath: .status.free + name: Free + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: GPUClass is one GPU offer of the hoster's catalog. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: GPUClassSpec defines one GPU offer of the hoster's catalog. + properties: + displayName: + description: DisplayName is the human-readable name of the offer. + maxLength: 256 + type: string + maxCount: + description: |- + MaxCount is how many GPUs of this class one VM may take. Unset means 1 + for mode vgpu and, for mode passthrough, no limit beyond the GPUs of + one node; read it through the shared model rather than the raw field. + format: int32 + minimum: 1 + type: integer + memory: + anyOf: + - type: integer + - type: string + description: Memory is the GPU memory of one unit of the class. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + mode: + description: GPUMode is how a class's GPUs reach a VM. + enum: + - passthrough + - vgpu + type: string + model: + maxLength: 63 + minLength: 1 + type: string + nodeSelector: + additionalProperties: + type: string + description: NodeSelector selects the nodes that serve this class. + type: object + onCordon: + default: Move + description: OnCordon decides whether an unattached device leaves + a cordoned node. + enum: + - Move + - Stay + type: string + profile: + description: Profile is the vGPU profile, such as L40S-24C. Set only + for mode vgpu. + maxLength: 63 + minLength: 1 + type: string + source: + description: |- + GPUClassSource is the backend that serves a class. Exactly one member is + set. The name "dra" is reserved for a DRA-backed member. + properties: + devicePlugin: + description: DevicePluginSource serves a class through a Kubernetes + device plugin. + properties: + resourceName: + description: |- + ResourceName is the extended resource the device plugin advertises for + this class's GPUs. Two classes may share it only when their node + selectors do not overlap. + maxLength: 317 + minLength: 3 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$ + type: string + required: + - resourceName + type: object + type: object + x-kubernetes-validations: + - message: exactly one source must be set; devicePlugin is the only + source supported, dra is reserved + rule: '[has(self.devicePlugin)].filter(x, x).size() == 1' + vendor: + maxLength: 63 + minLength: 1 + type: string + required: + - mode + - model + - source + - vendor + type: object + x-kubernetes-validations: + - message: profile is required for mode vgpu and not allowed for mode + passthrough + rule: 'self.mode == ''vgpu'' ? has(self.profile) : !has(self.profile)' + status: + description: |- + GPUClassStatus is readable by every authenticated user, so it carries no + node names and no counts beyond free capacity. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + free: + description: |- + Free is the number of free units of the class on eligible nodes. + Absent until the controller has computed it. + format: int32 + type: integer + maxFreePerNode: + description: MaxFreePerNode is the largest number of free units on + one node. + format: int32 + type: integer + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: GPUClass name must be a DNS label, since it becomes part of a quota + key + rule: size(self.metadata.name) <= 63 && self.metadata.name.matches('^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') + - message: GPUClass name total is reserved for the gpu.cozystack.io/total + quota key + rule: self.metadata.name != 'total' + served: true + storage: true + subresources: + status: {} diff --git a/packages/system/cozystack-controller/definitions/gpu.cozystack.io_gpudevices.yaml b/packages/system/cozystack-controller/definitions/gpu.cozystack.io_gpudevices.yaml new file mode 100644 index 000000000..8cc948558 --- /dev/null +++ b/packages/system/cozystack-controller/definitions/gpu.cozystack.io_gpudevices.yaml @@ -0,0 +1,232 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + helm.sh/resource-policy: keep + name: gpudevices.gpu.cozystack.io +spec: + group: gpu.cozystack.io + names: + kind: GPUDevice + listKind: GPUDeviceList + plural: gpudevices + singular: gpudevice + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.className + name: Class + type: string + - jsonPath: .spec.count + name: Count + type: integer + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.attachedTo.vmi + name: VMI + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + GPUDevice is a tenant's GPU, or a set of GPUs of one class used together + on one node. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: 'GPUDeviceSpec is immutable: a change is a new order.' + properties: + className: + description: ClassName is the GPUClass the device is ordered against. + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + count: + description: Count is the number of GPUs of the class, always on one + node. + format: int32 + minimum: 1 + type: integer + required: + - className + - count + type: object + x-kubernetes-validations: + - message: spec is immutable; order a new GPUDevice instead + rule: self == oldSelf + status: + description: |- + GPUDeviceStatus is written by the controller and the admission webhook; + tenants cannot write it. + properties: + attachedTo: + description: GPUDeviceAttachment names the VMI a device is attached + to. + properties: + vmi: + description: |- + VMI is the name of the VirtualMachineInstance in the device's namespace. + It is recorded at admission, before the VMI exists. + minLength: 1 + type: string + required: + - vmi + type: object + bindRequest: + description: |- + BindRequest asks the binder to place a set of devices onto one node. The + admission webhook writes ID, VMI and Devices; the binder writes Result and + Message. internal/gpu/doc.go carries the full contract. + properties: + devices: + description: |- + Devices names every device of the set, in the device's namespace, + this one included. + items: + type: string + minItems: 1 + type: array + x-kubernetes-list-type: set + id: + description: ID is shared by every device of one request and is + unique per request. + minLength: 1 + type: string + message: + description: Message explains a refusal and names each device + and its node. + type: string + requestedAt: + description: RequestedAt lets the controller expire a request + whose VMI never appears. + format: date-time + type: string + result: + description: BindResult is the binder's answer to a BindRequest. + enum: + - Succeeded + - Refused + type: string + vmi: + description: VMI is the VirtualMachineInstance the set is gathered + for. + minLength: 1 + type: string + required: + - devices + - id + - requestedAt + - vmi + type: object + boundAt: + description: BoundAt is when the device was last bound to a node. + format: date-time + type: string + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + nodeName: + description: NodeName is the node the device holds its units on. + type: string + phase: + description: |- + GPUDevicePhase decides what a device holds and costs; the shared model + package carries the table. + enum: + - Pending + - Bound + - Attached + - Lost + type: string + type: object + required: + - spec + type: object + selectableFields: + - jsonPath: .status.nodeName + served: true + storage: true + subresources: + status: {} diff --git a/packages/system/cozystack-controller/definitions/gpu.cozystack.io_gpunodes.yaml b/packages/system/cozystack-controller/definitions/gpu.cozystack.io_gpunodes.yaml new file mode 100644 index 000000000..1bfe04aa8 --- /dev/null +++ b/packages/system/cozystack-controller/definitions/gpu.cozystack.io_gpunodes.yaml @@ -0,0 +1,176 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + helm.sh/resource-policy: keep + name: gpunodes.gpu.cozystack.io +spec: + group: gpu.cozystack.io + names: + kind: GPUNode + listKind: GPUNodeList + plural: gpunodes + singular: gpunode + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + GPUNode is the GPU inventory of the node of the same name. It is owned by + that Node and goes away with it. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + status: + description: GPUNodeStatus is the GPU inventory of one node, written by + the node agent. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + devices: + items: + description: GPUNodeDevice is one PCI function of a GPU. + properties: + function: + description: PCIFunction is the role of a PCI function in GPU + virtualization. + enum: + - pf + - vf + - whole + type: string + healthy: + description: |- + Healthy is the device plugin's health of the function. Absent for a + function no device plugin advertises. + type: boolean + model: + type: string + pciAddress: + pattern: ^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$ + type: string + profile: + description: Profile is the vGPU profile written to a VF. + type: string + resourceName: + description: |- + ResourceName is the device-plugin resource that advertises this + function, as kubelet's pod-resources API reports it. + type: string + usedBy: + description: GPUPodReference names the pod that uses a device. + properties: + namespace: + type: string + pod: + type: string + required: + - namespace + - pod + type: object + vfs: + description: VFs is the number of VFs a physical function carries. + format: int32 + type: integer + required: + - function + - pciAddress + type: object + type: array + x-kubernetes-list-map-keys: + - pciAddress + x-kubernetes-list-type: map + resourceNames: + description: |- + ResourceNames are the device-plugin resource names on this node whose + devices are GPU PCI functions. They join the cluster's GPU + resource-name set. + items: + type: string + type: array + x-kubernetes-list-type: set + type: object + type: object + served: true + storage: true + subresources: + status: {}