diff --git a/internal/gpu/config.go b/internal/gpu/config.go new file mode 100644 index 000000000..4b3e6b9b8 --- /dev/null +++ b/internal/gpu/config.go @@ -0,0 +1,46 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gpu + +import "time" + +// Config is the GPU feature configuration cozystack-controller takes from +// its flags. +type Config struct { + // Enabled turns on everything that watches KubeVirt objects: binder + // placement, the VMI and pod webhooks, VMI garbage collection. The + // gpu.cozystack.io lifecycle paths (finalizers, usage-record cleanup) + // run regardless, so disabling the feature never hangs a deletion. + Enabled bool + + // ResourceNames is the hoster's gpu.resourceNames list. The effective + // set adds class and agent-reported names (model.ResourceNames). + ResourceNames []string + + RawResourceNames RawNamesPolicy + + // LostCapacityGrace is how long a device stays missing or unhealthy, or + // a node NotReady, before the node has lost capacity. kubelet reports + // zero allocatable after every restart, so it must outlast a plugin + // re-registration. + LostCapacityGrace time.Duration + + // AttachmentGrace is how long an attachment, or a template device, may + // wait for its VMI to appear. It must exceed the API server request + // timeout. + AttachmentGrace time.Duration +} diff --git a/internal/gpu/consts.go b/internal/gpu/consts.go new file mode 100644 index 000000000..c5bb6a968 --- /dev/null +++ b/internal/gpu/consts.go @@ -0,0 +1,96 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gpu + +const ( + // Group is the API group of the GPU kinds and the prefix of every key below. + Group = "gpu.cozystack.io" + + // LabelDeviceReference marks a VMI template, and through KubeVirt its + // virt-launcher pod, as carrying device references. The webhooks select + // on it, so only such VMIs and pods can be blocked by their outage. + LabelDeviceReference = Group + "/device-reference" + // AnnotationDevices holds the VMI's device references as a JSON list; + // model.ParseDeviceRefs reads it. + AnnotationDevices = Group + "/devices" + // VMIGPUNamePrefix starts the name of every gpus entry the webhook adds + // to a VMI, so a reinvocation recognises its own entries. + VMIGPUNamePrefix = "gpudev-" + + // LabelTemplateDevice marks a GPUDevice the webhook created from a + // deviceTemplate entry. Its value is the VMI name. Tenants cannot set it. + LabelTemplateDevice = Group + "/template-device" + + // LabelUsageMonitor is the reserved selector key of the controller-owned + // WorkloadMonitor. Admission refuses it on every object in a tenant + // namespace, so the monitor matches nothing and only the controller + // writes GPU usage records. + LabelUsageMonitor = Group + "/usage-monitor" + // UsageMonitorName is the controller-owned WorkloadMonitor in each + // tenant namespace that holds a GPUDevice. + UsageMonitorName = "gpu-devices" + // WorkloadKind is status.kind of a GPUDevice usage record. + WorkloadKind = "gpu-device" + + // FinalizerClass keeps a deleted GPUClass until its last Bound, + // Attached or Lost device is gone. + FinalizerClass = Group + "/class-protection" + // FinalizerDevice keeps a deleted GPUDevice, its binding and its usage + // record until no VMI uses it. + FinalizerDevice = Group + "/device-protection" + + // LabelNamespaceVGPU is the effective vGPU entitlement the tenant chart + // writes on every tenant namespace. Only LabelValueTrue entitles. + LabelNamespaceVGPU = "namespace.cozystack.io/vgpu" + LabelValueTrue = "true" + + // QuotaKeyPrefix starts every GPU budget key in a tenant's + // resourceQuotas: QuotaKeyPrefix+ per class and QuotaKeyTotal + // across classes. + QuotaKeyPrefix = Group + "/" + // QuotaTotal is the reserved class name of the cross-class budget. + QuotaTotal = "total" + QuotaKeyTotal = QuotaKeyPrefix + QuotaTotal + + // SettingsConfigMap is written by the controller and read by the + // ValidatingAdmissionPolicies (as paramRef) and by cozystack-api. + SettingsConfigMap = "cozystack-gpu-settings" + // SettingsNamespace is the namespace of SettingsConfigMap. + SettingsNamespace = "cozy-system" + // SettingsKeyResourceNames is the effective GPU resource-name set, + // sorted and comma-separated; ConfigMap values cannot hold a list, and a + // resource name cannot contain a comma. + SettingsKeyResourceNames = "resourceNames" + // SettingsKeyRawResourceNames is RawNamesAllowed or RawNamesForbidden. + SettingsKeyRawResourceNames = "rawResourceNames" + + // ConditionLostCapacity is set on a GPUClass and a GPUNode whose node + // has lost capacity for longer than the grace period. + ConditionLostCapacity = "LostCapacity" + // ConditionResourceNameConflict is set on two classes that share a + // resource name on a node they both select, and on that GPUNode. The + // node is not used for binding. + ConditionResourceNameConflict = "ResourceNameConflict" +) + +// RawNamesPolicy is the hoster's gpu.rawResourceNames setting. +type RawNamesPolicy string + +const ( + RawNamesAllowed RawNamesPolicy = "allowed" + RawNamesForbidden RawNamesPolicy = "forbidden" +) diff --git a/internal/gpu/controller/setup.go b/internal/gpu/controller/setup.go new file mode 100644 index 000000000..fcb9a2659 --- /dev/null +++ b/internal/gpu/controller/setup.go @@ -0,0 +1,31 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package controller runs the GPU binder, the class and device lifecycle, +// usage records and metrics in cozystack-controller. +package controller + +import ( + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/cozystack/cozystack/internal/gpu" +) + +// Setup adds the GPU reconcilers to mgr. With no reconciler implemented it +// adds nothing. +func Setup(mgr manager.Manager, cfg gpu.Config) error { + return nil +} diff --git a/internal/gpu/controller/setup_test.go b/internal/gpu/controller/setup_test.go new file mode 100644 index 000000000..6ec6b3cbb --- /dev/null +++ b/internal/gpu/controller/setup_test.go @@ -0,0 +1,31 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + "github.com/cozystack/cozystack/internal/gpu" +) + +// Setup is wired into cozystack-controller before any reconciler exists: +// it must succeed and touch nothing, so it accepts a nil manager. +func TestSetupWithoutReconcilersIsANoOp(t *testing.T) { + if err := Setup(nil, gpu.Config{Enabled: true}); err != nil { + t.Fatalf("Setup: %v", err) + } +} diff --git a/internal/gpu/doc.go b/internal/gpu/doc.go new file mode 100644 index 000000000..6832e12a2 --- /dev/null +++ b/internal/gpu/doc.go @@ -0,0 +1,75 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package gpu holds what the GPU components of cozystack-controller, +// cozystack-api and the node agent share: label, annotation, finalizer and +// quota keys, the settings ConfigMap, and the feature Config. The pure +// placement, budget and capacity logic lives in package model. +// +// # Binding +// +// The binder runs in the leader and is the only writer of a GPUDevice's +// status.phase, status.nodeName and status.boundAt. Every check of capacity +// and budget that decides a bind runs again in its single loop, so two +// admissions racing on one unit cannot both win. +// +// The VMI admission webhook runs in every replica and never binds. On a dry +// run it writes nothing. Otherwise, for a VMI, it: +// +// 1. creates the template devices, +// 2. writes status.attachedTo on every device of the VMI's set, held and +// template, with optimistic concurrency, so no other VMI can take one, +// 3. writes the same status.bindRequest (ID, VMI, Devices, RequestedAt) on +// each of them, with an ID unique to this admission, +// 4. waits on its informer until every device of the set carries that ID +// with a Result, and admits the VMI only if all of them are Succeeded +// and share one node. +// +// The binder, for a request whose devices all carry the same ID, either +// places the whole set on one eligible node with room for all of it, moving +// or binding devices as needed, and writes Result Succeeded on each; or +// changes no placement and writes Result Refused with a Message that names +// each device and its node. A partial request, where some device of +// Devices lacks the ID, is not acted on. A move that has started finishes +// even if the webhook stops waiting. +// +// The webhook can be called more than once for one VMI, so every step is +// idempotent: a repeated call finds its template devices and its +// attachments already in place, and admits without a new request when every +// device of the set carries a Succeeded request for that VMI on one node. +// +// When the webhook refuses or times out, it clears what it wrote. When it +// cannot (it crashed, or the VMI was refused by a later admission step), +// the controller clears an attachment and a request whose VMI has not +// appeared after Config.AttachmentGrace, and deletes such template devices. +// +// Status is a subresource that no tenant role grants, so a tenant cannot +// forge an attachment or a request. +// +// # Settings ConfigMap +// +// The controller writes SettingsConfigMap in SettingsNamespace: +// +// data: +// resourceNames: "nvidia.com/GH100_H100,nvidia.com/NVIDIA_L40S-24C" +// rawResourceNames: "allowed" +// +// resourceNames is the effective GPU resource-name set (model.ResourceNames, +// formatted by model.FormatResourceNames); rawResourceNames is a +// RawNamesPolicy. The ValidatingAdmissionPolicies take it as paramRef, and +// cozystack-api reads it for the option sources and the tenant quota gate. +// The controller owns the object; nothing else writes it. +package gpu diff --git a/internal/gpu/webhook/register.go b/internal/gpu/webhook/register.go new file mode 100644 index 000000000..ffc681779 --- /dev/null +++ b/internal/gpu/webhook/register.go @@ -0,0 +1,31 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package webhook serves the GPUDevice, VMI and virt-launcher pod admission +// webhooks. +package webhook + +import ( + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/cozystack/cozystack/internal/gpu" +) + +// Register adds the GPU admission webhooks to mgr's webhook server. With +// no webhook implemented it adds nothing. +func Register(mgr manager.Manager, cfg gpu.Config) error { + return nil +} diff --git a/internal/gpu/webhook/register_test.go b/internal/gpu/webhook/register_test.go new file mode 100644 index 000000000..29fbfc2f0 --- /dev/null +++ b/internal/gpu/webhook/register_test.go @@ -0,0 +1,31 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "testing" + + "github.com/cozystack/cozystack/internal/gpu" +) + +// Register is wired into cozystack-controller before any webhook exists: +// it must succeed and touch nothing, so it accepts a nil manager. +func TestRegisterWithoutWebhooksIsANoOp(t *testing.T) { + if err := Register(nil, gpu.Config{Enabled: true}); err != nil { + t.Fatalf("Register: %v", err) + } +}