mirror of
https://github.com/cozystack/cozystack.git
synced 2026-10-29 08:07:33 +09:00
Aleksei Sviridkin
9da24c2aa8
The binder is the only writer of a device's placement, so budget and capacity checks that admission already ran cannot be raced past by two replicas or two webhook calls. Every event collapses into one pass over all GPU objects, which serializes the decisions without per-object locks, and devices are read past the cache so a unit booked in the last pass is never booked again from a stale view. A device that carries an unanswered bind request also queues a pass that only answers requests, so the webhook, which waits seconds, is not held behind moves, usage records and status of the whole cluster. The full pass moves held sets off nodes that cannot keep them, rebinds Lost sets before new orders, and binds new orders onto the fullest node with room. Finalizers and usage records run with the feature disabled too, so turning GPU off never hangs a namespace or class deletion. The devices one VM names move, go Lost and are rebound as a set. Every pass restores that invariant before it places anything, gathering a set left partly Lost, by a write that failed half way or by a VMI that ended next to held members, onto one node or sending all of it to Lost, so each later step only ever changes whole sets. A Lost device already counts against its tenant's budgets, so the pod of its still running VMI is not counted again under a raw GPU key; for node capacity it is, since the device itself holds no units. Signed-off-by: Aleksei Sviridkin <f@lex.la> Assisted-by: LLM
84 lines
2.8 KiB
Go
84 lines
2.8 KiB
Go
/*
|
|
Copyright 2026 The Cozystack Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package controller
|
|
|
|
import (
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"sigs.k8s.io/controller-runtime/pkg/metrics"
|
|
|
|
gpuv1alpha1 "github.com/cozystack/cozystack/api/gpu/v1alpha1"
|
|
)
|
|
|
|
// The class's capacity and load show the hoster's inventory, which the
|
|
// GPUClass status, readable by every authenticated user, must not.
|
|
var (
|
|
classCapacity = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Name: "cozystack_gpu_class_capacity_units",
|
|
Help: "Allocatable GPU units of the class on nodes that serve it.",
|
|
}, []string{"class"})
|
|
classBound = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Name: "cozystack_gpu_class_bound_units",
|
|
Help: "GPU units of the class held by Bound or Attached devices.",
|
|
}, []string{"class"})
|
|
classLost = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Name: "cozystack_gpu_class_lost_units",
|
|
Help: "GPU units of the class whose devices are Lost.",
|
|
}, []string{"class"})
|
|
classWaiting = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Name: "cozystack_gpu_class_waiting_units",
|
|
Help: "GPU units of the class that wait for a node: Pending orders and Lost devices.",
|
|
}, []string{"class"})
|
|
)
|
|
|
|
func init() {
|
|
metrics.Registry.MustRegister(classCapacity, classBound, classLost, classWaiting)
|
|
}
|
|
|
|
func (r *Reconciler) exportMetrics(w *world) {
|
|
for _, g := range []*prometheus.GaugeVec{classCapacity, classBound, classLost, classWaiting} {
|
|
g.Reset()
|
|
}
|
|
for name := range w.classes {
|
|
rn := w.resourceName(name)
|
|
var capacity float64
|
|
for _, n := range w.snap.Nodes {
|
|
if w.snap.Serves(name, n.Name) {
|
|
capacity += float64(n.Allocatable[rn])
|
|
}
|
|
}
|
|
classCapacity.WithLabelValues(name).Set(capacity)
|
|
classBound.WithLabelValues(name).Set(0)
|
|
classLost.WithLabelValues(name).Set(0)
|
|
classWaiting.WithLabelValues(name).Set(0)
|
|
}
|
|
for _, d := range w.devices {
|
|
if w.classes[d.Spec.ClassName] == nil || released(d) {
|
|
continue
|
|
}
|
|
units := float64(d.Spec.Count)
|
|
switch d.Status.Phase {
|
|
case gpuv1alpha1.GPUDeviceBound, gpuv1alpha1.GPUDeviceAttached:
|
|
classBound.WithLabelValues(d.Spec.ClassName).Add(units)
|
|
case gpuv1alpha1.GPUDeviceLost:
|
|
classLost.WithLabelValues(d.Spec.ClassName).Add(units)
|
|
classWaiting.WithLabelValues(d.Spec.ClassName).Add(units)
|
|
default:
|
|
classWaiting.WithLabelValues(d.Spec.ClassName).Add(units)
|
|
}
|
|
}
|
|
}
|