mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-09 05:25:14 +09:00
feat(gpu): copy vGPU license Secrets into entitled namespaces
A VM of a licensed vGPU class mounts its license from a Secret in its own namespace, so the token reaches entitled tenants only and is rotated by a VM restart rather than by rebuilding a shared image. The copier keeps one copy per class in every namespace whose vGPU label is "true", with the token and a gridd.conf carrying the class's FeatureType, and deletes copies the label, the class or its license no longer call for. A missing hoster Secret, or one without the token, keeps the copies already written: deleting them would stop every restarting VM over what is usually a rotation in progress. New copies wait, and the class shows LicenseSecretMissing so the hoster sees why. Only Secrets carrying both the copy's name and the class label are ever written or deleted, so a tenant's own Secret of the same name is left alone. The token is read through the uncached reader and stays out of every cache. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
2 files changed
+671
No files matched your search
@@ -0,0 +1,250 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/equality"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
|
||||
"github.com/cozystack/cozystack/internal/gpu"
|
||||
)
|
||||
|
||||
const (
|
||||
reasonSecretFound = "SecretFound"
|
||||
reasonSecretNotFound = "SecretNotFound"
|
||||
reasonTokenKeyMissing = "TokenKeyMissing"
|
||||
)
|
||||
|
||||
// LicenseReconciler copies the hoster's license Secret of one GPUClass
|
||||
// into every namespace entitled to vGPU, and deletes the copies a
|
||||
// namespace, the class or its license no longer calls for.
|
||||
type LicenseReconciler struct {
|
||||
Client client.Client
|
||||
// Reader is uncached and reads the hoster's Secret and the copies:
|
||||
// the manager's Secret cache is scoped to other reconcilers' Secrets,
|
||||
// and the token stays out of every cache.
|
||||
Reader client.Reader
|
||||
// Secrets lists Secret metadata, without data, to find the copies.
|
||||
Secrets client.Reader
|
||||
// Enabled follows the desired GPU-for-VMs configuration, not whether
|
||||
// a retained GPU operator Package still runs. When false, it writes
|
||||
// no copy, deletes every copy and clears the computed class status.
|
||||
// Existing VMs must stop using the platform license before disabling
|
||||
// GPU-for-VMs: a retained Package does not retain their license copies.
|
||||
// Cleanup needs only the controller's Secret and GPUClass status writes.
|
||||
Enabled bool
|
||||
}
|
||||
|
||||
func (r *LicenseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
class := &cozyv1alpha1.GPUClass{}
|
||||
if err := r.Client.Get(ctx, req.NamespacedName, class); apierrors.IsNotFound(err) {
|
||||
class = nil
|
||||
} else if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
var license *cozyv1alpha1.GPUClassLicense
|
||||
if r.Enabled && class != nil {
|
||||
license = class.Spec.License
|
||||
}
|
||||
entitled := sets.New[string]()
|
||||
var data map[string][]byte
|
||||
var condition *metav1.Condition
|
||||
if license != nil {
|
||||
var nsList corev1.NamespaceList
|
||||
if err := r.Client.List(ctx, &nsList, client.MatchingLabels{gpu.LabelNamespaceVGPU: gpu.LabelValueTrue}); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
for i := range nsList.Items {
|
||||
// A terminating namespace refuses new objects until it is gone,
|
||||
// and the hoster's namespace never holds a copy.
|
||||
if ns := &nsList.Items[i]; ns.DeletionTimestamp == nil && ns.Name != gpu.LicenseSecretNamespace {
|
||||
entitled.Insert(ns.Name)
|
||||
}
|
||||
}
|
||||
var err error
|
||||
data, condition, err = r.desired(ctx, class)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
}
|
||||
|
||||
var errs []error
|
||||
if data != nil {
|
||||
for ns := range entitled {
|
||||
errs = append(errs, r.write(ctx, ns, req.Name, data))
|
||||
}
|
||||
}
|
||||
errs = append(errs, r.prune(ctx, req.Name, entitled))
|
||||
if class != nil {
|
||||
errs = append(errs, r.setStatus(ctx, class, condition))
|
||||
}
|
||||
return ctrl.Result{}, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// desired returns the data of every copy, or nil and a True
|
||||
// LicenseSecretMissing while the hoster's Secret cannot provide a token.
|
||||
func (r *LicenseReconciler) desired(ctx context.Context, class *cozyv1alpha1.GPUClass) (map[string][]byte, *metav1.Condition, error) {
|
||||
license := class.Spec.License
|
||||
cond := &metav1.Condition{Type: cozyv1alpha1.GPUClassLicenseSecretMissing, ObservedGeneration: class.Generation}
|
||||
src := &corev1.Secret{}
|
||||
err := r.Reader.Get(ctx, client.ObjectKey{Namespace: gpu.LicenseSecretNamespace, Name: license.SecretName}, src)
|
||||
switch {
|
||||
case apierrors.IsNotFound(err):
|
||||
cond.Status, cond.Reason = metav1.ConditionTrue, reasonSecretNotFound
|
||||
cond.Message = fmt.Sprintf("Secret %s/%s does not exist; existing license copies are kept and no new ones are written", gpu.LicenseSecretNamespace, license.SecretName)
|
||||
return nil, cond, nil
|
||||
case err != nil:
|
||||
return nil, nil, err
|
||||
case len(src.Data[gpu.LicenseTokenKey]) == 0:
|
||||
cond.Status, cond.Reason = metav1.ConditionTrue, reasonTokenKeyMissing
|
||||
cond.Message = fmt.Sprintf("Secret %s/%s has no %s; existing license copies are kept and no new ones are written", gpu.LicenseSecretNamespace, license.SecretName, gpu.LicenseTokenKey)
|
||||
return nil, cond, nil
|
||||
}
|
||||
cond.Status, cond.Reason = metav1.ConditionFalse, reasonSecretFound
|
||||
featureType := license.FeatureType
|
||||
if featureType == 0 {
|
||||
featureType = 1
|
||||
}
|
||||
return map[string][]byte{
|
||||
gpu.LicenseTokenKey: src.Data[gpu.LicenseTokenKey],
|
||||
gpu.LicenseGriddConfKey: fmt.Appendf(nil, "FeatureType=%d\n", featureType),
|
||||
}, cond, nil
|
||||
}
|
||||
|
||||
// write creates or refreshes the copy in ns, and leaves alone a Secret of
|
||||
// that name without the class's label, which is not a copy.
|
||||
func (r *LicenseReconciler) write(ctx context.Context, ns, class string, data map[string][]byte) error {
|
||||
name := gpu.LicenseSecretName(class)
|
||||
cur := &corev1.Secret{}
|
||||
err := r.Reader.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, cur)
|
||||
if apierrors.IsNotFound(err) {
|
||||
return r.Client.Create(ctx, &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name, Labels: map[string]string{gpu.LabelGPUClass: class}},
|
||||
Type: corev1.SecretTypeOpaque,
|
||||
Data: data,
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if cur.Labels[gpu.LabelGPUClass] != class {
|
||||
log.FromContext(ctx).Info("a Secret that is not a license copy has the copy's name; the namespace gets no license",
|
||||
"namespace", ns, "secret", name, "class", class)
|
||||
return nil
|
||||
}
|
||||
if maps.EqualFunc(cur.Data, data, func(a, b []byte) bool { return string(a) == string(b) }) {
|
||||
return nil
|
||||
}
|
||||
cur.Data = data
|
||||
return r.Client.Update(ctx, cur)
|
||||
}
|
||||
|
||||
// prune deletes the class's copies outside keep. A copy is a Secret with
|
||||
// both the class's label and the copy's name, outside the namespace of the
|
||||
// hoster's Secrets, which a copy is never written to.
|
||||
func (r *LicenseReconciler) prune(ctx context.Context, class string, keep sets.Set[string]) error {
|
||||
list := &metav1.PartialObjectMetadataList{}
|
||||
list.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("SecretList"))
|
||||
if err := r.Secrets.List(ctx, list, client.MatchingLabels{gpu.LabelGPUClass: class}); err != nil {
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
for i := range list.Items {
|
||||
s := &list.Items[i]
|
||||
if s.Name != gpu.LicenseSecretName(class) || s.Namespace == gpu.LicenseSecretNamespace || keep.Has(s.Namespace) {
|
||||
continue
|
||||
}
|
||||
del := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: s.Namespace, Name: s.Name}}
|
||||
errs = append(errs, client.IgnoreNotFound(r.Client.Delete(ctx, del, client.Preconditions{UID: &s.UID})))
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// setStatus sets cond, or removes LicenseSecretMissing when cond is nil.
|
||||
// Where GPU features are off it also clears what the status controller
|
||||
// wrote, which no longer follows the nodes once that controller stops.
|
||||
func (r *LicenseReconciler) setStatus(ctx context.Context, class *cozyv1alpha1.GPUClass, cond *metav1.Condition) error {
|
||||
before := class.Status.DeepCopy()
|
||||
if !r.Enabled {
|
||||
class.Status.Free, class.Status.MaxFreePerNode = nil, nil
|
||||
meta.RemoveStatusCondition(&class.Status.Conditions, cozyv1alpha1.GPUClassResourceNameConflict)
|
||||
meta.RemoveStatusCondition(&class.Status.Conditions, cozyv1alpha1.GPUClassResourceNameNotPermitted)
|
||||
}
|
||||
if cond == nil {
|
||||
meta.RemoveStatusCondition(&class.Status.Conditions, cozyv1alpha1.GPUClassLicenseSecretMissing)
|
||||
} else {
|
||||
meta.SetStatusCondition(&class.Status.Conditions, *cond)
|
||||
}
|
||||
if equality.Semantic.DeepEqual(before, &class.Status) {
|
||||
return nil
|
||||
}
|
||||
return client.IgnoreNotFound(r.Client.Status().Update(ctx, class))
|
||||
}
|
||||
|
||||
// secretToClasses maps a hoster Secret to the classes naming it, and a
|
||||
// copy to its class, so a rotation, a tampered copy and a copy left by a
|
||||
// class deleted while the controller was down are all reconciled.
|
||||
func (r *LicenseReconciler) secretToClasses(ctx context.Context, obj *metav1.PartialObjectMetadata) []reconcile.Request {
|
||||
var out []reconcile.Request
|
||||
if class := obj.GetLabels()[gpu.LabelGPUClass]; class != "" {
|
||||
out = append(out, reconcile.Request{NamespacedName: client.ObjectKey{Name: class}})
|
||||
}
|
||||
if obj.GetNamespace() != gpu.LicenseSecretNamespace {
|
||||
return out
|
||||
}
|
||||
var classes cozyv1alpha1.GPUClassList
|
||||
if err := r.Client.List(ctx, &classes); err != nil {
|
||||
log.FromContext(ctx).Error(err, "map a license Secret to its GPUClasses", "secret", obj.GetName())
|
||||
return out
|
||||
}
|
||||
for i := range classes.Items {
|
||||
if l := classes.Items[i].Spec.License; l != nil && l.SecretName == obj.GetName() {
|
||||
out = append(out, reconcile.Request{NamespacedName: client.ObjectKey{Name: classes.Items[i].Name}})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// allClasses maps a namespace event to every class: any of them may have
|
||||
// a copy to write or delete there.
|
||||
func (r *LicenseReconciler) allClasses(ctx context.Context, _ client.Object) []reconcile.Request {
|
||||
var classes cozyv1alpha1.GPUClassList
|
||||
if err := r.Client.List(ctx, &classes); err != nil {
|
||||
log.FromContext(ctx).Error(err, "list GPUClasses for a namespace event")
|
||||
return nil
|
||||
}
|
||||
out := make([]reconcile.Request, 0, len(classes.Items))
|
||||
for i := range classes.Items {
|
||||
out = append(out, reconcile.Request{NamespacedName: client.ObjectKey{Name: classes.Items[i].Name}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,421 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/utils/ptr"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
|
||||
"github.com/cozystack/cozystack/internal/gpu"
|
||||
)
|
||||
|
||||
const (
|
||||
tokenSecret = "nvidia-license-token"
|
||||
copyName = "gpuclass-l40s-license"
|
||||
)
|
||||
|
||||
func namespace(name string, vgpu string) *corev1.Namespace {
|
||||
ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}}
|
||||
if vgpu != "" {
|
||||
ns.Labels = map[string]string{gpu.LabelNamespaceVGPU: vgpu}
|
||||
}
|
||||
return ns
|
||||
}
|
||||
|
||||
func secret(ns, name string, labels map[string]string, data map[string]string) *corev1.Secret {
|
||||
s := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name, Labels: labels}, Data: map[string][]byte{}}
|
||||
for k, v := range data {
|
||||
s.Data[k] = []byte(v)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func hosterSecret(data map[string]string) *corev1.Secret {
|
||||
return secret(gpu.LicenseSecretNamespace, tokenSecret, nil, data)
|
||||
}
|
||||
|
||||
func licenseCopy(ns, class string, data map[string]string) *corev1.Secret {
|
||||
return secret(ns, gpu.LicenseSecretName(class), map[string]string{gpu.LabelGPUClass: class}, data)
|
||||
}
|
||||
|
||||
func licensed(secretName string, featureType int32) func(*cozyv1alpha1.GPUClass) {
|
||||
return func(c *cozyv1alpha1.GPUClass) {
|
||||
c.Spec.Mode, c.Spec.Profile = cozyv1alpha1.GPUModeVGPU, "L40S-24C"
|
||||
c.Spec.License = &cozyv1alpha1.GPUClassLicense{SecretName: secretName, FeatureType: featureType}
|
||||
}
|
||||
}
|
||||
|
||||
func runLicense(t *testing.T, c client.Client, enabled bool, class string) {
|
||||
t.Helper()
|
||||
r := &LicenseReconciler{Client: c, Reader: c, Secrets: c, Enabled: enabled}
|
||||
if _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Name: class}}); err != nil {
|
||||
t.Fatalf("Reconcile: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// getSecret returns nil when the Secret does not exist.
|
||||
func getSecret(t *testing.T, c client.Client, ns, name string) *corev1.Secret {
|
||||
t.Helper()
|
||||
s := &corev1.Secret{}
|
||||
err := c.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, s)
|
||||
if apierrors.IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func dataOf(s *corev1.Secret) map[string]string {
|
||||
out := map[string]string{}
|
||||
for k, v := range s.Data {
|
||||
out[k] = string(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func wantCopy(t *testing.T, c client.Client, ns string, want map[string]string) {
|
||||
t.Helper()
|
||||
s := getSecret(t, c, ns, copyName)
|
||||
if s == nil {
|
||||
t.Fatalf("%s: no license copy", ns)
|
||||
}
|
||||
if s.Labels[gpu.LabelGPUClass] != "l40s" {
|
||||
t.Errorf("%s: labels = %v, want %s=l40s", ns, s.Labels, gpu.LabelGPUClass)
|
||||
}
|
||||
if got := dataOf(s); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("%s: data = %v, want %v", ns, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func wantNoCopy(t *testing.T, c client.Client, ns string) {
|
||||
t.Helper()
|
||||
if s := getSecret(t, c, ns, copyName); s != nil {
|
||||
t.Errorf("%s: license copy exists: %v", ns, dataOf(s))
|
||||
}
|
||||
}
|
||||
|
||||
func licenseCondition(t *testing.T, c client.Client) *metav1.Condition {
|
||||
t.Helper()
|
||||
return meta.FindStatusCondition(getClass(t, c, "l40s").Status.Conditions, cozyv1alpha1.GPUClassLicenseSecretMissing)
|
||||
}
|
||||
|
||||
func TestLicenseCopiesIntoEntitledNamespacesOnly(t *testing.T) {
|
||||
c := newClient(t, nil,
|
||||
gpuClass("l40s", rnL40S, 0, licensed(tokenSecret, 4)),
|
||||
hosterSecret(map[string]string{gpu.LicenseTokenKey: "token-1", "notes.txt": "hoster only"}),
|
||||
namespace("tenant-a", "true"), namespace("tenant-b", "false"), namespace("tenant-c", ""),
|
||||
)
|
||||
runLicense(t, c, true, "l40s")
|
||||
|
||||
wantCopy(t, c, "tenant-a", map[string]string{gpu.LicenseTokenKey: "token-1", gpu.LicenseGriddConfKey: "FeatureType=4\n"})
|
||||
wantNoCopy(t, c, "tenant-b")
|
||||
wantNoCopy(t, c, "tenant-c")
|
||||
if cond := licenseCondition(t, c); cond == nil || cond.Status != metav1.ConditionFalse {
|
||||
t.Errorf("LicenseSecretMissing = %+v, want False", cond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLicenseFeatureTypeDefaultsToOne(t *testing.T) {
|
||||
c := newClient(t, nil,
|
||||
gpuClass("l40s", rnL40S, 0, licensed(tokenSecret, 0)),
|
||||
hosterSecret(map[string]string{gpu.LicenseTokenKey: "token-1"}),
|
||||
namespace("tenant-a", "true"),
|
||||
)
|
||||
runLicense(t, c, true, "l40s")
|
||||
wantCopy(t, c, "tenant-a", map[string]string{gpu.LicenseTokenKey: "token-1", gpu.LicenseGriddConfKey: "FeatureType=1\n"})
|
||||
}
|
||||
|
||||
func TestLicenseRefreshesAStaleCopy(t *testing.T) {
|
||||
stale := licenseCopy("tenant-a", "l40s", map[string]string{gpu.LicenseTokenKey: "token-0", gpu.LicenseGriddConfKey: "FeatureType=1\n", "extra": "x"})
|
||||
stale.Annotations = map[string]string{"kept": "yes"}
|
||||
c := newClient(t, nil,
|
||||
gpuClass("l40s", rnL40S, 0, licensed(tokenSecret, 4)),
|
||||
hosterSecret(map[string]string{gpu.LicenseTokenKey: "token-1"}),
|
||||
namespace("tenant-a", "true"),
|
||||
stale,
|
||||
)
|
||||
runLicense(t, c, true, "l40s")
|
||||
wantCopy(t, c, "tenant-a", map[string]string{gpu.LicenseTokenKey: "token-1", gpu.LicenseGriddConfKey: "FeatureType=4\n"})
|
||||
if got := getSecret(t, c, "tenant-a", copyName).Annotations["kept"]; got != "yes" {
|
||||
t.Errorf("annotation kept = %q, want the copy's own metadata left alone", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLicenseDeletesCopies(t *testing.T) {
|
||||
copies := func() []client.Object {
|
||||
return []client.Object{
|
||||
licenseCopy("tenant-a", "l40s", map[string]string{gpu.LicenseTokenKey: "token-1"}),
|
||||
licenseCopy("tenant-b", "l40s", map[string]string{gpu.LicenseTokenKey: "token-1"}),
|
||||
}
|
||||
}
|
||||
src := func() client.Object { return hosterSecret(map[string]string{gpu.LicenseTokenKey: "token-1"}) }
|
||||
tests := []struct {
|
||||
name string
|
||||
objs []client.Object
|
||||
enabled bool
|
||||
gone []string
|
||||
kept []string
|
||||
}{
|
||||
{
|
||||
name: "namespace loses the entitlement",
|
||||
objs: []client.Object{gpuClass("l40s", rnL40S, 0, licensed(tokenSecret, 1)), src(), namespace("tenant-a", "true"), namespace("tenant-b", "false")},
|
||||
enabled: true, gone: []string{"tenant-b"}, kept: []string{"tenant-a"},
|
||||
},
|
||||
{
|
||||
name: "class is gone",
|
||||
objs: []client.Object{src(), namespace("tenant-a", "true"), namespace("tenant-b", "true")},
|
||||
enabled: true, gone: []string{"tenant-a", "tenant-b"},
|
||||
},
|
||||
{
|
||||
name: "class drops its license",
|
||||
objs: []client.Object{gpuClass("l40s", rnL40S, 0), src(), namespace("tenant-a", "true"), namespace("tenant-b", "true")},
|
||||
enabled: true, gone: []string{"tenant-a", "tenant-b"},
|
||||
},
|
||||
{
|
||||
name: "GPU features disabled",
|
||||
objs: []client.Object{gpuClass("l40s", rnL40S, 0, licensed(tokenSecret, 1)), src(), namespace("tenant-a", "true"), namespace("tenant-b", "true")},
|
||||
enabled: false, gone: []string{"tenant-a", "tenant-b"},
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
c := newClient(t, nil, append(tc.objs, copies()...)...)
|
||||
runLicense(t, c, tc.enabled, "l40s")
|
||||
for _, ns := range tc.gone {
|
||||
wantNoCopy(t, c, ns)
|
||||
}
|
||||
for _, ns := range tc.kept {
|
||||
wantCopy(t, c, ns, map[string]string{gpu.LicenseTokenKey: "token-1", gpu.LicenseGriddConfKey: "FeatureType=1\n"})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLicenseConditionGoesWithTheLicense(t *testing.T) {
|
||||
class := gpuClass("l40s", rnL40S, 0, func(c *cozyv1alpha1.GPUClass) {
|
||||
c.Status.Conditions = []metav1.Condition{{Type: cozyv1alpha1.GPUClassLicenseSecretMissing, Status: metav1.ConditionTrue, Reason: reasonSecretNotFound, LastTransitionTime: metav1.NewTime(epoch)}}
|
||||
})
|
||||
c := newClient(t, nil, class)
|
||||
runLicense(t, c, true, "l40s")
|
||||
if cond := licenseCondition(t, c); cond != nil {
|
||||
t.Errorf("LicenseSecretMissing = %+v on a class without a license, want none", cond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLicenseDisabledClearsComputedStatus(t *testing.T) {
|
||||
class := gpuClass("l40s", rnL40S, 0, func(c *cozyv1alpha1.GPUClass) {
|
||||
c.Status.Free, c.Status.MaxFreePerNode = ptr.To[int32](3), ptr.To[int32](2)
|
||||
for _, typ := range []string{cozyv1alpha1.GPUClassResourceNameConflict, cozyv1alpha1.GPUClassResourceNameNotPermitted} {
|
||||
c.Status.Conditions = append(c.Status.Conditions, metav1.Condition{Type: typ, Status: metav1.ConditionFalse, Reason: "Computed", LastTransitionTime: metav1.NewTime(epoch)})
|
||||
}
|
||||
})
|
||||
for _, enabled := range []bool{true, false} {
|
||||
c := newClient(t, nil, class.DeepCopy())
|
||||
runLicense(t, c, enabled, "l40s")
|
||||
got := getClass(t, c, "l40s").Status
|
||||
cleared := got.Free == nil && got.MaxFreePerNode == nil && len(got.Conditions) == 0
|
||||
if cleared == enabled {
|
||||
t.Errorf("enabled=%v: status = %+v, want it cleared only while GPU features are off", enabled, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLicenseSourceUnusable(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
source *corev1.Secret
|
||||
reason string
|
||||
}{
|
||||
{"source missing", nil, reasonSecretNotFound},
|
||||
{"source without the token key", hosterSecret(map[string]string{"token": "misnamed"}), reasonTokenKeyMissing},
|
||||
{"source with an empty token", hosterSecret(map[string]string{gpu.LicenseTokenKey: ""}), reasonTokenKeyMissing},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
objs := []client.Object{
|
||||
gpuClass("l40s", rnL40S, 0, licensed(tokenSecret, 4)),
|
||||
namespace("tenant-a", "true"), namespace("tenant-new", "true"), namespace("tenant-gone", "false"),
|
||||
licenseCopy("tenant-a", "l40s", map[string]string{gpu.LicenseTokenKey: "token-0", gpu.LicenseGriddConfKey: "FeatureType=1\n"}),
|
||||
licenseCopy("tenant-gone", "l40s", map[string]string{gpu.LicenseTokenKey: "token-0"}),
|
||||
}
|
||||
if tc.source != nil {
|
||||
objs = append(objs, tc.source)
|
||||
}
|
||||
c := newClient(t, nil, objs...)
|
||||
runLicense(t, c, true, "l40s")
|
||||
|
||||
wantCopy(t, c, "tenant-a", map[string]string{gpu.LicenseTokenKey: "token-0", gpu.LicenseGriddConfKey: "FeatureType=1\n"})
|
||||
wantNoCopy(t, c, "tenant-new")
|
||||
wantNoCopy(t, c, "tenant-gone")
|
||||
cond := licenseCondition(t, c)
|
||||
if cond == nil || cond.Status != metav1.ConditionTrue || cond.Reason != tc.reason {
|
||||
t.Errorf("LicenseSecretMissing = %+v, want True with reason %s", cond, tc.reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLicenseTouchesOnlyItsLabelledCopies(t *testing.T) {
|
||||
foreign := secret("tenant-a", copyName, map[string]string{"app": "mine"}, map[string]string{"k": "tenant data"})
|
||||
unentitledForeign := secret("tenant-b", copyName, nil, map[string]string{"k": "tenant data"})
|
||||
otherName := secret("tenant-b", "my-secret", map[string]string{gpu.LabelGPUClass: "l40s"}, map[string]string{"k": "v"})
|
||||
otherClass := licenseCopy("tenant-b", "h100", map[string]string{gpu.LicenseTokenKey: "h"})
|
||||
hosterLabelled := licenseCopy(gpu.LicenseSecretNamespace, "l40s", map[string]string{gpu.LicenseTokenKey: "hoster"})
|
||||
terminating := namespace("tenant-d", "true")
|
||||
terminating.Finalizers = []string{"test"}
|
||||
terminating.DeletionTimestamp = ptr.To(metav1.NewTime(epoch))
|
||||
c := newClient(t, nil,
|
||||
gpuClass("l40s", rnL40S, 0, licensed(tokenSecret, 1)),
|
||||
hosterSecret(map[string]string{gpu.LicenseTokenKey: "token-1"}),
|
||||
namespace("tenant-a", "true"), namespace("tenant-b", "false"), namespace("tenant-c", "true"),
|
||||
namespace(gpu.LicenseSecretNamespace, "true"),
|
||||
foreign, unentitledForeign, otherName, otherClass, hosterLabelled, terminating,
|
||||
)
|
||||
runLicense(t, c, true, "l40s")
|
||||
|
||||
if got := getSecret(t, c, "tenant-a", copyName); got == nil || !reflect.DeepEqual(dataOf(got), map[string]string{"k": "tenant data"}) || got.Labels[gpu.LabelGPUClass] != "" {
|
||||
t.Errorf("foreign Secret changed: %+v", got)
|
||||
}
|
||||
if getSecret(t, c, "tenant-b", copyName) == nil {
|
||||
t.Error("deleted an unlabelled Secret with the copy's name")
|
||||
}
|
||||
if getSecret(t, c, "tenant-b", "my-secret") == nil {
|
||||
t.Error("deleted a labelled Secret that is not named as a copy")
|
||||
}
|
||||
if got := getSecret(t, c, gpu.LicenseSecretNamespace, copyName); got == nil || string(got.Data[gpu.LicenseTokenKey]) != "hoster" {
|
||||
t.Errorf("touched a hoster Secret in cozy-system that looks like a copy: %+v", got)
|
||||
}
|
||||
wantNoCopy(t, c, "tenant-d")
|
||||
if getSecret(t, c, "tenant-b", gpu.LicenseSecretName("h100")) == nil {
|
||||
t.Error("deleted another class's copy")
|
||||
}
|
||||
wantCopy(t, c, "tenant-c", map[string]string{gpu.LicenseTokenKey: "token-1", gpu.LicenseGriddConfKey: "FeatureType=1\n"})
|
||||
}
|
||||
|
||||
func TestLicenseSecretToClasses(t *testing.T) {
|
||||
c := newClient(t, nil,
|
||||
gpuClass("l40s", rnL40S, 0, licensed(tokenSecret, 1)),
|
||||
gpuClass("l40s-q", rnH100, 1, licensed(tokenSecret, 1)),
|
||||
gpuClass("h100", rnH100, 2),
|
||||
)
|
||||
r := &LicenseReconciler{Client: c}
|
||||
stub := func(ns, name string, labels map[string]string) *metav1.PartialObjectMetadata {
|
||||
return &metav1.PartialObjectMetadata{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name, Labels: labels}}
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
obj *metav1.PartialObjectMetadata
|
||||
want []string
|
||||
}{
|
||||
{"hoster Secret named by two classes", stub(gpu.LicenseSecretNamespace, tokenSecret, nil), []string{"l40s", "l40s-q"}},
|
||||
{"same name outside cozy-system", stub("tenant-a", tokenSecret, nil), nil},
|
||||
{"unrelated cozy-system Secret", stub(gpu.LicenseSecretNamespace, "other", nil), nil},
|
||||
{"a copy", stub("tenant-a", gpu.LicenseSecretName("gone"), map[string]string{gpu.LabelGPUClass: "gone"}), []string{"gone"}},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
var got []string
|
||||
for _, req := range r.secretToClasses(context.Background(), tc.obj) {
|
||||
got = append(got, req.Name)
|
||||
}
|
||||
slices.Sort(got)
|
||||
if !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("%s: classes = %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
var all []string
|
||||
for _, req := range r.allClasses(context.Background(), namespace("tenant-a", "true")) {
|
||||
all = append(all, req.Name)
|
||||
}
|
||||
slices.Sort(all)
|
||||
if want := []string{"h100", "l40s", "l40s-q"}; !reflect.DeepEqual(all, want) {
|
||||
t.Errorf("namespace event classes = %v, want %v", all, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLicenseSecondPassWritesNothing(t *testing.T) {
|
||||
writes := 0
|
||||
count := func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
|
||||
writes++
|
||||
return cl.Update(ctx, obj, opts...)
|
||||
}
|
||||
c := newClient(t, &interceptor.Funcs{
|
||||
Update: count,
|
||||
SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error {
|
||||
writes++
|
||||
return cl.SubResource(sub).Update(ctx, obj, opts...)
|
||||
},
|
||||
},
|
||||
gpuClass("l40s", rnL40S, 0, licensed(tokenSecret, 1)),
|
||||
hosterSecret(map[string]string{gpu.LicenseTokenKey: "token-1"}),
|
||||
namespace("tenant-a", "true"),
|
||||
)
|
||||
runLicense(t, c, true, "l40s")
|
||||
writes = 0
|
||||
runLicense(t, c, true, "l40s")
|
||||
if writes != 0 {
|
||||
t.Errorf("writes on an unchanged second pass = %d, want 0", writes)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLicensePruneDeletesOnlyTheListedCopy covers a copy replaced by a
|
||||
// tenant's own Secret of the same name after the metadata cache listed it.
|
||||
func TestLicensePruneDeletesOnlyTheListedCopy(t *testing.T) {
|
||||
stale := licenseCopy("tenant-b", "l40s", nil)
|
||||
stale.UID = "listed-copy"
|
||||
cached := newClient(t, nil, stale)
|
||||
replaced := secret("tenant-b", copyName, nil, map[string]string{"k": "tenant data"})
|
||||
replaced.UID = "tenant-secret"
|
||||
// The fake client checks only resourceVersion preconditions; this
|
||||
// applies the UID one as the API server does.
|
||||
c := newClient(t, &interceptor.Funcs{
|
||||
Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteOption) error {
|
||||
do := &client.DeleteOptions{}
|
||||
do.ApplyOptions(opts)
|
||||
cur := &corev1.Secret{}
|
||||
if err := cl.Get(ctx, client.ObjectKeyFromObject(obj), cur); err != nil {
|
||||
return err
|
||||
}
|
||||
if do.Preconditions != nil && do.Preconditions.UID != nil && *do.Preconditions.UID != cur.UID {
|
||||
return apierrors.NewConflict(corev1.Resource("secrets"), obj.GetName(), errors.New("UID precondition failed"))
|
||||
}
|
||||
return cl.Delete(ctx, obj, opts...)
|
||||
},
|
||||
}, gpuClass("l40s", rnL40S, 0), replaced)
|
||||
|
||||
r := &LicenseReconciler{Client: c, Reader: c, Secrets: cached, Enabled: true}
|
||||
_, _ = r.Reconcile(context.Background(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "l40s"}})
|
||||
if getSecret(t, c, "tenant-b", copyName) == nil {
|
||||
t.Error("deleted a Secret the metadata cache did not list")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user