From 8bdb93385debe809ee572f0d9369e689d3347a9d Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 18 Nov 2026 12:15:37 +0300 Subject: [PATCH] fix(credentials): serialize an application's seeding and pruning with a lock A create or update seeds and prunes account Secrets outside the HelmRelease write's optimistic concurrency, so two concurrent writes to one application could interleave: one read the live release, a second re-added an account and wrote it back, and the first then pruned the Secret the live release now referenced, leaving a dangling reference and a lost password. The API server runs with two replicas, so an in-process guard is not enough. A per-application coordination.k8s.io Lease in cozy-system, which tenants cannot write, now serializes the credential section of a create and an update across replicas: a write is refused with a retryable Conflict while another holds the lock, the lease is stolen once it expires so a crashed holder cannot wedge it, and a lock store that cannot be reached is logged and the request proceeds, since the lock only guards a rare race. Assisted-by: LLM Signed-off-by: IvanHunters --- .../system/cozystack-api/templates/rbac.yaml | 7 + .../system/cozystack-api/tests/rbac_test.yaml | 13 +- pkg/apiserver/apiserver.go | 5 + pkg/credentials/applock/applock.go | 151 ++++++++++++++++++ pkg/credentials/applock/applock_test.go | 85 ++++++++++ pkg/registry/apps/application/rest.go | 30 ++++ .../apps/application/rest_credentials.go | 22 +++ .../apps/application/rest_credentials_test.go | 78 +++++++++ 8 files changed, 388 insertions(+), 3 deletions(-) create mode 100644 pkg/credentials/applock/applock.go create mode 100644 pkg/credentials/applock/applock_test.go diff --git a/packages/system/cozystack-api/templates/rbac.yaml b/packages/system/cozystack-api/templates/rbac.yaml index 70f1bec97..1cc5d8e0e 100644 --- a/packages/system/cozystack-api/templates/rbac.yaml +++ b/packages/system/cozystack-api/templates/rbac.yaml @@ -90,3 +90,10 @@ rules: - apiGroups: ["cilium.io"] resources: ["ciliumnetworkpolicies"] verbs: ["get", "watch", "list", "create", "update", "patch", "delete"] +# The per-application credential lock (a Lease in cozy-system) serializes a +# create's or update's account-Secret seeding and pruning across the API +# server's replicas, so two concurrent writes cannot interleave into deleting a +# Secret the other reused. +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "update", "delete"] diff --git a/packages/system/cozystack-api/tests/rbac_test.yaml b/packages/system/cozystack-api/tests/rbac_test.yaml index a919fedb8..af24136b5 100644 --- a/packages/system/cozystack-api/tests/rbac_test.yaml +++ b/packages/system/cozystack-api/tests/rbac_test.yaml @@ -94,11 +94,18 @@ tests: resources: ["events"] verbs: ["create"] - - it: ships exactly the twenty-two rules this role is pinned to + - it: ships exactly the twenty-three rules this role is pinned to asserts: # Both directions. A dropped grant is the failure this suite is about: # the apiserver returns forbidden and List silently drops the dropdown. - exists: - path: rules[21] - - notExists: path: rules[22] + - notExists: + path: rules[23] + # The last rule is the credential lock's Lease access. + - equal: + path: rules[22] + value: + apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "update", "delete"] diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 55de032e9..92b081f9c 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -23,6 +23,7 @@ import ( cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" helmv2 "github.com/fluxcd/helm-controller/api/v2" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -93,6 +94,10 @@ func init() { if err := rbacv1.AddToScheme(mgrScheme); err != nil { panic(fmt.Errorf("failed to add RBAC types to scheme: %w", err)) } + // Register Lease types for the per-application credential lock. + if err := coordinationv1.AddToScheme(mgrScheme); err != nil { + panic(fmt.Errorf("failed to add coordination types to scheme: %w", err)) + } // Register Cozystack types for WorkloadMonitor queries. if err := cozyv1alpha1.AddToScheme(mgrScheme); err != nil { diff --git a/pkg/credentials/applock/applock.go b/pkg/credentials/applock/applock.go new file mode 100644 index 000000000..8a8b673ae --- /dev/null +++ b/pkg/credentials/applock/applock.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package applock serializes an application's credential operations across the +// API server's replicas. A create or update seeds and prunes account Secrets +// outside the HelmRelease write's optimistic concurrency, so two concurrent +// writes to one application can interleave: one deletes a Secret the other has +// just reused and referenced. The lock makes those operations take turns, keyed +// by the application, through a coordination.k8s.io Lease. +package applock + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ErrHeld means another operation holds the application's lock. The caller +// turns it into a retryable Conflict rather than waiting, so a stuck holder +// cannot block a write past its lease. +var ErrHeld = errors.New("another change to the application's credentials is in progress") + +// Locker takes and releases the per-application lock. +type Locker struct { + // c reads and writes Leases; it must be uncached so an acquire sees a Lease + // a concurrent operation wrote a moment ago. + c client.Client + // namespace holds the Leases; cozy-system, which tenants cannot write, so a + // tenant cannot hold or delete its own application's lock. + namespace string + // duration is how long a Lease is honored without renewal. An operation + // finishes well within it; a holder that crashed has its Lease stolen once + // it expires, so the lock never wedges. + duration time.Duration +} + +// New returns a Locker that keeps its Leases in namespace (cozy-system). +func New(c client.Client, namespace string) *Locker { + return &Locker{c: c, namespace: namespace, duration: 60 * time.Second} +} + +// leaseName is the Lease name for one application, a hash so any namespace and +// release name map to a valid, collision-free object name. +func leaseName(appNamespace, release string) string { + sum := sha256.Sum256([]byte(appNamespace + "/" + release)) + return "cred-" + hex.EncodeToString(sum[:])[:40] +} + +func holderID() (string, error) { + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + host, _ := os.Hostname() + return fmt.Sprintf("%s-%d-%s", host, os.Getpid(), hex.EncodeToString(b[:])), nil +} + +// Acquire takes the application's lock. On success it returns a release +// function the caller defers. It returns ErrHeld when another operation holds a +// live lock. Any other error is an infrastructure failure the caller may treat +// as fail-open, since the lock only guards a rare race and the Lease store being +// unreachable is no worse than not having the lock at all. +func (l *Locker) Acquire(ctx context.Context, appNamespace, release string) (func(), error) { + name := leaseName(appNamespace, release) + id, err := holderID() + if err != nil { + return nil, err + } + now := metav1.NewMicroTime(time.Now()) + dur := int32(l.duration / time.Second) + + cur := &coordinationv1.Lease{} + err = l.c.Get(ctx, client.ObjectKey{Namespace: l.namespace, Name: name}, cur) + switch { + case apierrors.IsNotFound(err): + lease := &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{Namespace: l.namespace, Name: name}, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: &id, + LeaseDurationSeconds: &dur, + AcquireTime: &now, + RenewTime: &now, + }, + } + if err := l.c.Create(ctx, lease); err != nil { + if apierrors.IsAlreadyExists(err) { + return nil, ErrHeld // another operation created it between the Get and here + } + return nil, err + } + return l.releaser(name, id), nil + case err != nil: + return nil, err + } + + // A Lease exists: steal it only if it has expired, under its resourceVersion + // so two stealers do not both win. + if held(cur) { + return nil, ErrHeld + } + cur.Spec.HolderIdentity = &id + cur.Spec.LeaseDurationSeconds = &dur + cur.Spec.AcquireTime = &now + cur.Spec.RenewTime = &now + if err := l.c.Update(ctx, cur); err != nil { + if apierrors.IsConflict(err) { + return nil, ErrHeld // another operation stole or renewed it first + } + return nil, err + } + return l.releaser(name, id), nil +} + +// held reports whether a Lease is still honored: it has a holder and its renew +// time plus its duration is in the future. +func held(lease *coordinationv1.Lease) bool { + if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity == "" || lease.Spec.RenewTime == nil { + return false + } + dur := time.Duration(60) * time.Second + if lease.Spec.LeaseDurationSeconds != nil { + dur = time.Duration(*lease.Spec.LeaseDurationSeconds) * time.Second + } + return lease.Spec.RenewTime.Add(dur).After(time.Now()) +} + +// releaser deletes the Lease if this operation still holds it. Best effort: a +// failed release leaves a Lease that the next acquire steals once it expires. +func (l *Locker) releaser(name, id string) func() { + return func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cur := &coordinationv1.Lease{} + if err := l.c.Get(ctx, client.ObjectKey{Namespace: l.namespace, Name: name}, cur); err != nil { + return + } + if cur.Spec.HolderIdentity == nil || *cur.Spec.HolderIdentity != id { + return // someone else holds it now; not ours to delete + } + _ = l.c.Delete(ctx, cur, client.Preconditions{UID: &cur.UID, ResourceVersion: &cur.ResourceVersion}) + } +} diff --git a/pkg/credentials/applock/applock_test.go b/pkg/credentials/applock/applock_test.go new file mode 100644 index 000000000..d63e91233 --- /dev/null +++ b/pkg/credentials/applock/applock_test.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 + +package applock + +import ( + "context" + "errors" + "testing" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func newClient(t *testing.T) client.Client { + t.Helper() + s := runtime.NewScheme() + if err := coordinationv1.AddToScheme(s); err != nil { + t.Fatal(err) + } + return fake.NewClientBuilder().WithScheme(s).Build() +} + +func TestAcquireBlocksAndReleases(t *testing.T) { + ctx := context.Background() + l := New(newClient(t), "cozy-system") + + unlock, err := l.Acquire(ctx, "tenant-foo", "postgres-orders") + if err != nil { + t.Fatalf("first acquire: %v", err) + } + // Held now: a second acquire of the same application is refused. + if _, err := l.Acquire(ctx, "tenant-foo", "postgres-orders"); !errors.Is(err, ErrHeld) { + t.Fatalf("a held lock must refuse, got %v", err) + } + // A different application locks independently. + if u2, err := l.Acquire(ctx, "tenant-foo", "postgres-other"); err != nil { + t.Fatalf("a different application must lock independently: %v", err) + } else { + u2() + } + // After release, the application can be locked again. + unlock() + if u3, err := l.Acquire(ctx, "tenant-foo", "postgres-orders"); err != nil { + t.Fatalf("acquire after release: %v", err) + } else { + u3() + } +} + +func seedLease(t *testing.T, c client.Client, appNamespace, release, holder string, renew time.Time) { + t.Helper() + rt := metav1.NewMicroTime(renew) + dur := int32(60) + if err := c.Create(context.Background(), &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{Namespace: "cozy-system", Name: leaseName(appNamespace, release)}, + Spec: coordinationv1.LeaseSpec{HolderIdentity: &holder, LeaseDurationSeconds: &dur, RenewTime: &rt}, + }); err != nil { + t.Fatal(err) + } +} + +func TestAcquireRefusesAFreshLeaseOfAnother(t *testing.T) { + ctx := context.Background() + c := newClient(t) + seedLease(t, c, "tenant-foo", "postgres-orders", "someone-else", time.Now()) + if _, err := New(c, "cozy-system").Acquire(ctx, "tenant-foo", "postgres-orders"); !errors.Is(err, ErrHeld) { + t.Fatalf("a fresh lease held by another must refuse, got %v", err) + } +} + +func TestAcquireStealsAnExpiredLease(t *testing.T) { + ctx := context.Background() + c := newClient(t) + // Held by a holder that last renewed longer than a lease duration ago. + seedLease(t, c, "tenant-foo", "postgres-orders", "crashed-holder", time.Now().Add(-5*time.Minute)) + unlock, err := New(c, "cozy-system").Acquire(ctx, "tenant-foo", "postgres-orders") + if err != nil { + t.Fatalf("an expired lease must be stealable: %v", err) + } + unlock() +} diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index cd5d1bbe8..a03c508e1 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -56,6 +56,7 @@ import ( appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" "github.com/cozystack/cozystack/pkg/apis/apps/validation" "github.com/cozystack/cozystack/pkg/config" + "github.com/cozystack/cozystack/pkg/credentials/applock" "github.com/cozystack/cozystack/pkg/registry" fieldfilter "github.com/cozystack/cozystack/pkg/registry/fields" "github.com/cozystack/cozystack/pkg/registry/sorting" @@ -105,6 +106,10 @@ type REST struct { // credentials is the kind's credentials block, nil for a kind whose // passwords the chart still generates. credentials *cozyv1alpha1.ApplicationDefinitionCredentials + // lock serializes one application's credential seeding and pruning across + // the API server's replicas, so two concurrent writes cannot interleave + // into deleting a Secret the other has just reused. + lock *applock.Locker } // buildSpecSchema parses an OpenAPI-v3 JSON schema string into the @@ -126,6 +131,13 @@ func NewREST(c client.Client, w client.WithWatch, config *config.Resource) *REST config.Application.Kind, appsv1alpha1.NameSchemaExtension, config.Application.Kind, nameSchemaErr) } + // The lock reads and writes its Leases uncached, and in cozy-system, which + // tenants cannot write. + lockClient := client.Client(w) + if lockClient == nil { + lockClient = c + } + return &REST{ c: c, w: w, @@ -145,6 +157,7 @@ func NewREST(c client.Client, w client.WithWatch, config *config.Resource) *REST nameSchema: nameSchema, nameSchemaErr: nameSchemaErr, credentials: config.Credentials, + lock: applock.New(lockClient, "cozy-system"), } } @@ -252,6 +265,14 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation helmRelease.Labels[ApplicationNameLabel] = app.Name // Note: Annotations from config are not handled as r.releaseConfig.Annotations is undefined + // Hold the application's credential lock across the seed, the release write + // and the owning, so a concurrent write cannot interleave with them. + unlock, err := r.lockCredentials(ctx, app) + if err != nil { + return nil, err + } + defer unlock() + // The account Secrets exist before the release that reads them, and the // release owns them as soon as it exists. seeded, err := r.seedBeforeCreate(ctx, app, helmRelease, accounts, options != nil && dryrun.IsDryRun(options.DryRun)) @@ -602,6 +623,15 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje return nil, false, fmt.Errorf("conversion error: %v", err) } + // Hold the application's credential lock across the read of the live + // release, the seed, the release write and the prune, so a concurrent write + // cannot re-add an account and have this request prune the Secret it reused. + unlock, err := r.lockCredentials(ctx, app) + if err != nil { + return nil, false, err + } + defer unlock() + cur := &helmv2.HelmRelease{} if err := r.c.Get(ctx, client.ObjectKey{Namespace: helmRelease.Namespace, Name: helmRelease.Name}, cur, &client.GetOptions{Raw: &metav1.GetOptions{}}); err != nil { return nil, false, registry.WrapPreservingStatus("failed to fetch current HelmRelease", err, r.gvr.GroupResource(), name) diff --git a/pkg/registry/apps/application/rest_credentials.go b/pkg/registry/apps/application/rest_credentials.go index 5ca90c882..74a084f9b 100644 --- a/pkg/registry/apps/application/rest_credentials.go +++ b/pkg/registry/apps/application/rest_credentials.go @@ -22,6 +22,7 @@ import ( cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" "github.com/cozystack/cozystack/pkg/credentials/account" + "github.com/cozystack/cozystack/pkg/credentials/applock" "github.com/cozystack/cozystack/pkg/credentials/store" ) @@ -78,6 +79,27 @@ func (r *REST) direct() client.Client { return r.c } +// lockCredentials takes the per-application lock so this request's seeding and +// pruning do not interleave with a concurrent write to the same application. It +// returns a release function the caller defers. A lock another request holds is +// a retryable Conflict; a lock store that cannot be reached is logged and the +// request proceeds without it, since the lock only guards a rare race and its +// store being unreachable is no worse than not having it. +func (r *REST) lockCredentials(ctx context.Context, app *appsv1alpha1.Application) (func(), error) { + if r.credentials == nil || r.lock == nil { + return func() {}, nil + } + unlock, err := r.lock.Acquire(ctx, app.Namespace, r.storeApp(app.Namespace, app.Name).Release) + switch { + case errors.Is(err, applock.ErrHeld): + return nil, apierrors.NewConflict(r.gvr.GroupResource(), app.Name, errors.New("another change to this application's credentials is in progress; retry")) + case err != nil: + klog.Warningf("credential lock unavailable for %s/%s, proceeding without it: %v", app.Namespace, app.Name, err) + return func() {}, nil + } + return unlock, nil +} + // seedBeforeCreate writes the account Secrets of a new application, with no // owner yet, and the valuesFrom entries that carry them into the chart. // It returns the Secrets it wrote, for the owner to be set once the diff --git a/pkg/registry/apps/application/rest_credentials_test.go b/pkg/registry/apps/application/rest_credentials_test.go index ebada7eb8..bf769ae23 100644 --- a/pkg/registry/apps/application/rest_credentials_test.go +++ b/pkg/registry/apps/application/rest_credentials_test.go @@ -10,6 +10,7 @@ import ( "testing" helmv2 "github.com/fluxcd/helm-controller/api/v2" + coordinationv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -55,6 +56,9 @@ func newCredentialsREST(t *testing.T, objects ...client.Object) (*REST, client.C if err := corev1.AddToScheme(scheme); err != nil { t.Fatal(err) } + if err := coordinationv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } cfg := &config.Resource{ Application: config.ApplicationConfig{Kind: postgresKind, Singular: "postgres", Plural: "postgreses", OpenAPISchema: credentialsSchema}, Release: config.ReleaseConfig{Prefix: "postgres-"}, @@ -449,6 +453,80 @@ func TestConcurrentCreateKeepsLiveSecret(t *testing.T) { // it, because its UID is not the one this create seeded. Deleting by name, or // only skipping an owned Secret, would take it and leave the live release with // no account Secret. +// A concurrent write to one application's credentials is refused while another +// holds the lock, so the two cannot interleave into deleting a Secret the other +// reused. Request A adds api; while A holds the lock and writes its release, the +// interceptor runs request B (which would remove web and prune its Secret). B +// is refused with Conflict, so web's Secret is untouched. +func TestConcurrentUpdateRefusedByCredentialLock(t *testing.T) { + scheme := runtime.NewScheme() + if err := helmv2.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := coordinationv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cfg := &config.Resource{ + Application: config.ApplicationConfig{Kind: postgresKind, Singular: "postgres", Plural: "postgreses", OpenAPISchema: credentialsSchema}, + Release: config.ReleaseConfig{Prefix: "postgres-"}, + Credentials: credentialsBlock(), + } + if err := appsv1alpha1.RegisterDynamicTypes(scheme, &config.ResourceConfig{Resources: []config.Resource{*cfg}}); err != nil { + t.Fatal(err) + } + + var r *REST + var raced atomic.Bool + var bErr error + var uids atomic.Int64 + c := fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if obj.GetUID() == "" { + obj.SetUID(types.UID(fmt.Sprintf("uid-%d", uids.Add(1)))) + } + return cl.Create(ctx, obj, opts...) + }, + Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + // Request A is writing its HelmRelease, so A holds the lock: a second + // update of the same application must be refused now. + if _, ok := obj.(*helmv2.HelmRelease); ok && raced.CompareAndSwap(false, true) { + _, _, bErr = r.Update(credentialsCtx(), "orders", newDefaultUpdatedObjectInfo(postgresApp(`{"users":{}}`)), nil, nil, false, &metav1.UpdateOptions{}) + } + if obj.GetUID() == "" { + cur := obj.DeepCopyObject().(client.Object) + if err := cl.Get(ctx, client.ObjectKeyFromObject(obj), cur); err == nil { + obj.SetUID(cur.GetUID()) + } + } + return cl.Update(ctx, obj, opts...) + }, + }).Build() + r = NewREST(c, c, cfg) + ctx := credentialsCtx() + + if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + webBefore := accountSecrets(t, c)["postgres-orders.web.account"] + + if _, _, err := r.Update(ctx, "orders", newDefaultUpdatedObjectInfo(postgresApp(`{"users":{"web":{},"api":{}}}`)), nil, nil, false, &metav1.UpdateOptions{}); err != nil { + t.Fatalf("request A: %v", err) + } + if !apierrors.IsConflict(bErr) { + t.Fatalf("a concurrent credential write must be refused with Conflict, got %v", bErr) + } + after := accountSecrets(t, c)["postgres-orders.web.account"] + if after == nil || after.UID != webBefore.UID { + t.Fatal("the refused concurrent update must not have pruned web's Secret") + } + if accountSecrets(t, c)["postgres-orders.api.account"] == nil { + t.Fatal("request A must have seeded api") + } +} + func TestUnseedLeavesAReplacedSecret(t *testing.T) { r, c := newCredentialsREST(t) ctx := credentialsCtx()