fix(credentials): serialize an application's seeding and pruning with a lock

A create or update seeds and prunes account Secrets outside the HelmRelease write's optimistic concurrency, so two concurrent writes to one application could interleave: one read the live release, a second re-added an account and wrote it back, and the first then pruned the Secret the live release now referenced, leaving a dangling reference and a lost password. The API server runs with two replicas, so an in-process guard is not enough. A per-application coordination.k8s.io Lease in cozy-system, which tenants cannot write, now serializes the credential section of a create and an update across replicas: a write is refused with a retryable Conflict while another holds the lock, the lease is stolen once it expires so a crashed holder cannot wedge it, and a lock store that cannot be reached is logged and the request proceeds, since the lock only guards a rare race.

Assisted-by: LLM
Signed-off-by: IvanHunters <xorokhotnikov@gmail.com>
This commit is contained in:
IvanHunters committed 2026-11-18 12:15:37 +03:00
1 parent 4772b8e760
commit 8bdb93385d
8 files changed
+388 -3

No files matched your search

@@ -90,3 +90,10 @@ rules:
- apiGroups: ["cilium.io"]
resources: ["ciliumnetworkpolicies"]
verbs: ["get", "watch", "list", "create", "update", "patch", "delete"]
# The per-application credential lock (a Lease in cozy-system) serializes a
# create's or update's account-Secret seeding and pruning across the API
# server's replicas, so two concurrent writes cannot interleave into deleting a
# Secret the other reused.
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "create", "update", "delete"]
@@ -94,11 +94,18 @@ tests:
resources: ["events"]
verbs: ["create"]
- it: ships exactly the twenty-two rules this role is pinned to
- it: ships exactly the twenty-three rules this role is pinned to
asserts:
# Both directions. A dropped grant is the failure this suite is about:
# the apiserver returns forbidden and List silently drops the dropdown.
- exists:
path: rules[21]
- notExists:
path: rules[22]
- notExists:
path: rules[23]
# The last rule is the credential lock's Lease access.
- equal:
path: rules[22]
value:
apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "create", "update", "delete"]
+5
View File
@@ -23,6 +23,7 @@ import (
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
helmv2 "github.com/fluxcd/helm-controller/api/v2"
coordinationv1 "k8s.io/api/coordination/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -93,6 +94,10 @@ func init() {
if err := rbacv1.AddToScheme(mgrScheme); err != nil {
panic(fmt.Errorf("failed to add RBAC types to scheme: %w", err))
}
// Register Lease types for the per-application credential lock.
if err := coordinationv1.AddToScheme(mgrScheme); err != nil {
panic(fmt.Errorf("failed to add coordination types to scheme: %w", err))
}
// Register Cozystack types for WorkloadMonitor queries.
if err := cozyv1alpha1.AddToScheme(mgrScheme); err != nil {
+151
View File
@@ -0,0 +1,151 @@
// SPDX-License-Identifier: Apache-2.0
// Package applock serializes an application's credential operations across the
// API server's replicas. A create or update seeds and prunes account Secrets
// outside the HelmRelease write's optimistic concurrency, so two concurrent
// writes to one application can interleave: one deletes a Secret the other has
// just reused and referenced. The lock makes those operations take turns, keyed
// by the application, through a coordination.k8s.io Lease.
package applock
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"time"
coordinationv1 "k8s.io/api/coordination/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// ErrHeld means another operation holds the application's lock. The caller
// turns it into a retryable Conflict rather than waiting, so a stuck holder
// cannot block a write past its lease.
var ErrHeld = errors.New("another change to the application's credentials is in progress")
// Locker takes and releases the per-application lock.
type Locker struct {
// c reads and writes Leases; it must be uncached so an acquire sees a Lease
// a concurrent operation wrote a moment ago.
c client.Client
// namespace holds the Leases; cozy-system, which tenants cannot write, so a
// tenant cannot hold or delete its own application's lock.
namespace string
// duration is how long a Lease is honored without renewal. An operation
// finishes well within it; a holder that crashed has its Lease stolen once
// it expires, so the lock never wedges.
duration time.Duration
}
// New returns a Locker that keeps its Leases in namespace (cozy-system).
func New(c client.Client, namespace string) *Locker {
return &Locker{c: c, namespace: namespace, duration: 60 * time.Second}
}
// leaseName is the Lease name for one application, a hash so any namespace and
// release name map to a valid, collision-free object name.
func leaseName(appNamespace, release string) string {
sum := sha256.Sum256([]byte(appNamespace + "/" + release))
return "cred-" + hex.EncodeToString(sum[:])[:40]
}
func holderID() (string, error) {
var b [8]byte
if _, err := rand.Read(b[:]); err != nil {
return "", err
}
host, _ := os.Hostname()
return fmt.Sprintf("%s-%d-%s", host, os.Getpid(), hex.EncodeToString(b[:])), nil
}
// Acquire takes the application's lock. On success it returns a release
// function the caller defers. It returns ErrHeld when another operation holds a
// live lock. Any other error is an infrastructure failure the caller may treat
// as fail-open, since the lock only guards a rare race and the Lease store being
// unreachable is no worse than not having the lock at all.
func (l *Locker) Acquire(ctx context.Context, appNamespace, release string) (func(), error) {
name := leaseName(appNamespace, release)
id, err := holderID()
if err != nil {
return nil, err
}
now := metav1.NewMicroTime(time.Now())
dur := int32(l.duration / time.Second)
cur := &coordinationv1.Lease{}
err = l.c.Get(ctx, client.ObjectKey{Namespace: l.namespace, Name: name}, cur)
switch {
case apierrors.IsNotFound(err):
lease := &coordinationv1.Lease{
ObjectMeta: metav1.ObjectMeta{Namespace: l.namespace, Name: name},
Spec: coordinationv1.LeaseSpec{
HolderIdentity: &id,
LeaseDurationSeconds: &dur,
AcquireTime: &now,
RenewTime: &now,
},
}
if err := l.c.Create(ctx, lease); err != nil {
if apierrors.IsAlreadyExists(err) {
return nil, ErrHeld // another operation created it between the Get and here
}
return nil, err
}
return l.releaser(name, id), nil
case err != nil:
return nil, err
}
// A Lease exists: steal it only if it has expired, under its resourceVersion
// so two stealers do not both win.
if held(cur) {
return nil, ErrHeld
}
cur.Spec.HolderIdentity = &id
cur.Spec.LeaseDurationSeconds = &dur
cur.Spec.AcquireTime = &now
cur.Spec.RenewTime = &now
if err := l.c.Update(ctx, cur); err != nil {
if apierrors.IsConflict(err) {
return nil, ErrHeld // another operation stole or renewed it first
}
return nil, err
}
return l.releaser(name, id), nil
}
// held reports whether a Lease is still honored: it has a holder and its renew
// time plus its duration is in the future.
func held(lease *coordinationv1.Lease) bool {
if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity == "" || lease.Spec.RenewTime == nil {
return false
}
dur := time.Duration(60) * time.Second
if lease.Spec.LeaseDurationSeconds != nil {
dur = time.Duration(*lease.Spec.LeaseDurationSeconds) * time.Second
}
return lease.Spec.RenewTime.Add(dur).After(time.Now())
}
// releaser deletes the Lease if this operation still holds it. Best effort: a
// failed release leaves a Lease that the next acquire steals once it expires.
func (l *Locker) releaser(name, id string) func() {
return func() {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
cur := &coordinationv1.Lease{}
if err := l.c.Get(ctx, client.ObjectKey{Namespace: l.namespace, Name: name}, cur); err != nil {
return
}
if cur.Spec.HolderIdentity == nil || *cur.Spec.HolderIdentity != id {
return // someone else holds it now; not ours to delete
}
_ = l.c.Delete(ctx, cur, client.Preconditions{UID: &cur.UID, ResourceVersion: &cur.ResourceVersion})
}
}
+85
View File
@@ -0,0 +1,85 @@
// SPDX-License-Identifier: Apache-2.0
package applock
import (
"context"
"errors"
"testing"
"time"
coordinationv1 "k8s.io/api/coordination/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
)
func newClient(t *testing.T) client.Client {
t.Helper()
s := runtime.NewScheme()
if err := coordinationv1.AddToScheme(s); err != nil {
t.Fatal(err)
}
return fake.NewClientBuilder().WithScheme(s).Build()
}
func TestAcquireBlocksAndReleases(t *testing.T) {
ctx := context.Background()
l := New(newClient(t), "cozy-system")
unlock, err := l.Acquire(ctx, "tenant-foo", "postgres-orders")
if err != nil {
t.Fatalf("first acquire: %v", err)
}
// Held now: a second acquire of the same application is refused.
if _, err := l.Acquire(ctx, "tenant-foo", "postgres-orders"); !errors.Is(err, ErrHeld) {
t.Fatalf("a held lock must refuse, got %v", err)
}
// A different application locks independently.
if u2, err := l.Acquire(ctx, "tenant-foo", "postgres-other"); err != nil {
t.Fatalf("a different application must lock independently: %v", err)
} else {
u2()
}
// After release, the application can be locked again.
unlock()
if u3, err := l.Acquire(ctx, "tenant-foo", "postgres-orders"); err != nil {
t.Fatalf("acquire after release: %v", err)
} else {
u3()
}
}
func seedLease(t *testing.T, c client.Client, appNamespace, release, holder string, renew time.Time) {
t.Helper()
rt := metav1.NewMicroTime(renew)
dur := int32(60)
if err := c.Create(context.Background(), &coordinationv1.Lease{
ObjectMeta: metav1.ObjectMeta{Namespace: "cozy-system", Name: leaseName(appNamespace, release)},
Spec: coordinationv1.LeaseSpec{HolderIdentity: &holder, LeaseDurationSeconds: &dur, RenewTime: &rt},
}); err != nil {
t.Fatal(err)
}
}
func TestAcquireRefusesAFreshLeaseOfAnother(t *testing.T) {
ctx := context.Background()
c := newClient(t)
seedLease(t, c, "tenant-foo", "postgres-orders", "someone-else", time.Now())
if _, err := New(c, "cozy-system").Acquire(ctx, "tenant-foo", "postgres-orders"); !errors.Is(err, ErrHeld) {
t.Fatalf("a fresh lease held by another must refuse, got %v", err)
}
}
func TestAcquireStealsAnExpiredLease(t *testing.T) {
ctx := context.Background()
c := newClient(t)
// Held by a holder that last renewed longer than a lease duration ago.
seedLease(t, c, "tenant-foo", "postgres-orders", "crashed-holder", time.Now().Add(-5*time.Minute))
unlock, err := New(c, "cozy-system").Acquire(ctx, "tenant-foo", "postgres-orders")
if err != nil {
t.Fatalf("an expired lease must be stealable: %v", err)
}
unlock()
}
+30
View File
@@ -56,6 +56,7 @@ import (
appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
"github.com/cozystack/cozystack/pkg/apis/apps/validation"
"github.com/cozystack/cozystack/pkg/config"
"github.com/cozystack/cozystack/pkg/credentials/applock"
"github.com/cozystack/cozystack/pkg/registry"
fieldfilter "github.com/cozystack/cozystack/pkg/registry/fields"
"github.com/cozystack/cozystack/pkg/registry/sorting"
@@ -105,6 +106,10 @@ type REST struct {
// credentials is the kind's credentials block, nil for a kind whose
// passwords the chart still generates.
credentials *cozyv1alpha1.ApplicationDefinitionCredentials
// lock serializes one application's credential seeding and pruning across
// the API server's replicas, so two concurrent writes cannot interleave
// into deleting a Secret the other has just reused.
lock *applock.Locker
}
// buildSpecSchema parses an OpenAPI-v3 JSON schema string into the
@@ -126,6 +131,13 @@ func NewREST(c client.Client, w client.WithWatch, config *config.Resource) *REST
config.Application.Kind, appsv1alpha1.NameSchemaExtension, config.Application.Kind, nameSchemaErr)
}
// The lock reads and writes its Leases uncached, and in cozy-system, which
// tenants cannot write.
lockClient := client.Client(w)
if lockClient == nil {
lockClient = c
}
return &REST{
c: c,
w: w,
@@ -145,6 +157,7 @@ func NewREST(c client.Client, w client.WithWatch, config *config.Resource) *REST
nameSchema: nameSchema,
nameSchemaErr: nameSchemaErr,
credentials: config.Credentials,
lock: applock.New(lockClient, "cozy-system"),
}
}
@@ -252,6 +265,14 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation
helmRelease.Labels[ApplicationNameLabel] = app.Name
// Note: Annotations from config are not handled as r.releaseConfig.Annotations is undefined
// Hold the application's credential lock across the seed, the release write
// and the owning, so a concurrent write cannot interleave with them.
unlock, err := r.lockCredentials(ctx, app)
if err != nil {
return nil, err
}
defer unlock()
// The account Secrets exist before the release that reads them, and the
// release owns them as soon as it exists.
seeded, err := r.seedBeforeCreate(ctx, app, helmRelease, accounts, options != nil && dryrun.IsDryRun(options.DryRun))
@@ -602,6 +623,15 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
return nil, false, fmt.Errorf("conversion error: %v", err)
}
// Hold the application's credential lock across the read of the live
// release, the seed, the release write and the prune, so a concurrent write
// cannot re-add an account and have this request prune the Secret it reused.
unlock, err := r.lockCredentials(ctx, app)
if err != nil {
return nil, false, err
}
defer unlock()
cur := &helmv2.HelmRelease{}
if err := r.c.Get(ctx, client.ObjectKey{Namespace: helmRelease.Namespace, Name: helmRelease.Name}, cur, &client.GetOptions{Raw: &metav1.GetOptions{}}); err != nil {
return nil, false, registry.WrapPreservingStatus("failed to fetch current HelmRelease", err, r.gvr.GroupResource(), name)
@@ -22,6 +22,7 @@ import (
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
"github.com/cozystack/cozystack/pkg/credentials/account"
"github.com/cozystack/cozystack/pkg/credentials/applock"
"github.com/cozystack/cozystack/pkg/credentials/store"
)
@@ -78,6 +79,27 @@ func (r *REST) direct() client.Client {
return r.c
}
// lockCredentials takes the per-application lock so this request's seeding and
// pruning do not interleave with a concurrent write to the same application. It
// returns a release function the caller defers. A lock another request holds is
// a retryable Conflict; a lock store that cannot be reached is logged and the
// request proceeds without it, since the lock only guards a rare race and its
// store being unreachable is no worse than not having it.
func (r *REST) lockCredentials(ctx context.Context, app *appsv1alpha1.Application) (func(), error) {
if r.credentials == nil || r.lock == nil {
return func() {}, nil
}
unlock, err := r.lock.Acquire(ctx, app.Namespace, r.storeApp(app.Namespace, app.Name).Release)
switch {
case errors.Is(err, applock.ErrHeld):
return nil, apierrors.NewConflict(r.gvr.GroupResource(), app.Name, errors.New("another change to this application's credentials is in progress; retry"))
case err != nil:
klog.Warningf("credential lock unavailable for %s/%s, proceeding without it: %v", app.Namespace, app.Name, err)
return func() {}, nil
}
return unlock, nil
}
// seedBeforeCreate writes the account Secrets of a new application, with no
// owner yet, and the valuesFrom entries that carry them into the chart.
// It returns the Secrets it wrote, for the owner to be set once the
@@ -10,6 +10,7 @@ import (
"testing"
helmv2 "github.com/fluxcd/helm-controller/api/v2"
coordinationv1 "k8s.io/api/coordination/v1"
corev1 "k8s.io/api/core/v1"
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -55,6 +56,9 @@ func newCredentialsREST(t *testing.T, objects ...client.Object) (*REST, client.C
if err := corev1.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
if err := coordinationv1.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
cfg := &config.Resource{
Application: config.ApplicationConfig{Kind: postgresKind, Singular: "postgres", Plural: "postgreses", OpenAPISchema: credentialsSchema},
Release: config.ReleaseConfig{Prefix: "postgres-"},
@@ -449,6 +453,80 @@ func TestConcurrentCreateKeepsLiveSecret(t *testing.T) {
// it, because its UID is not the one this create seeded. Deleting by name, or
// only skipping an owned Secret, would take it and leave the live release with
// no account Secret.
// A concurrent write to one application's credentials is refused while another
// holds the lock, so the two cannot interleave into deleting a Secret the other
// reused. Request A adds api; while A holds the lock and writes its release, the
// interceptor runs request B (which would remove web and prune its Secret). B
// is refused with Conflict, so web's Secret is untouched.
func TestConcurrentUpdateRefusedByCredentialLock(t *testing.T) {
scheme := runtime.NewScheme()
if err := helmv2.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
if err := corev1.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
if err := coordinationv1.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
cfg := &config.Resource{
Application: config.ApplicationConfig{Kind: postgresKind, Singular: "postgres", Plural: "postgreses", OpenAPISchema: credentialsSchema},
Release: config.ReleaseConfig{Prefix: "postgres-"},
Credentials: credentialsBlock(),
}
if err := appsv1alpha1.RegisterDynamicTypes(scheme, &config.ResourceConfig{Resources: []config.Resource{*cfg}}); err != nil {
t.Fatal(err)
}
var r *REST
var raced atomic.Bool
var bErr error
var uids atomic.Int64
c := fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(interceptor.Funcs{
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
if obj.GetUID() == "" {
obj.SetUID(types.UID(fmt.Sprintf("uid-%d", uids.Add(1))))
}
return cl.Create(ctx, obj, opts...)
},
Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
// Request A is writing its HelmRelease, so A holds the lock: a second
// update of the same application must be refused now.
if _, ok := obj.(*helmv2.HelmRelease); ok && raced.CompareAndSwap(false, true) {
_, _, bErr = r.Update(credentialsCtx(), "orders", newDefaultUpdatedObjectInfo(postgresApp(`{"users":{}}`)), nil, nil, false, &metav1.UpdateOptions{})
}
if obj.GetUID() == "" {
cur := obj.DeepCopyObject().(client.Object)
if err := cl.Get(ctx, client.ObjectKeyFromObject(obj), cur); err == nil {
obj.SetUID(cur.GetUID())
}
}
return cl.Update(ctx, obj, opts...)
},
}).Build()
r = NewREST(c, c, cfg)
ctx := credentialsCtx()
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
webBefore := accountSecrets(t, c)["postgres-orders.web.account"]
if _, _, err := r.Update(ctx, "orders", newDefaultUpdatedObjectInfo(postgresApp(`{"users":{"web":{},"api":{}}}`)), nil, nil, false, &metav1.UpdateOptions{}); err != nil {
t.Fatalf("request A: %v", err)
}
if !apierrors.IsConflict(bErr) {
t.Fatalf("a concurrent credential write must be refused with Conflict, got %v", bErr)
}
after := accountSecrets(t, c)["postgres-orders.web.account"]
if after == nil || after.UID != webBefore.UID {
t.Fatal("the refused concurrent update must not have pruned web's Secret")
}
if accountSecrets(t, c)["postgres-orders.api.account"] == nil {
t.Fatal("request A must have seeded api")
}
}
func TestUnseedLeavesAReplacedSecret(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()