mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(credentials): serialize an application's seeding and pruning with a lock
A create or update seeds and prunes account Secrets outside the HelmRelease write's optimistic concurrency, so two concurrent writes to one application could interleave: one read the live release, a second re-added an account and wrote it back, and the first then pruned the Secret the live release now referenced, leaving a dangling reference and a lost password. The API server runs with two replicas, so an in-process guard is not enough. A per-application coordination.k8s.io Lease in cozy-system, which tenants cannot write, now serializes the credential section of a create and an update across replicas: a write is refused with a retryable Conflict while another holds the lock, the lease is stolen once it expires so a crashed holder cannot wedge it, and a lock store that cannot be reached is logged and the request proceeds, since the lock only guards a rare race. Assisted-by: LLM Signed-off-by: IvanHunters <xorokhotnikov@gmail.com>
This commit is contained in:
8 files changed
+388
-3
No files matched your search
@@ -90,3 +90,10 @@ rules:
|
||||
- apiGroups: ["cilium.io"]
|
||||
resources: ["ciliumnetworkpolicies"]
|
||||
verbs: ["get", "watch", "list", "create", "update", "patch", "delete"]
|
||||
# The per-application credential lock (a Lease in cozy-system) serializes a
|
||||
# create's or update's account-Secret seeding and pruning across the API
|
||||
# server's replicas, so two concurrent writes cannot interleave into deleting a
|
||||
# Secret the other reused.
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["get", "create", "update", "delete"]
|
||||
@@ -94,11 +94,18 @@ tests:
|
||||
resources: ["events"]
|
||||
verbs: ["create"]
|
||||
|
||||
- it: ships exactly the twenty-two rules this role is pinned to
|
||||
- it: ships exactly the twenty-three rules this role is pinned to
|
||||
asserts:
|
||||
# Both directions. A dropped grant is the failure this suite is about:
|
||||
# the apiserver returns forbidden and List silently drops the dropdown.
|
||||
- exists:
|
||||
path: rules[21]
|
||||
- notExists:
|
||||
path: rules[22]
|
||||
- notExists:
|
||||
path: rules[23]
|
||||
# The last rule is the credential lock's Lease access.
|
||||
- equal:
|
||||
path: rules[22]
|
||||
value:
|
||||
apiGroups: ["coordination.k8s.io"]
|
||||
resources: ["leases"]
|
||||
verbs: ["get", "create", "update", "delete"]
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
|
||||
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
|
||||
helmv2 "github.com/fluxcd/helm-controller/api/v2"
|
||||
coordinationv1 "k8s.io/api/coordination/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -93,6 +94,10 @@ func init() {
|
||||
if err := rbacv1.AddToScheme(mgrScheme); err != nil {
|
||||
panic(fmt.Errorf("failed to add RBAC types to scheme: %w", err))
|
||||
}
|
||||
// Register Lease types for the per-application credential lock.
|
||||
if err := coordinationv1.AddToScheme(mgrScheme); err != nil {
|
||||
panic(fmt.Errorf("failed to add coordination types to scheme: %w", err))
|
||||
}
|
||||
|
||||
// Register Cozystack types for WorkloadMonitor queries.
|
||||
if err := cozyv1alpha1.AddToScheme(mgrScheme); err != nil {
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
// Package applock serializes an application's credential operations across the
|
||||
// API server's replicas. A create or update seeds and prunes account Secrets
|
||||
// outside the HelmRelease write's optimistic concurrency, so two concurrent
|
||||
// writes to one application can interleave: one deletes a Secret the other has
|
||||
// just reused and referenced. The lock makes those operations take turns, keyed
|
||||
// by the application, through a coordination.k8s.io Lease.
|
||||
package applock
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
coordinationv1 "k8s.io/api/coordination/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
)
|
||||
|
||||
// ErrHeld means another operation holds the application's lock. The caller
|
||||
// turns it into a retryable Conflict rather than waiting, so a stuck holder
|
||||
// cannot block a write past its lease.
|
||||
var ErrHeld = errors.New("another change to the application's credentials is in progress")
|
||||
|
||||
// Locker takes and releases the per-application lock.
|
||||
type Locker struct {
|
||||
// c reads and writes Leases; it must be uncached so an acquire sees a Lease
|
||||
// a concurrent operation wrote a moment ago.
|
||||
c client.Client
|
||||
// namespace holds the Leases; cozy-system, which tenants cannot write, so a
|
||||
// tenant cannot hold or delete its own application's lock.
|
||||
namespace string
|
||||
// duration is how long a Lease is honored without renewal. An operation
|
||||
// finishes well within it; a holder that crashed has its Lease stolen once
|
||||
// it expires, so the lock never wedges.
|
||||
duration time.Duration
|
||||
}
|
||||
|
||||
// New returns a Locker that keeps its Leases in namespace (cozy-system).
|
||||
func New(c client.Client, namespace string) *Locker {
|
||||
return &Locker{c: c, namespace: namespace, duration: 60 * time.Second}
|
||||
}
|
||||
|
||||
// leaseName is the Lease name for one application, a hash so any namespace and
|
||||
// release name map to a valid, collision-free object name.
|
||||
func leaseName(appNamespace, release string) string {
|
||||
sum := sha256.Sum256([]byte(appNamespace + "/" + release))
|
||||
return "cred-" + hex.EncodeToString(sum[:])[:40]
|
||||
}
|
||||
|
||||
func holderID() (string, error) {
|
||||
var b [8]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
host, _ := os.Hostname()
|
||||
return fmt.Sprintf("%s-%d-%s", host, os.Getpid(), hex.EncodeToString(b[:])), nil
|
||||
}
|
||||
|
||||
// Acquire takes the application's lock. On success it returns a release
|
||||
// function the caller defers. It returns ErrHeld when another operation holds a
|
||||
// live lock. Any other error is an infrastructure failure the caller may treat
|
||||
// as fail-open, since the lock only guards a rare race and the Lease store being
|
||||
// unreachable is no worse than not having the lock at all.
|
||||
func (l *Locker) Acquire(ctx context.Context, appNamespace, release string) (func(), error) {
|
||||
name := leaseName(appNamespace, release)
|
||||
id, err := holderID()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := metav1.NewMicroTime(time.Now())
|
||||
dur := int32(l.duration / time.Second)
|
||||
|
||||
cur := &coordinationv1.Lease{}
|
||||
err = l.c.Get(ctx, client.ObjectKey{Namespace: l.namespace, Name: name}, cur)
|
||||
switch {
|
||||
case apierrors.IsNotFound(err):
|
||||
lease := &coordinationv1.Lease{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: l.namespace, Name: name},
|
||||
Spec: coordinationv1.LeaseSpec{
|
||||
HolderIdentity: &id,
|
||||
LeaseDurationSeconds: &dur,
|
||||
AcquireTime: &now,
|
||||
RenewTime: &now,
|
||||
},
|
||||
}
|
||||
if err := l.c.Create(ctx, lease); err != nil {
|
||||
if apierrors.IsAlreadyExists(err) {
|
||||
return nil, ErrHeld // another operation created it between the Get and here
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return l.releaser(name, id), nil
|
||||
case err != nil:
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// A Lease exists: steal it only if it has expired, under its resourceVersion
|
||||
// so two stealers do not both win.
|
||||
if held(cur) {
|
||||
return nil, ErrHeld
|
||||
}
|
||||
cur.Spec.HolderIdentity = &id
|
||||
cur.Spec.LeaseDurationSeconds = &dur
|
||||
cur.Spec.AcquireTime = &now
|
||||
cur.Spec.RenewTime = &now
|
||||
if err := l.c.Update(ctx, cur); err != nil {
|
||||
if apierrors.IsConflict(err) {
|
||||
return nil, ErrHeld // another operation stole or renewed it first
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return l.releaser(name, id), nil
|
||||
}
|
||||
|
||||
// held reports whether a Lease is still honored: it has a holder and its renew
|
||||
// time plus its duration is in the future.
|
||||
func held(lease *coordinationv1.Lease) bool {
|
||||
if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity == "" || lease.Spec.RenewTime == nil {
|
||||
return false
|
||||
}
|
||||
dur := time.Duration(60) * time.Second
|
||||
if lease.Spec.LeaseDurationSeconds != nil {
|
||||
dur = time.Duration(*lease.Spec.LeaseDurationSeconds) * time.Second
|
||||
}
|
||||
return lease.Spec.RenewTime.Add(dur).After(time.Now())
|
||||
}
|
||||
|
||||
// releaser deletes the Lease if this operation still holds it. Best effort: a
|
||||
// failed release leaves a Lease that the next acquire steals once it expires.
|
||||
func (l *Locker) releaser(name, id string) func() {
|
||||
return func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
cur := &coordinationv1.Lease{}
|
||||
if err := l.c.Get(ctx, client.ObjectKey{Namespace: l.namespace, Name: name}, cur); err != nil {
|
||||
return
|
||||
}
|
||||
if cur.Spec.HolderIdentity == nil || *cur.Spec.HolderIdentity != id {
|
||||
return // someone else holds it now; not ours to delete
|
||||
}
|
||||
_ = l.c.Delete(ctx, cur, client.Preconditions{UID: &cur.UID, ResourceVersion: &cur.ResourceVersion})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package applock
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
coordinationv1 "k8s.io/api/coordination/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
)
|
||||
|
||||
func newClient(t *testing.T) client.Client {
|
||||
t.Helper()
|
||||
s := runtime.NewScheme()
|
||||
if err := coordinationv1.AddToScheme(s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return fake.NewClientBuilder().WithScheme(s).Build()
|
||||
}
|
||||
|
||||
func TestAcquireBlocksAndReleases(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
l := New(newClient(t), "cozy-system")
|
||||
|
||||
unlock, err := l.Acquire(ctx, "tenant-foo", "postgres-orders")
|
||||
if err != nil {
|
||||
t.Fatalf("first acquire: %v", err)
|
||||
}
|
||||
// Held now: a second acquire of the same application is refused.
|
||||
if _, err := l.Acquire(ctx, "tenant-foo", "postgres-orders"); !errors.Is(err, ErrHeld) {
|
||||
t.Fatalf("a held lock must refuse, got %v", err)
|
||||
}
|
||||
// A different application locks independently.
|
||||
if u2, err := l.Acquire(ctx, "tenant-foo", "postgres-other"); err != nil {
|
||||
t.Fatalf("a different application must lock independently: %v", err)
|
||||
} else {
|
||||
u2()
|
||||
}
|
||||
// After release, the application can be locked again.
|
||||
unlock()
|
||||
if u3, err := l.Acquire(ctx, "tenant-foo", "postgres-orders"); err != nil {
|
||||
t.Fatalf("acquire after release: %v", err)
|
||||
} else {
|
||||
u3()
|
||||
}
|
||||
}
|
||||
|
||||
func seedLease(t *testing.T, c client.Client, appNamespace, release, holder string, renew time.Time) {
|
||||
t.Helper()
|
||||
rt := metav1.NewMicroTime(renew)
|
||||
dur := int32(60)
|
||||
if err := c.Create(context.Background(), &coordinationv1.Lease{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "cozy-system", Name: leaseName(appNamespace, release)},
|
||||
Spec: coordinationv1.LeaseSpec{HolderIdentity: &holder, LeaseDurationSeconds: &dur, RenewTime: &rt},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquireRefusesAFreshLeaseOfAnother(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := newClient(t)
|
||||
seedLease(t, c, "tenant-foo", "postgres-orders", "someone-else", time.Now())
|
||||
if _, err := New(c, "cozy-system").Acquire(ctx, "tenant-foo", "postgres-orders"); !errors.Is(err, ErrHeld) {
|
||||
t.Fatalf("a fresh lease held by another must refuse, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAcquireStealsAnExpiredLease(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := newClient(t)
|
||||
// Held by a holder that last renewed longer than a lease duration ago.
|
||||
seedLease(t, c, "tenant-foo", "postgres-orders", "crashed-holder", time.Now().Add(-5*time.Minute))
|
||||
unlock, err := New(c, "cozy-system").Acquire(ctx, "tenant-foo", "postgres-orders")
|
||||
if err != nil {
|
||||
t.Fatalf("an expired lease must be stealable: %v", err)
|
||||
}
|
||||
unlock()
|
||||
}
|
||||
@@ -56,6 +56,7 @@ import (
|
||||
appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
|
||||
"github.com/cozystack/cozystack/pkg/apis/apps/validation"
|
||||
"github.com/cozystack/cozystack/pkg/config"
|
||||
"github.com/cozystack/cozystack/pkg/credentials/applock"
|
||||
"github.com/cozystack/cozystack/pkg/registry"
|
||||
fieldfilter "github.com/cozystack/cozystack/pkg/registry/fields"
|
||||
"github.com/cozystack/cozystack/pkg/registry/sorting"
|
||||
@@ -105,6 +106,10 @@ type REST struct {
|
||||
// credentials is the kind's credentials block, nil for a kind whose
|
||||
// passwords the chart still generates.
|
||||
credentials *cozyv1alpha1.ApplicationDefinitionCredentials
|
||||
// lock serializes one application's credential seeding and pruning across
|
||||
// the API server's replicas, so two concurrent writes cannot interleave
|
||||
// into deleting a Secret the other has just reused.
|
||||
lock *applock.Locker
|
||||
}
|
||||
|
||||
// buildSpecSchema parses an OpenAPI-v3 JSON schema string into the
|
||||
@@ -126,6 +131,13 @@ func NewREST(c client.Client, w client.WithWatch, config *config.Resource) *REST
|
||||
config.Application.Kind, appsv1alpha1.NameSchemaExtension, config.Application.Kind, nameSchemaErr)
|
||||
}
|
||||
|
||||
// The lock reads and writes its Leases uncached, and in cozy-system, which
|
||||
// tenants cannot write.
|
||||
lockClient := client.Client(w)
|
||||
if lockClient == nil {
|
||||
lockClient = c
|
||||
}
|
||||
|
||||
return &REST{
|
||||
c: c,
|
||||
w: w,
|
||||
@@ -145,6 +157,7 @@ func NewREST(c client.Client, w client.WithWatch, config *config.Resource) *REST
|
||||
nameSchema: nameSchema,
|
||||
nameSchemaErr: nameSchemaErr,
|
||||
credentials: config.Credentials,
|
||||
lock: applock.New(lockClient, "cozy-system"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -252,6 +265,14 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation
|
||||
helmRelease.Labels[ApplicationNameLabel] = app.Name
|
||||
// Note: Annotations from config are not handled as r.releaseConfig.Annotations is undefined
|
||||
|
||||
// Hold the application's credential lock across the seed, the release write
|
||||
// and the owning, so a concurrent write cannot interleave with them.
|
||||
unlock, err := r.lockCredentials(ctx, app)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// The account Secrets exist before the release that reads them, and the
|
||||
// release owns them as soon as it exists.
|
||||
seeded, err := r.seedBeforeCreate(ctx, app, helmRelease, accounts, options != nil && dryrun.IsDryRun(options.DryRun))
|
||||
@@ -602,6 +623,15 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
|
||||
return nil, false, fmt.Errorf("conversion error: %v", err)
|
||||
}
|
||||
|
||||
// Hold the application's credential lock across the read of the live
|
||||
// release, the seed, the release write and the prune, so a concurrent write
|
||||
// cannot re-add an account and have this request prune the Secret it reused.
|
||||
unlock, err := r.lockCredentials(ctx, app)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
cur := &helmv2.HelmRelease{}
|
||||
if err := r.c.Get(ctx, client.ObjectKey{Namespace: helmRelease.Namespace, Name: helmRelease.Name}, cur, &client.GetOptions{Raw: &metav1.GetOptions{}}); err != nil {
|
||||
return nil, false, registry.WrapPreservingStatus("failed to fetch current HelmRelease", err, r.gvr.GroupResource(), name)
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
|
||||
appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
|
||||
"github.com/cozystack/cozystack/pkg/credentials/account"
|
||||
"github.com/cozystack/cozystack/pkg/credentials/applock"
|
||||
"github.com/cozystack/cozystack/pkg/credentials/store"
|
||||
)
|
||||
|
||||
@@ -78,6 +79,27 @@ func (r *REST) direct() client.Client {
|
||||
return r.c
|
||||
}
|
||||
|
||||
// lockCredentials takes the per-application lock so this request's seeding and
|
||||
// pruning do not interleave with a concurrent write to the same application. It
|
||||
// returns a release function the caller defers. A lock another request holds is
|
||||
// a retryable Conflict; a lock store that cannot be reached is logged and the
|
||||
// request proceeds without it, since the lock only guards a rare race and its
|
||||
// store being unreachable is no worse than not having it.
|
||||
func (r *REST) lockCredentials(ctx context.Context, app *appsv1alpha1.Application) (func(), error) {
|
||||
if r.credentials == nil || r.lock == nil {
|
||||
return func() {}, nil
|
||||
}
|
||||
unlock, err := r.lock.Acquire(ctx, app.Namespace, r.storeApp(app.Namespace, app.Name).Release)
|
||||
switch {
|
||||
case errors.Is(err, applock.ErrHeld):
|
||||
return nil, apierrors.NewConflict(r.gvr.GroupResource(), app.Name, errors.New("another change to this application's credentials is in progress; retry"))
|
||||
case err != nil:
|
||||
klog.Warningf("credential lock unavailable for %s/%s, proceeding without it: %v", app.Namespace, app.Name, err)
|
||||
return func() {}, nil
|
||||
}
|
||||
return unlock, nil
|
||||
}
|
||||
|
||||
// seedBeforeCreate writes the account Secrets of a new application, with no
|
||||
// owner yet, and the valuesFrom entries that carry them into the chart.
|
||||
// It returns the Secrets it wrote, for the owner to be set once the
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"testing"
|
||||
|
||||
helmv2 "github.com/fluxcd/helm-controller/api/v2"
|
||||
coordinationv1 "k8s.io/api/coordination/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -55,6 +56,9 @@ func newCredentialsREST(t *testing.T, objects ...client.Object) (*REST, client.C
|
||||
if err := corev1.AddToScheme(scheme); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := coordinationv1.AddToScheme(scheme); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := &config.Resource{
|
||||
Application: config.ApplicationConfig{Kind: postgresKind, Singular: "postgres", Plural: "postgreses", OpenAPISchema: credentialsSchema},
|
||||
Release: config.ReleaseConfig{Prefix: "postgres-"},
|
||||
@@ -449,6 +453,80 @@ func TestConcurrentCreateKeepsLiveSecret(t *testing.T) {
|
||||
// it, because its UID is not the one this create seeded. Deleting by name, or
|
||||
// only skipping an owned Secret, would take it and leave the live release with
|
||||
// no account Secret.
|
||||
// A concurrent write to one application's credentials is refused while another
|
||||
// holds the lock, so the two cannot interleave into deleting a Secret the other
|
||||
// reused. Request A adds api; while A holds the lock and writes its release, the
|
||||
// interceptor runs request B (which would remove web and prune its Secret). B
|
||||
// is refused with Conflict, so web's Secret is untouched.
|
||||
func TestConcurrentUpdateRefusedByCredentialLock(t *testing.T) {
|
||||
scheme := runtime.NewScheme()
|
||||
if err := helmv2.AddToScheme(scheme); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := corev1.AddToScheme(scheme); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := coordinationv1.AddToScheme(scheme); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := &config.Resource{
|
||||
Application: config.ApplicationConfig{Kind: postgresKind, Singular: "postgres", Plural: "postgreses", OpenAPISchema: credentialsSchema},
|
||||
Release: config.ReleaseConfig{Prefix: "postgres-"},
|
||||
Credentials: credentialsBlock(),
|
||||
}
|
||||
if err := appsv1alpha1.RegisterDynamicTypes(scheme, &config.ResourceConfig{Resources: []config.Resource{*cfg}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var r *REST
|
||||
var raced atomic.Bool
|
||||
var bErr error
|
||||
var uids atomic.Int64
|
||||
c := fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(interceptor.Funcs{
|
||||
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
|
||||
if obj.GetUID() == "" {
|
||||
obj.SetUID(types.UID(fmt.Sprintf("uid-%d", uids.Add(1))))
|
||||
}
|
||||
return cl.Create(ctx, obj, opts...)
|
||||
},
|
||||
Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
|
||||
// Request A is writing its HelmRelease, so A holds the lock: a second
|
||||
// update of the same application must be refused now.
|
||||
if _, ok := obj.(*helmv2.HelmRelease); ok && raced.CompareAndSwap(false, true) {
|
||||
_, _, bErr = r.Update(credentialsCtx(), "orders", newDefaultUpdatedObjectInfo(postgresApp(`{"users":{}}`)), nil, nil, false, &metav1.UpdateOptions{})
|
||||
}
|
||||
if obj.GetUID() == "" {
|
||||
cur := obj.DeepCopyObject().(client.Object)
|
||||
if err := cl.Get(ctx, client.ObjectKeyFromObject(obj), cur); err == nil {
|
||||
obj.SetUID(cur.GetUID())
|
||||
}
|
||||
}
|
||||
return cl.Update(ctx, obj, opts...)
|
||||
},
|
||||
}).Build()
|
||||
r = NewREST(c, c, cfg)
|
||||
ctx := credentialsCtx()
|
||||
|
||||
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
webBefore := accountSecrets(t, c)["postgres-orders.web.account"]
|
||||
|
||||
if _, _, err := r.Update(ctx, "orders", newDefaultUpdatedObjectInfo(postgresApp(`{"users":{"web":{},"api":{}}}`)), nil, nil, false, &metav1.UpdateOptions{}); err != nil {
|
||||
t.Fatalf("request A: %v", err)
|
||||
}
|
||||
if !apierrors.IsConflict(bErr) {
|
||||
t.Fatalf("a concurrent credential write must be refused with Conflict, got %v", bErr)
|
||||
}
|
||||
after := accountSecrets(t, c)["postgres-orders.web.account"]
|
||||
if after == nil || after.UID != webBefore.UID {
|
||||
t.Fatal("the refused concurrent update must not have pruned web's Secret")
|
||||
}
|
||||
if accountSecrets(t, c)["postgres-orders.api.account"] == nil {
|
||||
t.Fatal("request A must have seeded api")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnseedLeavesAReplacedSecret(t *testing.T) {
|
||||
r, c := newCredentialsREST(t)
|
||||
ctx := credentialsCtx()
|
||||
|
||||
Reference in new issue
Block a user