Files
cozystack/pkg/registry/apps/application/rest_credentials_test.go
IvanHunters 8bdb93385d fix(credentials): serialize an application's seeding and pruning with a lock
A create or update seeds and prunes account Secrets outside the HelmRelease write's optimistic concurrency, so two concurrent writes to one application could interleave: one read the live release, a second re-added an account and wrote it back, and the first then pruned the Secret the live release now referenced, leaving a dangling reference and a lost password. The API server runs with two replicas, so an in-process guard is not enough. A per-application coordination.k8s.io Lease in cozy-system, which tenants cannot write, now serializes the credential section of a create and an update across replicas: a write is refused with a retryable Conflict while another holds the lock, the lease is stolen once it expires so a crashed holder cannot wedge it, and a lock store that cannot be reached is logged and the request proceeds, since the lock only guards a rare race.

Assisted-by: LLM
Signed-off-by: IvanHunters <xorokhotnikov@gmail.com>
2026-11-18 12:15:37 +03:00

596 lines
25 KiB
Go

// SPDX-License-Identifier: Apache-2.0
package application
import (
"context"
"fmt"
"strings"
"sync/atomic"
"testing"
helmv2 "github.com/fluxcd/helm-controller/api/v2"
coordinationv1 "k8s.io/api/coordination/v1"
corev1 "k8s.io/api/core/v1"
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/warning"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
"github.com/cozystack/cozystack/pkg/config"
"github.com/cozystack/cozystack/pkg/credentials/account"
"github.com/cozystack/cozystack/pkg/credentials/store"
)
// The schema defaults backup.enabled to true, so the Static backup account
// is seeded unless the application turns it off.
const credentialsSchema = `{"type":"object","properties":{"backup":{"type":"object","default":{},"properties":{"enabled":{"type":"boolean","default":true}}},"users":{"type":"object","additionalProperties":{"type":"object","properties":{"password":{"type":"string"}}}}}}`
func credentialsBlock() *cozyv1alpha1.ApplicationDefinitionCredentials {
return &cozyv1alpha1.ApplicationDefinitionCredentials{
Format: cozyv1alpha1.CredentialFormat{Name: "scram-sha-256"},
ValuesPath: "_accounts",
Accounts: []cozyv1alpha1.CredentialAccountSource{
{Kind: cozyv1alpha1.CredentialAccountsValuesMap, Path: "users"},
{Kind: cozyv1alpha1.CredentialAccountsStatic, Name: "backup", When: "backup.enabled"},
{Kind: cozyv1alpha1.CredentialAccountsStatic, Name: "postgres", Owner: cozyv1alpha1.CredentialAccountOwnerPlatform},
},
}
}
func newCredentialsREST(t *testing.T, objects ...client.Object) (*REST, client.Client) {
t.Helper()
scheme := runtime.NewScheme()
if err := helmv2.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
if err := corev1.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
if err := coordinationv1.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
cfg := &config.Resource{
Application: config.ApplicationConfig{Kind: postgresKind, Singular: "postgres", Plural: "postgreses", OpenAPISchema: credentialsSchema},
Release: config.ReleaseConfig{Prefix: "postgres-"},
Credentials: credentialsBlock(),
}
if err := appsv1alpha1.RegisterDynamicTypes(scheme, &config.ResourceConfig{Resources: []config.Resource{*cfg}}); err != nil {
t.Fatal(err)
}
// The fake client assigns no UIDs, and without them every ownership check
// compares two empty strings and passes.
var uids atomic.Int64
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).WithInterceptorFuncs(interceptor.Funcs{
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
if obj.GetUID() == "" {
obj.SetUID(types.UID(fmt.Sprintf("uid-%d", uids.Add(1))))
}
return cl.Create(ctx, obj, opts...)
},
// The API server keeps an object's UID on an update that sends none;
// the fake client would blank it.
Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
if obj.GetUID() == "" {
cur := obj.DeepCopyObject().(client.Object)
if err := cl.Get(ctx, client.ObjectKeyFromObject(obj), cur); err == nil {
obj.SetUID(cur.GetUID())
}
}
return cl.Update(ctx, obj, opts...)
},
}).Build()
return NewREST(c, c, cfg), c
}
func credentialsCtx() context.Context {
return request.WithUser(request.WithNamespace(context.Background(), "tenant-foo"), &user.DefaultInfo{Name: "jane@example.org"})
}
func postgresApp(spec string) *appsv1alpha1.Application {
return &appsv1alpha1.Application{
TypeMeta: metav1.TypeMeta{APIVersion: "apps.cozystack.io/v1alpha1", Kind: postgresKind},
ObjectMeta: metav1.ObjectMeta{Name: "orders", Namespace: "tenant-foo"},
Spec: &apiextv1.JSON{Raw: []byte(spec)},
}
}
func accountSecrets(t *testing.T, c client.Client) map[string]*corev1.Secret {
t.Helper()
list := &corev1.SecretList{}
if err := c.List(context.Background(), list, client.InNamespace("tenant-foo"), client.MatchingLabels{account.AccountLabel: "true"}); err != nil {
t.Fatal(err)
}
out := map[string]*corev1.Secret{}
for i := range list.Items {
out[list.Items[i].Name] = &list.Items[i]
}
return out
}
func getRelease(t *testing.T, c client.Client) *helmv2.HelmRelease {
t.Helper()
hr := &helmv2.HelmRelease{}
if err := c.Get(context.Background(), types.NamespacedName{Namespace: "tenant-foo", Name: "postgres-orders"}, hr); err != nil {
t.Fatal(err)
}
return hr
}
func TestCreateSeedsAccountsAndOwnsThem(t *testing.T) {
r, c := newCredentialsREST(t)
if _, err := r.Create(credentialsCtx(), postgresApp(`{"users":{"web":{},"postgres":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
hr := getRelease(t, c)
secrets := accountSecrets(t, c)
// web from the values, backup from the defaulted flag; postgres is the
// platform's and gets no account Secret.
if len(secrets) != 2 || secrets["postgres-orders.web.account"] == nil || secrets["postgres-orders.backup.account"] == nil {
t.Fatalf("secrets = %v", keys(secrets))
}
for name, sec := range secrets {
if len(sec.OwnerReferences) != 1 || sec.OwnerReferences[0].Name != hr.Name || sec.OwnerReferences[0].Kind != helmv2.HelmReleaseKind {
t.Fatalf("%s owners = %v", name, sec.OwnerReferences)
}
if account.ReadRecord(sec).By != "jane@example.org" {
t.Fatalf("%s record = %+v", name, account.ReadRecord(sec))
}
}
if len(hr.Spec.ValuesFrom) != 3 || hr.Spec.ValuesFrom[0].Name != "cozystack-values" || hr.Spec.ValuesFrom[1].Name != "postgres-orders.web.account" || !strings.HasPrefix(hr.Spec.ValuesFrom[1].TargetPath, "_accounts.u") {
t.Fatalf("valuesFrom = %+v", hr.Spec.ValuesFrom)
}
}
func TestCreateRefusesCollidingNames(t *testing.T) {
r, c := newCredentialsREST(t)
_, err := r.Create(credentialsCtx(), postgresApp(`{"users":{"web_app":{},"web-app":{}}}`), nil, &metav1.CreateOptions{})
if !apierrors.IsInvalid(err) || !strings.Contains(err.Error(), `both map to the name "web-app"`) {
t.Fatalf("expected Invalid naming the collision, got %v", err)
}
if n := len(accountSecrets(t, c)); n != 0 {
t.Fatalf("a refused write seeds nothing, got %d Secrets", n)
}
}
func TestCreateFailureRemovesSeeded(t *testing.T) {
existing := &helmv2.HelmRelease{ObjectMeta: metav1.ObjectMeta{Name: "postgres-orders", Namespace: "tenant-foo"}}
r, c := newCredentialsREST(t, existing)
if _, err := r.Create(credentialsCtx(), postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); !apierrors.IsAlreadyExists(err) {
t.Fatalf("expected AlreadyExists, got %v", err)
}
if n := len(accountSecrets(t, c)); n != 0 {
t.Fatalf("Secrets seeded for a release that was not created must go, got %d", n)
}
}
func TestUpdateSeedsAddedAndPrunesRemoved(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{},"api":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
webBefore := accountSecrets(t, c)["postgres-orders.web.account"]
updated := postgresApp(`{"users":{"web":{},"reports":{}},"backup":{"enabled":false}}`)
if _, _, err := r.Update(ctx, "orders", newDefaultUpdatedObjectInfo(updated), nil, nil, false, &metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
secrets := accountSecrets(t, c)
if len(secrets) != 2 || secrets["postgres-orders.web.account"] == nil || secrets["postgres-orders.reports.account"] == nil {
t.Fatalf("secrets = %v, want web kept, reports added, api and backup pruned", keys(secrets))
}
if string(secrets["postgres-orders.web.account"].Data[account.PasswordKey]) != string(webBefore.Data[account.PasswordKey]) {
t.Fatal("a kept account's Secret must survive the update untouched")
}
hr := getRelease(t, c)
reports := secrets["postgres-orders.reports.account"]
if len(reports.OwnerReferences) != 1 || reports.OwnerReferences[0].UID != hr.UID {
t.Fatalf("added account's owner = %v", reports.OwnerReferences)
}
if len(hr.Spec.ValuesFrom) != 3 {
t.Fatalf("valuesFrom = %+v", hr.Spec.ValuesFrom)
}
}
func TestUpdateRefusesACaseRename(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
before := accountSecrets(t, c)["postgres-orders.web.account"]
updated := postgresApp(`{"users":{"Web":{}}}`)
_, _, err := r.Update(ctx, "orders", newDefaultUpdatedObjectInfo(updated), nil, nil, false, &metav1.UpdateOptions{})
if !apierrors.IsInvalid(err) || !strings.Contains(err.Error(), "already belongs to account") {
t.Fatalf("expected Invalid for a case-only rename, got %v", err)
}
after := accountSecrets(t, c)["postgres-orders.web.account"]
if after == nil || after.UID != before.UID || string(after.Data[account.PasswordKey]) != string(before.Data[account.PasswordKey]) {
t.Fatal("a refused rename must leave the live account's Secret untouched")
}
}
func TestPruneAfterUpdatePrunesByTheLiveRelease(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{},"api":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
// The live release still declares web and api; a stale request whose app no
// longer lists api must not prune api, because a concurrent update may have
// written it back and the live release references it.
r.pruneAfterUpdate(ctx, postgresApp(`{"users":{"web":{}}}`), false)
if accountSecrets(t, c)["postgres-orders.api.account"] == nil {
t.Fatal("prune deleted a Secret the live release still declares")
}
// Once the live release drops api, pruning removes its Secret.
hr := getRelease(t, c)
hr.Spec.Values = &apiextv1.JSON{Raw: []byte(`{"users":{"web":{}}}`)}
if err := c.Update(ctx, hr); err != nil {
t.Fatal(err)
}
r.pruneAfterUpdate(ctx, postgresApp(`{"users":{"web":{}}}`), false)
if accountSecrets(t, c)["postgres-orders.api.account"] != nil {
t.Fatal("an account the live release no longer declares must be pruned")
}
}
func TestDeleteRefusesSuspendedRelease(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
hr := getRelease(t, c)
hr.Spec.Suspend = true
if err := c.Update(ctx, hr); err != nil {
t.Fatal(err)
}
_, _, err := r.Delete(ctx, "orders", nil, &metav1.DeleteOptions{})
if !apierrors.IsConflict(err) || !strings.Contains(err.Error(), "suspended") {
t.Fatalf("expected Conflict on a suspended release, got %v", err)
}
hr = getRelease(t, c)
hr.Spec.Suspend = false
if err := c.Update(ctx, hr); err != nil {
t.Fatal(err)
}
if _, _, err := r.Delete(ctx, "orders", nil, &metav1.DeleteOptions{}); err != nil {
t.Fatal(err)
}
}
func TestWarnMintedPasswords(t *testing.T) {
r, _ := newCredentialsREST(t)
rec := &fakeWarningRecorder{}
ctx := warning.WithWarningRecorder(credentialsCtx(), rec)
if _, err := r.Create(ctx, postgresApp(`{"users":{"Web":{"password":"hackme"},"api":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
if len(rec.warnings) != 1 || !strings.Contains(rec.warnings[0], `spec.users["Web"].password is ignored`) || !strings.Contains(rec.warnings[0], "postgres-orders.web") {
t.Fatalf("warnings = %v", rec.warnings)
}
}
func keys(m map[string]*corev1.Secret) []string {
var out []string
for k := range m {
out = append(out, k)
}
return out
}
func TestCreateDuplicateLeavesLiveSecretsAlone(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
before := accountSecrets(t, c)["postgres-orders.web.account"]
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); !apierrors.IsAlreadyExists(err) {
t.Fatalf("second create: %v, want AlreadyExists", err)
}
after := accountSecrets(t, c)["postgres-orders.web.account"]
if after == nil || after.UID != before.UID || string(after.Data[account.PasswordKey]) != string(before.Data[account.PasswordKey]) {
t.Fatal("a refused create must leave the live application's Secret untouched")
}
}
func TestCreateReplacesAnUnownedLeftover(t *testing.T) {
// Left by an earlier create whose owner was never set.
leftover, err := store.SeedSecret(store.App{Namespace: "tenant-foo", Release: "postgres-orders", Group: appsv1alpha1.GroupName, Kind: postgresKind, Name: "orders", Block: credentialsBlock()},
account.Account{Username: "web", Mapped: "web"}, nil, metav1.Now(), "earlier")
if err != nil {
t.Fatal(err)
}
leftover.UID = "uid-leftover"
r, c := newCredentialsREST(t, leftover)
if _, err := r.Create(credentialsCtx(), postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
sec := accountSecrets(t, c)["postgres-orders.web.account"]
hr := getRelease(t, c)
if sec.UID == "uid-leftover" || string(sec.Data[account.PasswordKey]) == string(leftover.Data[account.PasswordKey]) {
t.Fatal("a leftover must be replaced, so the account gets a new identity")
}
if len(sec.OwnerReferences) != 1 || sec.OwnerReferences[0].UID != hr.UID {
t.Fatalf("owners = %v, want the new release", sec.OwnerReferences)
}
}
func TestDryRunWritesNoAccountSecrets(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()
dry := []string{metav1.DryRunAll}
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{DryRun: dry}); err != nil {
t.Fatal(err)
}
if n := len(accountSecrets(t, c)); n != 0 {
t.Fatalf("a dry-run create wrote %d account Secrets", n)
}
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
before := accountSecrets(t, c)
updated := postgresApp(`{"users":{"api":{}}}`)
if _, _, err := r.Update(ctx, "orders", newDefaultUpdatedObjectInfo(updated), nil, nil, false, &metav1.UpdateOptions{DryRun: dry}); err != nil {
t.Fatal(err)
}
after := accountSecrets(t, c)
if len(after) != len(before) || after["postgres-orders.web.account"] == nil || after["postgres-orders.api.account"] != nil {
t.Fatalf("a dry-run update changed the account Secrets: %v -> %v", keys(before), keys(after))
}
}
// An account the release already declared and that has no Secret, as after a
// conversion that left the release out, is not seeded by an unrelated write:
// a seed would replace its working password with a discarded one.
func TestUpdateSeedsOnlyAddedAccounts(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}},"backup":{"enabled":false}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
if err := c.Delete(ctx, accountSecrets(t, c)["postgres-orders.web.account"]); err != nil {
t.Fatal(err)
}
updated := postgresApp(`{"users":{"web":{},"api":{}},"backup":{"enabled":false},"size":"20Gi"}`)
if _, _, err := r.Update(ctx, "orders", newDefaultUpdatedObjectInfo(updated), nil, nil, false, &metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
got := accountSecrets(t, c)
if got["postgres-orders.api.account"] == nil || got["postgres-orders.web.account"] != nil {
t.Fatalf("secrets = %v, want api seeded and web left alone", keys(got))
}
}
// Two creates of one application race. The interceptor runs the second create
// in full at the moment the first is about to write its HelmRelease: the
// second seeds over the first's unowned Secret, creates the release, and
// adopts the Secret. The first then fails AlreadyExists, and its cleanup must
// leave the Secret the live release now owns. Without the release check and
// the UID check in unseed the live release ends with no account Secret.
func TestConcurrentCreateKeepsLiveSecret(t *testing.T) {
scheme := runtime.NewScheme()
if err := helmv2.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
if err := corev1.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
cfg := &config.Resource{
Application: config.ApplicationConfig{Kind: postgresKind, Singular: "postgres", Plural: "postgreses", OpenAPISchema: credentialsSchema},
Release: config.ReleaseConfig{Prefix: "postgres-"},
Credentials: credentialsBlock(),
}
if err := appsv1alpha1.RegisterDynamicTypes(scheme, &config.ResourceConfig{Resources: []config.Resource{*cfg}}); err != nil {
t.Fatal(err)
}
var r *REST
var raced atomic.Bool
var uids atomic.Int64
c := fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(interceptor.Funcs{
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
// When the first create is about to write its HelmRelease, let a
// second create of the same application run to completion first, so
// it wins the release and adopts the account Secret the first one
// seeded.
if _, ok := obj.(*helmv2.HelmRelease); ok && raced.CompareAndSwap(false, true) {
if _, err := r.Create(credentialsCtx(), postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
return fmt.Errorf("concurrent create: %w", err)
}
}
if obj.GetUID() == "" {
obj.SetUID(types.UID(fmt.Sprintf("uid-%d", uids.Add(1))))
}
return cl.Create(ctx, obj, opts...)
},
Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
if obj.GetUID() == "" {
cur := obj.DeepCopyObject().(client.Object)
if err := cl.Get(ctx, client.ObjectKeyFromObject(obj), cur); err == nil {
obj.SetUID(cur.GetUID())
}
}
return cl.Update(ctx, obj, opts...)
},
}).Build()
r = NewREST(c, c, cfg)
// The create that loses the race.
if _, err := r.Create(credentialsCtx(), postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); !apierrors.IsAlreadyExists(err) {
t.Fatalf("the create that lost the race: %v, want AlreadyExists", err)
}
hr := getRelease(t, c)
sec := accountSecrets(t, c)["postgres-orders.web.account"]
if sec == nil {
t.Fatal("the live release lost its account Secret to the losing create's cleanup")
}
if len(sec.OwnerReferences) != 1 || sec.OwnerReferences[0].UID != hr.UID {
t.Fatalf("surviving Secret owners = %v, want the live release", sec.OwnerReferences)
}
}
// A concurrent create of the same application can replace a seeded Secret and
// win the release, but not yet have adopted the replacement when the losing
// create's cleanup runs: the replacement is still unowned. unseed must leave
// it, because its UID is not the one this create seeded. Deleting by name, or
// only skipping an owned Secret, would take it and leave the live release with
// no account Secret.
// A concurrent write to one application's credentials is refused while another
// holds the lock, so the two cannot interleave into deleting a Secret the other
// reused. Request A adds api; while A holds the lock and writes its release, the
// interceptor runs request B (which would remove web and prune its Secret). B
// is refused with Conflict, so web's Secret is untouched.
func TestConcurrentUpdateRefusedByCredentialLock(t *testing.T) {
scheme := runtime.NewScheme()
if err := helmv2.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
if err := corev1.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
if err := coordinationv1.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
cfg := &config.Resource{
Application: config.ApplicationConfig{Kind: postgresKind, Singular: "postgres", Plural: "postgreses", OpenAPISchema: credentialsSchema},
Release: config.ReleaseConfig{Prefix: "postgres-"},
Credentials: credentialsBlock(),
}
if err := appsv1alpha1.RegisterDynamicTypes(scheme, &config.ResourceConfig{Resources: []config.Resource{*cfg}}); err != nil {
t.Fatal(err)
}
var r *REST
var raced atomic.Bool
var bErr error
var uids atomic.Int64
c := fake.NewClientBuilder().WithScheme(scheme).WithInterceptorFuncs(interceptor.Funcs{
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
if obj.GetUID() == "" {
obj.SetUID(types.UID(fmt.Sprintf("uid-%d", uids.Add(1))))
}
return cl.Create(ctx, obj, opts...)
},
Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
// Request A is writing its HelmRelease, so A holds the lock: a second
// update of the same application must be refused now.
if _, ok := obj.(*helmv2.HelmRelease); ok && raced.CompareAndSwap(false, true) {
_, _, bErr = r.Update(credentialsCtx(), "orders", newDefaultUpdatedObjectInfo(postgresApp(`{"users":{}}`)), nil, nil, false, &metav1.UpdateOptions{})
}
if obj.GetUID() == "" {
cur := obj.DeepCopyObject().(client.Object)
if err := cl.Get(ctx, client.ObjectKeyFromObject(obj), cur); err == nil {
obj.SetUID(cur.GetUID())
}
}
return cl.Update(ctx, obj, opts...)
},
}).Build()
r = NewREST(c, c, cfg)
ctx := credentialsCtx()
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
webBefore := accountSecrets(t, c)["postgres-orders.web.account"]
if _, _, err := r.Update(ctx, "orders", newDefaultUpdatedObjectInfo(postgresApp(`{"users":{"web":{},"api":{}}}`)), nil, nil, false, &metav1.UpdateOptions{}); err != nil {
t.Fatalf("request A: %v", err)
}
if !apierrors.IsConflict(bErr) {
t.Fatalf("a concurrent credential write must be refused with Conflict, got %v", bErr)
}
after := accountSecrets(t, c)["postgres-orders.web.account"]
if after == nil || after.UID != webBefore.UID {
t.Fatal("the refused concurrent update must not have pruned web's Secret")
}
if accountSecrets(t, c)["postgres-orders.api.account"] == nil {
t.Fatal("request A must have seeded api")
}
}
func TestUnseedLeavesAReplacedSecret(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()
const name = "postgres-orders.web.account"
replacement := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tenant-foo", UID: "uid-winner",
Labels: map[string]string{account.AccountLabel: "true"}},
}
if err := c.Create(ctx, replacement); err != nil {
t.Fatal(err)
}
// This create seeded an object that the winner has since replaced, so the
// UID it recorded is gone and the live Secret carries the winner's UID. No
// release exists here, so the UID guard alone must keep the replacement.
r.unseed(ctx, "tenant-foo", "postgres-orders", []store.Seeded{{Name: name, UID: "uid-loser"}})
sec := accountSecrets(t, c)[name]
if sec == nil || sec.UID != "uid-winner" {
t.Fatalf("unseed deleted a Secret this create did not seed: %+v", sec)
}
}
// With no HelmRelease for the application, a failed create's seed is a genuine
// orphan and unseed deletes the exact object it wrote.
func TestUnseedDeletesOrphanWhenNoRelease(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()
const name = "postgres-orders.web.account"
orphan := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tenant-foo", UID: "uid-mine",
Labels: map[string]string{account.AccountLabel: "true"}},
}
if err := c.Create(ctx, orphan); err != nil {
t.Fatal(err)
}
r.unseed(ctx, "tenant-foo", "postgres-orders", []store.Seeded{{Name: name, UID: "uid-mine"}})
if accountSecrets(t, c)[name] != nil {
t.Fatal("unseed must delete its own orphaned seed when no release exists")
}
}
// Once a HelmRelease for the application exists, a concurrent create won it and
// owns the account Secrets, even one it adopted that this create had seeded, so
// unseed deletes nothing although the UID still matches.
func TestUnseedLeavesSecretsWhenReleaseExists(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()
const name = "postgres-orders.web.account"
hr := &helmv2.HelmRelease{ObjectMeta: metav1.ObjectMeta{Name: "postgres-orders", Namespace: "tenant-foo", UID: "hr-winner"}}
if err := c.Create(ctx, hr); err != nil {
t.Fatal(err)
}
adopted := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tenant-foo", UID: "uid-seeded",
Labels: map[string]string{account.AccountLabel: "true"},
OwnerReferences: []metav1.OwnerReference{{APIVersion: helmv2.GroupVersion.String(), Kind: helmv2.HelmReleaseKind, Name: "postgres-orders", UID: "hr-winner"}}},
}
if err := c.Create(ctx, adopted); err != nil {
t.Fatal(err)
}
// This create seeded the object the winner then adopted, so the UID matches;
// the live release is what must keep unseed from deleting it.
r.unseed(ctx, "tenant-foo", "postgres-orders", []store.Seeded{{Name: name, UID: "uid-seeded"}})
if accountSecrets(t, c)[name] == nil {
t.Fatal("unseed deleted an account Secret while the application's release exists")
}
}