fix(api): keep account Secrets of live apps intact

Three ordinary writes replaced or removed account Secrets. A duplicate
create seeded before learning the release existed, so it replaced the
live application's Secrets and then removed its own. Server dry-run
wrote and pruned for real, and a dry-run mint replaced the password.
An update seeded every declared account without a Secret, which after
the conversion swaps a working password for a discarded one.

A create now stops on an existing release before touching a Secret,
and replaces any leftover at an account's name unless a live release
owns it, so the new account always gets a fresh identity and an owner.
Dry-run writes nothing anywhere. An update seeds only the accounts it
adds. A mint requires a declared account even where the Secret
exists, and the write paths read through the uncached client, since a
stale read here is a replaced or recreated Secret.

Assisted-by: LLM
Signed-off-by: Timofei Larkin <lllamnyp@gmail.com>
This commit is contained in:
Timofei Larkin authored and IvanHunters committed 2026-11-17 12:07:14 +03:00
1 parent c96b91f67a
commit 19d183a1d1
9 files changed
+487 -57

No files matched your search

+1 -1
View File
@@ -247,7 +247,7 @@ func (c completedConfig) New() (*CozyServer, error) {
coreV1alpha1Storage["taps"] = cozyregistry.RESTInPeace(
tapstorage.NewREST(dyn),
)
credentials := credentialstorage.NewREST(cli, credentialKinds)
credentials := credentialstorage.NewREST(cli, watchCli, credentialKinds)
coreV1alpha1Storage["credentials"] = cozyregistry.RESTInPeace(credentials)
coreV1alpha1Storage["credentials/mint"] = credentialstorage.NewMintREST(credentials)
coreV1alpha1Storage["credentials/revoke"] = credentialstorage.NewRevokeREST(credentials)
+60 -10
View File
@@ -10,6 +10,7 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
helmv2 "github.com/fluxcd/helm-controller/api/v2"
@@ -85,30 +86,55 @@ func SeedSecret(app App, acc account.Account, owner *metav1.OwnerReference, now
return sec, nil
}
// ErrLiveOwner is returned by Ensure on a create when a Secret at an account's
// name is owned by a HelmRelease that still exists: the application exists
// already, and its Secrets must not be touched.
var ErrLiveOwner = errors.New("account Secret is owned by a live HelmRelease")
// Theirs reports whether an existing Secret is the application's own account
// Secret: marker, same username, owned by the live HelmRelease. With no
// HelmRelease yet (a create), one any HelmRelease owns is left from a deleted
// application of the same name and is replaced, so the account gets a new
// identity; one nobody owns is from a create that did not finish.
// Secret: the account marker, the same username, and owned by owner.
func Theirs(sec *corev1.Secret, acc account.Account, owner *metav1.OwnerReference) bool {
if !account.IsAccountSecret(sec) || string(sec.Data[account.UsernameKey]) != acc.Username {
if owner == nil || !account.IsAccountSecret(sec) || string(sec.Data[account.UsernameKey]) != acc.Username {
return false
}
for _, ref := range sec.OwnerReferences {
if owner == nil && ref.Kind == helmv2.HelmReleaseKind {
return false
}
if owner != nil && ref.UID == owner.UID {
if ref.UID == owner.UID {
return true
}
}
return owner == nil
return false
}
// liveOwner reports whether a HelmRelease the Secret names as its owner still
// exists under the same UID.
func liveOwner(ctx context.Context, c client.Reader, sec *corev1.Secret) (bool, error) {
for _, ref := range sec.OwnerReferences {
if ref.Kind != helmv2.HelmReleaseKind {
continue
}
hr := &helmv2.HelmRelease{}
err := c.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: ref.Name}, hr)
switch {
case apierrors.IsNotFound(err):
continue
case err != nil:
return false, err
case hr.UID == ref.UID:
return true, nil
}
}
return false, nil
}
// Ensure writes an account Secret for every account that has none, or whose
// Secret is not the application's, and returns the names of the Secrets it
// wrote. It never touches a Secret that is the application's own, so a
// password already minted survives every later write of the application.
//
// With owner nil (a create, before the HelmRelease exists) every existing
// Secret at an account's name is a leftover and is replaced, so the account
// gets a new identity and the Secret is among those to be owned; one a live
// HelmRelease owns stops the create with ErrLiveOwner.
func Ensure(ctx context.Context, c client.Client, app App, accounts []account.Account, owner *metav1.OwnerReference, now metav1.Time, by string) ([]string, error) {
var written []string
for _, acc := range accounts {
@@ -119,6 +145,15 @@ func Ensure(ctx context.Context, c client.Client, app App, accounts []account.Ac
case err == nil && Theirs(cur, acc, owner):
continue
case err == nil:
if owner == nil {
live, err := liveOwner(ctx, c, cur)
if err != nil {
return written, fmt.Errorf("read owner of account Secret %s: %w", name, err)
}
if live {
return written, fmt.Errorf("%s: %w", name, ErrLiveOwner)
}
}
if err := c.Delete(ctx, cur, client.Preconditions{UID: &cur.UID}); err != nil && !apierrors.IsNotFound(err) {
return written, fmt.Errorf("replace account Secret %s: %w", name, err)
}
@@ -137,6 +172,21 @@ func Ensure(ctx context.Context, c client.Client, app App, accounts []account.Ac
return written, nil
}
// Added returns the accounts of next whose username prev does not declare.
func Added(prev, next []account.Account) []account.Account {
had := make(map[string]bool, len(prev))
for _, acc := range prev {
had[acc.Username] = true
}
var added []account.Account
for _, acc := range next {
if !had[acc.Username] {
added = append(added, acc)
}
}
return added
}
// SetOwner adds the HelmRelease as owner of the named account Secrets. It is
// the second half of a create, once the HelmRelease exists.
func SetOwner(ctx context.Context, c client.Client, namespace string, names []string, owner metav1.OwnerReference) error {
+101 -2
View File
@@ -4,6 +4,7 @@ package store
import (
"context"
"errors"
"strings"
"testing"
@@ -101,8 +102,8 @@ func TestEnsureSeedsAndKeepsTheirs(t *testing.T) {
}
first := string(sec.Data[account.PasswordKey])
// A second create of the same application, as after an interrupted one,
// reuses the Secret nobody owns.
// An update, with the owner known, seeds only what it is given and leaves
// an owned Secret alone.
hr := release("uid-1")
if err := SetOwner(ctx, c, "tenant-foo", written, OwnerReference(hr)); err != nil {
t.Fatal(err)
@@ -246,3 +247,101 @@ func TestResolveAccountsAppliesDefaults(t *testing.T) {
t.Fatalf("no block, no accounts: %v %v", got, err)
}
}
func TestEnsureCreateReplacesLeftovers(t *testing.T) {
ctx := context.Background()
gone := OwnerReference(release("uid-gone"))
live := release("uid-live")
c := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(live).Build()
app := testApp()
now := metav1.Now()
// Left by an earlier create that never got its owner, and by a deleted
// application of the same name whose HelmRelease is gone.
if _, err := Ensure(ctx, c, app, accounts("web"), nil, now, "x"); err != nil {
t.Fatal(err)
}
if _, err := Ensure(ctx, c, app, accounts("api"), &gone, now, "x"); err != nil {
t.Fatal(err)
}
unowned := getSecret(t, c, "postgres-orders.web.account")
stale := getSecret(t, c, "postgres-orders.api.account")
written, err := Ensure(ctx, c, app, accounts("web", "api"), nil, now, "jane")
if err != nil {
t.Fatal(err)
}
if len(written) != 2 {
t.Fatalf("both leftovers must be replaced and handed to SetOwner, written = %v", written)
}
if string(getSecret(t, c, "postgres-orders.web.account").Data[account.PasswordKey]) == string(unowned.Data[account.PasswordKey]) {
t.Fatal("an unowned leftover kept its verifier")
}
if string(getSecret(t, c, "postgres-orders.api.account").Data[account.PasswordKey]) == string(stale.Data[account.PasswordKey]) {
t.Fatal("a Secret of a gone release kept its verifier")
}
}
func TestEnsureCreateRefusesLiveOwner(t *testing.T) {
ctx := context.Background()
live := release("uid-live")
c := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(live).Build()
app := testApp()
owner := OwnerReference(live)
if _, err := Ensure(ctx, c, app, accounts("web"), &owner, metav1.Now(), "x"); err != nil {
t.Fatal(err)
}
before := getSecret(t, c, "postgres-orders.web.account")
written, err := Ensure(ctx, c, app, accounts("web"), nil, metav1.Now(), "jane")
if !errors.Is(err, ErrLiveOwner) || len(written) != 0 {
t.Fatalf("written=%v err=%v, want ErrLiveOwner and nothing written", written, err)
}
after := getSecret(t, c, "postgres-orders.web.account")
if string(after.Data[account.PasswordKey]) != string(before.Data[account.PasswordKey]) || len(after.OwnerReferences) != 1 {
t.Fatal("the live application's Secret must be left untouched")
}
}
// The application labels alone do not make a Secret an account Secret: the
// lineage webhook puts them on every Secret of the application, and Prune
// must never delete the chart's own.
func TestListAndPruneSeeOnlyAccountSecrets(t *testing.T) {
ctx := context.Background()
app := testApp()
chartOwned := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{
Name: "postgres-orders-credentials", Namespace: "tenant-foo",
Labels: map[string]string{
"apps.cozystack.io/application.group": app.Group,
"apps.cozystack.io/application.kind": app.Kind,
"apps.cozystack.io/application.name": app.Name,
},
}}
c := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(chartOwned).Build()
if _, err := Ensure(ctx, c, app, accounts("web"), nil, metav1.Now(), "x"); err != nil {
t.Fatal(err)
}
listed, err := List(ctx, c, app)
if err != nil {
t.Fatal(err)
}
if len(listed) != 1 || listed[0].Name != "postgres-orders.web.account" {
t.Fatalf("listed = %v", listed)
}
if _, err := Prune(ctx, c, app, nil); err != nil {
t.Fatal(err)
}
getSecret(t, c, "postgres-orders-credentials")
}
func TestAdded(t *testing.T) {
got := Added(accounts("web", "api"), []account.Account{{Username: "web", Mapped: "web"}, {Username: "Api", Mapped: "api"}, {Username: "new", Mapped: "new"}})
var names []string
for _, a := range got {
names = append(names, a.Username)
}
// Api is a different user from api, however the name maps.
if strings.Join(names, ",") != "Api,new" {
t.Fatalf("added = %v", names)
}
}
+4 -3
View File
@@ -44,6 +44,7 @@ import (
"k8s.io/apimachinery/pkg/watch"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/util/dryrun"
"k8s.io/apiserver/pkg/warning"
"k8s.io/client-go/util/retry"
"k8s.io/klog/v2"
@@ -253,7 +254,7 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation
// The account Secrets exist before the release that reads them, and the
// release owns them as soon as it exists.
seeded, err := r.seedBeforeCreate(ctx, app, helmRelease, accounts)
seeded, err := r.seedBeforeCreate(ctx, app, helmRelease, accounts, options != nil && dryrun.IsDryRun(options.DryRun))
if err != nil {
return nil, err
}
@@ -607,7 +608,7 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
}
// Accounts the write adds get their Secrets before the release reads them.
if err := r.seedBeforeUpdate(ctx, app, helmRelease, cur, accounts); err != nil {
if err := r.seedBeforeUpdate(ctx, app, helmRelease, cur, accounts, options != nil && dryrun.IsDryRun(options.DryRun)); err != nil {
return nil, false, err
}
@@ -672,7 +673,7 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje
// Accounts the write removed are gone from the chart now, so their
// Secrets can go.
r.pruneAfterUpdate(ctx, app, accounts)
r.pruneAfterUpdate(ctx, app, accounts, options != nil && dryrun.IsDryRun(options.DryRun))
// Convert the updated HelmRelease back to Application
convertedApp, err := r.ConvertHelmReleaseToApplication(ctx, helmRelease)
@@ -5,6 +5,7 @@ package application
import (
"context"
"encoding/json"
"errors"
"fmt"
helmv2 "github.com/fluxcd/helm-controller/api/v2"
@@ -16,6 +17,7 @@ import (
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/warning"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client"
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
@@ -66,31 +68,56 @@ func actor(ctx context.Context) string {
return ""
}
// direct is the client the account Secret writes read through. The cached
// client can miss an object written a moment ago, and a stale read here
// replaces or recreates a Secret that exists.
func (r *REST) direct() client.Client {
if r.w != nil {
return r.w
}
return r.c
}
// seedBeforeCreate writes the account Secrets of a new application, with no
// owner yet, and the valuesFrom entries that carry them into the chart.
// It returns the Secrets it wrote, for the owner to be set once the
// HelmRelease exists, or for removal if the HelmRelease is not created.
func (r *REST) seedBeforeCreate(ctx context.Context, app *appsv1alpha1.Application, hr *helmv2.HelmRelease, accounts []account.Account) ([]string, error) {
// An existing release fails the create before any Secret is touched, since
// its own Secrets would otherwise be replaced.
func (r *REST) seedBeforeCreate(ctx context.Context, app *appsv1alpha1.Application, hr *helmv2.HelmRelease, accounts []account.Account, dryRun bool) ([]string, error) {
if r.credentials == nil {
return nil, nil
}
hr.Spec.ValuesFrom = store.ValuesFrom(r.credentials, hr.Name, accounts)
written, err := store.Ensure(ctx, r.c, r.storeApp(app.Namespace, app.Name), accounts, nil, metav1.Now(), actor(ctx))
existing := &helmv2.HelmRelease{}
err := r.direct().Get(ctx, client.ObjectKey{Namespace: hr.Namespace, Name: hr.Name}, existing)
switch {
case err == nil:
return nil, apierrors.NewAlreadyExists(r.gvr.GroupResource(), app.Name)
case !apierrors.IsNotFound(err):
return nil, apierrors.NewInternalError(fmt.Errorf("read HelmRelease %s: %w", hr.Name, err))
}
if dryRun {
return nil, nil
}
written, err := store.Ensure(ctx, r.direct(), r.storeApp(app.Namespace, app.Name), accounts, nil, metav1.Now(), actor(ctx))
if err != nil {
r.unseed(ctx, app.Namespace, written)
if errors.Is(err, store.ErrLiveOwner) {
return nil, apierrors.NewAlreadyExists(r.gvr.GroupResource(), app.Name)
}
return nil, apierrors.NewInternalError(fmt.Errorf("seed account Secrets: %w", err))
}
return written, nil
}
// unseed removes the Secrets a create wrote before its HelmRelease failed, so
// a retry starts clean. Best effort: a Secret nobody owns is reused by the
// retry in any case.
// a retry starts clean. Best effort: a retry replaces a leftover in any case.
func (r *REST) unseed(ctx context.Context, namespace string, names []string) {
for _, name := range names {
sec := &corev1.Secret{}
sec.Namespace, sec.Name = namespace, name
if err := r.c.Delete(ctx, sec); err != nil && !apierrors.IsNotFound(err) {
if err := r.direct().Delete(ctx, sec); err != nil && !apierrors.IsNotFound(err) {
klog.Warningf("failed to remove account Secret %s/%s after a failed create: %v", namespace, name, err)
}
}
@@ -98,26 +125,39 @@ func (r *REST) unseed(ctx context.Context, namespace string, names []string) {
// ownSeeded gives the created HelmRelease the account Secrets seeded before
// it. The application exists by now, so a failure is reported as a warning
// rather than an error: the next write of the application retries it.
// rather than an error.
func (r *REST) ownSeeded(ctx context.Context, hr *helmv2.HelmRelease, names []string) {
if len(names) == 0 {
return
}
if err := store.SetOwner(ctx, r.c, hr.Namespace, names, store.OwnerReference(hr)); err != nil {
if err := store.SetOwner(ctx, r.direct(), hr.Namespace, names, store.OwnerReference(hr)); err != nil {
klog.Errorf("failed to own account Secrets of %s/%s: %v", hr.Namespace, hr.Name, err)
warning.AddWarning(ctx, "", fmt.Sprintf("account Secrets were written but not yet owned by the release: %v", err))
}
}
// seedBeforeUpdate writes the account Secrets an update adds, owned by the
// live HelmRelease from the start, and the valuesFrom entries of the result.
func (r *REST) seedBeforeUpdate(ctx context.Context, app *appsv1alpha1.Application, hr, cur *helmv2.HelmRelease, accounts []account.Account) error {
// seedBeforeUpdate writes the account Secrets of the accounts the update adds,
// owned by the live HelmRelease from the start, and the valuesFrom entries of
// the result. An account the release already declared and that has no Secret
// is left alone: a seed would replace its working password with a discarded
// one, and a mint creates the Secret when someone needs it.
func (r *REST) seedBeforeUpdate(ctx context.Context, app *appsv1alpha1.Application, hr, cur *helmv2.HelmRelease, accounts []account.Account, dryRun bool) error {
if r.credentials == nil {
return nil
}
hr.Spec.ValuesFrom = store.ValuesFrom(r.credentials, hr.Name, accounts)
if dryRun {
return nil
}
prev, err := r.accountsOf(cur.Spec.Values)
if err != nil {
// The stored values were written around the API; nothing tells which
// accounts are new, so seed none and leave the rest to a mint.
klog.Warningf("accounts of %s/%s before the update cannot be resolved, seeding none: %v", cur.Namespace, cur.Name, err)
return nil
}
owner := store.OwnerReference(cur)
if _, err := store.Ensure(ctx, r.c, r.storeApp(app.Namespace, app.Name), accounts, &owner, metav1.Now(), actor(ctx)); err != nil {
if _, err := store.Ensure(ctx, r.direct(), r.storeApp(app.Namespace, app.Name), store.Added(prev, accounts), &owner, metav1.Now(), actor(ctx)); err != nil {
return apierrors.NewInternalError(fmt.Errorf("seed account Secrets: %w", err))
}
return nil
@@ -125,11 +165,11 @@ func (r *REST) seedBeforeUpdate(ctx context.Context, app *appsv1alpha1.Applicati
// pruneAfterUpdate deletes the account Secrets of the accounts the update
// removed, once the HelmRelease write has dropped them from the chart.
func (r *REST) pruneAfterUpdate(ctx context.Context, app *appsv1alpha1.Application, accounts []account.Account) {
if r.credentials == nil {
func (r *REST) pruneAfterUpdate(ctx context.Context, app *appsv1alpha1.Application, accounts []account.Account, dryRun bool) {
if r.credentials == nil || dryRun {
return
}
if _, err := store.Prune(ctx, r.c, r.storeApp(app.Namespace, app.Name), accounts); err != nil {
if _, err := store.Prune(ctx, r.direct(), r.storeApp(app.Namespace, app.Name), accounts); err != nil {
klog.Errorf("failed to prune account Secrets of %s/%s: %v", app.Namespace, app.Name, err)
warning.AddWarning(ctx, "", fmt.Sprintf("account Secrets of removed users were not all deleted: %v", err))
}
@@ -4,7 +4,9 @@ package application
import (
"context"
"fmt"
"strings"
"sync/atomic"
"testing"
helmv2 "github.com/fluxcd/helm-controller/api/v2"
@@ -19,11 +21,13 @@ import (
"k8s.io/apiserver/pkg/warning"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
"github.com/cozystack/cozystack/pkg/config"
"github.com/cozystack/cozystack/pkg/credentials/account"
"github.com/cozystack/cozystack/pkg/credentials/store"
)
// The schema defaults backup.enabled to true, so the Static backup account
@@ -59,7 +63,28 @@ func newCredentialsREST(t *testing.T, objects ...client.Object) (*REST, client.C
if err := appsv1alpha1.RegisterDynamicTypes(scheme, &config.ResourceConfig{Resources: []config.Resource{*cfg}}); err != nil {
t.Fatal(err)
}
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build()
// The fake client assigns no UIDs, and without them every ownership check
// compares two empty strings and passes.
var uids atomic.Int64
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).WithInterceptorFuncs(interceptor.Funcs{
Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
if obj.GetUID() == "" {
obj.SetUID(types.UID(fmt.Sprintf("uid-%d", uids.Add(1))))
}
return cl.Create(ctx, obj, opts...)
},
// The API server keeps an object's UID on an update that sends none;
// the fake client would blank it.
Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
if obj.GetUID() == "" {
cur := obj.DeepCopyObject().(client.Object)
if err := cl.Get(ctx, client.ObjectKeyFromObject(obj), cur); err == nil {
obj.SetUID(cur.GetUID())
}
}
return cl.Update(ctx, obj, opts...)
},
}).Build()
return NewREST(c, c, cfg), c
}
@@ -217,3 +242,90 @@ func keys(m map[string]*corev1.Secret) []string {
}
return out
}
func TestCreateDuplicateLeavesLiveSecretsAlone(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
before := accountSecrets(t, c)["postgres-orders.web.account"]
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); !apierrors.IsAlreadyExists(err) {
t.Fatalf("second create: %v, want AlreadyExists", err)
}
after := accountSecrets(t, c)["postgres-orders.web.account"]
if after == nil || after.UID != before.UID || string(after.Data[account.PasswordKey]) != string(before.Data[account.PasswordKey]) {
t.Fatal("a refused create must leave the live application's Secret untouched")
}
}
func TestCreateReplacesAnUnownedLeftover(t *testing.T) {
// Left by an earlier create whose owner was never set.
leftover, err := store.SeedSecret(store.App{Namespace: "tenant-foo", Release: "postgres-orders", Group: appsv1alpha1.GroupName, Kind: postgresKind, Name: "orders", Block: credentialsBlock()},
account.Account{Username: "web", Mapped: "web"}, nil, metav1.Now(), "earlier")
if err != nil {
t.Fatal(err)
}
leftover.UID = "uid-leftover"
r, c := newCredentialsREST(t, leftover)
if _, err := r.Create(credentialsCtx(), postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
sec := accountSecrets(t, c)["postgres-orders.web.account"]
hr := getRelease(t, c)
if sec.UID == "uid-leftover" || string(sec.Data[account.PasswordKey]) == string(leftover.Data[account.PasswordKey]) {
t.Fatal("a leftover must be replaced, so the account gets a new identity")
}
if len(sec.OwnerReferences) != 1 || sec.OwnerReferences[0].UID != hr.UID {
t.Fatalf("owners = %v, want the new release", sec.OwnerReferences)
}
}
func TestDryRunWritesNoAccountSecrets(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()
dry := []string{metav1.DryRunAll}
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{DryRun: dry}); err != nil {
t.Fatal(err)
}
if n := len(accountSecrets(t, c)); n != 0 {
t.Fatalf("a dry-run create wrote %d account Secrets", n)
}
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
before := accountSecrets(t, c)
updated := postgresApp(`{"users":{"api":{}}}`)
if _, _, err := r.Update(ctx, "orders", newDefaultUpdatedObjectInfo(updated), nil, nil, false, &metav1.UpdateOptions{DryRun: dry}); err != nil {
t.Fatal(err)
}
after := accountSecrets(t, c)
if len(after) != len(before) || after["postgres-orders.web.account"] == nil || after["postgres-orders.api.account"] != nil {
t.Fatalf("a dry-run update changed the account Secrets: %v -> %v", keys(before), keys(after))
}
}
// An account the release already declared and that has no Secret, as after a
// conversion that left the release out, is not seeded by an unrelated write:
// a seed would replace its working password with a discarded one.
func TestUpdateSeedsOnlyAddedAccounts(t *testing.T) {
r, c := newCredentialsREST(t)
ctx := credentialsCtx()
if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}},"backup":{"enabled":false}}`), nil, &metav1.CreateOptions{}); err != nil {
t.Fatal(err)
}
if err := c.Delete(ctx, accountSecrets(t, c)["postgres-orders.web.account"]); err != nil {
t.Fatal(err)
}
updated := postgresApp(`{"users":{"web":{},"api":{}},"backup":{"enabled":false},"size":"20Gi"}`)
if _, _, err := r.Update(ctx, "orders", newDefaultUpdatedObjectInfo(updated), nil, nil, false, &metav1.UpdateOptions{}); err != nil {
t.Fatal(err)
}
got := accountSecrets(t, c)
if got["postgres-orders.api.account"] == nil || got["postgres-orders.web.account"] != nil {
t.Fatalf("secrets = %v, want api seeded and web left alone", keys(got))
}
}
+12 -8
View File
@@ -60,8 +60,11 @@ type Kind struct {
// REST serves the credentials view.
type REST struct {
c client.Client
gvr schema.GroupVersionResource
c client.Client
// direct reads and writes for mint and revoke, which the cached client
// would answer with a Secret written a moment ago missing.
direct client.Client
gvr schema.GroupVersionResource
// kinds is keyed by the application kind the HelmRelease labels name.
kinds map[string]Kind
now func() time.Time
@@ -69,9 +72,10 @@ type REST struct {
// NewREST creates the credentials storage. kinds carries every converted
// application kind; a HelmRelease of a kind absent from it has no credentials.
func NewREST(c client.Client, kinds map[string]Kind) *REST {
func NewREST(c, direct client.Client, kinds map[string]Kind) *REST {
return &REST{
c: c,
c: c,
direct: direct,
gvr: schema.GroupVersionResource{
Group: corev1alpha1.GroupName,
Version: "v1alpha1",
@@ -117,13 +121,13 @@ type resolved struct {
// must exist and belong to a converted kind, or the credential does not exist.
// A Secret that is not the application's own, by marker or by owner, counts
// as absent.
func (r *REST) resolve(ctx context.Context, namespace, name string) (*resolved, error) {
func (r *REST) resolve(ctx context.Context, c client.Reader, namespace, name string) (*resolved, error) {
release, mapped, ok := account.SplitCredentialName(name)
if !ok {
return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name)
}
hr := &helmv2.HelmRelease{}
if err := r.c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: release}, hr); err != nil {
if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: release}, hr); err != nil {
if apierrors.IsNotFound(err) {
return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name)
}
@@ -135,7 +139,7 @@ func (r *REST) resolve(ctx context.Context, namespace, name string) (*resolved,
}
res := &resolved{hr: hr, kind: kind, mapped: mapped}
sec := &corev1.Secret{}
err := r.c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: account.SecretName(release, mapped)}, sec)
err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: account.SecretName(release, mapped)}, sec)
switch {
case err == nil && ownedBy(sec, hr):
res.secret = sec
@@ -197,7 +201,7 @@ func (r *REST) Get(ctx context.Context, name string, _ *metav1.GetOptions) (runt
if err != nil {
return nil, err
}
res, err := r.resolve(ctx, ns, name)
res, err := r.resolve(ctx, r.c, ns, name)
if err != nil {
return nil, err
}
+113 -3
View File
@@ -3,7 +3,10 @@
package credential
import (
"bytes"
"context"
"flag"
"io"
"strings"
"testing"
"time"
@@ -17,8 +20,10 @@ import (
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
@@ -49,6 +54,12 @@ type fixture struct {
}
func newFixture(t *testing.T, objs ...client.Object) *fixture {
t.Helper()
return newFixtureWith(t, nil, objs...)
}
// newFixtureWith builds the fixture on a client whose calls go through funcs.
func newFixtureWith(t *testing.T, funcs *interceptor.Funcs, objs ...client.Object) *fixture {
t.Helper()
scheme := runtime.NewScheme()
if err := corev1.AddToScheme(scheme); err != nil {
@@ -66,9 +77,13 @@ func newFixture(t *testing.T, objs ...client.Object) *fixture {
},
}}
f := &fixture{hr: hr}
c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(append(objs, hr)...).Build()
b := fake.NewClientBuilder().WithScheme(scheme).WithObjects(append(objs, hr)...)
if funcs != nil {
b = b.WithInterceptorFuncs(*funcs)
}
c := b.Build()
f.c = c
f.r = NewREST(c, map[string]Kind{"Postgres": {
f.r = NewREST(c, c, map[string]Kind{"Postgres": {
Block: testBlock(),
Accounts: func(context.Context, *helmv2.HelmRelease) ([]account.Account, error) { return f.declare, nil },
}})
@@ -81,11 +96,13 @@ func (f *fixture) app() store.App {
return store.App{Namespace: ns, Release: "postgres-orders", Group: appsv1alpha1.GroupName, Kind: "Postgres", Name: "orders", Block: testBlock()}
}
// seed writes an owned account Secret for user and returns it as stored.
// seed declares user and writes its owned account Secret, and returns the
// Secret as stored.
func (f *fixture) seed(t *testing.T, username string) *corev1.Secret {
t.Helper()
owner := store.OwnerReference(f.hr)
mapped, _ := account.MapName(username)
f.declare = append(f.declare, account.Account{Username: username, Mapped: mapped})
sec, err := store.SeedSecret(f.app(), account.Account{Username: username, Mapped: mapped}, &owner, metav1.NewTime(f.r.now()), "seeder")
if err != nil {
t.Fatal(err)
@@ -307,3 +324,96 @@ func TestUnconvertedKindHasNoCredentials(t *testing.T) {
}
func ptr[T any](v T) *T { return &v }
func TestMintRefusesAnUndeclaredAccountWithASecret(t *testing.T) {
f := newFixture(t)
before := f.seed(t, "web")
// The user left the values and the prune of its Secret failed.
f.declare = nil
if _, err := f.mint(t, "postgres-orders.web", ""); !apierrors.IsNotFound(err) {
t.Fatalf("mint on an undeclared account: %v, want NotFound", err)
}
// A Secret holding another user than the one the name maps to.
f.declare = []account.Account{{Username: "Web", Mapped: "web"}}
if _, err := f.mint(t, "postgres-orders.web", ""); !apierrors.IsConflict(err) {
t.Fatalf("mint on a Secret of another user: %v, want Conflict", err)
}
if string(f.secret(t, before.Name).Data[account.PasswordKey]) != string(before.Data[account.PasswordKey]) {
t.Fatal("a refused mint must not write")
}
}
func TestDryRunMintAndRevokeWriteNothing(t *testing.T) {
f := newFixture(t)
before := f.seed(t, "web")
dry := &metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}}
for _, op := range []*OperationREST{NewMintREST(f.r), NewRevokeREST(f.r)} {
out, err := op.Create(f.ctx, "postgres-orders.web", &corev1alpha1.CredentialRequest{}, nil, dry)
if err != nil {
t.Fatal(err)
}
if res := out.(*corev1alpha1.CredentialRequest); res.Status.Password != "" || res.Status.Username != "web" {
t.Fatalf("dry-run response = %+v", res.Status)
}
}
after := f.secret(t, before.Name)
if after.ResourceVersion != before.ResourceVersion || string(after.Data[account.PasswordKey]) != string(before.Data[account.PasswordKey]) {
t.Fatal("a dry run changed the account Secret")
}
if evs := f.events(t); len(evs) != 0 {
t.Fatalf("a dry run recorded events: %+v", evs)
}
}
// Two operations racing on one account cannot both land: the write carries
// the resourceVersion the operation read, so the slower one gets a conflict
// and no password.
func TestMintConflictsOnAConcurrentChange(t *testing.T) {
var raced bool
f := newFixtureWith(t, &interceptor.Funcs{
Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
if sec, ok := obj.(*corev1.Secret); ok && !raced {
raced = true
other := &corev1.Secret{}
if err := cl.Get(ctx, client.ObjectKeyFromObject(sec), other); err != nil {
return err
}
other.Annotations["concurrent"] = "write"
if err := cl.Update(ctx, other); err != nil {
return err
}
}
return cl.Update(ctx, obj, opts...)
},
})
f.seed(t, "web")
res, err := f.mint(t, "postgres-orders.web", "")
if !apierrors.IsConflict(err) || res != nil {
t.Fatalf("mint racing a concurrent write: res=%v err=%v, want a conflict and no password", res, err)
}
}
func TestMintLogsNoPassword(t *testing.T) {
var buf bytes.Buffer
fs := flag.NewFlagSet("klog", flag.ContinueOnError)
klog.InitFlags(fs)
_ = fs.Set("v", "10")
_ = fs.Set("logtostderr", "false")
klog.SetOutput(&buf)
defer func() {
klog.SetOutput(io.Discard)
_ = fs.Set("logtostderr", "true")
_ = fs.Set("v", "0")
}()
f := newFixture(t)
f.seed(t, "web")
res, err := f.mint(t, "postgres-orders.web", "")
if err != nil {
t.Fatal(err)
}
klog.Flush()
if strings.Contains(buf.String(), res.Status.Password) {
t.Fatal("the minted password reached the log")
}
}
+29 -15
View File
@@ -12,6 +12,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/util/dryrun"
"sigs.k8s.io/controller-runtime/pkg/client"
appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1"
@@ -55,7 +56,7 @@ func (o *OperationREST) GroupVersionKind(schema.GroupVersion) schema.GroupVersio
}
// Create performs the operation on the named Credential.
func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Object, createValidation rest.ValidateObjectFunc, _ *metav1.CreateOptions) (runtime.Object, error) {
func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
req, ok := obj.(*corev1alpha1.CredentialRequest)
if !ok {
return nil, apierrors.NewBadRequest(fmt.Sprintf("expected CredentialRequest, got %T", obj))
@@ -75,10 +76,21 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj
}
r := o.parent
res, err := r.resolve(ctx, ns, name)
res, err := r.resolve(ctx, r.direct, ns, name)
if err != nil {
return nil, err
}
// Only a declared account is minted, whether or not its Secret exists: a
// Secret left behind by a removed user, or by values written around the
// API, does not make the account real.
acc, err := o.declared(ctx, res)
if err != nil {
return nil, err
}
if res.secret != nil && string(res.secret.Data[account.UsernameKey]) != acc.Username {
return nil, apierrors.NewConflict(r.gvr.GroupResource(), name, fmt.Errorf(
"credential %s holds user %q, the application declares %q; write the application to reseed it", name, res.secret.Data[account.UsernameKey], acc.Username))
}
var precondition string
if req.Spec.Preconditions != nil {
@@ -95,6 +107,15 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj
"credential %s changed: password %s by %s at %s", name, describe(rec.Origin), rec.By, rec.At.UTC().Format("2006-01-02T15:04:05Z")))
}
if options != nil && dryrun.IsDryRun(options.DryRun) {
// A dry run answers as the operation would, without a password: one
// that is never stored would not log in.
return &corev1alpha1.CredentialRequest{
TypeMeta: metav1.TypeMeta{APIVersion: corev1alpha1.SchemeGroupVersion.String(), Kind: "CredentialRequest"},
Status: corev1alpha1.CredentialRequestStatus{Username: acc.Username, IssuedAt: metav1.NewTime(r.now())},
}, nil
}
password, err := format.GeneratePassword()
if err != nil {
return nil, apierrors.NewInternalError(err)
@@ -106,15 +127,14 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj
now := metav1.NewTime(r.now())
by := user.GetName()
var username string
username := acc.Username
if res.secret != nil {
username = string(res.secret.Data[account.UsernameKey])
sec := res.secret.DeepCopy()
sec.Data[account.PasswordKey] = []byte(stored)
account.WriteRecord(sec, account.ReadRecord(sec).Apply(o.op, now, by))
// The update carries the Secret's own resourceVersion, so two concurrent
// operations cannot both land: the second gets a conflict and no password.
if err := r.c.Update(ctx, sec); err != nil {
if err := r.direct.Update(ctx, sec); err != nil {
if apierrors.IsConflict(err) {
return nil, apierrors.NewConflict(r.gvr.GroupResource(), name, fmt.Errorf("credential %s changed while the request was in flight", name))
}
@@ -122,13 +142,7 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj
}
} else {
// A declared account whose Secret is missing, as after a conversion
// that left the release out, gets a Secret with a new identity. One that
// is not declared does not exist.
acc, err := o.declared(ctx, res)
if err != nil {
return nil, err
}
username = acc.Username
// that left the release out, gets a Secret with a new identity.
app := store.App{
Namespace: ns, Release: res.hr.Name,
Group: res.hr.Labels[appsv1alpha1.ApplicationGroupLabel],
@@ -137,7 +151,7 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj
Block: res.kind.Block,
}
if res.stale != nil {
if err := r.c.Delete(ctx, res.stale, client.Preconditions{UID: &res.stale.UID}); err != nil && !apierrors.IsNotFound(err) {
if err := r.direct.Delete(ctx, res.stale, client.Preconditions{UID: &res.stale.UID}); err != nil && !apierrors.IsNotFound(err) {
return nil, err
}
}
@@ -148,7 +162,7 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj
}
sec.Data[account.PasswordKey] = []byte(stored)
account.WriteRecord(sec, account.Record{}.Apply(o.op, now, by))
if err := r.c.Create(ctx, sec); err != nil {
if err := r.direct.Create(ctx, sec); err != nil {
if apierrors.IsAlreadyExists(err) {
return nil, apierrors.NewConflict(r.gvr.GroupResource(), name, fmt.Errorf("credential %s was created while the request was in flight", name))
}
@@ -161,7 +175,7 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj
reason = store.ReasonCredentialRevoked
password = ""
}
store.Emit(ctx, r.c, res.hr, reason, username, by)
store.Emit(ctx, r.direct, res.hr, reason, username, by)
return &corev1alpha1.CredentialRequest{
TypeMeta: metav1.TypeMeta{