diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 7af717527..55de032e9 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -247,7 +247,7 @@ func (c completedConfig) New() (*CozyServer, error) { coreV1alpha1Storage["taps"] = cozyregistry.RESTInPeace( tapstorage.NewREST(dyn), ) - credentials := credentialstorage.NewREST(cli, credentialKinds) + credentials := credentialstorage.NewREST(cli, watchCli, credentialKinds) coreV1alpha1Storage["credentials"] = cozyregistry.RESTInPeace(credentials) coreV1alpha1Storage["credentials/mint"] = credentialstorage.NewMintREST(credentials) coreV1alpha1Storage["credentials/revoke"] = credentialstorage.NewRevokeREST(credentials) diff --git a/pkg/credentials/store/store.go b/pkg/credentials/store/store.go index 95a83a339..63ccd26a0 100644 --- a/pkg/credentials/store/store.go +++ b/pkg/credentials/store/store.go @@ -10,6 +10,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" helmv2 "github.com/fluxcd/helm-controller/api/v2" @@ -85,30 +86,55 @@ func SeedSecret(app App, acc account.Account, owner *metav1.OwnerReference, now return sec, nil } +// ErrLiveOwner is returned by Ensure on a create when a Secret at an account's +// name is owned by a HelmRelease that still exists: the application exists +// already, and its Secrets must not be touched. +var ErrLiveOwner = errors.New("account Secret is owned by a live HelmRelease") + // Theirs reports whether an existing Secret is the application's own account -// Secret: marker, same username, owned by the live HelmRelease. With no -// HelmRelease yet (a create), one any HelmRelease owns is left from a deleted -// application of the same name and is replaced, so the account gets a new -// identity; one nobody owns is from a create that did not finish. +// Secret: the account marker, the same username, and owned by owner. func Theirs(sec *corev1.Secret, acc account.Account, owner *metav1.OwnerReference) bool { - if !account.IsAccountSecret(sec) || string(sec.Data[account.UsernameKey]) != acc.Username { + if owner == nil || !account.IsAccountSecret(sec) || string(sec.Data[account.UsernameKey]) != acc.Username { return false } for _, ref := range sec.OwnerReferences { - if owner == nil && ref.Kind == helmv2.HelmReleaseKind { - return false - } - if owner != nil && ref.UID == owner.UID { + if ref.UID == owner.UID { return true } } - return owner == nil + return false +} + +// liveOwner reports whether a HelmRelease the Secret names as its owner still +// exists under the same UID. +func liveOwner(ctx context.Context, c client.Reader, sec *corev1.Secret) (bool, error) { + for _, ref := range sec.OwnerReferences { + if ref.Kind != helmv2.HelmReleaseKind { + continue + } + hr := &helmv2.HelmRelease{} + err := c.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: ref.Name}, hr) + switch { + case apierrors.IsNotFound(err): + continue + case err != nil: + return false, err + case hr.UID == ref.UID: + return true, nil + } + } + return false, nil } // Ensure writes an account Secret for every account that has none, or whose // Secret is not the application's, and returns the names of the Secrets it // wrote. It never touches a Secret that is the application's own, so a // password already minted survives every later write of the application. +// +// With owner nil (a create, before the HelmRelease exists) every existing +// Secret at an account's name is a leftover and is replaced, so the account +// gets a new identity and the Secret is among those to be owned; one a live +// HelmRelease owns stops the create with ErrLiveOwner. func Ensure(ctx context.Context, c client.Client, app App, accounts []account.Account, owner *metav1.OwnerReference, now metav1.Time, by string) ([]string, error) { var written []string for _, acc := range accounts { @@ -119,6 +145,15 @@ func Ensure(ctx context.Context, c client.Client, app App, accounts []account.Ac case err == nil && Theirs(cur, acc, owner): continue case err == nil: + if owner == nil { + live, err := liveOwner(ctx, c, cur) + if err != nil { + return written, fmt.Errorf("read owner of account Secret %s: %w", name, err) + } + if live { + return written, fmt.Errorf("%s: %w", name, ErrLiveOwner) + } + } if err := c.Delete(ctx, cur, client.Preconditions{UID: &cur.UID}); err != nil && !apierrors.IsNotFound(err) { return written, fmt.Errorf("replace account Secret %s: %w", name, err) } @@ -137,6 +172,21 @@ func Ensure(ctx context.Context, c client.Client, app App, accounts []account.Ac return written, nil } +// Added returns the accounts of next whose username prev does not declare. +func Added(prev, next []account.Account) []account.Account { + had := make(map[string]bool, len(prev)) + for _, acc := range prev { + had[acc.Username] = true + } + var added []account.Account + for _, acc := range next { + if !had[acc.Username] { + added = append(added, acc) + } + } + return added +} + // SetOwner adds the HelmRelease as owner of the named account Secrets. It is // the second half of a create, once the HelmRelease exists. func SetOwner(ctx context.Context, c client.Client, namespace string, names []string, owner metav1.OwnerReference) error { diff --git a/pkg/credentials/store/store_test.go b/pkg/credentials/store/store_test.go index f82df48cf..b6b97f786 100644 --- a/pkg/credentials/store/store_test.go +++ b/pkg/credentials/store/store_test.go @@ -4,6 +4,7 @@ package store import ( "context" + "errors" "strings" "testing" @@ -101,8 +102,8 @@ func TestEnsureSeedsAndKeepsTheirs(t *testing.T) { } first := string(sec.Data[account.PasswordKey]) - // A second create of the same application, as after an interrupted one, - // reuses the Secret nobody owns. + // An update, with the owner known, seeds only what it is given and leaves + // an owned Secret alone. hr := release("uid-1") if err := SetOwner(ctx, c, "tenant-foo", written, OwnerReference(hr)); err != nil { t.Fatal(err) @@ -246,3 +247,101 @@ func TestResolveAccountsAppliesDefaults(t *testing.T) { t.Fatalf("no block, no accounts: %v %v", got, err) } } + +func TestEnsureCreateReplacesLeftovers(t *testing.T) { + ctx := context.Background() + gone := OwnerReference(release("uid-gone")) + live := release("uid-live") + c := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(live).Build() + app := testApp() + now := metav1.Now() + + // Left by an earlier create that never got its owner, and by a deleted + // application of the same name whose HelmRelease is gone. + if _, err := Ensure(ctx, c, app, accounts("web"), nil, now, "x"); err != nil { + t.Fatal(err) + } + if _, err := Ensure(ctx, c, app, accounts("api"), &gone, now, "x"); err != nil { + t.Fatal(err) + } + unowned := getSecret(t, c, "postgres-orders.web.account") + stale := getSecret(t, c, "postgres-orders.api.account") + + written, err := Ensure(ctx, c, app, accounts("web", "api"), nil, now, "jane") + if err != nil { + t.Fatal(err) + } + if len(written) != 2 { + t.Fatalf("both leftovers must be replaced and handed to SetOwner, written = %v", written) + } + if string(getSecret(t, c, "postgres-orders.web.account").Data[account.PasswordKey]) == string(unowned.Data[account.PasswordKey]) { + t.Fatal("an unowned leftover kept its verifier") + } + if string(getSecret(t, c, "postgres-orders.api.account").Data[account.PasswordKey]) == string(stale.Data[account.PasswordKey]) { + t.Fatal("a Secret of a gone release kept its verifier") + } +} + +func TestEnsureCreateRefusesLiveOwner(t *testing.T) { + ctx := context.Background() + live := release("uid-live") + c := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(live).Build() + app := testApp() + owner := OwnerReference(live) + if _, err := Ensure(ctx, c, app, accounts("web"), &owner, metav1.Now(), "x"); err != nil { + t.Fatal(err) + } + before := getSecret(t, c, "postgres-orders.web.account") + + written, err := Ensure(ctx, c, app, accounts("web"), nil, metav1.Now(), "jane") + if !errors.Is(err, ErrLiveOwner) || len(written) != 0 { + t.Fatalf("written=%v err=%v, want ErrLiveOwner and nothing written", written, err) + } + after := getSecret(t, c, "postgres-orders.web.account") + if string(after.Data[account.PasswordKey]) != string(before.Data[account.PasswordKey]) || len(after.OwnerReferences) != 1 { + t.Fatal("the live application's Secret must be left untouched") + } +} + +// The application labels alone do not make a Secret an account Secret: the +// lineage webhook puts them on every Secret of the application, and Prune +// must never delete the chart's own. +func TestListAndPruneSeeOnlyAccountSecrets(t *testing.T) { + ctx := context.Background() + app := testApp() + chartOwned := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: "postgres-orders-credentials", Namespace: "tenant-foo", + Labels: map[string]string{ + "apps.cozystack.io/application.group": app.Group, + "apps.cozystack.io/application.kind": app.Kind, + "apps.cozystack.io/application.name": app.Name, + }, + }} + c := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(chartOwned).Build() + if _, err := Ensure(ctx, c, app, accounts("web"), nil, metav1.Now(), "x"); err != nil { + t.Fatal(err) + } + listed, err := List(ctx, c, app) + if err != nil { + t.Fatal(err) + } + if len(listed) != 1 || listed[0].Name != "postgres-orders.web.account" { + t.Fatalf("listed = %v", listed) + } + if _, err := Prune(ctx, c, app, nil); err != nil { + t.Fatal(err) + } + getSecret(t, c, "postgres-orders-credentials") +} + +func TestAdded(t *testing.T) { + got := Added(accounts("web", "api"), []account.Account{{Username: "web", Mapped: "web"}, {Username: "Api", Mapped: "api"}, {Username: "new", Mapped: "new"}}) + var names []string + for _, a := range got { + names = append(names, a.Username) + } + // Api is a different user from api, however the name maps. + if strings.Join(names, ",") != "Api,new" { + t.Fatalf("added = %v", names) + } +} diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 3e30a79ef..9827e5107 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -44,6 +44,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" + "k8s.io/apiserver/pkg/util/dryrun" "k8s.io/apiserver/pkg/warning" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" @@ -253,7 +254,7 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation // The account Secrets exist before the release that reads them, and the // release owns them as soon as it exists. - seeded, err := r.seedBeforeCreate(ctx, app, helmRelease, accounts) + seeded, err := r.seedBeforeCreate(ctx, app, helmRelease, accounts, options != nil && dryrun.IsDryRun(options.DryRun)) if err != nil { return nil, err } @@ -607,7 +608,7 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje } // Accounts the write adds get their Secrets before the release reads them. - if err := r.seedBeforeUpdate(ctx, app, helmRelease, cur, accounts); err != nil { + if err := r.seedBeforeUpdate(ctx, app, helmRelease, cur, accounts, options != nil && dryrun.IsDryRun(options.DryRun)); err != nil { return nil, false, err } @@ -672,7 +673,7 @@ func (r *REST) Update(ctx context.Context, name string, objInfo rest.UpdatedObje // Accounts the write removed are gone from the chart now, so their // Secrets can go. - r.pruneAfterUpdate(ctx, app, accounts) + r.pruneAfterUpdate(ctx, app, accounts, options != nil && dryrun.IsDryRun(options.DryRun)) // Convert the updated HelmRelease back to Application convertedApp, err := r.ConvertHelmReleaseToApplication(ctx, helmRelease) diff --git a/pkg/registry/apps/application/rest_credentials.go b/pkg/registry/apps/application/rest_credentials.go index 4ae281fcd..95bb69297 100644 --- a/pkg/registry/apps/application/rest_credentials.go +++ b/pkg/registry/apps/application/rest_credentials.go @@ -5,6 +5,7 @@ package application import ( "context" "encoding/json" + "errors" "fmt" helmv2 "github.com/fluxcd/helm-controller/api/v2" @@ -16,6 +17,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/warning" "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" @@ -66,31 +68,56 @@ func actor(ctx context.Context) string { return "" } +// direct is the client the account Secret writes read through. The cached +// client can miss an object written a moment ago, and a stale read here +// replaces or recreates a Secret that exists. +func (r *REST) direct() client.Client { + if r.w != nil { + return r.w + } + return r.c +} + // seedBeforeCreate writes the account Secrets of a new application, with no // owner yet, and the valuesFrom entries that carry them into the chart. // It returns the Secrets it wrote, for the owner to be set once the // HelmRelease exists, or for removal if the HelmRelease is not created. -func (r *REST) seedBeforeCreate(ctx context.Context, app *appsv1alpha1.Application, hr *helmv2.HelmRelease, accounts []account.Account) ([]string, error) { +// An existing release fails the create before any Secret is touched, since +// its own Secrets would otherwise be replaced. +func (r *REST) seedBeforeCreate(ctx context.Context, app *appsv1alpha1.Application, hr *helmv2.HelmRelease, accounts []account.Account, dryRun bool) ([]string, error) { if r.credentials == nil { return nil, nil } hr.Spec.ValuesFrom = store.ValuesFrom(r.credentials, hr.Name, accounts) - written, err := store.Ensure(ctx, r.c, r.storeApp(app.Namespace, app.Name), accounts, nil, metav1.Now(), actor(ctx)) + existing := &helmv2.HelmRelease{} + err := r.direct().Get(ctx, client.ObjectKey{Namespace: hr.Namespace, Name: hr.Name}, existing) + switch { + case err == nil: + return nil, apierrors.NewAlreadyExists(r.gvr.GroupResource(), app.Name) + case !apierrors.IsNotFound(err): + return nil, apierrors.NewInternalError(fmt.Errorf("read HelmRelease %s: %w", hr.Name, err)) + } + if dryRun { + return nil, nil + } + written, err := store.Ensure(ctx, r.direct(), r.storeApp(app.Namespace, app.Name), accounts, nil, metav1.Now(), actor(ctx)) if err != nil { r.unseed(ctx, app.Namespace, written) + if errors.Is(err, store.ErrLiveOwner) { + return nil, apierrors.NewAlreadyExists(r.gvr.GroupResource(), app.Name) + } return nil, apierrors.NewInternalError(fmt.Errorf("seed account Secrets: %w", err)) } return written, nil } // unseed removes the Secrets a create wrote before its HelmRelease failed, so -// a retry starts clean. Best effort: a Secret nobody owns is reused by the -// retry in any case. +// a retry starts clean. Best effort: a retry replaces a leftover in any case. func (r *REST) unseed(ctx context.Context, namespace string, names []string) { for _, name := range names { sec := &corev1.Secret{} sec.Namespace, sec.Name = namespace, name - if err := r.c.Delete(ctx, sec); err != nil && !apierrors.IsNotFound(err) { + if err := r.direct().Delete(ctx, sec); err != nil && !apierrors.IsNotFound(err) { klog.Warningf("failed to remove account Secret %s/%s after a failed create: %v", namespace, name, err) } } @@ -98,26 +125,39 @@ func (r *REST) unseed(ctx context.Context, namespace string, names []string) { // ownSeeded gives the created HelmRelease the account Secrets seeded before // it. The application exists by now, so a failure is reported as a warning -// rather than an error: the next write of the application retries it. +// rather than an error. func (r *REST) ownSeeded(ctx context.Context, hr *helmv2.HelmRelease, names []string) { if len(names) == 0 { return } - if err := store.SetOwner(ctx, r.c, hr.Namespace, names, store.OwnerReference(hr)); err != nil { + if err := store.SetOwner(ctx, r.direct(), hr.Namespace, names, store.OwnerReference(hr)); err != nil { klog.Errorf("failed to own account Secrets of %s/%s: %v", hr.Namespace, hr.Name, err) warning.AddWarning(ctx, "", fmt.Sprintf("account Secrets were written but not yet owned by the release: %v", err)) } } -// seedBeforeUpdate writes the account Secrets an update adds, owned by the -// live HelmRelease from the start, and the valuesFrom entries of the result. -func (r *REST) seedBeforeUpdate(ctx context.Context, app *appsv1alpha1.Application, hr, cur *helmv2.HelmRelease, accounts []account.Account) error { +// seedBeforeUpdate writes the account Secrets of the accounts the update adds, +// owned by the live HelmRelease from the start, and the valuesFrom entries of +// the result. An account the release already declared and that has no Secret +// is left alone: a seed would replace its working password with a discarded +// one, and a mint creates the Secret when someone needs it. +func (r *REST) seedBeforeUpdate(ctx context.Context, app *appsv1alpha1.Application, hr, cur *helmv2.HelmRelease, accounts []account.Account, dryRun bool) error { if r.credentials == nil { return nil } hr.Spec.ValuesFrom = store.ValuesFrom(r.credentials, hr.Name, accounts) + if dryRun { + return nil + } + prev, err := r.accountsOf(cur.Spec.Values) + if err != nil { + // The stored values were written around the API; nothing tells which + // accounts are new, so seed none and leave the rest to a mint. + klog.Warningf("accounts of %s/%s before the update cannot be resolved, seeding none: %v", cur.Namespace, cur.Name, err) + return nil + } owner := store.OwnerReference(cur) - if _, err := store.Ensure(ctx, r.c, r.storeApp(app.Namespace, app.Name), accounts, &owner, metav1.Now(), actor(ctx)); err != nil { + if _, err := store.Ensure(ctx, r.direct(), r.storeApp(app.Namespace, app.Name), store.Added(prev, accounts), &owner, metav1.Now(), actor(ctx)); err != nil { return apierrors.NewInternalError(fmt.Errorf("seed account Secrets: %w", err)) } return nil @@ -125,11 +165,11 @@ func (r *REST) seedBeforeUpdate(ctx context.Context, app *appsv1alpha1.Applicati // pruneAfterUpdate deletes the account Secrets of the accounts the update // removed, once the HelmRelease write has dropped them from the chart. -func (r *REST) pruneAfterUpdate(ctx context.Context, app *appsv1alpha1.Application, accounts []account.Account) { - if r.credentials == nil { +func (r *REST) pruneAfterUpdate(ctx context.Context, app *appsv1alpha1.Application, accounts []account.Account, dryRun bool) { + if r.credentials == nil || dryRun { return } - if _, err := store.Prune(ctx, r.c, r.storeApp(app.Namespace, app.Name), accounts); err != nil { + if _, err := store.Prune(ctx, r.direct(), r.storeApp(app.Namespace, app.Name), accounts); err != nil { klog.Errorf("failed to prune account Secrets of %s/%s: %v", app.Namespace, app.Name, err) warning.AddWarning(ctx, "", fmt.Sprintf("account Secrets of removed users were not all deleted: %v", err)) } diff --git a/pkg/registry/apps/application/rest_credentials_test.go b/pkg/registry/apps/application/rest_credentials_test.go index 125d74498..ce78ca7f5 100644 --- a/pkg/registry/apps/application/rest_credentials_test.go +++ b/pkg/registry/apps/application/rest_credentials_test.go @@ -4,7 +4,9 @@ package application import ( "context" + "fmt" "strings" + "sync/atomic" "testing" helmv2 "github.com/fluxcd/helm-controller/api/v2" @@ -19,11 +21,13 @@ import ( "k8s.io/apiserver/pkg/warning" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" "github.com/cozystack/cozystack/pkg/config" "github.com/cozystack/cozystack/pkg/credentials/account" + "github.com/cozystack/cozystack/pkg/credentials/store" ) // The schema defaults backup.enabled to true, so the Static backup account @@ -59,7 +63,28 @@ func newCredentialsREST(t *testing.T, objects ...client.Object) (*REST, client.C if err := appsv1alpha1.RegisterDynamicTypes(scheme, &config.ResourceConfig{Resources: []config.Resource{*cfg}}); err != nil { t.Fatal(err) } - c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + // The fake client assigns no UIDs, and without them every ownership check + // compares two empty strings and passes. + var uids atomic.Int64 + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if obj.GetUID() == "" { + obj.SetUID(types.UID(fmt.Sprintf("uid-%d", uids.Add(1)))) + } + return cl.Create(ctx, obj, opts...) + }, + // The API server keeps an object's UID on an update that sends none; + // the fake client would blank it. + Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if obj.GetUID() == "" { + cur := obj.DeepCopyObject().(client.Object) + if err := cl.Get(ctx, client.ObjectKeyFromObject(obj), cur); err == nil { + obj.SetUID(cur.GetUID()) + } + } + return cl.Update(ctx, obj, opts...) + }, + }).Build() return NewREST(c, c, cfg), c } @@ -217,3 +242,90 @@ func keys(m map[string]*corev1.Secret) []string { } return out } + +func TestCreateDuplicateLeavesLiveSecretsAlone(t *testing.T) { + r, c := newCredentialsREST(t) + ctx := credentialsCtx() + if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + before := accountSecrets(t, c)["postgres-orders.web.account"] + + if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); !apierrors.IsAlreadyExists(err) { + t.Fatalf("second create: %v, want AlreadyExists", err) + } + after := accountSecrets(t, c)["postgres-orders.web.account"] + if after == nil || after.UID != before.UID || string(after.Data[account.PasswordKey]) != string(before.Data[account.PasswordKey]) { + t.Fatal("a refused create must leave the live application's Secret untouched") + } +} + +func TestCreateReplacesAnUnownedLeftover(t *testing.T) { + // Left by an earlier create whose owner was never set. + leftover, err := store.SeedSecret(store.App{Namespace: "tenant-foo", Release: "postgres-orders", Group: appsv1alpha1.GroupName, Kind: postgresKind, Name: "orders", Block: credentialsBlock()}, + account.Account{Username: "web", Mapped: "web"}, nil, metav1.Now(), "earlier") + if err != nil { + t.Fatal(err) + } + leftover.UID = "uid-leftover" + r, c := newCredentialsREST(t, leftover) + if _, err := r.Create(credentialsCtx(), postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + sec := accountSecrets(t, c)["postgres-orders.web.account"] + hr := getRelease(t, c) + if sec.UID == "uid-leftover" || string(sec.Data[account.PasswordKey]) == string(leftover.Data[account.PasswordKey]) { + t.Fatal("a leftover must be replaced, so the account gets a new identity") + } + if len(sec.OwnerReferences) != 1 || sec.OwnerReferences[0].UID != hr.UID { + t.Fatalf("owners = %v, want the new release", sec.OwnerReferences) + } +} + +func TestDryRunWritesNoAccountSecrets(t *testing.T) { + r, c := newCredentialsREST(t) + ctx := credentialsCtx() + dry := []string{metav1.DryRunAll} + + if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{DryRun: dry}); err != nil { + t.Fatal(err) + } + if n := len(accountSecrets(t, c)); n != 0 { + t.Fatalf("a dry-run create wrote %d account Secrets", n) + } + + if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}}}`), nil, &metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + before := accountSecrets(t, c) + updated := postgresApp(`{"users":{"api":{}}}`) + if _, _, err := r.Update(ctx, "orders", newDefaultUpdatedObjectInfo(updated), nil, nil, false, &metav1.UpdateOptions{DryRun: dry}); err != nil { + t.Fatal(err) + } + after := accountSecrets(t, c) + if len(after) != len(before) || after["postgres-orders.web.account"] == nil || after["postgres-orders.api.account"] != nil { + t.Fatalf("a dry-run update changed the account Secrets: %v -> %v", keys(before), keys(after)) + } +} + +// An account the release already declared and that has no Secret, as after a +// conversion that left the release out, is not seeded by an unrelated write: +// a seed would replace its working password with a discarded one. +func TestUpdateSeedsOnlyAddedAccounts(t *testing.T) { + r, c := newCredentialsREST(t) + ctx := credentialsCtx() + if _, err := r.Create(ctx, postgresApp(`{"users":{"web":{}},"backup":{"enabled":false}}`), nil, &metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if err := c.Delete(ctx, accountSecrets(t, c)["postgres-orders.web.account"]); err != nil { + t.Fatal(err) + } + updated := postgresApp(`{"users":{"web":{},"api":{}},"backup":{"enabled":false},"size":"20Gi"}`) + if _, _, err := r.Update(ctx, "orders", newDefaultUpdatedObjectInfo(updated), nil, nil, false, &metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + got := accountSecrets(t, c) + if got["postgres-orders.api.account"] == nil || got["postgres-orders.web.account"] != nil { + t.Fatalf("secrets = %v, want api seeded and web left alone", keys(got)) + } +} diff --git a/pkg/registry/core/credential/rest.go b/pkg/registry/core/credential/rest.go index f09c43cf7..090eac360 100644 --- a/pkg/registry/core/credential/rest.go +++ b/pkg/registry/core/credential/rest.go @@ -60,8 +60,11 @@ type Kind struct { // REST serves the credentials view. type REST struct { - c client.Client - gvr schema.GroupVersionResource + c client.Client + // direct reads and writes for mint and revoke, which the cached client + // would answer with a Secret written a moment ago missing. + direct client.Client + gvr schema.GroupVersionResource // kinds is keyed by the application kind the HelmRelease labels name. kinds map[string]Kind now func() time.Time @@ -69,9 +72,10 @@ type REST struct { // NewREST creates the credentials storage. kinds carries every converted // application kind; a HelmRelease of a kind absent from it has no credentials. -func NewREST(c client.Client, kinds map[string]Kind) *REST { +func NewREST(c, direct client.Client, kinds map[string]Kind) *REST { return &REST{ - c: c, + c: c, + direct: direct, gvr: schema.GroupVersionResource{ Group: corev1alpha1.GroupName, Version: "v1alpha1", @@ -117,13 +121,13 @@ type resolved struct { // must exist and belong to a converted kind, or the credential does not exist. // A Secret that is not the application's own, by marker or by owner, counts // as absent. -func (r *REST) resolve(ctx context.Context, namespace, name string) (*resolved, error) { +func (r *REST) resolve(ctx context.Context, c client.Reader, namespace, name string) (*resolved, error) { release, mapped, ok := account.SplitCredentialName(name) if !ok { return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) } hr := &helmv2.HelmRelease{} - if err := r.c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: release}, hr); err != nil { + if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: release}, hr); err != nil { if apierrors.IsNotFound(err) { return nil, apierrors.NewNotFound(r.gvr.GroupResource(), name) } @@ -135,7 +139,7 @@ func (r *REST) resolve(ctx context.Context, namespace, name string) (*resolved, } res := &resolved{hr: hr, kind: kind, mapped: mapped} sec := &corev1.Secret{} - err := r.c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: account.SecretName(release, mapped)}, sec) + err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: account.SecretName(release, mapped)}, sec) switch { case err == nil && ownedBy(sec, hr): res.secret = sec @@ -197,7 +201,7 @@ func (r *REST) Get(ctx context.Context, name string, _ *metav1.GetOptions) (runt if err != nil { return nil, err } - res, err := r.resolve(ctx, ns, name) + res, err := r.resolve(ctx, r.c, ns, name) if err != nil { return nil, err } diff --git a/pkg/registry/core/credential/rest_test.go b/pkg/registry/core/credential/rest_test.go index ba2f264b8..95e4d19a7 100644 --- a/pkg/registry/core/credential/rest_test.go +++ b/pkg/registry/core/credential/rest_test.go @@ -3,7 +3,10 @@ package credential import ( + "bytes" "context" + "flag" + "io" "strings" "testing" "time" @@ -17,8 +20,10 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" @@ -49,6 +54,12 @@ type fixture struct { } func newFixture(t *testing.T, objs ...client.Object) *fixture { + t.Helper() + return newFixtureWith(t, nil, objs...) +} + +// newFixtureWith builds the fixture on a client whose calls go through funcs. +func newFixtureWith(t *testing.T, funcs *interceptor.Funcs, objs ...client.Object) *fixture { t.Helper() scheme := runtime.NewScheme() if err := corev1.AddToScheme(scheme); err != nil { @@ -66,9 +77,13 @@ func newFixture(t *testing.T, objs ...client.Object) *fixture { }, }} f := &fixture{hr: hr} - c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(append(objs, hr)...).Build() + b := fake.NewClientBuilder().WithScheme(scheme).WithObjects(append(objs, hr)...) + if funcs != nil { + b = b.WithInterceptorFuncs(*funcs) + } + c := b.Build() f.c = c - f.r = NewREST(c, map[string]Kind{"Postgres": { + f.r = NewREST(c, c, map[string]Kind{"Postgres": { Block: testBlock(), Accounts: func(context.Context, *helmv2.HelmRelease) ([]account.Account, error) { return f.declare, nil }, }}) @@ -81,11 +96,13 @@ func (f *fixture) app() store.App { return store.App{Namespace: ns, Release: "postgres-orders", Group: appsv1alpha1.GroupName, Kind: "Postgres", Name: "orders", Block: testBlock()} } -// seed writes an owned account Secret for user and returns it as stored. +// seed declares user and writes its owned account Secret, and returns the +// Secret as stored. func (f *fixture) seed(t *testing.T, username string) *corev1.Secret { t.Helper() owner := store.OwnerReference(f.hr) mapped, _ := account.MapName(username) + f.declare = append(f.declare, account.Account{Username: username, Mapped: mapped}) sec, err := store.SeedSecret(f.app(), account.Account{Username: username, Mapped: mapped}, &owner, metav1.NewTime(f.r.now()), "seeder") if err != nil { t.Fatal(err) @@ -307,3 +324,96 @@ func TestUnconvertedKindHasNoCredentials(t *testing.T) { } func ptr[T any](v T) *T { return &v } + +func TestMintRefusesAnUndeclaredAccountWithASecret(t *testing.T) { + f := newFixture(t) + before := f.seed(t, "web") + // The user left the values and the prune of its Secret failed. + f.declare = nil + if _, err := f.mint(t, "postgres-orders.web", ""); !apierrors.IsNotFound(err) { + t.Fatalf("mint on an undeclared account: %v, want NotFound", err) + } + // A Secret holding another user than the one the name maps to. + f.declare = []account.Account{{Username: "Web", Mapped: "web"}} + if _, err := f.mint(t, "postgres-orders.web", ""); !apierrors.IsConflict(err) { + t.Fatalf("mint on a Secret of another user: %v, want Conflict", err) + } + if string(f.secret(t, before.Name).Data[account.PasswordKey]) != string(before.Data[account.PasswordKey]) { + t.Fatal("a refused mint must not write") + } +} + +func TestDryRunMintAndRevokeWriteNothing(t *testing.T) { + f := newFixture(t) + before := f.seed(t, "web") + dry := &metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}} + for _, op := range []*OperationREST{NewMintREST(f.r), NewRevokeREST(f.r)} { + out, err := op.Create(f.ctx, "postgres-orders.web", &corev1alpha1.CredentialRequest{}, nil, dry) + if err != nil { + t.Fatal(err) + } + if res := out.(*corev1alpha1.CredentialRequest); res.Status.Password != "" || res.Status.Username != "web" { + t.Fatalf("dry-run response = %+v", res.Status) + } + } + after := f.secret(t, before.Name) + if after.ResourceVersion != before.ResourceVersion || string(after.Data[account.PasswordKey]) != string(before.Data[account.PasswordKey]) { + t.Fatal("a dry run changed the account Secret") + } + if evs := f.events(t); len(evs) != 0 { + t.Fatalf("a dry run recorded events: %+v", evs) + } +} + +// Two operations racing on one account cannot both land: the write carries +// the resourceVersion the operation read, so the slower one gets a conflict +// and no password. +func TestMintConflictsOnAConcurrentChange(t *testing.T) { + var raced bool + f := newFixtureWith(t, &interceptor.Funcs{ + Update: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if sec, ok := obj.(*corev1.Secret); ok && !raced { + raced = true + other := &corev1.Secret{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(sec), other); err != nil { + return err + } + other.Annotations["concurrent"] = "write" + if err := cl.Update(ctx, other); err != nil { + return err + } + } + return cl.Update(ctx, obj, opts...) + }, + }) + f.seed(t, "web") + res, err := f.mint(t, "postgres-orders.web", "") + if !apierrors.IsConflict(err) || res != nil { + t.Fatalf("mint racing a concurrent write: res=%v err=%v, want a conflict and no password", res, err) + } +} + +func TestMintLogsNoPassword(t *testing.T) { + var buf bytes.Buffer + fs := flag.NewFlagSet("klog", flag.ContinueOnError) + klog.InitFlags(fs) + _ = fs.Set("v", "10") + _ = fs.Set("logtostderr", "false") + klog.SetOutput(&buf) + defer func() { + klog.SetOutput(io.Discard) + _ = fs.Set("logtostderr", "true") + _ = fs.Set("v", "0") + }() + + f := newFixture(t) + f.seed(t, "web") + res, err := f.mint(t, "postgres-orders.web", "") + if err != nil { + t.Fatal(err) + } + klog.Flush() + if strings.Contains(buf.String(), res.Status.Password) { + t.Fatal("the minted password reached the log") + } +} diff --git a/pkg/registry/core/credential/subresources.go b/pkg/registry/core/credential/subresources.go index 5dbf2b8e4..a4f9e2229 100644 --- a/pkg/registry/core/credential/subresources.go +++ b/pkg/registry/core/credential/subresources.go @@ -12,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" + "k8s.io/apiserver/pkg/util/dryrun" "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" @@ -55,7 +56,7 @@ func (o *OperationREST) GroupVersionKind(schema.GroupVersion) schema.GroupVersio } // Create performs the operation on the named Credential. -func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Object, createValidation rest.ValidateObjectFunc, _ *metav1.CreateOptions) (runtime.Object, error) { +func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) { req, ok := obj.(*corev1alpha1.CredentialRequest) if !ok { return nil, apierrors.NewBadRequest(fmt.Sprintf("expected CredentialRequest, got %T", obj)) @@ -75,10 +76,21 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj } r := o.parent - res, err := r.resolve(ctx, ns, name) + res, err := r.resolve(ctx, r.direct, ns, name) if err != nil { return nil, err } + // Only a declared account is minted, whether or not its Secret exists: a + // Secret left behind by a removed user, or by values written around the + // API, does not make the account real. + acc, err := o.declared(ctx, res) + if err != nil { + return nil, err + } + if res.secret != nil && string(res.secret.Data[account.UsernameKey]) != acc.Username { + return nil, apierrors.NewConflict(r.gvr.GroupResource(), name, fmt.Errorf( + "credential %s holds user %q, the application declares %q; write the application to reseed it", name, res.secret.Data[account.UsernameKey], acc.Username)) + } var precondition string if req.Spec.Preconditions != nil { @@ -95,6 +107,15 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj "credential %s changed: password %s by %s at %s", name, describe(rec.Origin), rec.By, rec.At.UTC().Format("2006-01-02T15:04:05Z"))) } + if options != nil && dryrun.IsDryRun(options.DryRun) { + // A dry run answers as the operation would, without a password: one + // that is never stored would not log in. + return &corev1alpha1.CredentialRequest{ + TypeMeta: metav1.TypeMeta{APIVersion: corev1alpha1.SchemeGroupVersion.String(), Kind: "CredentialRequest"}, + Status: corev1alpha1.CredentialRequestStatus{Username: acc.Username, IssuedAt: metav1.NewTime(r.now())}, + }, nil + } + password, err := format.GeneratePassword() if err != nil { return nil, apierrors.NewInternalError(err) @@ -106,15 +127,14 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj now := metav1.NewTime(r.now()) by := user.GetName() - var username string + username := acc.Username if res.secret != nil { - username = string(res.secret.Data[account.UsernameKey]) sec := res.secret.DeepCopy() sec.Data[account.PasswordKey] = []byte(stored) account.WriteRecord(sec, account.ReadRecord(sec).Apply(o.op, now, by)) // The update carries the Secret's own resourceVersion, so two concurrent // operations cannot both land: the second gets a conflict and no password. - if err := r.c.Update(ctx, sec); err != nil { + if err := r.direct.Update(ctx, sec); err != nil { if apierrors.IsConflict(err) { return nil, apierrors.NewConflict(r.gvr.GroupResource(), name, fmt.Errorf("credential %s changed while the request was in flight", name)) } @@ -122,13 +142,7 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj } } else { // A declared account whose Secret is missing, as after a conversion - // that left the release out, gets a Secret with a new identity. One that - // is not declared does not exist. - acc, err := o.declared(ctx, res) - if err != nil { - return nil, err - } - username = acc.Username + // that left the release out, gets a Secret with a new identity. app := store.App{ Namespace: ns, Release: res.hr.Name, Group: res.hr.Labels[appsv1alpha1.ApplicationGroupLabel], @@ -137,7 +151,7 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj Block: res.kind.Block, } if res.stale != nil { - if err := r.c.Delete(ctx, res.stale, client.Preconditions{UID: &res.stale.UID}); err != nil && !apierrors.IsNotFound(err) { + if err := r.direct.Delete(ctx, res.stale, client.Preconditions{UID: &res.stale.UID}); err != nil && !apierrors.IsNotFound(err) { return nil, err } } @@ -148,7 +162,7 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj } sec.Data[account.PasswordKey] = []byte(stored) account.WriteRecord(sec, account.Record{}.Apply(o.op, now, by)) - if err := r.c.Create(ctx, sec); err != nil { + if err := r.direct.Create(ctx, sec); err != nil { if apierrors.IsAlreadyExists(err) { return nil, apierrors.NewConflict(r.gvr.GroupResource(), name, fmt.Errorf("credential %s was created while the request was in flight", name)) } @@ -161,7 +175,7 @@ func (o *OperationREST) Create(ctx context.Context, name string, obj runtime.Obj reason = store.ReasonCredentialRevoked password = "" } - store.Emit(ctx, r.c, res.hr, reason, username, by) + store.Emit(ctx, r.direct, res.hr, reason, username, by) return &corev1alpha1.CredentialRequest{ TypeMeta: metav1.TypeMeta{