diff --git a/pkg/registry/apps/application/quota.go b/pkg/registry/apps/application/quota.go index 47c41b464..ffd8169f8 100644 --- a/pkg/registry/apps/application/quota.go +++ b/pkg/registry/apps/application/quota.go @@ -24,6 +24,7 @@ import ( "strings" helmv2 "github.com/fluxcd/helm-controller/api/v2" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/labels" @@ -211,6 +212,20 @@ func (r *REST) validateTenantResourceQuotas(ctx context.Context, app *appsv1alph return append(allErrs, field.InternalError(fldPath, err)) } + // Current usage of the parent's pool (its own namespace plus its unbounded + // descendants). Counting it keeps the invariant — a sub-tree may never, via + // quota, be promised more than its root owns — holding at declaration time: + // without it a parent already using most of its budget could still carve a + // full-budget child and the two would collectively exceed the root. This is + // best-effort (a snapshot; concurrent writes can briefly overshoot and + // already-running pods are never evicted) — the tenant-quota controller is + // the runtime backstop. + childNamespace := r.computeTenantNamespace(app.Namespace, app.Name) + poolUsed, err := r.parentPoolUsage(ctx, app.Namespace, childNamespace) + if err != nil { + return append(allErrs, field.InternalError(fldPath, err)) + } + for _, res := range sortedQuotaKeys(parentQuota) { want, requested := childQuota[res] if !requested { @@ -218,12 +233,14 @@ func (r *REST) validateTenantResourceQuotas(ctx context.Context, app *appsv1alph } parentLimit := parentQuota[res] allocated := siblings[res] // zero value when no sibling bounds this resource + used := poolUsed[renderedLimitKey(res)] remaining := parentLimit.DeepCopy() remaining.Sub(allocated) + remaining.Sub(used) if want.Cmp(remaining) > 0 { allErrs = append(allErrs, field.Forbidden(fldPath.Key(res), - fmt.Sprintf("requested %s exceeds the parent tenant's remaining %q quota of %s (parent allows %s, %s already allocated to sibling tenants); a child tenant may not be granted more quota than its parent has left", - want.String(), res, remaining.String(), parentLimit.String(), allocated.String()))) + fmt.Sprintf("requested %s exceeds the parent tenant's remaining %q quota of %s (parent allows %s, %s already allocated to sibling tenants, %s currently in use by the parent pool); a child tenant may not be granted more quota than its parent has left", + want.String(), res, remaining.String(), parentLimit.String(), allocated.String(), used.String()))) } } return allErrs @@ -251,3 +268,162 @@ func sortedQuotaKeys(q map[string]resource.Quantity) []string { sort.Strings(keys) return keys } + +// rawQuotaKeys are the shorthand resourceQuotas keys that the tenant chart emits +// verbatim as ResourceQuota hard keys (no limits./requests. prefix). Kept in +// sync with cozy-lib.resources.flattenResource ($rawQuotaKeys). +var rawQuotaKeys = map[string]struct{}{ + "pods": {}, + "services": {}, + "services.loadbalancers": {}, + "services.nodeports": {}, + "services.clusterip": {}, + "configmaps": {}, + "secrets": {}, + "persistentvolumeclaims": {}, + "replicationcontrollers": {}, + "resourcequotas": {}, +} + +// renderedLimitKey maps a shorthand resourceQuotas key (the vocabulary admission +// works in) to the rendered ResourceQuota key that carries its limit-side +// budget, mirroring cozy-lib.resources.flatten. The mapping is allocation-ratio +// independent: ratios only scale the requests.* side that flatten derives from +// the same shorthand value, so the limit-side key always holds the operator's +// declared amount. This lets admission read a tenant's current usage from its +// rendered ResourceQuota.status.used without replicating the chart's ratio math. +// +// cpu, memory, ephemeral-storage, custom resources -> "limits." +// storage -> "requests.storage" (no hard limits.storage) +// pods, services, services.loadbalancers, ... -> "" (verbatim) +func renderedLimitKey(raw string) string { + if raw == "storage" { + return "requests.storage" + } + if _, ok := rawQuotaKeys[raw]; ok { + return raw + } + return "limits." + raw +} + +// parentNamespaceOf returns the namespace owned by the parent of the tenant that +// owns ns, or "" for the root tenant and non-tenant namespaces. It mirrors +// tenantquota.parentNamespace / the inverse of computeTenantNamespace: the +// hierarchy is encoded in the namespace name, so the parent is recovered by +// stripping the trailing "-" segment. +func (r *REST) parentNamespaceOf(ns string) string { + if ns == rootTenantNamespace || !strings.HasPrefix(ns, r.releaseConfig.Prefix) { + return "" + } + segments := strings.Split(ns, "-") + if len(segments) == 2 { + return rootTenantNamespace + } + return strings.Join(segments[:len(segments)-1], "-") +} + +// poolRootOf returns the nearest ancestor namespace (inclusive of ns) that is +// bounded — declares its own quota — or "" when none is. That ancestor is the +// pool root governing ns. +func (r *REST) poolRootOf(ns string, bounded map[string]bool) string { + for cur := ns; cur != ""; cur = r.parentNamespaceOf(cur) { + if bounded[cur] { + return cur + } + } + return "" +} + +// parentPoolUsage sums the current usage of the pool rooted at parentNS: the +// parent's own namespace plus every unbounded descendant whose nearest bounded +// ancestor is parentNS. Bounded sub-tenants are excluded — their reservations +// are already charged as sibling carve-outs — as is excludeNS (the tenant being +// created/updated). Usage is read from each member's ResourceQuota.status.used, +// taking a per-resource max within a namespace to avoid double-counting the +// chart-rendered and controller-allocated quotas, and is keyed in the rendered +// quota key space (see renderedLimitKey). Boundedness is decided by declared +// resourceQuotas so the result does not depend on whether Flux has rendered a +// freshly-created tenant's quota yet. +func (r *REST) parentPoolUsage(ctx context.Context, parentNS, excludeNS string) (map[string]resource.Quantity, error) { + releases := &helmv2.HelmReleaseList{} + selector := labels.SelectorFromSet(labels.Set{ + ApplicationKindLabel: r.kindName, + ApplicationGroupLabel: r.gvk.Group, + }) + if err := r.c.List(ctx, releases, &client.ListOptions{LabelSelector: selector}); err != nil { + return nil, err + } + + bounded := map[string]bool{} + owned := make([]string, 0, len(releases.Items)) + for i := range releases.Items { + hr := &releases.Items[i] + name := strings.TrimPrefix(hr.Name, r.releaseConfig.Prefix) + ns := r.computeTenantNamespace(hr.Namespace, name) + owned = append(owned, ns) + quotas, err := declaredQuotasFromHelmRelease(hr) + if err != nil { + klog.Warningf("skipping tenant %s/%s with unparseable quotas while summing pool usage: %v", hr.Namespace, hr.Name, err) + continue + } + if len(quotas) > 0 { + bounded[ns] = true + } + } + + members := map[string]bool{parentNS: true} + for _, ns := range owned { + if ns == excludeNS { + continue + } + if r.poolRootOf(ns, bounded) == parentNS { + members[ns] = true + } + } + delete(members, excludeNS) + + quotas := &corev1.ResourceQuotaList{} + if err := r.c.List(ctx, quotas); err != nil { + return nil, err + } + perNamespace := map[string]map[string]resource.Quantity{} + for i := range quotas.Items { + rq := "as.Items[i] + if !members[rq.Namespace] { + continue + } + perNamespace[rq.Namespace] = maxQuotas(perNamespace[rq.Namespace], resourceListToQuotas(rq.Status.Used)) + } + + total := map[string]resource.Quantity{} + for _, used := range perNamespace { + addQuotas(total, used) + } + return total, nil +} + +// resourceListToQuotas converts a corev1.ResourceList to the string-keyed quota +// map the admission arithmetic uses. +func resourceListToQuotas(rl corev1.ResourceList) map[string]resource.Quantity { + out := make(map[string]resource.Quantity, len(rl)) + for k, v := range rl { + out[string(k)] = v + } + return out +} + +// maxQuotas returns the per-resource maximum of dst and src as a new map. +// Multiple ResourceQuotas in one namespace each report the same usage, so usage +// is merged with a max (not a sum) to avoid double counting. +func maxQuotas(dst, src map[string]resource.Quantity) map[string]resource.Quantity { + out := map[string]resource.Quantity{} + for k, v := range dst { + out[k] = v.DeepCopy() + } + for k, v := range src { + if cur, ok := out[k]; !ok || v.Cmp(cur) > 0 { + out[k] = v.DeepCopy() + } + } + return out +} diff --git a/pkg/registry/apps/application/quota_test.go b/pkg/registry/apps/application/quota_test.go index 9b5320e85..e03294559 100644 --- a/pkg/registry/apps/application/quota_test.go +++ b/pkg/registry/apps/application/quota_test.go @@ -22,7 +22,9 @@ import ( "testing" helmv2 "github.com/fluxcd/helm-controller/api/v2" + corev1 "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -79,6 +81,9 @@ func newTenantREST(t *testing.T, objects ...client.Object) *REST { if err := helmv2.AddToScheme(scheme); err != nil { t.Fatalf("register helmv2 scheme: %v", err) } + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("register corev1 scheme: %v", err) + } fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() return &REST{ c: fakeClient, @@ -116,6 +121,37 @@ func childApplication(t *testing.T, name, namespace string, quotas map[string]st } } +// usageQuota builds a ResourceQuota reporting current usage (status.used) in a +// namespace, keyed in the rendered quota key space (e.g. "limits.cpu"). +func usageQuota(t *testing.T, namespace, name string, used map[string]string) *corev1.ResourceQuota { + t.Helper() + rl := corev1.ResourceList{} + for k, v := range used { + rl[corev1.ResourceName(k)] = resource.MustParse(v) + } + return &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Status: corev1.ResourceQuotaStatus{Used: rl}, + } +} + +func TestRenderedLimitKey(t *testing.T) { + cases := map[string]string{ + "cpu": "limits.cpu", + "memory": "limits.memory", + "ephemeral-storage": "limits.ephemeral-storage", + "devices.com/nvidia": "limits.devices.com/nvidia", + "storage": "requests.storage", + "pods": "pods", + "services.loadbalancers": "services.loadbalancers", + } + for raw, want := range cases { + if got := renderedLimitKey(raw); got != want { + t.Errorf("renderedLimitKey(%q) = %q, want %q", raw, got, want) + } + } +} + func TestValidateTenantResourceQuotas(t *testing.T) { tests := []struct { name string @@ -265,6 +301,54 @@ func TestValidateTenantResourceQuotas_UpdateExcludesSelf(t *testing.T) { } } +// TestValidateTenantResourceQuotas_PoolUsage verifies that the parent pool's +// current usage is carved out of the budget available to a new child, so the +// sub-tree invariant holds at declaration time. +func TestValidateTenantResourceQuotas_PoolUsage(t *testing.T) { + // foo budget cpu=10, already using 8 in its own namespace. A child asking + // for 10 must be denied (only 2 left); a child asking for 2 is allowed. + t.Run("parent usage shrinks remaining", func(t *testing.T) { + parent := tenantHelmRelease(t, "foo", "tenant-root", map[string]string{"cpu": "10"}) + used := usageQuota(t, "tenant-foo", "tenant-quota", map[string]string{"limits.cpu": "8"}) + r := newTenantREST(t, parent, used) + + denied := r.validateTenantResourceQuotas(context.Background(), childApplication(t, "bar", "tenant-foo", map[string]string{"cpu": "10"})) + if len(denied) == 0 { + t.Fatalf("child cpu=10 should be denied when parent pool already uses 8 of 10") + } + allowed := r.validateTenantResourceQuotas(context.Background(), childApplication(t, "bar", "tenant-foo", map[string]string{"cpu": "2"})) + if len(allowed) > 0 { + t.Fatalf("child cpu=2 should be allowed (2 left), got: %v", allowed) + } + }) + + // Usage in a bounded child's own namespace (its own pool) must not count + // against the parent pool's remaining budget. + t.Run("bounded child usage excluded from parent pool", func(t *testing.T) { + parent := tenantHelmRelease(t, "foo", "tenant-root", map[string]string{"cpu": "10"}) + child := tenantHelmRelease(t, "bar", "tenant-foo", map[string]string{"cpu": "4"}) + // bar uses its full 4 in its own namespace. + childUsed := usageQuota(t, "tenant-foo-bar", "tenant-quota", map[string]string{"limits.cpu": "4"}) + r := newTenantREST(t, parent, child, childUsed) + + // Raising bar to 9 must still be allowed: bar's own usage is in its own + // pool, and as the tenant being updated it is excluded anyway. + if errs := r.validateTenantResourceQuotas(context.Background(), childApplication(t, "bar", "tenant-foo", map[string]string{"cpu": "9"})); len(errs) > 0 { + t.Fatalf("raising own quota within budget must be allowed, got: %v", errs) + } + }) + + // No rendered usage yet (freshly-created parent, Flux has not rendered the + // quota): usage degrades to zero and behaviour matches the pre-usage gate. + t.Run("absent usage degrades to budget-only check", func(t *testing.T) { + parent := tenantHelmRelease(t, "foo", "tenant-root", map[string]string{"cpu": "10"}) + r := newTenantREST(t, parent) + if errs := r.validateTenantResourceQuotas(context.Background(), childApplication(t, "bar", "tenant-foo", map[string]string{"cpu": "10"})); len(errs) > 0 { + t.Fatalf("with no recorded usage a full-budget child must be allowed, got: %v", errs) + } + }) +} + func TestParentTenantHelmReleaseRef(t *testing.T) { tests := []struct { owned string