From c368c48fd8a0dbb858fdf1c1136f20448f0facd8 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 10 Nov 2026 09:03:55 +0100 Subject: [PATCH 1/3] fix(controller, platform): give unbounded pool members the root's LimitRange The tenant chart renders the tenant-range-limits LimitRange only next to the tenant-quota of a tenant that declares resourceQuotas. Since quotas became hierarchical, an unbounded sub-tenant shares its parent's pool and gets a controller-owned tenant-quota-allocated ResourceQuota, but still no LimitRange. A ResourceQuota on limits.cpu or limits.memory makes admission reject any container that omits that limit, and with no LimitRange nothing fills it in. Pods that ran before the quota was introduced fail as soon as they are recreated; the barman-cloud plugin sidecar, which sets no CPU limit, takes every backed-up postgres instance in such a namespace down on its next restart. The controller already knows which namespaces it clamps, so it copies the pool root's chart LimitRange into each unbounded member, under its own name so that Helm can still create the chart object if the tenant later declares a quota of its own, and removes the copy once the namespace leaves the pool. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- internal/controller/tenantquota/reconciler.go | 85 ++++++++++- .../controller/tenantquota/reconciler_test.go | 136 ++++++++++++++++++ .../cozystack-controller/templates/rbac.yaml | 8 +- 3 files changed, 224 insertions(+), 5 deletions(-) diff --git a/internal/controller/tenantquota/reconciler.go b/internal/controller/tenantquota/reconciler.go index dddd0fbac..6c77eff6d 100644 --- a/internal/controller/tenantquota/reconciler.go +++ b/internal/controller/tenantquota/reconciler.go @@ -52,6 +52,17 @@ const ( // whenever it is below the chart quota, without the controller fighting Flux // over the chart-owned object. allocatedQuotaName = "tenant-quota-allocated" + // chartLimitRangeName is the LimitRange the tenant chart renders next to + // chartQuotaName, giving containers default requests and limits. The chart + // renders both only for a tenant that declares resourceQuotas. + chartLimitRangeName = "tenant-range-limits" + // allocatedLimitRangeName is the controller-owned copy of the pool root's + // chart LimitRange, written into every unbounded pool member. Those members + // render no chart LimitRange, yet carry an allocated quota; without + // container defaults, any pod omitting a limit the quota bounds is rejected + // at admission. A distinct name keeps the copy from colliding with the + // chart-owned object if the member later declares its own quota. + allocatedLimitRangeName = "tenant-range-limits-allocated" // managedByLabel marks the controller-owned ResourceQuotas so they can be // listed and garbage-collected. managedByLabel = "quota.cozystack.io/managed-by" @@ -102,6 +113,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result pools := ComputePools(tenants) desired := map[string]corev1.ResourceList{} + // limitRangeSource maps each unbounded pool member to the pool root whose + // chart LimitRange it must inherit. + limitRangeSource := map[string]string{} for _, p := range pools { if over := p.Overcommitted(); len(over) > 0 { logger.Info("tenant quota pool is overcommitted by sub-tenant carve-outs", "pool", p.Root, "overcommit", over) @@ -120,6 +134,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result for _, ns := range p.Members { if existing[ns] { desired[ns] = buffered.EnforcedHard(ns, usedByNS) + if ns != p.Root { + limitRangeSource[ns] = p.Root + } } } } @@ -134,6 +151,22 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result logger.Error(err, "failed to garbage-collect stale allocated quotas") } + written := map[string]bool{} + for ns, root := range limitRangeSource { + ok, err := r.upsertAllocatedLimitRange(ctx, ns, root) + if err != nil { + logger.Error(err, "failed to apply allocated limit range", "namespace", ns, "poolRoot", root) + } + // Keep the existing copy on a transient error rather than dropping the + // defaults the namespace's pods rely on. + if ok || err != nil { + written[ns] = true + } + } + if err := r.gcAllocatedLimitRanges(ctx, written); err != nil { + logger.Error(err, "failed to garbage-collect stale allocated limit ranges") + } + return ctrl.Result{RequeueAfter: resyncInterval}, nil } @@ -224,6 +257,50 @@ func (r *Reconciler) gcAllocatedQuotas(ctx context.Context, desired map[string]c return nil } +// upsertAllocatedLimitRange copies the pool root's chart LimitRange into an +// unbounded member namespace. It reports false, without error, when the root +// has no chart LimitRange to copy. +func (r *Reconciler) upsertAllocatedLimitRange(ctx context.Context, namespace, poolRoot string) (bool, error) { + source := &corev1.LimitRange{} + if err := r.Get(ctx, types.NamespacedName{Namespace: poolRoot, Name: chartLimitRangeName}, source); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + lr := &corev1.LimitRange{ObjectMeta: metav1.ObjectMeta{Name: allocatedLimitRangeName, Namespace: namespace}} + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, lr, func() error { + if lr.Labels == nil { + lr.Labels = map[string]string{} + } + lr.Labels[managedByLabel] = managedByValue + lr.Labels["internal.cozystack.io/managed-by-cozystack"] = "" + lr.Spec = *source.Spec.DeepCopy() + return nil + }) + return err == nil, err +} + +// gcAllocatedLimitRanges deletes controller-owned LimitRanges in namespaces +// that are no longer unbounded pool members (e.g. the tenant gained its own +// quota and now renders the chart LimitRange, or was deleted). +func (r *Reconciler) gcAllocatedLimitRanges(ctx context.Context, keep map[string]bool) error { + managed := &corev1.LimitRangeList{} + if err := r.List(ctx, managed, client.MatchingLabels{managedByLabel: managedByValue}); err != nil { + return err + } + for i := range managed.Items { + m := &managed.Items[i] + if keep[m.Namespace] { + continue + } + if err := r.Delete(ctx, m); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + return nil +} + func (r *Reconciler) recordOvercommit(ctx context.Context, rootNamespace string, over corev1.ResourceList) { if r.Recorder == nil { return @@ -237,8 +314,8 @@ func (r *Reconciler) recordOvercommit(ctx context.Context, rootNamespace string, } // SetupWithManager wires the controller. Every relevant change (a tenant -// HelmRelease, a namespace, or a tenant ResourceQuota whose usage moved) -// coalesces into one full-tree sweep. +// HelmRelease, a namespace, a tenant ResourceQuota whose usage moved, or a +// tenant LimitRange) coalesces into one full-tree sweep. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { toSweep := handler.EnqueueRequestsFromMapFunc(func(context.Context, client.Object) []reconcile.Request { return []reconcile.Request{sweepKey} @@ -249,11 +326,15 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { tenantQuotas := predicate.NewPredicateFuncs(func(o client.Object) bool { return o.GetName() == chartQuotaName || o.GetName() == allocatedQuotaName }) + tenantLimitRanges := predicate.NewPredicateFuncs(func(o client.Object) bool { + return o.GetName() == chartLimitRangeName || o.GetName() == allocatedLimitRangeName + }) return ctrl.NewControllerManagedBy(mgr). Named("tenantquota-controller"). Watches(&helmv2.HelmRelease{}, toSweep, builder.WithPredicates(tenantReleases)). Watches(&corev1.Namespace{}, toSweep). Watches(&corev1.ResourceQuota{}, toSweep, builder.WithPredicates(tenantQuotas)). + Watches(&corev1.LimitRange{}, toSweep, builder.WithPredicates(tenantLimitRanges)). Complete(r) } diff --git a/internal/controller/tenantquota/reconciler_test.go b/internal/controller/tenantquota/reconciler_test.go index 5f19ed1b1..4624b2e36 100644 --- a/internal/controller/tenantquota/reconciler_test.go +++ b/internal/controller/tenantquota/reconciler_test.go @@ -227,3 +227,139 @@ func TestReconcile_GarbageCollect(t *testing.T) { t.Fatalf("stale allocated quota in tenant-gone should have been garbage-collected") } } + +// chartLimitRange builds the chart-rendered "tenant-range-limits" LimitRange, +// which the tenant chart emits alongside its "tenant-quota". +func chartLimitRange(namespace, defaultCPU string) *corev1.LimitRange { + return &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{Name: chartLimitRangeName, Namespace: namespace}, + Spec: corev1.LimitRangeSpec{Limits: []corev1.LimitRangeItem{{ + Type: corev1.LimitTypeContainer, + Default: resourceList(map[string]string{"cpu": defaultCPU, "memory": "128Mi"}), + DefaultRequest: resourceList(map[string]string{"cpu": "25m", "memory": "128Mi"}), + }}}, + } +} + +func getLimitRange(c client.Client, namespace, name string) (*corev1.LimitRange, bool) { + lr := &corev1.LimitRange{} + if err := c.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: name}, lr); err != nil { + return nil, false + } + return lr, true +} + +// TestReconcile_UnboundedMemberGetsPoolLimitRange: an unbounded child renders +// no chart quota and therefore no chart LimitRange, yet the controller clamps +// it with an allocated quota. Without container defaults, every pod that omits +// a limit the quota bounds (e.g. a sidecar without a CPU limit) is rejected at +// admission, so the controller must copy the pool root's LimitRange into it. +func TestReconcile_UnboundedMemberGetsPoolLimitRange(t *testing.T) { + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + chartLimitRange("tenant-foo", "250m"), + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + lr, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName) + if !ok { + t.Fatalf("unbounded member tenant-foo-bar has an allocated quota but no LimitRange") + } + if len(lr.Spec.Limits) != 1 { + t.Fatalf("tenant-foo-bar LimitRange limits = %d, want 1", len(lr.Spec.Limits)) + } + if got := lr.Spec.Limits[0].Default[corev1.ResourceCPU]; got.String() != "250m" { + t.Fatalf("tenant-foo-bar default cpu = %q, want 250m copied from the pool root", got.String()) + } + if lr.Labels[managedByLabel] != managedByValue { + t.Fatalf("tenant-foo-bar LimitRange is not marked as controller-owned") + } + if _, ok := getLimitRange(c, "tenant-foo", allocatedLimitRangeName); ok { + t.Fatalf("pool root tenant-foo already has the chart LimitRange and must not get a second one") + } +} + +// TestReconcile_PoolLimitRangeFollowsRoot: a change to the root's chart +// LimitRange propagates to the members on the next sweep. +func TestReconcile_PoolLimitRangeFollowsRoot(t *testing.T) { + root := chartLimitRange("tenant-foo", "250m") + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + root, + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + root.Spec.Limits[0].Default[corev1.ResourceCPU] = resource.MustParse("500m") + if err := c.Update(context.Background(), root); err != nil { + t.Fatalf("update root LimitRange: %v", err) + } + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + lr, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName) + if !ok { + t.Fatalf("tenant-foo-bar lost its LimitRange") + } + if got := lr.Spec.Limits[0].Default[corev1.ResourceCPU]; got.String() != "500m" { + t.Fatalf("tenant-foo-bar default cpu = %q, want 500m after the root changed", got.String()) + } +} + +// TestReconcile_PoolLimitRangeGarbageCollected: once the child declares its own +// quota, the chart renders its LimitRange and the controller-owned copy goes. +func TestReconcile_PoolLimitRangeGarbageCollected(t *testing.T) { + stale := &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: allocatedLimitRangeName, + Namespace: "tenant-foo-bar", + Labels: map[string]string{managedByLabel: managedByValue}, + }, + } + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + chartLimitRange("tenant-foo", "250m"), + chartQuota("tenant-foo-bar", map[string]string{"limits.cpu": "4"}, nil), + chartLimitRange("tenant-foo-bar", "250m"), + stale, + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { + t.Fatalf("bounded child tenant-foo-bar renders its own LimitRange; the controller copy must be removed") + } + if _, ok := getLimitRange(c, "tenant-foo-bar", chartLimitRangeName); !ok { + t.Fatalf("the chart-owned LimitRange must never be touched") + } +} + +// TestReconcile_PoolRootWithoutLimitRange: if the root has no chart LimitRange +// there is nothing to copy; the quota is still enforced and the sweep succeeds. +func TestReconcile_PoolRootWithoutLimitRange(t *testing.T) { + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"cpu": "10"}, nil), + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, ok := allocatedHard(t, c, "tenant-foo-bar", "cpu"); !ok { + t.Fatalf("tenant-foo-bar must still get its allocated quota") + } + if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { + t.Fatalf("no LimitRange can be copied from a root that has none") + } +} diff --git a/packages/system/cozystack-controller/templates/rbac.yaml b/packages/system/cozystack-controller/templates/rbac.yaml index e40ab9080..da8f71f49 100644 --- a/packages/system/cozystack-controller/templates/rbac.yaml +++ b/packages/system/cozystack-controller/templates/rbac.yaml @@ -49,10 +49,12 @@ rules: resources: ["namespaces"] verbs: ["get", "list", "watch", "patch", "update"] # TenantQuotaReconciler manages the per-namespace "tenant-quota-allocated" -# ResourceQuota that enforces each tenant's share of its parent pool, and emits -# events on the tenant namespace when sub-tenant quotas overcommit the budget. +# ResourceQuota that enforces each tenant's share of its parent pool, copies the +# pool root's "tenant-range-limits" LimitRange into unbounded members as +# "tenant-range-limits-allocated", and emits events on the tenant namespace when +# sub-tenant quotas overcommit the budget. - apiGroups: [""] - resources: ["resourcequotas"] + resources: ["resourcequotas", "limitranges"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: [""] resources: ["events"] From e6127f7e423f4839d5d22262f98e2b91fbc8865f Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 13 Nov 2026 10:42:20 +0100 Subject: [PATCH 2/3] fix(controller, platform): copy the LimitRange only under a compute-limit quota The tenant chart renders tenant-range-limits under any non-empty resourceQuotas, including one that bounds no compute, such as services.loadbalancers alone. Copying it into an unbounded member whose allocated quota bounds neither limits.cpu nor limits.memory imposes a 128Mi default memory limit the quota never asked for: a container that requests more and sets no limit, virt-launcher among them, is refused where it was admitted before. Only a quota on limits.cpu or limits.memory makes admission reject a container that omits a limit, so the copy is now limited to that case, and removed otherwise. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- internal/controller/tenantquota/reconciler.go | 18 ++++++++-- .../controller/tenantquota/reconciler_test.go | 33 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/internal/controller/tenantquota/reconciler.go b/internal/controller/tenantquota/reconciler.go index 6c77eff6d..ed18499ce 100644 --- a/internal/controller/tenantquota/reconciler.go +++ b/internal/controller/tenantquota/reconciler.go @@ -113,8 +113,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result pools := ComputePools(tenants) desired := map[string]corev1.ResourceList{} - // limitRangeSource maps each unbounded pool member to the pool root whose - // chart LimitRange it must inherit. + // limitRangeSource maps each unbounded pool member whose quota bounds a + // compute limit to the pool root whose chart LimitRange it must inherit. limitRangeSource := map[string]string{} for _, p := range pools { if over := p.Overcommitted(); len(over) > 0 { @@ -134,7 +134,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result for _, ns := range p.Members { if existing[ns] { desired[ns] = buffered.EnforcedHard(ns, usedByNS) - if ns != p.Root { + if ns != p.Root && boundsComputeLimits(desired[ns]) { limitRangeSource[ns] = p.Root } } @@ -352,6 +352,18 @@ func ownedNamespace(hrNamespace, appName string) string { } } +// boundsComputeLimits reports whether a quota bounds limits.cpu or +// limits.memory, the only case in which admission rejects a container that +// omits a limit. The chart renders its LimitRange under any non-empty +// resourceQuotas, so the root may carry one under a quota that bounds no +// compute at all; copying it then would impose default limits the quota +// never asked for. +func boundsComputeLimits(hard corev1.ResourceList) bool { + _, cpu := hard[corev1.ResourceLimitsCPU] + _, memory := hard[corev1.ResourceLimitsMemory] + return cpu || memory +} + // maxResourceList returns the per-resource maximum of a and b. func maxResourceList(a, b corev1.ResourceList) corev1.ResourceList { out := corev1.ResourceList{} diff --git a/internal/controller/tenantquota/reconciler_test.go b/internal/controller/tenantquota/reconciler_test.go index 4624b2e36..5c5778e63 100644 --- a/internal/controller/tenantquota/reconciler_test.go +++ b/internal/controller/tenantquota/reconciler_test.go @@ -363,3 +363,36 @@ func TestReconcile_PoolRootWithoutLimitRange(t *testing.T) { t.Fatalf("no LimitRange can be copied from a root that has none") } } + +// TestReconcile_NoLimitRangeWithoutComputeLimitQuota: the chart renders its +// LimitRange for any non-empty resourceQuotas, compute or not. A member whose +// allocated quota bounds neither limits.cpu nor limits.memory is admitted +// without limits today; copying the root's defaults there would cap or refuse +// pods (one requesting more than the default memory limit, for instance) +// that the quota itself never constrains. +func TestReconcile_NoLimitRangeWithoutComputeLimitQuota(t *testing.T) { + stale := &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: allocatedLimitRangeName, + Namespace: "tenant-foo-bar", + Labels: map[string]string{managedByLabel: managedByValue}, + }, + } + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"services.loadbalancers": "1"}, nil), + chartLimitRange("tenant-foo", "250m"), + stale, + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, ok := allocatedHard(t, c, "tenant-foo-bar", "services.loadbalancers"); !ok { + t.Fatalf("tenant-foo-bar must still get its allocated quota") + } + if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { + t.Fatalf("tenant-foo-bar's quota bounds no compute limit; it must carry no LimitRange copy") + } +} From cc412d44f2f5dbb7f0ed6686feb0e331c6338bcd Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 13 Nov 2026 13:07:08 +0100 Subject: [PATCH 3/3] test(controller): exercise the missing-root-LimitRange path under a compute-limit quota TestReconcile_PoolRootWithoutLimitRange declared its root quota on cpu, which the compute-limit gate does not count, so the sweep never looked for the root's LimitRange and the test passed whatever that path did. Declaring limits.cpu makes it reach the lookup, and it now fails if a missing root LimitRange produces an empty copy. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- internal/controller/tenantquota/reconciler_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/controller/tenantquota/reconciler_test.go b/internal/controller/tenantquota/reconciler_test.go index 5c5778e63..b869825dc 100644 --- a/internal/controller/tenantquota/reconciler_test.go +++ b/internal/controller/tenantquota/reconciler_test.go @@ -351,12 +351,12 @@ func TestReconcile_PoolRootWithoutLimitRange(t *testing.T) { tenantHR("foo", "tenant-root"), tenantHR("bar", "tenant-foo"), ns("tenant-foo"), ns("tenant-foo-bar"), - chartQuota("tenant-foo", map[string]string{"cpu": "10"}, nil), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), ) if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { t.Fatalf("reconcile: %v", err) } - if _, ok := allocatedHard(t, c, "tenant-foo-bar", "cpu"); !ok { + if _, ok := allocatedHard(t, c, "tenant-foo-bar", "limits.cpu"); !ok { t.Fatalf("tenant-foo-bar must still get its allocated quota") } if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok {