From c368c48fd8a0dbb858fdf1c1136f20448f0facd8 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 10 Nov 2026 09:03:55 +0100 Subject: [PATCH] fix(controller, platform): give unbounded pool members the root's LimitRange The tenant chart renders the tenant-range-limits LimitRange only next to the tenant-quota of a tenant that declares resourceQuotas. Since quotas became hierarchical, an unbounded sub-tenant shares its parent's pool and gets a controller-owned tenant-quota-allocated ResourceQuota, but still no LimitRange. A ResourceQuota on limits.cpu or limits.memory makes admission reject any container that omits that limit, and with no LimitRange nothing fills it in. Pods that ran before the quota was introduced fail as soon as they are recreated; the barman-cloud plugin sidecar, which sets no CPU limit, takes every backed-up postgres instance in such a namespace down on its next restart. The controller already knows which namespaces it clamps, so it copies the pool root's chart LimitRange into each unbounded member, under its own name so that Helm can still create the chart object if the tenant later declares a quota of its own, and removes the copy once the namespace leaves the pool. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- internal/controller/tenantquota/reconciler.go | 85 ++++++++++- .../controller/tenantquota/reconciler_test.go | 136 ++++++++++++++++++ .../cozystack-controller/templates/rbac.yaml | 8 +- 3 files changed, 224 insertions(+), 5 deletions(-) diff --git a/internal/controller/tenantquota/reconciler.go b/internal/controller/tenantquota/reconciler.go index dddd0fbac..6c77eff6d 100644 --- a/internal/controller/tenantquota/reconciler.go +++ b/internal/controller/tenantquota/reconciler.go @@ -52,6 +52,17 @@ const ( // whenever it is below the chart quota, without the controller fighting Flux // over the chart-owned object. allocatedQuotaName = "tenant-quota-allocated" + // chartLimitRangeName is the LimitRange the tenant chart renders next to + // chartQuotaName, giving containers default requests and limits. The chart + // renders both only for a tenant that declares resourceQuotas. + chartLimitRangeName = "tenant-range-limits" + // allocatedLimitRangeName is the controller-owned copy of the pool root's + // chart LimitRange, written into every unbounded pool member. Those members + // render no chart LimitRange, yet carry an allocated quota; without + // container defaults, any pod omitting a limit the quota bounds is rejected + // at admission. A distinct name keeps the copy from colliding with the + // chart-owned object if the member later declares its own quota. + allocatedLimitRangeName = "tenant-range-limits-allocated" // managedByLabel marks the controller-owned ResourceQuotas so they can be // listed and garbage-collected. managedByLabel = "quota.cozystack.io/managed-by" @@ -102,6 +113,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result pools := ComputePools(tenants) desired := map[string]corev1.ResourceList{} + // limitRangeSource maps each unbounded pool member to the pool root whose + // chart LimitRange it must inherit. + limitRangeSource := map[string]string{} for _, p := range pools { if over := p.Overcommitted(); len(over) > 0 { logger.Info("tenant quota pool is overcommitted by sub-tenant carve-outs", "pool", p.Root, "overcommit", over) @@ -120,6 +134,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result for _, ns := range p.Members { if existing[ns] { desired[ns] = buffered.EnforcedHard(ns, usedByNS) + if ns != p.Root { + limitRangeSource[ns] = p.Root + } } } } @@ -134,6 +151,22 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result logger.Error(err, "failed to garbage-collect stale allocated quotas") } + written := map[string]bool{} + for ns, root := range limitRangeSource { + ok, err := r.upsertAllocatedLimitRange(ctx, ns, root) + if err != nil { + logger.Error(err, "failed to apply allocated limit range", "namespace", ns, "poolRoot", root) + } + // Keep the existing copy on a transient error rather than dropping the + // defaults the namespace's pods rely on. + if ok || err != nil { + written[ns] = true + } + } + if err := r.gcAllocatedLimitRanges(ctx, written); err != nil { + logger.Error(err, "failed to garbage-collect stale allocated limit ranges") + } + return ctrl.Result{RequeueAfter: resyncInterval}, nil } @@ -224,6 +257,50 @@ func (r *Reconciler) gcAllocatedQuotas(ctx context.Context, desired map[string]c return nil } +// upsertAllocatedLimitRange copies the pool root's chart LimitRange into an +// unbounded member namespace. It reports false, without error, when the root +// has no chart LimitRange to copy. +func (r *Reconciler) upsertAllocatedLimitRange(ctx context.Context, namespace, poolRoot string) (bool, error) { + source := &corev1.LimitRange{} + if err := r.Get(ctx, types.NamespacedName{Namespace: poolRoot, Name: chartLimitRangeName}, source); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + lr := &corev1.LimitRange{ObjectMeta: metav1.ObjectMeta{Name: allocatedLimitRangeName, Namespace: namespace}} + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, lr, func() error { + if lr.Labels == nil { + lr.Labels = map[string]string{} + } + lr.Labels[managedByLabel] = managedByValue + lr.Labels["internal.cozystack.io/managed-by-cozystack"] = "" + lr.Spec = *source.Spec.DeepCopy() + return nil + }) + return err == nil, err +} + +// gcAllocatedLimitRanges deletes controller-owned LimitRanges in namespaces +// that are no longer unbounded pool members (e.g. the tenant gained its own +// quota and now renders the chart LimitRange, or was deleted). +func (r *Reconciler) gcAllocatedLimitRanges(ctx context.Context, keep map[string]bool) error { + managed := &corev1.LimitRangeList{} + if err := r.List(ctx, managed, client.MatchingLabels{managedByLabel: managedByValue}); err != nil { + return err + } + for i := range managed.Items { + m := &managed.Items[i] + if keep[m.Namespace] { + continue + } + if err := r.Delete(ctx, m); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + return nil +} + func (r *Reconciler) recordOvercommit(ctx context.Context, rootNamespace string, over corev1.ResourceList) { if r.Recorder == nil { return @@ -237,8 +314,8 @@ func (r *Reconciler) recordOvercommit(ctx context.Context, rootNamespace string, } // SetupWithManager wires the controller. Every relevant change (a tenant -// HelmRelease, a namespace, or a tenant ResourceQuota whose usage moved) -// coalesces into one full-tree sweep. +// HelmRelease, a namespace, a tenant ResourceQuota whose usage moved, or a +// tenant LimitRange) coalesces into one full-tree sweep. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { toSweep := handler.EnqueueRequestsFromMapFunc(func(context.Context, client.Object) []reconcile.Request { return []reconcile.Request{sweepKey} @@ -249,11 +326,15 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { tenantQuotas := predicate.NewPredicateFuncs(func(o client.Object) bool { return o.GetName() == chartQuotaName || o.GetName() == allocatedQuotaName }) + tenantLimitRanges := predicate.NewPredicateFuncs(func(o client.Object) bool { + return o.GetName() == chartLimitRangeName || o.GetName() == allocatedLimitRangeName + }) return ctrl.NewControllerManagedBy(mgr). Named("tenantquota-controller"). Watches(&helmv2.HelmRelease{}, toSweep, builder.WithPredicates(tenantReleases)). Watches(&corev1.Namespace{}, toSweep). Watches(&corev1.ResourceQuota{}, toSweep, builder.WithPredicates(tenantQuotas)). + Watches(&corev1.LimitRange{}, toSweep, builder.WithPredicates(tenantLimitRanges)). Complete(r) } diff --git a/internal/controller/tenantquota/reconciler_test.go b/internal/controller/tenantquota/reconciler_test.go index 5f19ed1b1..4624b2e36 100644 --- a/internal/controller/tenantquota/reconciler_test.go +++ b/internal/controller/tenantquota/reconciler_test.go @@ -227,3 +227,139 @@ func TestReconcile_GarbageCollect(t *testing.T) { t.Fatalf("stale allocated quota in tenant-gone should have been garbage-collected") } } + +// chartLimitRange builds the chart-rendered "tenant-range-limits" LimitRange, +// which the tenant chart emits alongside its "tenant-quota". +func chartLimitRange(namespace, defaultCPU string) *corev1.LimitRange { + return &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{Name: chartLimitRangeName, Namespace: namespace}, + Spec: corev1.LimitRangeSpec{Limits: []corev1.LimitRangeItem{{ + Type: corev1.LimitTypeContainer, + Default: resourceList(map[string]string{"cpu": defaultCPU, "memory": "128Mi"}), + DefaultRequest: resourceList(map[string]string{"cpu": "25m", "memory": "128Mi"}), + }}}, + } +} + +func getLimitRange(c client.Client, namespace, name string) (*corev1.LimitRange, bool) { + lr := &corev1.LimitRange{} + if err := c.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: name}, lr); err != nil { + return nil, false + } + return lr, true +} + +// TestReconcile_UnboundedMemberGetsPoolLimitRange: an unbounded child renders +// no chart quota and therefore no chart LimitRange, yet the controller clamps +// it with an allocated quota. Without container defaults, every pod that omits +// a limit the quota bounds (e.g. a sidecar without a CPU limit) is rejected at +// admission, so the controller must copy the pool root's LimitRange into it. +func TestReconcile_UnboundedMemberGetsPoolLimitRange(t *testing.T) { + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + chartLimitRange("tenant-foo", "250m"), + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + lr, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName) + if !ok { + t.Fatalf("unbounded member tenant-foo-bar has an allocated quota but no LimitRange") + } + if len(lr.Spec.Limits) != 1 { + t.Fatalf("tenant-foo-bar LimitRange limits = %d, want 1", len(lr.Spec.Limits)) + } + if got := lr.Spec.Limits[0].Default[corev1.ResourceCPU]; got.String() != "250m" { + t.Fatalf("tenant-foo-bar default cpu = %q, want 250m copied from the pool root", got.String()) + } + if lr.Labels[managedByLabel] != managedByValue { + t.Fatalf("tenant-foo-bar LimitRange is not marked as controller-owned") + } + if _, ok := getLimitRange(c, "tenant-foo", allocatedLimitRangeName); ok { + t.Fatalf("pool root tenant-foo already has the chart LimitRange and must not get a second one") + } +} + +// TestReconcile_PoolLimitRangeFollowsRoot: a change to the root's chart +// LimitRange propagates to the members on the next sweep. +func TestReconcile_PoolLimitRangeFollowsRoot(t *testing.T) { + root := chartLimitRange("tenant-foo", "250m") + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + root, + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + root.Spec.Limits[0].Default[corev1.ResourceCPU] = resource.MustParse("500m") + if err := c.Update(context.Background(), root); err != nil { + t.Fatalf("update root LimitRange: %v", err) + } + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + lr, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName) + if !ok { + t.Fatalf("tenant-foo-bar lost its LimitRange") + } + if got := lr.Spec.Limits[0].Default[corev1.ResourceCPU]; got.String() != "500m" { + t.Fatalf("tenant-foo-bar default cpu = %q, want 500m after the root changed", got.String()) + } +} + +// TestReconcile_PoolLimitRangeGarbageCollected: once the child declares its own +// quota, the chart renders its LimitRange and the controller-owned copy goes. +func TestReconcile_PoolLimitRangeGarbageCollected(t *testing.T) { + stale := &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: allocatedLimitRangeName, + Namespace: "tenant-foo-bar", + Labels: map[string]string{managedByLabel: managedByValue}, + }, + } + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + chartLimitRange("tenant-foo", "250m"), + chartQuota("tenant-foo-bar", map[string]string{"limits.cpu": "4"}, nil), + chartLimitRange("tenant-foo-bar", "250m"), + stale, + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { + t.Fatalf("bounded child tenant-foo-bar renders its own LimitRange; the controller copy must be removed") + } + if _, ok := getLimitRange(c, "tenant-foo-bar", chartLimitRangeName); !ok { + t.Fatalf("the chart-owned LimitRange must never be touched") + } +} + +// TestReconcile_PoolRootWithoutLimitRange: if the root has no chart LimitRange +// there is nothing to copy; the quota is still enforced and the sweep succeeds. +func TestReconcile_PoolRootWithoutLimitRange(t *testing.T) { + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"cpu": "10"}, nil), + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, ok := allocatedHard(t, c, "tenant-foo-bar", "cpu"); !ok { + t.Fatalf("tenant-foo-bar must still get its allocated quota") + } + if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { + t.Fatalf("no LimitRange can be copied from a root that has none") + } +} diff --git a/packages/system/cozystack-controller/templates/rbac.yaml b/packages/system/cozystack-controller/templates/rbac.yaml index e40ab9080..da8f71f49 100644 --- a/packages/system/cozystack-controller/templates/rbac.yaml +++ b/packages/system/cozystack-controller/templates/rbac.yaml @@ -49,10 +49,12 @@ rules: resources: ["namespaces"] verbs: ["get", "list", "watch", "patch", "update"] # TenantQuotaReconciler manages the per-namespace "tenant-quota-allocated" -# ResourceQuota that enforces each tenant's share of its parent pool, and emits -# events on the tenant namespace when sub-tenant quotas overcommit the budget. +# ResourceQuota that enforces each tenant's share of its parent pool, copies the +# pool root's "tenant-range-limits" LimitRange into unbounded members as +# "tenant-range-limits-allocated", and emits events on the tenant namespace when +# sub-tenant quotas overcommit the budget. - apiGroups: [""] - resources: ["resourcequotas"] + resources: ["resourcequotas", "limitranges"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: [""] resources: ["events"]