diff --git a/internal/controller/tenantquota/reconciler.go b/internal/controller/tenantquota/reconciler.go index dddd0fbac..6c77eff6d 100644 --- a/internal/controller/tenantquota/reconciler.go +++ b/internal/controller/tenantquota/reconciler.go @@ -52,6 +52,17 @@ const ( // whenever it is below the chart quota, without the controller fighting Flux // over the chart-owned object. allocatedQuotaName = "tenant-quota-allocated" + // chartLimitRangeName is the LimitRange the tenant chart renders next to + // chartQuotaName, giving containers default requests and limits. The chart + // renders both only for a tenant that declares resourceQuotas. + chartLimitRangeName = "tenant-range-limits" + // allocatedLimitRangeName is the controller-owned copy of the pool root's + // chart LimitRange, written into every unbounded pool member. Those members + // render no chart LimitRange, yet carry an allocated quota; without + // container defaults, any pod omitting a limit the quota bounds is rejected + // at admission. A distinct name keeps the copy from colliding with the + // chart-owned object if the member later declares its own quota. + allocatedLimitRangeName = "tenant-range-limits-allocated" // managedByLabel marks the controller-owned ResourceQuotas so they can be // listed and garbage-collected. managedByLabel = "quota.cozystack.io/managed-by" @@ -102,6 +113,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result pools := ComputePools(tenants) desired := map[string]corev1.ResourceList{} + // limitRangeSource maps each unbounded pool member to the pool root whose + // chart LimitRange it must inherit. + limitRangeSource := map[string]string{} for _, p := range pools { if over := p.Overcommitted(); len(over) > 0 { logger.Info("tenant quota pool is overcommitted by sub-tenant carve-outs", "pool", p.Root, "overcommit", over) @@ -120,6 +134,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result for _, ns := range p.Members { if existing[ns] { desired[ns] = buffered.EnforcedHard(ns, usedByNS) + if ns != p.Root { + limitRangeSource[ns] = p.Root + } } } } @@ -134,6 +151,22 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result logger.Error(err, "failed to garbage-collect stale allocated quotas") } + written := map[string]bool{} + for ns, root := range limitRangeSource { + ok, err := r.upsertAllocatedLimitRange(ctx, ns, root) + if err != nil { + logger.Error(err, "failed to apply allocated limit range", "namespace", ns, "poolRoot", root) + } + // Keep the existing copy on a transient error rather than dropping the + // defaults the namespace's pods rely on. + if ok || err != nil { + written[ns] = true + } + } + if err := r.gcAllocatedLimitRanges(ctx, written); err != nil { + logger.Error(err, "failed to garbage-collect stale allocated limit ranges") + } + return ctrl.Result{RequeueAfter: resyncInterval}, nil } @@ -224,6 +257,50 @@ func (r *Reconciler) gcAllocatedQuotas(ctx context.Context, desired map[string]c return nil } +// upsertAllocatedLimitRange copies the pool root's chart LimitRange into an +// unbounded member namespace. It reports false, without error, when the root +// has no chart LimitRange to copy. +func (r *Reconciler) upsertAllocatedLimitRange(ctx context.Context, namespace, poolRoot string) (bool, error) { + source := &corev1.LimitRange{} + if err := r.Get(ctx, types.NamespacedName{Namespace: poolRoot, Name: chartLimitRangeName}, source); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + lr := &corev1.LimitRange{ObjectMeta: metav1.ObjectMeta{Name: allocatedLimitRangeName, Namespace: namespace}} + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, lr, func() error { + if lr.Labels == nil { + lr.Labels = map[string]string{} + } + lr.Labels[managedByLabel] = managedByValue + lr.Labels["internal.cozystack.io/managed-by-cozystack"] = "" + lr.Spec = *source.Spec.DeepCopy() + return nil + }) + return err == nil, err +} + +// gcAllocatedLimitRanges deletes controller-owned LimitRanges in namespaces +// that are no longer unbounded pool members (e.g. the tenant gained its own +// quota and now renders the chart LimitRange, or was deleted). +func (r *Reconciler) gcAllocatedLimitRanges(ctx context.Context, keep map[string]bool) error { + managed := &corev1.LimitRangeList{} + if err := r.List(ctx, managed, client.MatchingLabels{managedByLabel: managedByValue}); err != nil { + return err + } + for i := range managed.Items { + m := &managed.Items[i] + if keep[m.Namespace] { + continue + } + if err := r.Delete(ctx, m); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + return nil +} + func (r *Reconciler) recordOvercommit(ctx context.Context, rootNamespace string, over corev1.ResourceList) { if r.Recorder == nil { return @@ -237,8 +314,8 @@ func (r *Reconciler) recordOvercommit(ctx context.Context, rootNamespace string, } // SetupWithManager wires the controller. Every relevant change (a tenant -// HelmRelease, a namespace, or a tenant ResourceQuota whose usage moved) -// coalesces into one full-tree sweep. +// HelmRelease, a namespace, a tenant ResourceQuota whose usage moved, or a +// tenant LimitRange) coalesces into one full-tree sweep. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { toSweep := handler.EnqueueRequestsFromMapFunc(func(context.Context, client.Object) []reconcile.Request { return []reconcile.Request{sweepKey} @@ -249,11 +326,15 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { tenantQuotas := predicate.NewPredicateFuncs(func(o client.Object) bool { return o.GetName() == chartQuotaName || o.GetName() == allocatedQuotaName }) + tenantLimitRanges := predicate.NewPredicateFuncs(func(o client.Object) bool { + return o.GetName() == chartLimitRangeName || o.GetName() == allocatedLimitRangeName + }) return ctrl.NewControllerManagedBy(mgr). Named("tenantquota-controller"). Watches(&helmv2.HelmRelease{}, toSweep, builder.WithPredicates(tenantReleases)). Watches(&corev1.Namespace{}, toSweep). Watches(&corev1.ResourceQuota{}, toSweep, builder.WithPredicates(tenantQuotas)). + Watches(&corev1.LimitRange{}, toSweep, builder.WithPredicates(tenantLimitRanges)). Complete(r) } diff --git a/internal/controller/tenantquota/reconciler_test.go b/internal/controller/tenantquota/reconciler_test.go index 5f19ed1b1..4624b2e36 100644 --- a/internal/controller/tenantquota/reconciler_test.go +++ b/internal/controller/tenantquota/reconciler_test.go @@ -227,3 +227,139 @@ func TestReconcile_GarbageCollect(t *testing.T) { t.Fatalf("stale allocated quota in tenant-gone should have been garbage-collected") } } + +// chartLimitRange builds the chart-rendered "tenant-range-limits" LimitRange, +// which the tenant chart emits alongside its "tenant-quota". +func chartLimitRange(namespace, defaultCPU string) *corev1.LimitRange { + return &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{Name: chartLimitRangeName, Namespace: namespace}, + Spec: corev1.LimitRangeSpec{Limits: []corev1.LimitRangeItem{{ + Type: corev1.LimitTypeContainer, + Default: resourceList(map[string]string{"cpu": defaultCPU, "memory": "128Mi"}), + DefaultRequest: resourceList(map[string]string{"cpu": "25m", "memory": "128Mi"}), + }}}, + } +} + +func getLimitRange(c client.Client, namespace, name string) (*corev1.LimitRange, bool) { + lr := &corev1.LimitRange{} + if err := c.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: name}, lr); err != nil { + return nil, false + } + return lr, true +} + +// TestReconcile_UnboundedMemberGetsPoolLimitRange: an unbounded child renders +// no chart quota and therefore no chart LimitRange, yet the controller clamps +// it with an allocated quota. Without container defaults, every pod that omits +// a limit the quota bounds (e.g. a sidecar without a CPU limit) is rejected at +// admission, so the controller must copy the pool root's LimitRange into it. +func TestReconcile_UnboundedMemberGetsPoolLimitRange(t *testing.T) { + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + chartLimitRange("tenant-foo", "250m"), + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + lr, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName) + if !ok { + t.Fatalf("unbounded member tenant-foo-bar has an allocated quota but no LimitRange") + } + if len(lr.Spec.Limits) != 1 { + t.Fatalf("tenant-foo-bar LimitRange limits = %d, want 1", len(lr.Spec.Limits)) + } + if got := lr.Spec.Limits[0].Default[corev1.ResourceCPU]; got.String() != "250m" { + t.Fatalf("tenant-foo-bar default cpu = %q, want 250m copied from the pool root", got.String()) + } + if lr.Labels[managedByLabel] != managedByValue { + t.Fatalf("tenant-foo-bar LimitRange is not marked as controller-owned") + } + if _, ok := getLimitRange(c, "tenant-foo", allocatedLimitRangeName); ok { + t.Fatalf("pool root tenant-foo already has the chart LimitRange and must not get a second one") + } +} + +// TestReconcile_PoolLimitRangeFollowsRoot: a change to the root's chart +// LimitRange propagates to the members on the next sweep. +func TestReconcile_PoolLimitRangeFollowsRoot(t *testing.T) { + root := chartLimitRange("tenant-foo", "250m") + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + root, + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + root.Spec.Limits[0].Default[corev1.ResourceCPU] = resource.MustParse("500m") + if err := c.Update(context.Background(), root); err != nil { + t.Fatalf("update root LimitRange: %v", err) + } + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + lr, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName) + if !ok { + t.Fatalf("tenant-foo-bar lost its LimitRange") + } + if got := lr.Spec.Limits[0].Default[corev1.ResourceCPU]; got.String() != "500m" { + t.Fatalf("tenant-foo-bar default cpu = %q, want 500m after the root changed", got.String()) + } +} + +// TestReconcile_PoolLimitRangeGarbageCollected: once the child declares its own +// quota, the chart renders its LimitRange and the controller-owned copy goes. +func TestReconcile_PoolLimitRangeGarbageCollected(t *testing.T) { + stale := &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: allocatedLimitRangeName, + Namespace: "tenant-foo-bar", + Labels: map[string]string{managedByLabel: managedByValue}, + }, + } + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + chartLimitRange("tenant-foo", "250m"), + chartQuota("tenant-foo-bar", map[string]string{"limits.cpu": "4"}, nil), + chartLimitRange("tenant-foo-bar", "250m"), + stale, + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { + t.Fatalf("bounded child tenant-foo-bar renders its own LimitRange; the controller copy must be removed") + } + if _, ok := getLimitRange(c, "tenant-foo-bar", chartLimitRangeName); !ok { + t.Fatalf("the chart-owned LimitRange must never be touched") + } +} + +// TestReconcile_PoolRootWithoutLimitRange: if the root has no chart LimitRange +// there is nothing to copy; the quota is still enforced and the sweep succeeds. +func TestReconcile_PoolRootWithoutLimitRange(t *testing.T) { + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"cpu": "10"}, nil), + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, ok := allocatedHard(t, c, "tenant-foo-bar", "cpu"); !ok { + t.Fatalf("tenant-foo-bar must still get its allocated quota") + } + if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { + t.Fatalf("no LimitRange can be copied from a root that has none") + } +} diff --git a/packages/system/cozystack-controller/templates/rbac.yaml b/packages/system/cozystack-controller/templates/rbac.yaml index e40ab9080..da8f71f49 100644 --- a/packages/system/cozystack-controller/templates/rbac.yaml +++ b/packages/system/cozystack-controller/templates/rbac.yaml @@ -49,10 +49,12 @@ rules: resources: ["namespaces"] verbs: ["get", "list", "watch", "patch", "update"] # TenantQuotaReconciler manages the per-namespace "tenant-quota-allocated" -# ResourceQuota that enforces each tenant's share of its parent pool, and emits -# events on the tenant namespace when sub-tenant quotas overcommit the budget. +# ResourceQuota that enforces each tenant's share of its parent pool, copies the +# pool root's "tenant-range-limits" LimitRange into unbounded members as +# "tenant-range-limits-allocated", and emits events on the tenant namespace when +# sub-tenant quotas overcommit the budget. - apiGroups: [""] - resources: ["resourcequotas"] + resources: ["resourcequotas", "limitranges"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: [""] resources: ["events"]