diff --git a/go.mod b/go.mod index 34352e583..cb49f9f24 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/go-logr/logr v1.4.4 github.com/go-logr/zapr v1.3.0 github.com/go-task/slim-sprig/v3 v3.0.0 + github.com/google/cel-go v0.29.2 github.com/minio/minio-go/v7 v7.0.98 github.com/onsi/ginkgo/v2 v2.33.0 github.com/onsi/gomega v1.43.1 @@ -83,7 +84,6 @@ require ( github.com/go-openapi/testify/enable/yaml/v2 v2.8.0 // indirect github.com/go-openapi/testify/v2 v2.8.0 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/cel-go v0.29.2 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260906184651-6331bc6350fe // indirect diff --git a/pkg/apis/apps/defaulting/defaulting.go b/pkg/apis/apps/defaulting/defaulting.go new file mode 100644 index 000000000..bbec2fa88 --- /dev/null +++ b/pkg/apis/apps/defaulting/defaulting.go @@ -0,0 +1,201 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package defaulting applies an ApplicationDefinition's structural-schema +// defaults to application values. It is shared by the aggregated apiserver, +// which defaults on the read path, and by anything that has to reason about +// stored values, which carry only the keys a tenant actually wrote. +package defaulting + +import ( + "encoding/json" + "fmt" + "strings" + + internalapiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apimachinery/pkg/runtime" +) + +// BuildSchema parses an OpenAPI-v3 JSON schema string and returns the +// structural-schema form used for defaulting. Returns (nil, nil) when raw +// is empty after trimming. Returns (nil, error) for malformed JSON, +// v1→internal conversion failures, or structural-schema construction +// errors — callers can decide whether to log-and-continue or fail hard. +func BuildSchema(raw string) (*structuralschema.Structural, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } + var v1js apiextv1.JSONSchemaProps + if err := json.Unmarshal([]byte(raw), &v1js); err != nil { + return nil, fmt.Errorf("unmarshal v1 OpenAPI schema: %w", err) + } + scheme := runtime.NewScheme() + _ = internalapiext.AddToScheme(scheme) + _ = apiextv1.AddToScheme(scheme) + var ijs internalapiext.JSONSchemaProps + if err := scheme.Convert(&v1js, &ijs, nil); err != nil { + return nil, fmt.Errorf("convert v1->internal JSONSchemaProps: %w", err) + } + s, err := structuralschema.NewStructural(&ijs) + if err != nil { + return nil, fmt.Errorf("build structural schema: %w", err) + } + return s, nil +} + +// Apply defaults root in place against s the way the Kubernetes apiserver +// defaults a custom resource. A nil schema leaves root untouched. +func Apply(root *map[string]any, s *structuralschema.Structural) error { + if s == nil { + return nil + } + v := any(*root) + nv, err := applyDefaults(v, s, true) + if err != nil { + return err + } + obj, ok := nv.(map[string]any) + if !ok && nv != nil { + return fmt.Errorf("internal error: applyDefaults returned non-map type %T for object root", nv) + } + if obj == nil { + obj = map[string]any{} + } + *root = obj + return nil +} + +func applyDefaults(v any, s *structuralschema.Structural, top bool) (any, error) { + if s == nil { + return v, nil + } + + effType := s.Type + if effType == "" { + switch { + case len(s.Properties) > 0 || (s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil): + effType = "object" + case s.Items != nil: + effType = "array" + default: + // scalar + } + } + + switch effType { + case "object": + mv, isMap := v.(map[string]any) + if !isMap || v == nil { + if s.Default.Object != nil && !top { + if dm, ok := s.Default.Object.(map[string]any); ok { + mv = cloneMap(dm) + } + } + if mv == nil { + mv = map[string]any{} + } + } + + for name, ps := range s.Properties { + if _, ok := mv[name]; !ok { + if ps.Default.Object != nil { + mv[name] = clone(ps.Default.Object) + } + } + if cur, ok := mv[name]; ok { + cv, err := applyDefaults(cur, &ps, false) + if err != nil { + return nil, err + } + mv[name] = cv + } + } + + if s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil { + ap := s.AdditionalProperties.Structural + for k, cur := range mv { + if _, isKnown := s.Properties[k]; isKnown { + continue + } + cv, err := applyDefaults(cur, ap, false) + if err != nil { + return nil, err + } + mv[k] = cv + } + } + return mv, nil + + case "array": + sl, isSlice := v.([]any) + if !isSlice || v == nil { + if s.Default.Object != nil { + if ds, ok := s.Default.Object.([]any); ok { + sl = cloneSlice(ds) + } + } + if sl == nil { + sl = []any{} + } + } + if s.Items != nil { + for i := range sl { + cv, err := applyDefaults(sl[i], s.Items, false) + if err != nil { + return nil, err + } + sl[i] = cv + } + } + return sl, nil + + default: + if v == nil && s.Default.Object != nil { + return clone(s.Default.Object), nil + } + return v, nil + } +} + +func clone(x any) any { + switch t := x.(type) { + case map[string]any: + return cloneMap(t) + case []any: + return cloneSlice(t) + default: + return t + } +} + +func cloneMap(m map[string]any) map[string]any { + out := make(map[string]any, len(m)) + for k, v := range m { + out[k] = clone(v) + } + return out +} + +func cloneSlice(s []any) []any { + out := make([]any, len(s)) + for i := range s { + out[i] = clone(s[i]) + } + return out +} diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index 5f618b4e3..aa0e24a62 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -50,6 +50,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" fluxshard "github.com/cozystack/cozystack/internal/fluxshardoperator" + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" "github.com/cozystack/cozystack/pkg/apis/apps/presets" appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" "github.com/cozystack/cozystack/pkg/apis/apps/validation" @@ -57,7 +58,6 @@ import ( "github.com/cozystack/cozystack/pkg/registry" fieldfilter "github.com/cozystack/cozystack/pkg/registry/fields" "github.com/cozystack/cozystack/pkg/registry/sorting" - internalapiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" @@ -110,26 +110,7 @@ type REST struct { // errors — callers can decide whether to log-and-continue (NewREST does) // or fail hard (tests do). func buildSpecSchema(raw string) (*structuralschema.Structural, error) { - raw = strings.TrimSpace(raw) - if raw == "" { - return nil, nil - } - var v1js apiextv1.JSONSchemaProps - if err := json.Unmarshal([]byte(raw), &v1js); err != nil { - return nil, fmt.Errorf("unmarshal v1 OpenAPI schema: %w", err) - } - scheme := runtime.NewScheme() - _ = internalapiext.AddToScheme(scheme) - _ = apiextv1.AddToScheme(scheme) - var ijs internalapiext.JSONSchemaProps - if err := scheme.Convert(&v1js, &ijs, nil); err != nil { - return nil, fmt.Errorf("convert v1->internal JSONSchemaProps: %w", err) - } - s, err := structuralschema.NewStructural(&ijs) - if err != nil { - return nil, fmt.Errorf("build structural schema: %w", err) - } - return s, nil + return defaulting.BuildSchema(raw) } // NewREST creates a new REST storage for Application with specific configuration diff --git a/pkg/registry/apps/application/rest_defaulting.go b/pkg/registry/apps/application/rest_defaulting.go index 36518af69..4fe7409dd 100644 --- a/pkg/registry/apps/application/rest_defaulting.go +++ b/pkg/registry/apps/application/rest_defaulting.go @@ -18,8 +18,8 @@ package application import ( "encoding/json" - "fmt" + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" appsv1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" @@ -51,137 +51,5 @@ func (r *REST) applySpecDefaults(app *appsv1alpha1.Application) error { } func defaultLikeKubernetes(root *map[string]any, s *structuralschema.Structural) error { - v := any(*root) - nv, err := applyDefaults(v, s, true) - if err != nil { - return err - } - obj, ok := nv.(map[string]any) - if !ok && nv != nil { - return fmt.Errorf("internal error: applyDefaults returned non-map type %T for object root", nv) - } - if obj == nil { - obj = map[string]any{} - } - *root = obj - return nil -} - -func applyDefaults(v any, s *structuralschema.Structural, top bool) (any, error) { - if s == nil { - return v, nil - } - - effType := s.Type - if effType == "" { - switch { - case len(s.Properties) > 0 || (s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil): - effType = "object" - case s.Items != nil: - effType = "array" - default: - // scalar - } - } - - switch effType { - case "object": - mv, isMap := v.(map[string]any) - if !isMap || v == nil { - if s.Default.Object != nil && !top { - if dm, ok := s.Default.Object.(map[string]any); ok { - mv = cloneMap(dm) - } - } - if mv == nil { - mv = map[string]any{} - } - } - - for name, ps := range s.Properties { - if _, ok := mv[name]; !ok { - if ps.Default.Object != nil { - mv[name] = clone(ps.Default.Object) - } - } - if cur, ok := mv[name]; ok { - cv, err := applyDefaults(cur, &ps, false) - if err != nil { - return nil, err - } - mv[name] = cv - } - } - - if s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil { - ap := s.AdditionalProperties.Structural - for k, cur := range mv { - if _, isKnown := s.Properties[k]; isKnown { - continue - } - cv, err := applyDefaults(cur, ap, false) - if err != nil { - return nil, err - } - mv[k] = cv - } - } - return mv, nil - - case "array": - sl, isSlice := v.([]any) - if !isSlice || v == nil { - if s.Default.Object != nil { - if ds, ok := s.Default.Object.([]any); ok { - sl = cloneSlice(ds) - } - } - if sl == nil { - sl = []any{} - } - } - if s.Items != nil { - for i := range sl { - cv, err := applyDefaults(sl[i], s.Items, false) - if err != nil { - return nil, err - } - sl[i] = cv - } - } - return sl, nil - - default: - if v == nil && s.Default.Object != nil { - return clone(s.Default.Object), nil - } - return v, nil - } -} - -func clone(x any) any { - switch t := x.(type) { - case map[string]any: - return cloneMap(t) - case []any: - return cloneSlice(t) - default: - return t - } -} - -func cloneMap(m map[string]any) map[string]any { - out := make(map[string]any, len(m)) - for k, v := range m { - out[k] = clone(v) - } - return out -} - -func cloneSlice(s []any) []any { - out := make([]any, len(s)) - for i := range s { - out[i] = clone(s[i]) - } - return out + return defaulting.Apply(root, s) } diff --git a/pkg/reservation/aggregate.go b/pkg/reservation/aggregate.go new file mode 100644 index 000000000..635c41284 --- /dev/null +++ b/pkg/reservation/aggregate.go @@ -0,0 +1,76 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import "fmt" + +// Application is one stored application as the aggregator sees it: the +// labelled HelmRelease the aggregated apiserver persisted, reduced to what +// identifies its definition and its undefaulted values. +type Application struct { + Namespace string + Name string + Kind string + Group string + Values []byte +} + +// EvaluatorLookup returns the evaluator of a kind, found by its kind and +// group so that an out-of-tree kind resolves through the same path as an +// in-tree one. +type EvaluatorLookup func(kind, group string) (*Evaluator, error) + +// Failure is an application whose reservation could not be computed. +type Failure struct { + Application Application + Err error +} + +func (f Failure) Error() string { + return fmt.Sprintf("%s %s/%s: %v", f.Application.Kind, f.Application.Namespace, f.Application.Name, f.Err) +} + +// Aggregate sums the reservations of apps per namespace. +// +// An application that cannot be evaluated is returned as a Failure and is +// left out of the sums. The sums are therefore a lower bound whenever a +// failure is returned, and the caller must not treat them as complete: a +// stored application that stops evaluating has to keep its last known +// reservation rather than hand its share of the pool to the next writer. +func Aggregate(apps []Application, lookup EvaluatorLookup, env Environment) (map[string]ResourceList, []Failure) { + perNamespace := map[string]ResourceList{} + var failures []Failure + for _, app := range apps { + ev, err := lookup(app.Kind, app.Group) + if err != nil { + failures = append(failures, Failure{Application: app, Err: err}) + continue + } + rl, err := ev.EvaluateJSON(app.Values, env) + if err != nil { + failures = append(failures, Failure{Application: app, Err: err}) + continue + } + sum, ok := perNamespace[app.Namespace] + if !ok { + sum = ResourceList{} + perNamespace[app.Namespace] = sum + } + sum.Add(rl) + } + return perNamespace, failures +} diff --git a/pkg/reservation/environment.go b/pkg/reservation/environment.go new file mode 100644 index 000000000..f6727c867 --- /dev/null +++ b/pkg/reservation/environment.go @@ -0,0 +1,84 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "errors" + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// ErrNotFound is wrapped by an Environment when a named instance type or +// preset does not exist. The evaluator fails closed on it rather than +// reserving nothing. +var ErrNotFound = errors.New("not found") + +// Environment resolves the cluster objects a declaration may reference by +// name. Implementations must be safe for concurrent use and return a list +// the caller may keep: the evaluator does not copy it. +type Environment interface { + Instancetype(name string) (ResourceList, error) + Preset(name string) (ResourceList, error) +} + +// StaticEnvironment is an Environment over fixed tables. +type StaticEnvironment struct { + Instancetypes map[string]ResourceList + Presets map[string]ResourceList +} + +var _ Environment = (*StaticEnvironment)(nil) + +// NewStaticEnvironment classifies every VirtualMachineClusterInstancetype in +// instancetypes and pairs the result with presets. One unclassifiable type +// fails the whole environment, so that a broken catalog is noticed when it is +// loaded rather than when a tenant first orders that type. +func NewStaticEnvironment(instancetypes []map[string]any, presets map[string]ResourceList) (*StaticEnvironment, error) { + env := &StaticEnvironment{ + Instancetypes: make(map[string]ResourceList, len(instancetypes)), + Presets: presets, + } + for _, obj := range instancetypes { + rl, err := ClassifyInstancetype(obj) + if err != nil { + return nil, err + } + name, _, _ := unstructured.NestedString(obj, "metadata", "name") + if name == "" { + return nil, fmt.Errorf("instance type without metadata.name") + } + env.Instancetypes[name] = rl + } + return env, nil +} + +func (e *StaticEnvironment) Instancetype(name string) (ResourceList, error) { + rl, ok := e.Instancetypes[name] + if !ok { + return nil, fmt.Errorf("instance type %q: %w", name, ErrNotFound) + } + return rl, nil +} + +func (e *StaticEnvironment) Preset(name string) (ResourceList, error) { + rl, ok := e.Presets[name] + if !ok { + return nil, fmt.Errorf("resource preset %q: %w", name, ErrNotFound) + } + return rl, nil +} diff --git a/pkg/reservation/evaluator.go b/pkg/reservation/evaluator.go new file mode 100644 index 000000000..6927b0bf2 --- /dev/null +++ b/pkg/reservation/evaluator.go @@ -0,0 +1,453 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "maps" + "math" + "regexp" + "slices" + + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common/functions" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apimachinery/pkg/api/resource" + apiservercel "k8s.io/apiserver/pkg/cel" + "k8s.io/apiserver/pkg/cel/library" + + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" +) + +// ErrNoDeclaration is returned for a kind whose definition declares neither +// its consumption nor an exemption. Such a kind is refused rather than +// treated as free: for an out-of-tree catalog an omitted declaration would +// otherwise be a quota-escalation vector no in-tree test can reach. +var ErrNoDeclaration = errors.New("the application definition declares neither its consumption nor an exemption") + +const ( + valuesVar = "values" + instancetypeFunc = "instanceType" + presetFunc = "preset" + + // costLimit bounds one expression. Declarations are platform-authored, + // but they run on the admission path, so a runaway comprehension must + // fail rather than stall a write. + costLimit = 1_000_000 +) + +var bindingName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + +// Evaluator computes the reservation of one kind's applications. It is built +// once per declaration and is safe for concurrent use. +type Evaluator struct { + exempt bool + schema *structuralschema.Structural + env *cel.Env + lets []compiled + resources []compiled +} + +type compiled struct { + name string + ast *cel.Ast +} + +// Compile checks decl and prepares it for evaluation. schema is the kind's +// structural OpenAPI schema; stored values are defaulted against it before +// any expression reads them, because the aggregated apiserver persists a +// tenant's spec without its defaults. A nil decl yields ErrNoDeclaration. +func Compile(decl *Declaration, schema *structuralschema.Structural) (*Evaluator, error) { + if decl == nil { + return nil, ErrNoDeclaration + } + if decl.Exempt { + if len(decl.Let) > 0 || len(decl.Resources) > 0 { + return nil, fmt.Errorf("an exempt declaration must not carry let or resources") + } + return &Evaluator{exempt: true}, nil + } + if len(decl.Resources) == 0 { + return nil, ErrNoDeclaration + } + + opts := []cel.EnvOption{ + cel.Variable(valuesVar, cel.DynType), + library.Quantity(), + cel.Function(instancetypeFunc, + cel.Overload("instanceType_string", []*cel.Type{cel.StringType}, cel.MapType(cel.StringType, cel.DynType), + cel.LateFunctionBinding())), + cel.Function(presetFunc, + cel.Overload("preset_string", []*cel.Type{cel.StringType}, cel.MapType(cel.StringType, cel.DynType), + cel.LateFunctionBinding())), + cel.Function("quantity", + cel.Overload("quantity_int", []*cel.Type{cel.IntType}, apiservercel.QuantityType, + cel.UnaryBinding(func(v ref.Val) ref.Val { + return apiservercel.Quantity{Quantity: resource.NewQuantity(int64(v.(types.Int)), resource.DecimalSI)} + }))), + // The standard _*_ is a singleton dispatching on the left operand's + // traits, so it cannot take a quantity overload; scaling is a method. + cel.Function("mul", + cel.MemberOverload("quantity_mul_int", []*cel.Type{apiservercel.QuantityType, cel.IntType}, apiservercel.QuantityType, + cel.BinaryBinding(mulQuantity))), + } + seen := map[string]bool{valuesVar: true} + for _, b := range decl.Let { + if !bindingName.MatchString(b.Name) { + return nil, fmt.Errorf("let %q: not a valid identifier", b.Name) + } + if seen[b.Name] { + return nil, fmt.Errorf("let %q: name already bound", b.Name) + } + seen[b.Name] = true + opts = append(opts, cel.Variable(b.Name, cel.DynType)) + } + env, err := cel.NewEnv(opts...) + if err != nil { + return nil, fmt.Errorf("build CEL environment: %w", err) + } + + ev := &Evaluator{schema: schema, env: env} + for _, b := range decl.Let { + ast, iss := env.Compile(b.Expression) + if iss.Err() != nil { + return nil, fmt.Errorf("let %q: %w", b.Name, iss.Err()) + } + ev.lets = append(ev.lets, compiled{name: b.Name, ast: ast}) + } + for _, key := range slices.Sorted(maps.Keys(decl.Resources)) { + if key == "" { + return nil, fmt.Errorf("resources: empty quota key") + } + ast, iss := env.Compile(decl.Resources[key]) + if iss.Err() != nil { + return nil, fmt.Errorf("resources[%q]: %w", key, iss.Err()) + } + ev.resources = append(ev.resources, compiled{name: key, ast: ast}) + } + return ev, nil +} + +// Exempt reports whether the kind declared that it consumes nothing. +func (e *Evaluator) Exempt() bool { return e.exempt } + +// EvaluateJSON evaluates the reservation of an application from its values +// as stored, a JSON object. +func (e *Evaluator) EvaluateJSON(raw []byte, env Environment) (ResourceList, error) { + values := map[string]any{} + if len(bytes.TrimSpace(raw)) > 0 { + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, fmt.Errorf("decode values: %w", err) + } + m, ok := v.(map[string]any) + if !ok && v != nil { + return nil, fmt.Errorf("decode values: expected an object, got %T", v) + } + if m != nil { + values = m + } + } + return e.Evaluate(values, env) +} + +// Evaluate returns the reservation of an application with the given values. +// The values are defaulted first and are not modified. Anything that cannot +// be resolved — an unknown instance type, a value of the wrong type, a key an +// expression reads without guarding it with has() — is an error, never a +// smaller figure. Keys that evaluate to zero are omitted. +func (e *Evaluator) Evaluate(values map[string]any, env Environment) (ResourceList, error) { + if e.exempt { + return ResourceList{}, nil + } + if env == nil { + return nil, fmt.Errorf("no environment to resolve instance types and presets against") + } + + defaulted := normalizeNumbers(values).(map[string]any) + // Defaulting replaces a value of the wrong type with the schema's + // default, which would turn a malformed spec into a smaller one. + if err := checkTypes(defaulted, e.schema, ""); err != nil { + return nil, err + } + if err := defaulting.Apply(&defaulted, e.schema); err != nil { + return nil, fmt.Errorf("default values: %w", err) + } + + // The environment differs per call, so instanceType and preset are + // declared with a late binding and bound here, per program; cel.Functions + // is deprecated for declaring functions but is still how cel-go supplies + // a late binding. + bindings := cel.Functions( + &functions.Overload{Operator: "instanceType_string", Unary: lookup(env.Instancetype, instancetypeZeroKeys)}, + &functions.Overload{Operator: "preset_string", Unary: lookup(env.Preset, nil)}, + ) + activation := map[string]any{valuesVar: defaulted} + + for _, c := range e.lets { + out, err := e.run(c, bindings, activation) + if err != nil { + return nil, fmt.Errorf("let %q: %w", c.name, err) + } + activation[c.name] = out + } + + result := ResourceList{} + for _, c := range e.resources { + out, err := e.run(c, bindings, activation) + if err != nil { + return nil, fmt.Errorf("resources[%q]: %w", c.name, err) + } + q, err := toQuantity(out) + if err != nil { + return nil, fmt.Errorf("resources[%q]: %w", c.name, err) + } + if q.Sign() < 0 { + return nil, fmt.Errorf("resources[%q]: negative reservation %s", c.name, q.String()) + } + if q.IsZero() { + continue + } + result[c.name] = q + } + return result, nil +} + +func (e *Evaluator) run(c compiled, bindings cel.ProgramOption, activation map[string]any) (ref.Val, error) { + prg, err := e.env.Program(c.ast, bindings, cel.CostLimit(costLimit)) + if err != nil { + return nil, err + } + out, _, err := prg.Eval(activation) + if err != nil { + return nil, err + } + return out, nil +} + +// instancetypeZeroKeys are present, at zero, in every instanceType() result, +// so that a declaration can read any class of a type without guarding it. +var instancetypeZeroKeys = []string{ + KeyCPU, KeyDedicatedCPU, KeyMemory, KeyOvercommittedMemory, + KeyHugepagesPrefix + "2Mi", KeyHugepagesPrefix + "1Gi", +} + +func lookup(resolve func(string) (ResourceList, error), zeroKeys []string) functions.UnaryOp { + return func(arg ref.Val) ref.Val { + name, ok := arg.(types.String) + if !ok { + return types.MaybeNoSuchOverloadErr(arg) + } + rl, err := resolve(string(name)) + if err != nil { + return types.WrapErr(err) + } + out := make(map[string]any, len(rl)+len(zeroKeys)) + for _, k := range zeroKeys { + out[k] = apiservercel.Quantity{Quantity: resource.NewQuantity(0, resource.DecimalSI)} + } + for k, v := range rl { + q := v.DeepCopy() + out[k] = apiservercel.Quantity{Quantity: &q} + } + return types.DefaultTypeAdapter.NativeToValue(out) + } +} + +func mulQuantity(q, n ref.Val) ref.Val { + qq, ok := q.(apiservercel.Quantity) + if !ok { + return types.MaybeNoSuchOverloadErr(q) + } + nn, ok := n.(types.Int) + if !ok { + return types.MaybeNoSuchOverloadErr(n) + } + out := qq.Quantity.DeepCopy() + if !out.Mul(int64(nn)) { + return types.NewErr("quantity %s * %d overflows", qq.Quantity.String(), int64(nn)) + } + return apiservercel.Quantity{Quantity: &out} +} + +func toQuantity(v ref.Val) (resource.Quantity, error) { + switch t := v.(type) { + case apiservercel.Quantity: + return t.Quantity.DeepCopy(), nil + case types.Int: + return *resource.NewQuantity(int64(t), resource.DecimalSI), nil + case types.Uint: + if uint64(t) > math.MaxInt64 { + return resource.Quantity{}, fmt.Errorf("%d overflows a quantity", uint64(t)) + } + return *resource.NewQuantity(int64(t), resource.DecimalSI), nil + case types.Double: + f := float64(t) + if math.IsNaN(f) || math.IsInf(f, 0) { + return resource.Quantity{}, fmt.Errorf("not a finite number: %v", f) + } + // Rounding up would carry a negative figure smaller than a milli to + // zero, past the sign check. + if f < 0 { + return resource.Quantity{}, fmt.Errorf("negative reservation %v", f) + } + // float64(math.MaxInt64) rounds up to 2^63, so the bound is + // exclusive; past it the conversion would saturate into a smaller + // figure. + milli := math.Ceil(f * 1000) + if milli >= math.MaxInt64 { + return resource.Quantity{}, fmt.Errorf("%v overflows a quantity", f) + } + return *resource.NewMilliQuantity(int64(milli), resource.DecimalSI), nil + case types.String: + q, err := resource.ParseQuantity(string(t)) + if err != nil { + return resource.Quantity{}, fmt.Errorf("parse %q as a quantity: %w", string(t), err) + } + return q, nil + default: + return resource.Quantity{}, fmt.Errorf("expression returned %s, expected an int, a double, a string or a quantity", v.Type().TypeName()) + } +} + +// checkTypes refuses a value whose JSON type the schema does not allow. It +// runs on normalized values: an integer is an int64, any other number a +// float64. A null is left to defaulting, which treats it as absent. +func checkTypes(v any, s *structuralschema.Structural, path string) error { + if s == nil || v == nil { + return nil + } + if s.XIntOrString { + switch v.(type) { + case int64, string: + return nil + } + return typeError(path, "an integer or a string", v) + } + // An untyped schema is defaulted as the shape its properties or items + // imply, so it is checked as that shape too. + effType := s.Type + if effType == "" { + switch { + case len(s.Properties) > 0 || (s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil): + effType = "object" + case s.Items != nil: + effType = "array" + } + } + switch effType { + case "object": + m, ok := v.(map[string]any) + if !ok { + return typeError(path, "an object", v) + } + for k, vv := range m { + ps, known := s.Properties[k] + switch { + case known: + if err := checkTypes(vv, &ps, path+"."+k); err != nil { + return err + } + case s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil: + if err := checkTypes(vv, s.AdditionalProperties.Structural, path+"."+k); err != nil { + return err + } + } + } + case "array": + sl, ok := v.([]any) + if !ok { + return typeError(path, "an array", v) + } + for i, vv := range sl { + if err := checkTypes(vv, s.Items, fmt.Sprintf("%s[%d]", path, i)); err != nil { + return err + } + } + case "string": + if _, ok := v.(string); !ok { + return typeError(path, "a string", v) + } + case "integer": + if _, ok := v.(int64); !ok { + return typeError(path, "an integer", v) + } + case "number": + switch v.(type) { + case int64, float64: + default: + return typeError(path, "a number", v) + } + case "boolean": + if _, ok := v.(bool); !ok { + return typeError(path, "a boolean", v) + } + } + return nil +} + +func typeError(path, want string, got any) error { + if path == "" { + path = "." + } + return fmt.Errorf("values%s: expected %s, got %T", path, want, got) +} + +// normalizeNumbers returns a copy of v in which every JSON number is an int64 +// when it is integral and a float64 otherwise. Without it every number in a +// stored spec reaches CEL as a double, and replicas * quantity(...) would not +// type-check at runtime. +func normalizeNumbers(v any) any { + switch t := v.(type) { + case map[string]any: + out := make(map[string]any, len(t)) + for k, vv := range t { + out[k] = normalizeNumbers(vv) + } + return out + case []any: + out := make([]any, len(t)) + for i, vv := range t { + out[i] = normalizeNumbers(vv) + } + return out + case json.Number: + if i, err := t.Int64(); err == nil { + return i + } + // 2.0 and 2e0 are integers to the schema validator; spelled that + // way they fail Int64 and are classified as a float64 would be. + f, _ := t.Float64() + return normalizeNumbers(f) + case float64: + if t == math.Trunc(t) && math.Abs(t) < 1<<53 { + return int64(t) + } + return t + case int: + return int64(t) + default: + return v + } +} diff --git a/pkg/reservation/evaluator_test.go b/pkg/reservation/evaluator_test.go new file mode 100644 index 000000000..7aa7ef9a7 --- /dev/null +++ b/pkg/reservation/evaluator_test.go @@ -0,0 +1,525 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "errors" + "os" + "sync" + "testing" + + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/yaml" + + "github.com/cozystack/cozystack/pkg/apis/apps/defaulting" +) + +// The declarations below are test fixtures in the shape a kind would carry +// on its ApplicationDefinition. They are evaluated against the kinds' real +// shipped schemas so that defaulting is exercised with the defaults tenants +// actually get. + +var postgresDecl = &Declaration{ + Let: []Binding{{ + Name: "count", + // The autoscaler writes the Cluster's scale subresource, never the + // application, so the ceiling is what has to be reserved. + Expression: `values.autoscaling.enabled && values.autoscaling.maxReplicas > values.replicas + ? values.autoscaling.maxReplicas : values.replicas`, + }}, + Resources: map[string]string{ + // cozy-lib fills a preset per key, so each key is overridden on its own. + "cpu": `(has(values.resources.cpu) ? quantity(values.resources.cpu) : preset(values.resourcesPreset).cpu).mul(count)`, + "memory": `(has(values.resources.memory) ? quantity(values.resources.memory) : preset(values.resourcesPreset).memory).mul(count)`, + "storage": `quantity(values.size).mul(count)`, + }, +} + +var vmInstanceDecl = &Declaration{ + Let: []Binding{ + // vm-instance treats resources as all-or-nothing, and cpu is per socket. + {Name: "sized", Expression: `has(values.resources.cpu) && has(values.resources.sockets) && has(values.resources.memory)`}, + {Name: "it", Expression: `instanceType(values.instanceType)`}, + }, + Resources: map[string]string{ + "cpu": `sized ? int(values.resources.cpu) * int(values.resources.sockets) : it.cpu`, + "dedicated-cpu": `sized ? 0 : it['dedicated-cpu']`, + "memory": `sized ? quantity(values.resources.memory) : it.memory`, + "hugepages-2Mi": `sized ? 0 : it['hugepages-2Mi']`, + "hugepages-1Gi": `sized ? 0 : it['hugepages-1Gi']`, + "overcommitted-memory": `sized ? 0 : it['overcommitted-memory']`, + "services.loadbalancers": `values.external ? 1 : 0`, + }, +} + +var kubernetesNodesDecl = &Declaration{ + Let: []Binding{{Name: "it", Expression: `instanceType(values.instanceType)`}}, + Resources: map[string]string{ + "cpu": `it.cpu.mul(values.maxReplicas)`, + "memory": `it.memory.mul(values.maxReplicas)`, + }, +} + +func shippedSchema(t *testing.T, kind string) *structuralschema.Structural { + t.Helper() + raw, err := os.ReadFile("../../packages/system/" + kind + "-rd/cozyrds/" + kind + ".yaml") + if err != nil { + t.Fatal(err) + } + var def struct { + Spec struct { + Application struct { + OpenAPISchema string `json:"openAPISchema"` + } `json:"application"` + } `json:"spec"` + } + if err := yaml.Unmarshal(raw, &def); err != nil { + t.Fatal(err) + } + s, err := defaulting.BuildSchema(def.Spec.Application.OpenAPISchema) + if err != nil || s == nil { + t.Fatalf("%s schema: %v", kind, err) + } + return s +} + +func testEnvironment(t *testing.T) Environment { + t.Helper() + env, err := NewStaticEnvironment(shippedInstancetypes(t), map[string]ResourceList{ + "t1.nano": {"cpu": resource.MustParse("250m"), "memory": resource.MustParse("128Mi"), "ephemeral-storage": resource.MustParse("2Gi")}, + "t1.micro": {"cpu": resource.MustParse("500m"), "memory": resource.MustParse("256Mi"), "ephemeral-storage": resource.MustParse("2Gi")}, + }) + if err != nil { + t.Fatal(err) + } + return env +} + +func mustCompile(t *testing.T, decl *Declaration, kind string) *Evaluator { + t.Helper() + ev, err := Compile(decl, shippedSchema(t, kind)) + if err != nil { + t.Fatalf("compile %s: %v", kind, err) + } + return ev +} + +func rl(kv ...string) ResourceList { + out := ResourceList{} + for i := 0; i < len(kv); i += 2 { + out[kv[i]] = resource.MustParse(kv[i+1]) + } + return out +} + +func TestEvaluateKinds(t *testing.T) { + env := testEnvironment(t) + postgres := mustCompile(t, postgresDecl, "postgres") + vm := mustCompile(t, vmInstanceDecl, "vm-instance") + + cases := []struct { + name string + ev *Evaluator + values string + want ResourceList + }{ + { + // Two replicas of t1.micro with 10Gi each, all from schema defaults. + name: "postgres defaults", ev: postgres, values: `{}`, + want: rl("cpu", "1", "memory", "512Mi", "storage", "20Gi"), + }, + { + // The per-key fill: cpu from resources, memory from the preset. + name: "postgres partial resources on a preset", ev: postgres, + values: `{"replicas": 1, "resourcesPreset": "t1.nano", "resources": {"cpu": "1"}}`, + want: rl("cpu", "1", "memory", "128Mi", "storage", "10Gi"), + }, + { + name: "postgres integer cpu", ev: postgres, + values: `{"replicas": 1, "resources": {"cpu": 2, "memory": "1Gi"}}`, + want: rl("cpu", "2", "memory", "1Gi", "storage", "10Gi"), + }, + { + name: "postgres autoscaling reserves the ceiling", ev: postgres, + values: `{"autoscaling": {"enabled": true}}`, + want: rl("cpu", "3", "memory", "1536Mi", "storage", "60Gi"), + }, + { + name: "postgres autoscaling off reserves replicas", ev: postgres, + values: `{"autoscaling": {"enabled": false, "maxReplicas": 6}}`, + want: rl("cpu", "1", "memory", "512Mi", "storage", "20Gi"), + }, + { + name: "vm-instance default instance type", ev: vm, values: `{}`, + want: rl("cpu", "1", "memory", "4Gi"), + }, + { + name: "vm-instance dedicated type pins the emulator core", ev: vm, + values: `{"instanceType": "cx1.2xlarge"}`, + want: rl("dedicated-cpu", "9", "hugepages-2Mi", "16Gi"), + }, + { + name: "vm-instance overcommitted type", ev: vm, + values: `{"instanceType": "o1.large"}`, + want: rl("cpu", "2", "overcommitted-memory", "8Gi"), + }, + { + // cpu is cores per socket. + name: "vm-instance complete resources block", ev: vm, + values: `{"instanceType": "u1.small", "resources": {"cpu": 4, "sockets": 2, "memory": "8Gi"}}`, + want: rl("cpu", "8", "memory", "8Gi"), + }, + { + // Without sockets the chart sizes the VM from its instance type. + name: "vm-instance cpu without sockets", ev: vm, + values: `{"instanceType": "u1.small", "resources": {"cpu": 4, "memory": "8Gi"}}`, + want: rl("cpu", "1", "memory", "2Gi"), + }, + { + name: "vm-instance external", ev: vm, + values: `{"instanceType": "u1.small", "external": true}`, + want: rl("cpu", "1", "memory", "2Gi", "services.loadbalancers", "1"), + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := tc.ev.EvaluateJSON([]byte(tc.values), env) + if err != nil { + t.Fatal(err) + } + assertList(t, tc.name, got, tc.want) + }) + } +} + +// A release a platform chart renders, such as ComputePlane's default md0 +// pool, carries only the keys the chart wrote. It has to reserve exactly what +// the same pool with its defaults written out reserves. +func TestEvaluateDefaultsStoredValues(t *testing.T) { + env := testEnvironment(t) + ev := mustCompile(t, kubernetesNodesDecl, "kubernetes-nodes") + + sparse, err := ev.EvaluateJSON([]byte(`{"roles": ["ingress-nginx"], "minReplicas": 0}`), env) + if err != nil { + t.Fatal(err) + } + explicit, err := ev.EvaluateJSON([]byte(`{"roles": ["ingress-nginx"], "minReplicas": 0, "maxReplicas": 10, "instanceType": "u1.medium"}`), env) + if err != nil { + t.Fatal(err) + } + assertList(t, "sparse vs explicit", sparse, explicit) + assertList(t, "sparse", sparse, rl("cpu", "10", "memory", "40Gi")) +} + +func TestEvaluateFailsClosed(t *testing.T) { + env := testEnvironment(t) + postgres := mustCompile(t, postgresDecl, "postgres") + vm := mustCompile(t, vmInstanceDecl, "vm-instance") + + if _, err := vm.EvaluateJSON([]byte(`{"instanceType": "u9.imaginary"}`), env); err == nil { + t.Error("unknown instance type: expected an error") + } else if !errors.Is(err, ErrNotFound) { + t.Errorf("unknown instance type: expected ErrNotFound, got %v", err) + } + if _, err := postgres.EvaluateJSON([]byte(`{"resourcesPreset": "t9.huge"}`), env); !errors.Is(err, ErrNotFound) { + t.Errorf("unknown preset: expected ErrNotFound, got %v", err) + } + if _, err := postgres.EvaluateJSON([]byte(`{"replicas": "two"}`), env); err == nil { + t.Error("wrong value type: expected an error") + } + if _, err := postgres.EvaluateJSON([]byte(`{"size": "lots"}`), env); err == nil { + t.Error("unparseable quantity: expected an error") + } + if _, err := postgres.EvaluateJSON([]byte(`[1, 2]`), env); err == nil { + t.Error("non-object values: expected an error") + } + if _, err := postgres.EvaluateJSON([]byte(`{}`), nil); err == nil { + t.Error("no environment: expected an error") + } + + unguarded, err := Compile(&Declaration{Resources: map[string]string{"cpu": `quantity(values.notInTheSchema)`}}, shippedSchema(t, "postgres")) + if err != nil { + t.Fatal(err) + } + if _, err := unguarded.EvaluateJSON([]byte(`{}`), env); err == nil { + t.Error("unguarded absent key: expected an error, not a zero") + } + + negative, err := Compile(&Declaration{Resources: map[string]string{"cpu": `-1`}}, nil) + if err != nil { + t.Fatal(err) + } + if _, err := negative.EvaluateJSON(nil, env); err == nil { + t.Error("negative reservation: expected an error") + } + + // A negative double too small to survive rounding to millis is still + // negative. + tiny, err := Compile(&Declaration{Resources: map[string]string{"cpu": `-0.0001`}}, nil) + if err != nil { + t.Fatal(err) + } + if _, err := tiny.EvaluateJSON(nil, env); err == nil { + t.Error("small negative reservation: expected an error, not a zero") + } + + // A double past the range of an int64 in millis must not be clamped into + // a smaller figure. + for _, expr := range []string{`1e100`, `1e308`, `9.3e15`} { + huge, err := Compile(&Declaration{Resources: map[string]string{"cpu": expr}}, nil) + if err != nil { + t.Fatal(err) + } + if got, err := huge.EvaluateJSON(nil, env); err == nil { + t.Errorf("%s: expected an error, got %v", expr, got) + } + } + finite, err := Compile(&Declaration{Resources: map[string]string{"cpu": `1e3`, "memory": `0.25`}}, nil) + if err != nil { + t.Fatal(err) + } + got, err := finite.EvaluateJSON(nil, env) + if err != nil { + t.Fatal(err) + } + assertList(t, "finite doubles", got, rl("cpu", "1k", "memory", "250m")) +} + +func TestEvaluateRejectsValuesOfTheWrongType(t *testing.T) { + env := testEnvironment(t) + // Defaulting replaces a value whose type does not match the schema with + // the schema's own default, so the type is checked before it runs: + // otherwise this guard would read an erased resources object and + // reserve its fallback. + ev, err := Compile(&Declaration{Resources: map[string]string{ + "cpu": `has(values.resources.cpu) ? values.resources.cpu : 1`, + }}, shippedSchema(t, "postgres")) + if err != nil { + t.Fatal(err) + } + for _, raw := range []string{ + `{"resources": "invalid"}`, + `{"resources": [1]}`, + `{"resources": true}`, + `{"resources": {"cpu": true}}`, + `{"replicas": 2.5}`, + `{"replicas": "2"}`, + `{"external": "yes"}`, + `{"postgresql": {"sharedPreloadLibraries": "pg_prewarm"}}`, + `{"users": {"app": "admin"}}`, + } { + if got, err := ev.EvaluateJSON([]byte(raw), env); err == nil { + t.Errorf("%s: expected an error, got %v", raw, got) + } + } + for raw, want := range map[string]ResourceList{ + `{}`: rl("cpu", "1"), + `{"resources": {}}`: rl("cpu", "1"), + `{"resources": {"cpu": 2}}`: rl("cpu", "2"), + `{"resources": {"cpu": "500m"}}`: rl("cpu", "500m"), + `{"resources": null, "replicas": 3}`: rl("cpu", "1"), + `{"users": {"app": {"replication": true}}}`: rl("cpu", "1"), + } { + got, err := ev.EvaluateJSON([]byte(raw), env) + if err != nil { + t.Errorf("%s: %v", raw, err) + continue + } + assertList(t, raw, got, want) + } +} + +func TestEvaluateAcceptsIntegralSpellings(t *testing.T) { + env := testEnvironment(t) + // The Kubernetes schema validator accepts an integral number however it + // is spelled, so the evaluator must too. + ev, err := Compile(&Declaration{Resources: map[string]string{"cpu": `values.replicas`}}, shippedSchema(t, "postgres")) + if err != nil { + t.Fatal(err) + } + for _, raw := range []string{`{"replicas": 2}`, `{"replicas": 2.0}`, `{"replicas": 2e0}`, `{"replicas": 0.2e1}`} { + got, err := ev.EvaluateJSON([]byte(raw), env) + if err != nil { + t.Errorf("%s: %v", raw, err) + continue + } + assertList(t, raw, got, rl("cpu", "2")) + } + for _, raw := range []string{`{"replicas": 2.5}`, `{"replicas": 25e-1}`} { + if got, err := ev.EvaluateJSON([]byte(raw), env); err == nil { + t.Errorf("%s: expected an error, got %v", raw, got) + } + } +} + +func TestEvaluateChecksUntypedSchemasByShape(t *testing.T) { + env := testEnvironment(t) + // BuildSchema does not require a structural schema to be typed, and + // defaulting treats an untyped node with properties as an object and one + // with items as an array, so the type check must as well. + s, err := defaulting.BuildSchema(`{"type":"object","properties":{ + "resources":{"properties":{"cpu":{"type":"integer"}}}, + "disks":{"items":{"type":"string"}}, + "labels":{"additionalProperties":{"type":"string"}}}}`) + if err != nil { + t.Fatal(err) + } + ev, err := Compile(&Declaration{Resources: map[string]string{ + "cpu": `has(values.resources) && has(values.resources.cpu) ? values.resources.cpu : 1`, + }}, s) + if err != nil { + t.Fatal(err) + } + for _, raw := range []string{`{"resources": "invalid"}`, `{"resources": {"cpu": "2"}}`, `{"disks": "a"}`, `{"disks": [1]}`, `{"labels": "a"}`, `{"labels": {"a": 1}}`} { + if got, err := ev.EvaluateJSON([]byte(raw), env); err == nil { + t.Errorf("%s: expected an error, got %v", raw, got) + } + } + for raw, want := range map[string]ResourceList{ + `{"resources": {"cpu": 2}}`: rl("cpu", "2"), + `{"disks": ["a"]}`: rl("cpu", "1"), + `{"labels": {"a": "b"}}`: rl("cpu", "1"), + } { + got, err := ev.EvaluateJSON([]byte(raw), env) + if err != nil { + t.Errorf("%s: %v", raw, err) + continue + } + assertList(t, raw, got, want) + } +} + +func TestCompile(t *testing.T) { + if _, err := Compile(nil, nil); !errors.Is(err, ErrNoDeclaration) { + t.Errorf("nil declaration: got %v, want ErrNoDeclaration", err) + } + if _, err := Compile(&Declaration{}, nil); !errors.Is(err, ErrNoDeclaration) { + t.Errorf("empty declaration: got %v, want ErrNoDeclaration", err) + } + + exempt, err := Compile(&Declaration{Exempt: true}, nil) + if err != nil { + t.Fatal(err) + } + got, err := exempt.EvaluateJSON([]byte(`{"anything": 1}`), nil) + if err != nil || len(got) != 0 || !exempt.Exempt() { + t.Errorf("exempt: got %v, %v", got, err) + } + + invalid := map[string]*Declaration{ + "exempt with resources": {Exempt: true, Resources: map[string]string{"cpu": "1"}}, + "syntax error": {Resources: map[string]string{"cpu": "values.("}}, + "undeclared variable": {Resources: map[string]string{"cpu": "count"}}, + "empty key": {Resources: map[string]string{"": "1"}}, + "bad let name": {Let: []Binding{{Name: "a-b", Expression: "1"}}, Resources: map[string]string{"cpu": "1"}}, + "let shadows values": {Let: []Binding{{Name: "values", Expression: "1"}}, Resources: map[string]string{"cpu": "1"}}, + "duplicate let": {Let: []Binding{{Name: "a", Expression: "1"}, {Name: "a", Expression: "2"}}, Resources: map[string]string{"cpu": "a"}}, + } + for name, decl := range invalid { + if _, err := Compile(decl, nil); err == nil { + t.Errorf("%s: expected a compile error", name) + } + } +} + +func TestEvaluateIsConcurrencySafe(t *testing.T) { + env := testEnvironment(t) + vm := mustCompile(t, vmInstanceDecl, "vm-instance") + catalog := shippedInstancetypes(t) + var wg sync.WaitGroup + for _, it := range []string{"u1.small", "cx1.2xlarge", "o1.large", "u1.medium"} { + want, err := ClassifyInstancetype(findInstancetype(t, catalog, it)) + if err != nil { + t.Fatal(err) + } + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + got, err := vm.EvaluateJSON([]byte(`{"instanceType": "`+it+`"}`), env) + if err != nil { + t.Error(err) + return + } + assertList(t, it, got, want) + }() + } + } + wg.Wait() +} + +func findInstancetype(t *testing.T, catalog []map[string]any, name string) map[string]any { + t.Helper() + for _, obj := range catalog { + if obj["metadata"].(map[string]any)["name"] == name { + return obj + } + } + t.Fatalf("%s is not shipped", name) + return nil +} + +func TestAggregate(t *testing.T) { + env := testEnvironment(t) + evaluators := map[string]*Evaluator{ + "VMInstance": mustCompile(t, vmInstanceDecl, "vm-instance"), + "Postgres": mustCompile(t, postgresDecl, "postgres"), + } + lookup := func(kind, group string) (*Evaluator, error) { + if group != "apps.cozystack.io" { + return nil, errors.New("unknown group") + } + if ev, ok := evaluators[kind]; ok { + return ev, nil + } + return nil, ErrNoDeclaration + } + apps := []Application{ + {Namespace: "tenant-a", Name: "vm1", Kind: "VMInstance", Group: "apps.cozystack.io", Values: []byte(`{"instanceType": "u1.small"}`)}, + {Namespace: "tenant-a", Name: "vm2", Kind: "VMInstance", Group: "apps.cozystack.io", Values: []byte(`{"instanceType": "u1.small", "external": true}`)}, + {Namespace: "tenant-a", Name: "db", Kind: "Postgres", Group: "apps.cozystack.io", Values: []byte(`{}`)}, + {Namespace: "tenant-b", Name: "vm3", Kind: "VMInstance", Group: "apps.cozystack.io", Values: []byte(`{"instanceType": "u1.small"}`)}, + {Namespace: "tenant-b", Name: "broken", Kind: "VMInstance", Group: "apps.cozystack.io", Values: []byte(`{"instanceType": "gone"}`)}, + {Namespace: "tenant-b", Name: "undeclared", Kind: "Redis", Group: "apps.cozystack.io", Values: []byte(`{}`)}, + } + sums, failures := Aggregate(apps, lookup, env) + assertList(t, "tenant-a", sums["tenant-a"], rl("cpu", "3", "memory", "4608Mi", "storage", "20Gi", "services.loadbalancers", "1")) + assertList(t, "tenant-b", sums["tenant-b"], rl("cpu", "1", "memory", "2Gi")) + if len(failures) != 2 { + t.Fatalf("expected 2 failures, got %v", failures) + } + if failures[0].Application.Name != "broken" || !errors.Is(failures[0].Err, ErrNotFound) { + t.Errorf("first failure: %v", failures[0]) + } + if failures[1].Application.Name != "undeclared" || !errors.Is(failures[1].Err, ErrNoDeclaration) { + t.Errorf("second failure: %v", failures[1]) + } +} + +func TestResourceListArithmetic(t *testing.T) { + a := rl("cpu", "2", "memory", "4Gi") + b := a.DeepCopy() + b.Add(rl("cpu", "1", "storage", "10Gi")) + assertList(t, "copy unaffected", a, rl("cpu", "2", "memory", "4Gi")) + assertList(t, "add", b, rl("cpu", "3", "memory", "4Gi", "storage", "10Gi")) + b.Sub(rl("cpu", "4", "services.loadbalancers", "1")) + assertList(t, "sub", b, rl("cpu", "-1", "memory", "4Gi", "storage", "10Gi", "services.loadbalancers", "-1")) + if keys := b.Keys(); len(keys) != 4 || keys[0] != "cpu" || keys[3] != "storage" { + t.Errorf("keys not sorted: %v", keys) + } +} diff --git a/pkg/reservation/instancetype.go b/pkg/reservation/instancetype.go new file mode 100644 index 000000000..b4373164f --- /dev/null +++ b/pkg/reservation/instancetype.go @@ -0,0 +1,148 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// EmulatorThreadCompleteToEvenParityAnnotation, set in an instance type's +// spec.annotations, makes KubeVirt give the isolated emulator thread one or +// two cores, whichever makes the pinned total even. KubeVirt copies those +// annotations onto the VMI and tests the key for presence, so its value is +// irrelevant. +const EmulatorThreadCompleteToEvenParityAnnotation = "alpha.kubevirt.io/EmulatorThreadCompleteToEvenParity" + +// ClassifyInstancetype resolves a VirtualMachineClusterInstancetype into the +// class-keyed reservation of one VM of that type. +// +// The class is read from the type's spec, never from its name, so a type an +// operator adds classifies itself. Cores include the emulator and +// supplemental I/O threads a dedicated type pins, because those are fixed +// properties of the type a tenant can see before ordering; the launcher's +// memory overhead is computed by virt-controller at runtime and is platform +// margin, not part of the reservation. +func ClassifyInstancetype(obj map[string]any) (ResourceList, error) { + name, _, _ := unstructured.NestedString(obj, "metadata", "name") + + guestCPU, err := nestedInt(obj, "spec", "cpu", "guest") + if err != nil { + return nil, fmt.Errorf("instance type %q: spec.cpu.guest: %w", name, err) + } + if guestCPU <= 0 { + return nil, fmt.Errorf("instance type %q: spec.cpu.guest must be positive, got %d", name, guestCPU) + } + guestMemoryRaw, found, err := unstructured.NestedFieldNoCopy(obj, "spec", "memory", "guest") + if err != nil || !found { + return nil, fmt.Errorf("instance type %q: spec.memory.guest is required", name) + } + guestMemory, err := parseQuantity(guestMemoryRaw) + if err != nil { + return nil, fmt.Errorf("instance type %q: spec.memory.guest: %w", name, err) + } + + out := ResourceList{} + + dedicated, _, _ := unstructured.NestedBool(obj, "spec", "cpu", "dedicatedCPUPlacement") + if dedicated { + cores := guestCPU + if isolate, _, _ := unstructured.NestedBool(obj, "spec", "cpu", "isolateEmulatorThread"); isolate { + cores++ + annotations, _, _ := unstructured.NestedMap(obj, "spec", "annotations") + if _, parity := annotations[EmulatorThreadCompleteToEvenParityAnnotation]; parity && cores%2 == 1 { + cores++ + } + } + if policy, _, _ := unstructured.NestedString(obj, "spec", "ioThreadsPolicy"); policy == "supplementalPool" { + extra, err := nestedInt(obj, "spec", "ioThreads", "supplementalPoolThreadCount") + if err != nil { + return nil, fmt.Errorf("instance type %q: ioThreadsPolicy supplementalPool: spec.ioThreads.supplementalPoolThreadCount: %w", name, err) + } + cores += extra + } + out[KeyDedicatedCPU] = *resource.NewQuantity(cores, resource.DecimalSI) + } else { + out[KeyCPU] = *resource.NewQuantity(guestCPU, resource.DecimalSI) + } + + pageSize, hasHugepages, _ := unstructured.NestedString(obj, "spec", "memory", "hugepages", "pageSize") + overcommit, hasOvercommit, _ := unstructured.NestedFieldNoCopy(obj, "spec", "memory", "overcommitPercent") + overcommitted := hasOvercommit && overcommit != nil && fmt.Sprint(overcommit) != "0" + switch { + case hasHugepages && overcommitted: + return nil, fmt.Errorf("instance type %q: hugepages and overcommitPercent are mutually exclusive", name) + case hasHugepages: + if pageSize == "" { + return nil, fmt.Errorf("instance type %q: spec.memory.hugepages.pageSize is empty", name) + } + out[KeyHugepagesPrefix+pageSize] = guestMemory + case overcommitted: + // The launcher requests less than the guest here, but the tenant + // declared the guest figure, and the same page cannot be granted as + // both overcommitted and ordinary memory. + out[KeyOvercommittedMemory] = guestMemory + default: + out[KeyMemory] = guestMemory + } + return out, nil +} + +func nestedInt(obj map[string]any, fields ...string) (int64, error) { + v, found, err := unstructured.NestedFieldNoCopy(obj, fields...) + if err != nil { + return 0, err + } + if !found || v == nil { + return 0, fmt.Errorf("is required") + } + switch n := v.(type) { + case int64: + return n, nil + case int: + return int64(n), nil + case int32: + return int64(n), nil + case float64: + if n != float64(int64(n)) { + return 0, fmt.Errorf("must be an integer, got %v", n) + } + return int64(n), nil + default: + return 0, fmt.Errorf("must be an integer, got %T", v) + } +} + +func parseQuantity(v any) (resource.Quantity, error) { + switch q := v.(type) { + case string: + return resource.ParseQuantity(q) + case int64: + return *resource.NewQuantity(q, resource.BinarySI), nil + case int: + return *resource.NewQuantity(int64(q), resource.BinarySI), nil + case float64: + if q != float64(int64(q)) { + return resource.Quantity{}, fmt.Errorf("fractional byte count %v", q) + } + return *resource.NewQuantity(int64(q), resource.BinarySI), nil + default: + return resource.Quantity{}, fmt.Errorf("must be a quantity, got %T", v) + } +} diff --git a/pkg/reservation/instancetype_test.go b/pkg/reservation/instancetype_test.go new file mode 100644 index 000000000..0d072b548 --- /dev/null +++ b/pkg/reservation/instancetype_test.go @@ -0,0 +1,259 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reservation + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" +) + +// shippedInstancetypes decodes every VirtualMachineClusterInstancetype the +// platform ships, so that the classification is pinned against the real +// catalog rather than against hand-written fixtures. +func shippedInstancetypes(t *testing.T) []map[string]any { + t.Helper() + files, err := filepath.Glob("../../packages/system/kubevirt-instancetypes/templates/*.yaml") + if err != nil || len(files) == 0 { + t.Fatalf("no instance type templates found: %v", err) + } + var out []map[string]any + for _, f := range files { + fh, err := os.Open(f) + if err != nil { + t.Fatal(err) + } + dec := utilyaml.NewYAMLOrJSONDecoder(fh, 4096) + for { + var obj map[string]any + if err := dec.Decode(&obj); err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatalf("%s: %v", f, err) + } + if obj["kind"] == "VirtualMachineClusterInstancetype" { + out = append(out, obj) + } + } + fh.Close() + } + return out +} + +func TestClassifyShippedInstancetypes(t *testing.T) { + // The expected class per series is written from what each series is, + // not derived from the spec, so a name-based rule and a spec-based rule + // disagreeing shows up here. m1 is shared CPU with hugepages, n1 a + // dedicated series with hugepages, o1 overcommits memory. + type class struct{ cpu, memory string } + want := map[string]class{ + "u1": {KeyCPU, KeyMemory}, + "gn1": {KeyCPU, KeyMemory}, + "m1": {KeyCPU, "hugepages"}, + "o1": {KeyCPU, KeyOvercommittedMemory}, + "d1": {KeyDedicatedCPU, KeyMemory}, + "cx1": {KeyDedicatedCPU, "hugepages"}, + "n1": {KeyDedicatedCPU, "hugepages"}, + "rt1": {KeyDedicatedCPU, "hugepages"}, + } + + types := shippedInstancetypes(t) + if len(types) < 60 { + t.Fatalf("expected the full shipped catalog, decoded only %d instance types", len(types)) + } + seen := map[string]bool{} + for _, obj := range types { + name := obj["metadata"].(map[string]any)["name"].(string) + series := strings.SplitN(name, ".", 2)[0] + seen[series] = true + w, ok := want[series] + if !ok { + t.Errorf("%s: series %q has no expected class; add it to this table", name, series) + continue + } + rl, err := ClassifyInstancetype(obj) + if err != nil { + t.Errorf("%s: %v", name, err) + continue + } + if len(rl) != 2 { + t.Errorf("%s: expected one CPU class and one memory class, got %v", name, rl) + } + if _, ok := rl[w.cpu]; !ok { + t.Errorf("%s: expected CPU class %q, got %v", name, w.cpu, rl) + } + if w.memory == "hugepages" { + found := false + for k := range rl { + found = found || strings.HasPrefix(k, KeyHugepagesPrefix) + } + if !found { + t.Errorf("%s: expected a hugepages class, got %v", name, rl) + } + } else if _, ok := rl[w.memory]; !ok { + t.Errorf("%s: expected memory class %q, got %v", name, w.memory, rl) + } + } + for series := range want { + if !seen[series] { + t.Errorf("series %q is no longer shipped; drop it from this table", series) + } + } +} + +func TestClassifyShippedFigures(t *testing.T) { + byName := map[string]map[string]any{} + for _, obj := range shippedInstancetypes(t) { + byName[obj["metadata"].(map[string]any)["name"].(string)] = obj + } + cases := []struct { + name string + want ResourceList + }{ + // The KubeVirt u1.small is 2Gi; the cozy-lib preset of the same name + // is 4Gi. The two catalogs share names, not figures. + {"u1.small", ResourceList{KeyCPU: resource.MustParse("1"), KeyMemory: resource.MustParse("2Gi")}}, + // Eight guest vCPUs on nine pinned cores: the isolated emulator + // thread takes a core nobody else can be given. + {"cx1.2xlarge", ResourceList{KeyDedicatedCPU: resource.MustParse("9"), KeyHugepagesPrefix + "2Mi": resource.MustParse("16Gi")}}, + {"d1.large", ResourceList{KeyDedicatedCPU: resource.MustParse("3"), KeyMemory: resource.MustParse("8Gi")}}, + {"m1.large", ResourceList{KeyCPU: resource.MustParse("2"), KeyHugepagesPrefix + "2Mi": resource.MustParse("16Gi")}}, + } + for _, tc := range cases { + obj, ok := byName[tc.name] + if !ok { + t.Fatalf("%s is not shipped", tc.name) + } + got, err := ClassifyInstancetype(obj) + if err != nil { + t.Fatalf("%s: %v", tc.name, err) + } + assertList(t, tc.name, got, tc.want) + } +} + +func TestClassifyEmulatorThreadParity(t *testing.T) { + // KubeVirt copies an instance type's spec.annotations onto the VMI and + // pins the extra parity core when the VMI carries the annotation, so the + // figure is a property of each type, not of the catalog. + dedicated := func(guest int64, annotations map[string]any) map[string]any { + spec := map[string]any{ + "cpu": map[string]any{"guest": guest, "dedicatedCPUPlacement": true, "isolateEmulatorThread": true}, + "memory": map[string]any{"guest": "1Gi"}, + } + if annotations != nil { + spec["annotations"] = annotations + } + return map[string]any{"metadata": map[string]any{"name": "test"}, "spec": spec} + } + parity := map[string]any{EmulatorThreadCompleteToEvenParityAnnotation: ""} + cases := []struct { + guest int64 + annotations map[string]any + want int64 + }{ + {guest: 8, want: 9}, + {guest: 8, annotations: parity, want: 10}, + {guest: 8, annotations: map[string]any{EmulatorThreadCompleteToEvenParityAnnotation: "false"}, want: 10}, + {guest: 8, annotations: map[string]any{"example.com/unrelated": ""}, want: 9}, + {guest: 3, want: 4}, + {guest: 3, annotations: parity, want: 4}, + } + for _, tc := range cases { + got, err := ClassifyInstancetype(dedicated(tc.guest, tc.annotations)) + if err != nil { + t.Fatal(err) + } + cores := got[KeyDedicatedCPU] + if cores.Value() != tc.want { + t.Errorf("guest %d annotations %v: got %s dedicated cores, want %d", tc.guest, tc.annotations, cores.String(), tc.want) + } + } + + // Only the type's own annotations reach the VMI; metadata.annotations + // describe the object and are not copied. + onMetadata := dedicated(8, nil) + onMetadata["metadata"].(map[string]any)["annotations"] = parity + got, err := ClassifyInstancetype(onMetadata) + if err != nil { + t.Fatal(err) + } + if cores := got[KeyDedicatedCPU]; cores.Value() != 9 { + t.Errorf("annotation on metadata: got %s dedicated cores, want 9", cores.String()) + } +} + +func TestClassifySupplementalIOThreads(t *testing.T) { + obj := map[string]any{ + "metadata": map[string]any{"name": "test"}, + "spec": map[string]any{ + "cpu": map[string]any{"guest": int64(4), "dedicatedCPUPlacement": true}, + "ioThreadsPolicy": "supplementalPool", + "ioThreads": map[string]any{"supplementalPoolThreadCount": int64(2)}, + "memory": map[string]any{"guest": "1Gi"}, + }, + } + got, err := ClassifyInstancetype(obj) + if err != nil { + t.Fatal(err) + } + if cores := got[KeyDedicatedCPU]; cores.Value() != 6 { + t.Errorf("got %s dedicated cores, want 6", cores.String()) + } +} + +func TestClassifyRejectsIncompleteTypes(t *testing.T) { + cases := map[string]map[string]any{ + "no guest cpu": {"spec": map[string]any{"memory": map[string]any{"guest": "1Gi"}}}, + "no guest memory": {"spec": map[string]any{"cpu": map[string]any{"guest": int64(1)}}}, + "hugepages and overcommit": {"spec": map[string]any{ + "cpu": map[string]any{"guest": int64(1)}, + "memory": map[string]any{"guest": "1Gi", "overcommitPercent": int64(50), "hugepages": map[string]any{"pageSize": "2Mi"}}, + }}, + "supplementalPool without a count": {"spec": map[string]any{ + "cpu": map[string]any{"guest": int64(1), "dedicatedCPUPlacement": true}, + "ioThreadsPolicy": "supplementalPool", + "memory": map[string]any{"guest": "1Gi"}, + }}, + } + for name, obj := range cases { + if got, err := ClassifyInstancetype(obj); err == nil { + t.Errorf("%s: expected an error, got %v", name, got) + } + } +} + +func assertList(t *testing.T, label string, got, want ResourceList) { + t.Helper() + if len(got) != len(want) { + t.Errorf("%s: got %v, want %v", label, got, want) + return + } + for k, w := range want { + g, ok := got[k] + if !ok || g.Cmp(w) != 0 { + t.Errorf("%s: %s = %s, want %s (full: %v)", label, k, g.String(), w.String(), got) + } + } +} diff --git a/pkg/reservation/types.go b/pkg/reservation/types.go new file mode 100644 index 000000000..1a01a0763 --- /dev/null +++ b/pkg/reservation/types.go @@ -0,0 +1,113 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package reservation computes what an application consumes from a tenant's +// quota budget, from its declared values, before anything runs. +// +// A reservation is denominated in the shorthand keys a Tenant's +// resourceQuotas uses (cpu, memory, storage, services.loadbalancers, ...), +// never in rendered pod units: no allocation ratio is applied and no +// virtualization overhead is added. The evaluator holds no per-kind +// knowledge; each kind describes its consumption in a Declaration carried by +// its ApplicationDefinition, so a kind shipped from an out-of-tree catalog +// participates without a change here. +package reservation + +import ( + "sort" + + "k8s.io/apimachinery/pkg/api/resource" +) + +// Quota keys for KubeVirt instance-type classes. A shared vCPU and a pinned +// core are not the same capacity, nor are ordinary, hugepage-backed and +// overcommitted memory, so each class counts against its own key. +const ( + KeyCPU = "cpu" + KeyDedicatedCPU = "dedicated-cpu" + KeyMemory = "memory" + KeyOvercommittedMemory = "overcommitted-memory" + // KeyHugepagesPrefix is followed by the page size, e.g. "hugepages-2Mi", + // matching the corev1 resource name for the same pages. + KeyHugepagesPrefix = "hugepages-" +) + +// Declaration is how a kind's reservation is computed from its values. +// +// Exactly one of Exempt or Resources is meaningful: a kind that consumes +// nothing says so with Exempt rather than with an empty Resources map, so +// that a zero reservation is always a reviewed statement and never an +// omission. +type Declaration struct { + // Exempt declares that the kind consumes nothing from a tenant's budget. + Exempt bool `json:"exempt,omitempty"` + + // Let binds intermediate CEL expressions, evaluated in order, each one + // visible to the later bindings and to every Resources expression. + Let []Binding `json:"let,omitempty"` + + // Resources maps a quota key to the CEL expression computing the + // amount of it the application reserves. An expression may return an + // int, a double, a quantity string or a quantity. + Resources map[string]string `json:"resources,omitempty"` +} + +// Binding names one intermediate CEL expression of a Declaration. +type Binding struct { + Name string `json:"name"` + Expression string `json:"expression"` +} + +// ResourceList is a reservation, keyed by quota key. +type ResourceList map[string]resource.Quantity + +// Add adds other into l in place. +func (l ResourceList) Add(other ResourceList) { + for k, v := range other { + cur := l[k] + cur.Add(v) + l[k] = cur + } +} + +// Sub subtracts other from l in place. A key absent from l goes negative, +// which is what the delta of an Update needs. +func (l ResourceList) Sub(other ResourceList) { + for k, v := range other { + cur := l[k] + cur.Sub(v) + l[k] = cur + } +} + +// DeepCopy returns an independent copy of l. +func (l ResourceList) DeepCopy() ResourceList { + out := make(ResourceList, len(l)) + for k, v := range l { + out[k] = v.DeepCopy() + } + return out +} + +// Keys returns the keys of l in a stable order, for deterministic messages. +func (l ResourceList) Keys() []string { + keys := make([]string, 0, len(l)) + for k := range l { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +}