From ecb5de6051b9ece64dacf12dfa0a1aa9b29634d8 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:57:50 +0700 Subject: [PATCH 01/41] feat(observability-proxy): map application plurals to their kinds Paths and RBAC name an application by its plural, while the lineage labels that scope its logs and series carry the kind. The mapping comes from ApplicationDefinitions, kept in one place so both application roots scope by the same kind. A plural declared for two kinds is refused rather than resolved by cache order. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appkinds.go | 97 ++++++++++++++++++++ internal/observabilityproxy/appkinds_test.go | 77 ++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 internal/observabilityproxy/appkinds.go create mode 100644 internal/observabilityproxy/appkinds_test.go diff --git a/internal/observabilityproxy/appkinds.go b/internal/observabilityproxy/appkinds.go new file mode 100644 index 000000000..5e545f390 --- /dev/null +++ b/internal/observabilityproxy/appkinds.go @@ -0,0 +1,97 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "errors" + "fmt" + "slices" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/cache" +) + +// ApplicationDefinitionsResource is where the plural of every application +// kind is declared (api/v1alpha1/applicationdefinitions_types.go). +var ApplicationDefinitionsResource = schema.GroupVersionResource{Group: "cozystack.io", Version: "v1alpha1", Resource: "applicationdefinitions"} + +// ErrKindsNotSynced means the ApplicationDefinitions cache has not been +// filled yet, so an unknown plural cannot be told from a missing one. +var ErrKindsNotSynced = errors.New("application definitions not synced") + +const pluralIndex = "plural" + +// KindLookup maps an application plural, as it appears in paths and RBAC, +// to the kind the lineage labels carry. An unknown plural is "", nil. +type KindLookup interface { + Kind(plural string) (string, error) +} + +// ApplicationPluralIndexers indexes ApplicationDefinitions by +// spec.application.plural; the informer feeding ApplicationKinds needs it. +func ApplicationPluralIndexers() cache.Indexers { + return cache.Indexers{pluralIndex: func(obj any) ([]string, error) { + u, ok := obj.(*unstructured.Unstructured) + if !ok { + return nil, nil + } + plural, _, _ := unstructured.NestedString(u.Object, "spec", "application", "plural") + if plural == "" { + return nil, nil + } + return []string{plural}, nil + }} +} + +// ApplicationKinds is the one place the proxy maps a plural to its kind; +// both application roots use it. +type ApplicationKinds struct { + indexer cache.Indexer + hasSynced func() bool +} + +func NewApplicationKinds(indexer cache.Indexer, hasSynced func() bool) *ApplicationKinds { + return &ApplicationKinds{indexer: indexer, hasSynced: hasSynced} +} + +func (k *ApplicationKinds) Kind(plural string) (string, error) { + if !k.hasSynced() { + return "", ErrKindsNotSynced + } + objs, err := k.indexer.ByIndex(pluralIndex, plural) + if err != nil { + return "", err + } + var kinds []string + for _, obj := range objs { + kind, _, _ := unstructured.NestedString(obj.(*unstructured.Unstructured).Object, "spec", "application", "kind") + if kind != "" && !slices.Contains(kinds, kind) { + kinds = append(kinds, kind) + } + } + switch len(kinds) { + case 0: + return "", nil + case 1: + return kinds[0], nil + } + // Picking one would scope the request to whichever definition the + // cache happens to list first. + slices.Sort(kinds) + return "", fmt.Errorf("plural %q is declared for kinds %v", plural, kinds) +} diff --git a/internal/observabilityproxy/appkinds_test.go b/internal/observabilityproxy/appkinds_test.go new file mode 100644 index 000000000..6eb8d60bc --- /dev/null +++ b/internal/observabilityproxy/appkinds_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "errors" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/tools/cache" +) + +func appDefinition(name, kind, plural string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "cozystack.io/v1alpha1", + "kind": "ApplicationDefinition", + "metadata": map[string]any{"name": name}, + "spec": map[string]any{"application": map[string]any{"kind": kind, "plural": plural, "singular": name}}, + }} +} + +func newTestKinds(t *testing.T, synced bool, defs ...*unstructured.Unstructured) *ApplicationKinds { + t.Helper() + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, ApplicationPluralIndexers()) + for _, d := range defs { + if err := indexer.Add(d); err != nil { + t.Fatal(err) + } + } + return NewApplicationKinds(indexer, func() bool { return synced }) +} + +func TestApplicationKinds(t *testing.T) { + kinds := newTestKinds(t, true, + appDefinition("postgres", "Postgres", "postgreses"), + appDefinition("vm-instance", "VMInstance", "vminstances"), + appDefinition("broken", "", "brokens"), + ) + for plural, want := range map[string]string{"postgreses": "Postgres", "vminstances": "VMInstance", "postgres": "", "brokens": "", "": ""} { + got, err := kinds.Kind(plural) + if err != nil || got != want { + t.Errorf("Kind(%q) = %q, %v; want %q", plural, got, err, want) + } + } +} + +func TestApplicationKindsNotSynced(t *testing.T) { + kinds := newTestKinds(t, false, appDefinition("postgres", "Postgres", "postgreses")) + if _, err := kinds.Kind("postgreses"); !errors.Is(err, ErrKindsNotSynced) { + t.Fatalf("Kind before sync: err = %v, want ErrKindsNotSynced", err) + } +} + +func TestApplicationKindsAmbiguous(t *testing.T) { + kinds := newTestKinds(t, true, appDefinition("a", "Postgres", "postgreses"), appDefinition("b", "Postgresql", "postgreses")) + if kind, err := kinds.Kind("postgreses"); err == nil { + t.Fatalf("two kinds for one plural resolved to %q", kind) + } + same := newTestKinds(t, true, appDefinition("a", "Postgres", "postgreses"), appDefinition("b", "Postgres", "postgreses")) + if kind, err := same.Kind("postgreses"); err != nil || kind != "Postgres" { + t.Fatalf("duplicate definitions of one kind: %q, %v", kind, err) + } +} From 2f600d5354f672b5e53d382c248d867624298dc2 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:59:44 +0700 Subject: [PATCH 02/41] feat(observability-proxy): serve the application logs root A viewer of one application gets that application's lines without writing a namespace query and filtering by hand, and a grant narrowed to one application by resourceNames is honoured on every logs endpoint, tail and the pickers included. The namespace filter stays and the application's identity is ANDed next to it as a second stream filter, so the result can only be narrower than the namespace root. The plural is mapped to its kind only after the caller is authorized, so an anonymous client cannot probe which applications exist, and the proxy is not ready until the ApplicationDefinitions cache is. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 20 ++- internal/observabilityproxy/server.go | 43 +++++-- .../observabilityproxy/server_app_test.go | 114 ++++++++++++++++++ .../observabilityproxy/server_metrics_test.go | 2 +- internal/observabilityproxy/server_test.go | 21 +++- .../observability-proxy/templates/rbac.yaml | 5 + .../observability-proxy/tests/rbac_test.yaml | 5 +- 7 files changed, 192 insertions(+), 18 deletions(-) create mode 100644 internal/observabilityproxy/server_app_test.go diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index a3687fe08..1de6e6e14 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -39,6 +39,8 @@ import ( "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" @@ -178,13 +180,24 @@ func run(o options) error { return fmt.Errorf("building client: %w", err) } + dynClient, err := dynamic.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("building dynamic client: %w", err) + } + factory := informers.NewSharedInformerFactory(client, 0) namespaces := factory.Core().V1().Namespaces() lister := namespaces.Lister() + appDefs := dynamicinformer.NewDynamicSharedInformerFactory(dynClient, 0). + ForResource(observabilityproxy.ApplicationDefinitionsResource).Informer() + if err := appDefs.AddIndexers(observabilityproxy.ApplicationPluralIndexers()); err != nil { + return fmt.Errorf("indexing application definitions: %w", err) + } factory.Start(ctx.Done()) + go appDefs.Run(ctx.Done()) var ready atomic.Bool go func() { - if cache.WaitForCacheSync(ctx.Done(), namespaces.Informer().HasSynced) { + if cache.WaitForCacheSync(ctx.Done(), namespaces.Informer().HasSynced, appDefs.HasSynced) { ready.Store(true) } }() @@ -198,13 +211,14 @@ func run(o options) error { Limiter: o.limiter, RequestTimeout: o.requestTimeout, TailIdleTimeout: o.tailIdleTimeout, - }, observabilityproxy.NewAuthorizer(client, o.authCacheTTL), observabilityproxy.NewStoreResolver(lister, namespaces.Informer().HasSynced, o.rootTenant), reg) + }, observabilityproxy.NewAuthorizer(client, o.authCacheTTL), observabilityproxy.NewStoreResolver(lister, namespaces.Informer().HasSynced, o.rootTenant), + observabilityproxy.NewApplicationKinds(appDefs.GetIndexer(), appDefs.HasSynced), reg) probes := http.NewServeMux() probes.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "ok") }) probes.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { if !ready.Load() { - http.Error(w, "namespace cache not synced", http.StatusServiceUnavailable) + http.Error(w, "namespace or application definition cache not synced", http.StatusServiceUnavailable) return } _, _ = io.WriteString(w, "ok") diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index eb540a271..617971bf2 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -73,13 +73,14 @@ type Server struct { cfg Config auth *Authorizer stores StoreLookup + kinds KindLookup client *http.Client limits *limiter now func() time.Time requests *prometheus.CounterVec } -func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus.Registerer) *Server { +func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, kinds KindLookup, reg prometheus.Registerer) *Server { if cfg.UpstreamURL == nil { domain := cfg.ClusterDomain cfg.UpstreamURL = func(store string) string { @@ -96,6 +97,7 @@ func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus. cfg: cfg, auth: auth, stores: stores, + kinds: kinds, // No client timeout: it would cut live tails. Non-tail requests are // bounded by a context deadline instead. client: &http.Client{Transport: http.DefaultTransport.(*http.Transport).Clone()}, @@ -172,12 +174,7 @@ func writeError(w http.ResponseWriter, e *HTTPError) { http.Error(w, e.Message, e.Code) } -// serve answers a root it has no handler for before authenticating, so an -// unknown path costs no review calls. func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTTPError { - if route.Kind != NamespaceLogs && route.Kind != NamespaceMetrics { - return errorf(http.StatusNotFound, "not found") - } user, herr := s.authorize(r, route) if herr != nil { return herr @@ -190,8 +187,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT w.Header().Set("Retry-After", strconv.FormatInt(int64(math.Ceil(wait.Seconds())), 10)) return errorf(http.StatusTooManyRequests, "request rate limit exceeded") } - if route.Kind == NamespaceMetrics { + var kind string + if route.Kind == ApplicationLog || route.Kind == ApplicationMetrics { + if kind, herr = s.applicationKind(route.Plural); herr != nil { + return herr + } + } + switch route.Kind { + case NamespaceMetrics: return s.serveMetrics(w, r, route, user) + case ApplicationMetrics: + return errorf(http.StatusNotFound, "not found") } endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/") if !ok { @@ -214,6 +220,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT if herr != nil { return herr } + if kind != "" { + params.Add("extra_stream_filters", applicationStreamFilter(kind, route.Name)) + } store, err := s.stores.Resolve(route.Namespace) if err != nil { @@ -322,6 +331,26 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout return nil } +func (s *Server) applicationKind(plural string) (string, *HTTPError) { + kind, err := s.kinds.Kind(plural) + if err != nil { + slog.Error("mapping application plural to kind", "plural", plural, "error", err) + return "", errorf(http.StatusServiceUnavailable, "cannot resolve the application kind") + } + if kind == "" { + return "", errorf(http.StatusNotFound, "unknown application resource %q", plural) + } + return kind, nil +} + +// applicationStreamFilter selects the streams Fluent Bit labels with the +// pod's lineage. vlselect ANDs each extra_stream_filters value into the +// query and every subquery (Query.AddExtraFilters in lib/logstorage), so +// it narrows the namespace filter sent next to it. +func applicationStreamFilter(kind, name string) string { + return "{apps_cozystack_io_application_kind=" + strconv.Quote(kind) + ",apps_cozystack_io_application_name=" + strconv.Quote(name) + "}" +} + // post sends params as a form with only the given headers, so nothing the // client sent reaches the store, and tells an unreachable store (502) from // one that missed its deadline or, for a tail, its idle timeout (504). diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go new file mode 100644 index 000000000..0a76ab9a5 --- /dev/null +++ b/internal/observabilityproxy/server_app_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "io" + "net/http" + "net/url" + "slices" + "testing" + + authorizationv1 "k8s.io/api/authorization/v1" +) + +const appRoot = "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/" + +// appPolicy lets alice read every postgres in tenant-a, as the view role's +// apps.cozystack.io wildcard does, and bob only "a", as a grant narrowed +// with resourceNames does. +func appPolicy(user string, a *authorizationv1.ResourceAttributes) bool { + if a.Namespace != "tenant-a" || a.Verb != "get" || a.Group != "apps.cozystack.io" || a.Resource != "postgreses" || + (a.Subresource != "log" && a.Subresource != "metrics") { + return false + } + return user == "alice" || a.Name == "a" +} + +func TestServerApplicationLogsScope(t *testing.T) { + h := newPolicyHarness(t, Config{}, nil, appPolicy, nil) + hostile := url.Values{ + "query": {"error"}, + "field": {"level"}, + "extra_filters": {`{kubernetes_namespace_name="tenant-b"}`}, + "extra_stream_filters": {`{apps_cozystack_io_application_name=~".*"}`}, + "AccountID": {"9"}, + } + headers := map[string]string{"AccountID": "9", "ProjectID": "9"} + endpoints := []string{"query", "hits", "stats_query_range", "tail", "field_values", "stream_field_values", "facets"} + for _, ep := range endpoints { + resp := h.do(http.MethodPost, appRoot+"db.main/log/select/logsql/"+ep+"?extra_stream_filters=x", "good", hostile, headers) + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("%s answered %d", ep, resp.StatusCode) + } + } + want := authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "log", Name: "db.main", Namespace: "tenant-a"} + if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want { + t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want) + } + reqs := h.requests() + if len(reqs) != len(endpoints) { + t.Fatalf("upstream saw %d requests, want %d", len(reqs), len(endpoints)) + } + // Two values: vlselect ANDs every extra_stream_filters value into the + // query and each of its subqueries. + wantFilters := []string{wantScope, `{apps_cozystack_io_application_kind="Postgres",apps_cozystack_io_application_name="db.main"}`} + for i, r := range reqs { + if r.path != "/select/logsql/"+endpoints[i] { + t.Errorf("upstream path = %s, want %s", r.path, endpoints[i]) + } + if got := r.form["extra_stream_filters"]; !slices.Equal(got, wantFilters) { + t.Errorf("%s: extra_stream_filters = %q, want %q", endpoints[i], got, wantFilters) + } + if r.form.Has("extra_filters") || r.form.Has("AccountID") || r.headers.Get("AccountID") != "0" || r.headers.Get("ProjectID") != "0" { + t.Errorf("%s: client scope reached the store: %v %v", endpoints[i], r.form, r.headers) + } + } +} + +func TestServerApplicationKinds(t *testing.T) { + tests := []struct { + name string + kinds KindLookup + path string + want int + }{ + {"unknown plural logs", nil, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/mysqls/a/log/select/logsql/query", http.StatusNotFound}, + {"unknown plural metrics", nil, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/mysqls/a/metrics/api/v1/query?query=up", http.StatusNotFound}, + {"not synced logs", newTestKinds(t, false), appRoot + "a/log/select/logsql/query", http.StatusServiceUnavailable}, + {"not synced metrics", newTestKinds(t, false), appRoot + "a/metrics/api/v1/query?query=up", http.StatusServiceUnavailable}, + {"ambiguous plural", newTestKinds(t, true, appDefinition("x", "Postgres", "postgreses"), appDefinition("y", "Pg", "postgreses")), appRoot + "a/log/select/logsql/query", http.StatusServiceUnavailable}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := newPolicyHarness(t, Config{}, nil, func(string, *authorizationv1.ResourceAttributes) bool { return true }, tt.kinds) + if got := h.status(http.MethodGet, tt.path, "good"); got != tt.want { + t.Fatalf("status = %d, want %d", got, tt.want) + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests", n) + } + }) + } + // The kind is looked up only for an authorized caller. + h := newPolicyHarness(t, Config{}, nil, appPolicy, newTestKinds(t, false)) + if got := h.status(http.MethodGet, appRoot+"b/log/select/logsql/query", "bob-token"); got != http.StatusForbidden { + t.Fatalf("denied caller answered %d, want 403", got) + } +} diff --git a/internal/observabilityproxy/server_metrics_test.go b/internal/observabilityproxy/server_metrics_test.go index 1ef5e761e..1a3d1de10 100644 --- a/internal/observabilityproxy/server_metrics_test.go +++ b/internal/observabilityproxy/server_metrics_test.go @@ -203,7 +203,7 @@ func TestServerMetricsConcurrency(t *testing.T) { } func TestDefaultMetricsUpstreamURL(t *testing.T) { - s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil) + s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil, nil) if got, want := s.cfg.Metrics.UpstreamURL("shortterm"), "http://vmselect-shortterm.tenant-root.svc.cozy.local:8481/select/0/prometheus"; got != want { t.Fatalf("upstream = %s, want %s", got, want) } diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index 2670d0e3e..b541a18e1 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -55,10 +55,19 @@ func (m mapStores) Resolve(ns string) (string, error) { return m[ns], nil } // (alice) and "bob-token" (bob), both allowed list logs and metrics in // tenant-a only, and to an upstream served by handler. func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { - h := &harness{t: t} - h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, func(_ string, a *authorizationv1.ResourceAttributes) bool { + return newPolicyHarness(t, cfg, handler, func(_ string, a *authorizationv1.ResourceAttributes) bool { return a.Namespace == "tenant-a" && (a.Resource == "logs" || a.Resource == "metrics") - }) + }, nil) +} + +// newPolicyHarness is newHarness with the given SubjectAccessReview policy +// and plural-to-kind mapping; nil kinds knows only postgreses. +func newPolicyHarness(t *testing.T, cfg Config, handler http.HandlerFunc, allow func(string, *authorizationv1.ResourceAttributes) bool, kinds KindLookup) *harness { + h := &harness{t: t} + h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, allow) + if kinds == nil { + kinds = newTestKinds(t, true, appDefinition("postgres", "Postgres", "postgreses")) + } h.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { t.Errorf("upstream ParseForm: %v", err) @@ -90,7 +99,7 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { if cfg.Logs == (LogsLimits{}) { cfg.Logs = testLimits() } - h.server = NewServer(cfg, NewAuthorizer(h.kube.client, 10*time.Second), mapStores{"tenant-a": "tenant-a-store"}, nil) + h.server = NewServer(cfg, NewAuthorizer(h.kube.client, 10*time.Second), mapStores{"tenant-a": "tenant-a-store"}, kinds, nil) h.server.now = func() time.Time { return testNow } h.proxy = httptest.NewServer(h.server) t.Cleanup(h.proxy.Close) @@ -154,8 +163,8 @@ func TestServerStatusCodes(t *testing.T) { {"unknown metrics endpoint", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/status/tsdb", "good", http.StatusNotFound}, {"metrics in another namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/metrics/api/v1/query?query=up", "good", http.StatusForbidden}, {"metrics without token", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query?query=up", "", http.StatusUnauthorized}, - {"application log not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusNotFound}, - {"application metrics not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusNotFound}, + {"application log without a grant", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusForbidden}, + {"application metrics without a grant", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusForbidden}, {"tenant_ids not proxied", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/tenant_ids", "good", http.StatusNotFound}, {"unknown logsql endpoint", http.MethodGet, logsRoot + "stats_query", "good", http.StatusNotFound}, {"logs root itself", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/", "good", http.StatusNotFound}, diff --git a/packages/system/observability-proxy/templates/rbac.yaml b/packages/system/observability-proxy/templates/rbac.yaml index 3c5392903..f3f67127c 100644 --- a/packages/system/observability-proxy/templates/rbac.yaml +++ b/packages/system/observability-proxy/templates/rbac.yaml @@ -15,6 +15,11 @@ rules: - apiGroups: [""] resources: ["namespaces"] verbs: ["list", "watch"] +# Application roots map the plural in the path to the kind the lineage +# labels carry, from an informer cache as well. +- apiGroups: ["cozystack.io"] + resources: ["applicationdefinitions"] + verbs: ["list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/packages/system/observability-proxy/tests/rbac_test.yaml b/packages/system/observability-proxy/tests/rbac_test.yaml index ef96c65d7..5382c2ab6 100644 --- a/packages/system/observability-proxy/tests/rbac_test.yaml +++ b/packages/system/observability-proxy/tests/rbac_test.yaml @@ -12,7 +12,7 @@ release: namespace: cozy-observability-proxy tests: - - it: lets the proxy review tokens and access, and list and watch namespaces + - it: lets the proxy review tokens and access, and list and watch namespaces and application definitions template: templates/rbac.yaml documentSelector: path: kind @@ -30,6 +30,9 @@ tests: - apiGroups: [""] resources: ["namespaces"] verbs: ["list", "watch"] + - apiGroups: ["cozystack.io"] + resources: ["applicationdefinitions"] + verbs: ["list", "watch"] - it: binds the proxy role to the proxy ServiceAccount in the release namespace template: templates/rbac.yaml From dcdf0adab3171c0139766f0999cfb975d0ff97c1 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 06:03:40 +0700 Subject: [PATCH 03/41] feat(observability-proxy): serve the application metrics root Graphs of one application need its series without the caller knowing which pods and volumes it ran on. Exporter series carry the identity the scrape class stamps, but cAdvisor and kubelet volume series do not, so the scope ORs three branches: the identity, and the application's pods and PVCs as kube-state-metrics labelled them over the window. Each branch also carries the namespace, so the scope is never wider than the namespace root. The lookup reaches back by the query's own lookback, bounded by a flag, so pods replaced inside an expression's range still count. A failed lookup only drops its branch. A list over its cap answers 422 instead of being truncated, which would hide series without saying so. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 5 + cmd/observability-proxy/main_test.go | 10 +- internal/observabilityproxy/appscope.go | 158 ++++++++++++++++ internal/observabilityproxy/appscope_test.go | 116 ++++++++++++ internal/observabilityproxy/metricsparams.go | 20 ++- internal/observabilityproxy/server.go | 18 +- .../observabilityproxy/server_app_test.go | 170 ++++++++++++++++++ internal/observabilityproxy/server_test.go | 3 + 8 files changed, 492 insertions(+), 8 deletions(-) create mode 100644 internal/observabilityproxy/appscope.go create mode 100644 internal/observabilityproxy/appscope_test.go diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 1de6e6e14..4a99aa4cc 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -94,6 +94,9 @@ func parseFlags(args []string) (options, error) { fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use") fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return") fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") + fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs at most") + fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") + fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables") fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user") @@ -134,6 +137,8 @@ func parseFlags(args []string) (options, error) { return o, errors.New("--metrics-shortterm-retention, --metrics-max-range and --metrics-max-timeout must be positive") case o.metrics.Limits.MaxPoints < 2: return o, errors.New("--metrics-max-points must be at least 2") + case o.metrics.App.Lookback < 0 || o.metrics.App.MaxPods < 1 || o.metrics.App.MaxPVCs < 1: + return o, errors.New("--metrics-app-lookback must not be negative and --metrics-app-max-pods and --metrics-app-max-pvcs must be positive") case o.logs.MaxTailResume < 0: return o, errors.New("--logs-max-tail-resume must not be negative") // The proxy's deadline starts before the request reaches vmselect, so diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go index bf7ac396b..6b5c60d7d 100644 --- a/cmd/observability-proxy/main_test.go +++ b/cmd/observability-proxy/main_test.go @@ -40,6 +40,9 @@ func TestParseFlagsDefaults(t *testing.T) { if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second { t.Errorf("metrics limit defaults changed: %+v", m.Limits) } + if m.App.Lookback != 24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 { + t.Errorf("application scope defaults changed: %+v", m.App) + } if m.RootTenant != o.rootTenant { t.Errorf("metrics root tenant = %q, want --root-tenant", m.RootTenant) } @@ -63,13 +66,15 @@ func TestParseFlagsMetricsOverrides(t *testing.T) { o, err := parseFlags([]string{ "--root-tenant=tenant-x", "--metrics-shortterm-store=fast", "--metrics-longterm-store=slow", "--metrics-shortterm-retention=24h", "--metrics-max-range=168h", "--metrics-max-points=500", "--metrics-max-timeout=10s", + "--metrics-app-lookback=0s", "--metrics-app-max-pods=7", "--metrics-app-max-pvcs=3", }) if err != nil { t.Fatal(err) } m := o.metrics if m.RootTenant != "tenant-x" || m.ShortTermStore != "fast" || m.LongTermStore != "slow" || m.ShortTermRetention != 24*time.Hour || - m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second { + m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second || + m.App.Lookback != 0 || m.App.MaxPods != 7 || m.App.MaxPVCs != 3 { t.Errorf("overrides not applied: %+v", m) } } @@ -101,6 +106,9 @@ func TestParseFlagsInvalid(t *testing.T) { {"--metrics-max-range=0s"}, {"--metrics-max-points=1"}, {"--metrics-max-timeout=0s"}, + {"--metrics-app-lookback=-1h"}, + {"--metrics-app-max-pods=0"}, + {"--metrics-app-max-pvcs=0"}, {"--unknown"}, // The proxy would cut the request while vmselect is still within // the timeout it was given. diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go new file mode 100644 index 000000000..b969cc502 --- /dev/null +++ b/internal/observabilityproxy/appscope.go @@ -0,0 +1,158 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +// AppScopeLimits bound the pod and PVC lookup of the application metrics +// root. +type AppScopeLimits struct { + // Lookback is how far before the window the lookup reaches at most, so + // pods gone before the window but inside an expression's range count. + Lookback time.Duration + MaxPods int + MaxPVCs int +} + +// maxLookupBytes bounds a lookup answer; the caps keep a valid one far +// below it. +const maxLookupBytes = 8 << 20 + +// The lineage labels as the scrape class writes them on series, and as +// kube-state-metrics writes them on kube_pod_labels and +// kube_persistentvolumeclaim_labels. +const ( + seriesKindLabel = "apps_cozystack_io_application_kind" + seriesNameLabel = "apps_cozystack_io_application_name" + ksmKindLabel = "label_" + seriesKindLabel + ksmNameLabel = "label_" + seriesNameLabel +) + +func identitySelector(ns, kindLabel, nameLabel, kind, name string) string { + return "{namespace=" + strconv.Quote(ns) + "," + kindLabel + "=" + strconv.Quote(kind) + "," + nameLabel + "=" + strconv.Quote(name) + "}" +} + +// nameBranch matches label against any of names within ns. With no names +// it is "": a branch of pod=~"^()$" would match every series without a pod +// label. +func nameBranch(ns, label string, names []string) string { + if len(names) == 0 { + return "" + } + quoted := make([]string, len(names)) + for i, n := range names { + quoted[i] = regexp.QuoteMeta(n) + } + return "{namespace=" + strconv.Quote(ns) + "," + label + "=~" + strconv.Quote("^("+strings.Join(quoted, "|")+")$") + "}" +} + +// applicationSelectors is the application scope: series stamped with the +// application's identity, plus series of its pods and PVCs for those not +// stamped (cAdvisor, kubelet volume stats). Every branch carries the +// namespace, so their OR stays inside it. +// +// Within the namespace, both the identity and the pod and PVC lists come +// from lineage labels on pods and PVCs. They separate applications for a +// grant narrowed by resourceNames only while whoever creates a pod or PVC +// cannot choose those labels; the lineage webhook keeps them as set on an +// object it finds no owner for, and the logs scope rests on the same. +func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { + out := []string{identitySelector(ns, seriesKindLabel, seriesNameLabel, kind, name)} + for _, b := range []string{nameBranch(ns, "pod", pods), nameBranch(ns, "persistentvolumeclaim", pvcs)} { + if b != "" { + out = append(out, b) + } + } + return out +} + +// applicationScope looks the application's pods and PVCs up in store over +// the request window, reaching back by the query's own lookback up to the +// configured limit. A failed lookup drops its branch, which only narrows +// the answer; a list over its cap is refused, because truncating it would +// silently drop series. +func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { + lim := s.cfg.Metrics.App + start := req.from.Add(-lim.Lookback) + if req.start.After(start) { + start = req.start + } + var lists [2][]string + for i, l := range []struct { + label, metric string + max int + }{{"pod", "kube_pod_labels", lim.MaxPods}, {"persistentvolumeclaim", "kube_persistentvolumeclaim_labels", lim.MaxPVCs}} { + match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name) + names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.to, l.max) + if err != nil { + slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) + continue + } + if len(names) > l.max { + return nil, errorf(http.StatusUnprocessableEntity, "the application has more than %d %ss in this window; use a shorter range", l.max, l.label) + } + lists[i] = names + } + return applicationSelectors(ns, kind, name, lists[0], lists[1]), nil +} + +// lookupNames asks for at most max+1 values, enough to tell a list over +// the cap without reading all of it. +func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, match, ns string, start, end time.Time, max int) ([]string, error) { + params := url.Values{} + params.Set("match[]", match) + params.Set("extra_filters[]", namespaceSelector(ns)) + params.Set("start", formatTime(start)) + params.Set("end", formatTime(end)) + params.Set("limit", strconv.Itoa(max+1)) + resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/api/v1/label/"+label+"/values", params, "metrics store", nil) + if herr != nil { + return nil, herr + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status) + } + var body struct { + Status string `json:"status"` + Data []string `json:"data"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, maxLookupBytes)).Decode(&body); err != nil { + return nil, err + } + if body.Status != "success" { + return nil, errorf(http.StatusBadGateway, "metrics store answered status %q", body.Status) + } + names := body.Data[:0] + for _, n := range body.Data { + if n != "" { + names = append(names, n) + } + } + return names, nil +} diff --git a/internal/observabilityproxy/appscope_test.go b/internal/observabilityproxy/appscope_test.go new file mode 100644 index 000000000..af028b867 --- /dev/null +++ b/internal/observabilityproxy/appscope_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "net/url" + "regexp" + "slices" + "strconv" + "testing" + "time" +) + +const wantIdentityBranch = `{namespace="tenant-a",apps_cozystack_io_application_kind="Postgres",apps_cozystack_io_application_name="db.main"}` + +func TestApplicationSelectors(t *testing.T) { + tests := []struct { + name string + pods, pvcs []string + want []string + }{ + {"identity only", nil, nil, []string{wantIdentityBranch}}, + {"pods", []string{"db-main-1", "db.main-2"}, nil, []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(db-main-1|db\\.main-2)$"}`, + }}, + {"pvcs", nil, []string{"data-db-main-1"}, []string{ + wantIdentityBranch, + `{namespace="tenant-a",persistentvolumeclaim=~"^(data-db-main-1)$"}`, + }}, + {"all three", []string{"p"}, []string{"v"}, []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(p)$"}`, + `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := applicationSelectors("tenant-a", "Postgres", "db.main", tt.pods, tt.pvcs); !slices.Equal(got, tt.want) { + t.Fatalf("got %q\nwant %q", got, tt.want) + } + }) + } +} + +// DNS-1123 keeps most metacharacters out of names, but the names come +// back from the store, so the encoder must not rely on that. +func TestNameBranchEscapes(t *testing.T) { + names := []string{`a.b`, `x|.*`, `)|(.*`, `q"} or {namespace=~".+`, `back\slash`, `$^+?[]{}`} + sel := nameBranch("tenant-a", "pod", names) + m := regexp.MustCompile(`^\{namespace="tenant-a",pod=~(".*")\}$`).FindStringSubmatch(sel) + if m == nil { + t.Fatalf("selector %s is not one namespace and one pod matcher", sel) + } + pattern, err := strconv.Unquote(m[1]) + if err != nil { + t.Fatalf("pod matcher %s does not unquote: %v", m[1], err) + } + re := regexp.MustCompile(pattern) + for _, n := range names { + if !re.MatchString(n) { + t.Errorf("%q does not match its own branch", n) + } + } + for _, other := range []string{"axb", "x", "anything", "tenant-b", "", `back\\slash`} { + if re.MatchString(other) { + t.Errorf("branch matches %q", other) + } + } + if nameBranch("tenant-a", "pod", nil) != "" { + t.Error("an empty list produced a branch, which would match series without the label") + } +} + +func TestMetricsRequestScope(t *testing.T) { + branches := []string{"{a}", "{b}"} + labels := buildMetrics(t, "api/v1/labels", url.Values{}) + labels.setScope(branches) + if got := labels.params["extra_filters[]"]; !slices.Equal(got, branches) { + t.Errorf("extra_filters[] = %q", got) + } + if got := labels.params["match[]"]; !slices.Equal(got, branches) { + t.Errorf("default match[] = %q, want the branches", got) + } + client := buildMetrics(t, "api/v1/labels", url.Values{"match[]": {"up"}}) + client.setScope(branches) + if got := client.params["match[]"]; !slices.Equal(got, []string{"up"}) { + t.Errorf("client match[] replaced: %q", got) + } + query := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}) + query.setScope(branches) + if query.params.Has("match[]") || !slices.Equal(query.params["extra_filters[]"], branches) { + t.Errorf("query params = %v", query.params) + } + if !query.from.Equal(testNow) || !query.to.Equal(testNow) { + t.Errorf("instant window = [%s, %s], want now", query.from, query.to) + } + rng := buildMetrics(t, "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}) + if !rng.from.Equal(testNow.Add(-time.Hour)) || !rng.to.Equal(testNow) { + t.Errorf("range window = [%s, %s]", rng.from, rng.to) + } +} diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index 8beb347d0..a0bb42f0d 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -46,6 +46,7 @@ type MetricsConfig struct { LongTermStore string ShortTermRetention time.Duration Limits MetricsLimits + App AppScopeLimits // UpstreamURL maps a store name to its Prometheus API base URL, without // a trailing slash; nil means the in-cluster vmselect Service. UpstreamURL func(store string) string @@ -77,6 +78,20 @@ type metricsRequest struct { // start is the earliest instant the request reads, for the store choice; // zero when that is unknown, which picks the long-term store. start time.Time + // from and to are the window the query is evaluated over, without the + // lookback of its expression. + from, to time.Time + // scopedMatch is set when match[] holds the scope, not client matchers. + scopedMatch bool +} + +// setScope replaces the namespace scope with the given selectors, which +// vmselect ORs. +func (req *metricsRequest) setScope(selectors []string) { + req.params["extra_filters[]"] = selectors + if req.scopedMatch { + req.params["match[]"] = selectors + } } // namespaceSelector quotes ns the way MetricsQL unquotes a label value @@ -127,6 +142,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti out.Set("step", formatSeconds(step)) } req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step) + req.from, req.to = at, at return req, nil case "api/v1/query_range": @@ -150,6 +166,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints) } req.start = queryStart(out.Get("query"), start, step) + req.from, req.to = start, end out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) out.Set("step", formatSeconds(step)) @@ -172,7 +189,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti if herr != nil { return req, herr } - req.start = start + req.start, req.from, req.to = start, start, end out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) @@ -194,6 +211,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti // extra_filters[] on the label APIs unless a match[] is present // (getCommonParamsInternal), so the scope is repeated here. out.Set("match[]", namespaceSelector(namespace)) + req.scopedMatch = true } return req, nil } diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index 617971bf2..fd74491b5 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -193,11 +193,8 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT return herr } } - switch route.Kind { - case NamespaceMetrics: - return s.serveMetrics(w, r, route, user) - case ApplicationMetrics: - return errorf(http.StatusNotFound, "not found") + if route.Kind == NamespaceMetrics || route.Kind == ApplicationMetrics { + return s.serveMetrics(w, r, route, user, kind) } endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/") if !ok { @@ -288,7 +285,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT return nil } -func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError { +// serveMetrics scopes to the namespace, or with kind set to the application +// route.Name of that kind within it. +func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user, kind string) *HTTPError { release, ok := s.limits.acquire(user) if !ok { return tooManyConcurrent(w) @@ -310,6 +309,13 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout defer cancel() } store := s.cfg.Metrics.store(req.start, now) + if kind != "" { + selectors, herr := s.applicationScope(ctx, r, store, req, route.Namespace, kind, route.Name) + if herr != nil { + return herr + } + req.setScope(selectors) + } resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/"+req.path, req.params, "metrics store", nil) if herr != nil { return herr diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 0a76ab9a5..14807637e 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -17,11 +17,14 @@ limitations under the License. package observabilityproxy import ( + "encoding/json" "io" "net/http" "net/url" "slices" + "strings" "testing" + "time" authorizationv1 "k8s.io/api/authorization/v1" ) @@ -112,3 +115,170 @@ func TestServerApplicationKinds(t *testing.T) { t.Fatalf("denied caller answered %d, want 403", got) } } + +const wantPodLookup = `kube_pod_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}` + +// appMetricsUpstream answers the pod and PVC lookups with the given names, +// or with status when it is not 200, and every other path like vmselect. +func appMetricsUpstream(pods, pvcs []string, podStatus int) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + names := map[string][]string{"/api/v1/label/pod/values": pods, "/api/v1/label/persistentvolumeclaim/values": pvcs} + list, lookup := names[r.URL.Path] + if !lookup { + promUpstream(w, r) + return + } + if r.URL.Path == "/api/v1/label/pod/values" && podStatus != http.StatusOK { + http.Error(w, "unavailable", podStatus) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"status": "success", "data": list}) + } +} + +func TestServerApplicationMetricsScope(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"db-main-0", "db.main-1"}, []string{"data-db-main-0"}, http.StatusOK), appPolicy, nil) + hostile := url.Values{ + "query": {"sum(rate(container_cpu_usage_seconds_total[5m]))"}, + "start": {unix(testNow.Add(-time.Hour))}, + "end": {unix(testNow)}, + "step": {"60"}, + "extra_filters": {`{namespace=~".+"}`}, + "extra_filters[]": {`{namespace="tenant-a"}`}, + "extra_label": {"apps_cozystack_io_application_name=b"}, + } + resp := h.do(http.MethodPost, appRoot+"db.main/metrics/api/v1/query_range", "good", hostile, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("answered %d %q", resp.StatusCode, body) + } + want := authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "metrics", Name: "db.main", Namespace: "tenant-a"} + if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want { + t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want) + } + reqs := h.requests() + if len(reqs) != 3 { + t.Fatalf("upstream saw %d requests, want two lookups and the query", len(reqs)) + } + lookups := map[string]string{ + "/api/v1/label/pod/values": wantPodLookup, + "/api/v1/label/persistentvolumeclaim/values": `kube_persistentvolumeclaim_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}`, + } + for _, r := range reqs[:2] { + if got := r.form["match[]"]; len(got) != 1 || got[0] != lookups[r.path] { + t.Errorf("%s match[] = %q", r.path, got) + } + if got := r.form["extra_filters[]"]; !slices.Equal(got, []string{wantMetricsScope}) { + t.Errorf("%s extra_filters[] = %q", r.path, got) + } + start, herr := parseMetricsTime("start", r.form.Get("start")) + if herr != nil || start.After(testNow.Add(-time.Hour)) || start.Before(testNow.Add(-25*time.Hour)) { + t.Errorf("%s start = %s, want within the lookback before the window", r.path, r.form.Get("start")) + } + if r.form.Get("end") != formatTime(testNow) || r.form.Get("limit") != "101" { + t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit")) + } + } + main := reqs[2] + wantBranches := []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(db-main-0|db\\.main-1)$"}`, + `{namespace="tenant-a",persistentvolumeclaim=~"^(data-db-main-0)$"}`, + } + if main.path != "/api/v1/query_range" || !slices.Equal(main.form["extra_filters[]"], wantBranches) { + t.Fatalf("query %s extra_filters[] = %q, want %q", main.path, main.form["extra_filters[]"], wantBranches) + } + if main.form.Has("extra_filters") || main.form.Has("extra_label") { + t.Fatalf("client scope reached the store: %v", main.form) + } + if !slices.Equal(h.stores, []string{"shortterm", "shortterm", "shortterm"}) { + t.Fatalf("stores = %q, want the lookups on the query's store", h.stores) + } +} + +func TestServerApplicationMetricsBranches(t *testing.T) { + podBranch := `{namespace="tenant-a",pod=~"^(p)$"}` + tests := []struct { + name string + pods, pvcs []string + podStatus int + cfg Config + path string + wantCode int + want []string + }{ + {name: "no pods or pvcs", path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, + {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusOK, + want: []string{wantIdentityBranch, `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`}}, + {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, + path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, + {name: "pvcs over the cap", pods: []string{"p"}, pvcs: []string{"v", "w"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, + path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, + {name: "labels get the branches as match[]", pods: []string{"p"}, path: "api/v1/labels", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, + {name: "series needs match[]", path: "api/v1/series", wantCode: http.StatusBadRequest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.podStatus == 0 { + tt.podStatus = http.StatusOK + } + h := newPolicyHarness(t, tt.cfg, appMetricsUpstream(tt.pods, tt.pvcs, tt.podStatus), appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tt.path, "good"); got != tt.wantCode { + t.Fatalf("status = %d, want %d", got, tt.wantCode) + } + var main *upstreamRequest + for _, r := range h.requests() { + if !strings.HasPrefix(r.path, "/api/v1/label/pod/") && !strings.HasPrefix(r.path, "/api/v1/label/persistentvolumeclaim/") { + main = &r + } + } + if tt.want == nil { + if main != nil { + t.Fatalf("query reached the store: %s %v", main.path, main.form) + } + return + } + if main == nil || !slices.Equal(main.form["extra_filters[]"], tt.want) { + t.Fatalf("extra_filters[] = %v, want %q", main, tt.want) + } + if strings.HasSuffix(tt.path, "labels") && !slices.Equal(main.form["match[]"], tt.want) { + t.Fatalf("match[] = %q, want the branches", main.form["match[]"]) + } + }) + } +} + +// A grant narrowed to application "a" by resourceNames must not reach "b" +// through any endpoint of either root. +func TestServerApplicationNarrowedGrant(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"a-0"}, nil, http.StatusOK), appPolicy, nil) + paths := []string{ + "metrics/api/v1/query?query=up", + "metrics/api/v1/query_range?query=up&start=" + unix(testNow.Add(-time.Hour)) + "&end=" + unix(testNow) + "&step=60", + "metrics/api/v1/series?match[]=up", + "metrics/api/v1/labels", + "metrics/api/v1/label/pod/values", + "log/select/logsql/query", + "log/select/logsql/hits", + "log/select/logsql/stats_query_range?query=" + url.QueryEscape("* | stats count()"), + "log/select/logsql/tail", + "log/select/logsql/field_values?field=level", + "log/select/logsql/stream_field_values?field=level", + "log/select/logsql/facets", + } + for _, p := range paths { + if got := h.status(http.MethodGet, appRoot+"b/"+p, "bob-token"); got != http.StatusForbidden { + t.Errorf("b/%s answered %d, want 403", p, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for a denied application", n) + } + for _, p := range []string{"metrics/api/v1/labels", "log/select/logsql/query"} { + if got := h.status(http.MethodGet, appRoot+"a/"+p, "bob-token"); got != http.StatusOK { + t.Errorf("a/%s answered %d, want 200", p, got) + } + } +} diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index b541a18e1..168c402fe 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -96,6 +96,9 @@ func newPolicyHarness(t *testing.T, cfg Config, handler http.HandlerFunc, allow if cfg.Metrics.Limits == (MetricsLimits{}) { cfg.Metrics.Limits = testMetricsLimits() } + if cfg.Metrics.App == (AppScopeLimits{}) { + cfg.Metrics.App = AppScopeLimits{Lookback: 24 * time.Hour, MaxPods: 100, MaxPVCs: 100} + } if cfg.Logs == (LogsLimits{}) { cfg.Logs = testLimits() } From e6697ea4f176d1b059a40bab9bb87b31340d4e08 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 06:11:18 +0700 Subject: [PATCH 04/41] test(observability-proxy): cover malformed pod lookups and the lookup window An empty name in a lookup answer would turn the pod branch into a pattern that matches every series without a pod label, widening the application scope to the namespace; nothing pinned the guard against it. Malformed answers must only drop their branch, and the lookup must start at the query's own lookback, or at the configured limit when that lookback is unknown. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appkinds.go | 6 ++- internal/observabilityproxy/appscope.go | 6 +-- .../observabilityproxy/server_app_test.go | 39 +++++++++++++++++-- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/internal/observabilityproxy/appkinds.go b/internal/observabilityproxy/appkinds.go index 5e545f390..d58368a13 100644 --- a/internal/observabilityproxy/appkinds.go +++ b/internal/observabilityproxy/appkinds.go @@ -79,7 +79,11 @@ func (k *ApplicationKinds) Kind(plural string) (string, error) { } var kinds []string for _, obj := range objs { - kind, _, _ := unstructured.NestedString(obj.(*unstructured.Unstructured).Object, "spec", "application", "kind") + u, ok := obj.(*unstructured.Unstructured) + if !ok { + continue + } + kind, _, _ := unstructured.NestedString(u.Object, "spec", "application", "kind") if kind != "" && !slices.Contains(kinds, kind) { kinds = append(kinds, kind) } diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index b969cc502..3b192d056 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -121,15 +121,15 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st return applicationSelectors(ns, kind, name, lists[0], lists[1]), nil } -// lookupNames asks for at most max+1 values, enough to tell a list over +// lookupNames asks for at most limit+1 values, enough to tell a list over // the cap without reading all of it. -func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, match, ns string, start, end time.Time, max int) ([]string, error) { +func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, match, ns string, start, end time.Time, limit int) ([]string, error) { params := url.Values{} params.Set("match[]", match) params.Set("extra_filters[]", namespaceSelector(ns)) params.Set("start", formatTime(start)) params.Set("end", formatTime(end)) - params.Set("limit", strconv.Itoa(max+1)) + params.Set("limit", strconv.Itoa(limit+1)) resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/api/v1/label/"+label+"/values", params, "metrics store", nil) if herr != nil { return nil, herr diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 14807637e..25d8f2a7f 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -173,9 +173,9 @@ func TestServerApplicationMetricsScope(t *testing.T) { if got := r.form["extra_filters[]"]; !slices.Equal(got, []string{wantMetricsScope}) { t.Errorf("%s extra_filters[] = %q", r.path, got) } - start, herr := parseMetricsTime("start", r.form.Get("start")) - if herr != nil || start.After(testNow.Add(-time.Hour)) || start.Before(testNow.Add(-25*time.Hour)) { - t.Errorf("%s start = %s, want within the lookback before the window", r.path, r.form.Get("start")) + // The query's own lookback is known and shorter than the 24h cap. + if want := formatTime(buildMetrics(t, "api/v1/query_range", hostile).start); r.form.Get("start") != want { + t.Errorf("%s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } if r.form.Get("end") != formatTime(testNow) || r.form.Get("limit") != "101" { t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit")) @@ -204,6 +204,7 @@ func TestServerApplicationMetricsBranches(t *testing.T) { name string pods, pvcs []string podStatus int + podBody string cfg Config path string wantCode int @@ -218,13 +219,29 @@ func TestServerApplicationMetricsBranches(t *testing.T) { path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "labels get the branches as match[]", pods: []string{"p"}, path: "api/v1/labels", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, {name: "series needs match[]", path: "api/v1/series", wantCode: http.StatusBadRequest}, + // An empty name would make pod=~"^(|p)$", which matches every + // series without a pod label. + {name: "empty pod name dropped", pods: []string{"", "p"}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, + {name: "only empty pod names", pods: []string{""}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, + {name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, + {name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if tt.podStatus == 0 { tt.podStatus = http.StatusOK } - h := newPolicyHarness(t, tt.cfg, appMetricsUpstream(tt.pods, tt.pvcs, tt.podStatus), appPolicy, nil) + upstream := appMetricsUpstream(tt.pods, tt.pvcs, tt.podStatus) + if tt.podBody != "" { + upstream = func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/label/pod/values" { + _, _ = io.WriteString(w, tt.podBody) + return + } + appMetricsUpstream(nil, nil, http.StatusOK)(w, r) + } + } + h := newPolicyHarness(t, tt.cfg, upstream, appPolicy, nil) if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tt.path, "good"); got != tt.wantCode { t.Fatalf("status = %d, want %d", got, tt.wantCode) } @@ -282,3 +299,17 @@ func TestServerApplicationNarrowedGrant(t *testing.T) { } } } + +// An expression whose lookback cannot be bounded looks pods up from the +// configured lookback before the window. +func TestServerApplicationLookupUnknownLookback(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + q := url.Values{"query": {"up @ end()"}, "time": {unix(testNow)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + r := h.requests()[0] + if want := formatTime(testNow.Add(-24 * time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { + t.Fatalf("lookup %s start = %s, want %s", r.path, r.form.Get("start"), want) + } +} From dd7116ea451d11e782852a545dee03b5e83c81e4 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 06:14:16 +0700 Subject: [PATCH 05/41] fix(observability-proxy): check the lookup cap before dropping empty names The store cuts a lookup at the cap plus one value. Dropping empty values first could bring such a cut list back under the cap and serve it as complete, hiding the application's other pods without the 422 that asks for a shorter range. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 5 +++++ internal/observabilityproxy/server_app_test.go | 2 ++ 2 files changed, 7 insertions(+) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 3b192d056..e259831da 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -148,6 +148,11 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, if body.Status != "success" { return nil, errorf(http.StatusBadGateway, "metrics store answered status %q", body.Status) } + if len(body.Data) > limit { + // Over the cap before dropping empty values: the store cut the + // list at limit+1, so a shorter filtered list could hide names. + return body.Data, nil + } names := body.Data[:0] for _, n := range body.Data { if n != "" { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 25d8f2a7f..6d353d00f 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -215,6 +215,8 @@ func TestServerApplicationMetricsBranches(t *testing.T) { want: []string{wantIdentityBranch, `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`}}, {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, + {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, + path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "pvcs over the cap", pods: []string{"p"}, pvcs: []string{"v", "w"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "labels get the branches as match[]", pods: []string{"p"}, path: "api/v1/labels", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, From c4593d4c3edcb4b999be72f6bace5dc707b8af50 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:34:41 +0700 Subject: [PATCH 06/41] fix(observability-proxy): require namespace access on application roots The application roots asked only for the application subresource. Their pod and PVC branches select by namespace and name across the whole lookup window, so a caller granted only application "a" received the older series of application "b" whenever "b" had used one of the same pod or PVC names. The lineage labels the scope rests on can also be set by whoever creates an unowned pod. Until the scope carries application identity on every branch, the application roots also require read access to the namespace root, the same review the namespace root asks. Every tenant role that reads applications already holds it, so no current grant loses access; a grant narrowed by resourceNames alone is refused instead of trusted. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 11 ++++---- internal/observabilityproxy/route.go | 15 ++++++++++ internal/observabilityproxy/route_test.go | 24 ++++++++++++++++ internal/observabilityproxy/server.go | 16 ++++++----- .../observabilityproxy/server_app_test.go | 28 +++++++++++++------ internal/observabilityproxy/server_test.go | 2 +- 6 files changed, 75 insertions(+), 21 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index e259831da..55b51afe6 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -76,11 +76,12 @@ func nameBranch(ns, label string, names []string) string { // stamped (cAdvisor, kubelet volume stats). Every branch carries the // namespace, so their OR stays inside it. // -// Within the namespace, both the identity and the pod and PVC lists come -// from lineage labels on pods and PVCs. They separate applications for a -// grant narrowed by resourceNames only while whoever creates a pod or PVC -// cannot choose those labels; the lineage webhook keeps them as set on an -// object it finds no owner for, and the logs scope rests on the same. +// Within the namespace this narrows, it does not isolate: the identity and +// the pod and PVC lists come from lineage labels that the lineage webhook +// keeps as set on an object it finds no owner for, and the pod and PVC +// branches match by name, so a name another application held within the +// lookup window pulls that application's series in. Route.Reviews makes +// the application roots require read access to the namespace root too. func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { out := []string{identitySelector(ns, seriesKindLabel, seriesNameLabel, kind, name)} for _, b := range []string{nameBranch(ns, "pod", pods), nameBranch(ns, "persistentvolumeclaim", pvcs)} { diff --git a/internal/observabilityproxy/route.go b/internal/observabilityproxy/route.go index 56ee9e496..1baff4578 100644 --- a/internal/observabilityproxy/route.go +++ b/internal/observabilityproxy/route.go @@ -133,6 +133,21 @@ func ParseRoute(escapedPath string) (Route, *HTTPError) { // ResourceAttributes is the single SubjectAccessReview a route requires. It // depends on the path only, so a POST query authorizes exactly like a GET. +// Reviews lists every access a request on the route needs. An application +// root also needs read access to its namespace root: its pod and PVC +// branches match by name across the window, so a name another application +// held earlier reaches that application's series, and a grant narrowed by +// resourceNames would read more than it names. +func (r Route) Reviews() []authorizationv1.ResourceAttributes { + switch r.Kind { + case ApplicationLog: + return []authorizationv1.ResourceAttributes{Route{Kind: NamespaceLogs, Namespace: r.Namespace}.ResourceAttributes(), r.ResourceAttributes()} + case ApplicationMetrics: + return []authorizationv1.ResourceAttributes{Route{Kind: NamespaceMetrics, Namespace: r.Namespace}.ResourceAttributes(), r.ResourceAttributes()} + } + return []authorizationv1.ResourceAttributes{r.ResourceAttributes()} +} + func (r Route) ResourceAttributes() authorizationv1.ResourceAttributes { switch r.Kind { case NamespaceLogs: diff --git a/internal/observabilityproxy/route_test.go b/internal/observabilityproxy/route_test.go index f34311e0f..a8e85b377 100644 --- a/internal/observabilityproxy/route_test.go +++ b/internal/observabilityproxy/route_test.go @@ -18,6 +18,7 @@ package observabilityproxy import ( "net/http" + "slices" "testing" authorizationv1 "k8s.io/api/authorization/v1" @@ -130,3 +131,26 @@ func TestRouteResourceAttributes(t *testing.T) { } } } + +func TestRouteReviews(t *testing.T) { + nsLogs := authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "logs", Namespace: "a"} + nsMetrics := authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "metrics", Namespace: "a"} + tests := []struct { + route Route + want []authorizationv1.ResourceAttributes + }{ + {Route{Kind: NamespaceLogs, Namespace: "a"}, []authorizationv1.ResourceAttributes{nsLogs}}, + {Route{Kind: NamespaceMetrics, Namespace: "a"}, []authorizationv1.ResourceAttributes{nsMetrics}}, + {Route{Kind: ApplicationLog, Namespace: "a", Plural: "postgreses", Name: "db"}, []authorizationv1.ResourceAttributes{ + nsLogs, {Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "log", Name: "db", Namespace: "a"}, + }}, + {Route{Kind: ApplicationMetrics, Namespace: "a", Plural: "postgreses", Name: "db"}, []authorizationv1.ResourceAttributes{ + nsMetrics, {Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "metrics", Name: "db", Namespace: "a"}, + }}, + } + for _, tt := range tests { + if got := tt.route.Reviews(); !slices.Equal(got, tt.want) { + t.Errorf("%v.Reviews() = %+v, want %+v", tt.route.Kind, got, tt.want) + } + } +} diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index fd74491b5..cf873af72 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -451,13 +451,15 @@ func (s *Server) authorize(r *http.Request, route Route) (string, *HTTPError) { slog.Error("token review failed", "error", err) return "", errorf(http.StatusServiceUnavailable, "authentication unavailable") } - allowed, err := s.auth.Authorize(ctx, user, route.ResourceAttributes()) - if err != nil { - slog.Error("subject access review failed", "error", err) - return "", errorf(http.StatusServiceUnavailable, "authorization unavailable") - } - if !allowed { - return "", errorf(http.StatusForbidden, "forbidden") + for _, attrs := range route.Reviews() { + allowed, err := s.auth.Authorize(ctx, user, attrs) + if err != nil { + slog.Error("subject access review failed", "error", err) + return "", errorf(http.StatusServiceUnavailable, "authorization unavailable") + } + if !allowed { + return "", errorf(http.StatusForbidden, "forbidden") + } } return user.Username, nil } diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 6d353d00f..a767e7f9e 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -31,11 +31,18 @@ import ( const appRoot = "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/" -// appPolicy lets alice read every postgres in tenant-a, as the view role's -// apps.cozystack.io wildcard does, and bob only "a", as a grant narrowed -// with resourceNames does. +// appPolicy lets alice read every postgres in tenant-a and the namespace +// roots, as the view role does. bob and carol may read only postgres "a", +// as a grant narrowed with resourceNames does; carol may also read the +// namespace roots. func appPolicy(user string, a *authorizationv1.ResourceAttributes) bool { - if a.Namespace != "tenant-a" || a.Verb != "get" || a.Group != "apps.cozystack.io" || a.Resource != "postgreses" || + if a.Namespace != "tenant-a" { + return false + } + if a.Verb == "list" && a.Group == "cozystack.io" && (a.Resource == "logs" || a.Resource == "metrics") && a.Name == "" && a.Subresource == "" { + return user == "alice" || user == "carol" + } + if a.Verb != "get" || a.Group != "apps.cozystack.io" || a.Resource != "postgreses" || (a.Subresource != "log" && a.Subresource != "metrics") { return false } @@ -270,7 +277,10 @@ func TestServerApplicationMetricsBranches(t *testing.T) { } // A grant narrowed to application "a" by resourceNames must not reach "b" -// through any endpoint of either root. +// through any endpoint of either root. Without read access to the +// namespace roots it does not reach "a" either: the pod and PVC branches +// match by name, and a name another application used earlier in the +// window would pull its series in. func TestServerApplicationNarrowedGrant(t *testing.T) { h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"a-0"}, nil, http.StatusOK), appPolicy, nil) paths := []string{ @@ -288,15 +298,17 @@ func TestServerApplicationNarrowedGrant(t *testing.T) { "log/select/logsql/facets", } for _, p := range paths { - if got := h.status(http.MethodGet, appRoot+"b/"+p, "bob-token"); got != http.StatusForbidden { - t.Errorf("b/%s answered %d, want 403", p, got) + for _, denied := range []struct{ app, token string }{{"b", "bob-token"}, {"b", "carol-token"}, {"a", "bob-token"}} { + if got := h.status(http.MethodGet, appRoot+denied.app+"/"+p, denied.token); got != http.StatusForbidden { + t.Errorf("%s on %s/%s answered %d, want 403", denied.token, denied.app, p, got) + } } } if n := len(h.requests()); n != 0 { t.Fatalf("upstream saw %d requests for a denied application", n) } for _, p := range []string{"metrics/api/v1/labels", "log/select/logsql/query"} { - if got := h.status(http.MethodGet, appRoot+"a/"+p, "bob-token"); got != http.StatusOK { + if got := h.status(http.MethodGet, appRoot+"a/"+p, "carol-token"); got != http.StatusOK { t.Errorf("a/%s answered %d, want 200", p, got) } } diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index 168c402fe..ce2ba669b 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -64,7 +64,7 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { // and plural-to-kind mapping; nil kinds knows only postgreses. func newPolicyHarness(t *testing.T, cfg Config, handler http.HandlerFunc, allow func(string, *authorizationv1.ResourceAttributes) bool, kinds KindLookup) *harness { h := &harness{t: t} - h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, allow) + h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob", "carol-token": "carol"}, allow) if kinds == nil { kinds = newTestKinds(t, true, appDefinition("postgres", "Postgres", "postgreses")) } From 07f4f187cef1a96e6f39247eeb2bf8ae7ff2fd79 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:36:03 +0700 Subject: [PATCH 07/41] fix(observability-proxy): cover forward-shifted samples in the app lookup The pod and PVC lookup of the application metrics root ended at the evaluation time, but a negative offset reads samples after it. An instant query evaluated two hours ago with "offset -1h" reads samples from one hour ago, so a pod created in between was missing from the scope and its series were dropped while the request succeeded. The lookup now ends at the latest instant the query reads, its window end plus the sum of its negative offsets. A query whose latest sample cannot be bounded, an @ modifier other than start() or end() or an offset that is not a plain duration, is refused with 422 on the application root; the namespace root has no lookup and still serves it. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 12 ++-- .../observabilityproxy/metricslookback.go | 64 +++++++++++++++---- internal/observabilityproxy/metricsparams.go | 11 +++- .../observabilityproxy/metricsparams_test.go | 37 +++++++++++ .../observabilityproxy/server_app_test.go | 36 +++++++++++ 5 files changed, 142 insertions(+), 18 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 55b51afe6..88acb4ba4 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -94,10 +94,14 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // applicationScope looks the application's pods and PVCs up in store over // the request window, reaching back by the query's own lookback up to the -// configured limit. A failed lookup drops its branch, which only narrows -// the answer; a list over its cap is refused, because truncating it would -// silently drop series. +// configured limit and forward to the latest sample a negative offset +// reads. A failed lookup drops its branch, which only narrows the answer; +// a list over its cap, or a query whose latest sample is unknown, is +// refused, because the answer would silently miss series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { + if req.last.IsZero() { + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root") + } lim := s.cfg.Metrics.App start := req.from.Add(-lim.Lookback) if req.start.After(start) { @@ -109,7 +113,7 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st max int }{{"pod", "kube_pod_labels", lim.MaxPods}, {"persistentvolumeclaim", "kube_persistentvolumeclaim_labels", lim.MaxPVCs}} { match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name) - names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.to, l.max) + names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max) if err != nil { slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) continue diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index e55d189a5..442e9c8d9 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -113,10 +113,55 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { sum += d } lower := strings.ToLower(query) + back, _, ok := scanOffsets(lower) + if !ok { + return 0, false + } + sum += back + if hasKeyword(lower, "with") { + return 0, false + } + n := float64(subqueries + strings.Count(query, "(")) + sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval) + if sum > float64(maxMetricsDuration) { + return 0, false + } + return time.Duration(sum), true +} + +// queryLookahead is an upper bound on how far after its evaluation window +// a query reads, from the same lexical scan. Only a negative offset reads +// forward; the negative offsets are summed, which covers nesting. An @ +// modifier pins samples to any instant unless it names start() or end(), +// which lie in the window, and WITH can apply one offset several times; +// either makes the bound unknown. +func queryLookahead(query string) (time.Duration, bool) { + lower := strings.ToLower(query) + for rest := lower; ; { + at := strings.IndexByte(rest, '@') + if at < 0 { + break + } + rest = strings.TrimLeft(rest[at+1:], " \t\n\r") + if !strings.HasPrefix(rest, "start()") && !strings.HasPrefix(rest, "end()") { + return 0, false + } + } + _, ahead, ok := scanOffsets(lower) + if !ok || ahead > 0 && hasKeyword(lower, "with") || ahead > float64(maxMetricsDuration) { + return 0, false + } + return time.Duration(ahead), true +} + +// scanOffsets sums every offset of the lowercased query, the positive ones +// into back and the negative ones into ahead, in nanoseconds; ok is false +// when one is not a plain duration. +func scanOffsets(lower string) (back, ahead float64, ok bool) { for i := 0; ; { k := strings.Index(lower[i:], "offset") if k < 0 { - break + return back, ahead, true } k += i i = k + len("offset") @@ -134,21 +179,14 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { } d, ok := parseQueryDuration(arg[:end]) if !ok { - return 0, false + return 0, 0, false } - if !negative { - sum += d + if negative { + ahead += d + } else { + back += d } } - if hasKeyword(lower, "with") { - return 0, false - } - n := float64(subqueries + strings.Count(query, "(")) - sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval) - if sum > float64(maxMetricsDuration) { - return 0, false - } - return time.Duration(sum), true } // isKeyword reports whether lower[start:end] stands on its own rather than diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index a0bb42f0d..b55f09318 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -81,6 +81,9 @@ type metricsRequest struct { // from and to are the window the query is evaluated over, without the // lookback of its expression. from, to time.Time + // last is the latest instant the request reads, past to by a negative + // offset; zero when that is unknown. + last time.Time // scopedMatch is set when match[] holds the scope, not client matchers. scopedMatch bool } @@ -143,6 +146,9 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step) req.from, req.to = at, at + if ahead, ok := queryLookahead(out.Get("query")); ok { + req.last = at.Add(ahead) + } return req, nil case "api/v1/query_range": @@ -167,6 +173,9 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } req.start = queryStart(out.Get("query"), start, step) req.from, req.to = start, end + if ahead, ok := queryLookahead(out.Get("query")); ok { + req.last = end.Add(ahead) + } out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) out.Set("step", formatSeconds(step)) @@ -189,7 +198,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti if herr != nil { return req, herr } - req.start, req.from, req.to = start, start, end + req.start, req.from, req.to, req.last = start, start, end, end out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index a4a435567..5c39fdfff 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -429,3 +429,40 @@ func TestBuildMetricsParamsPointsAsVmselectCounts(t *testing.T) { t.Fatalf("11001 points answered %d, want 422", got) } } + +func TestQueryLookahead(t *testing.T) { + tests := []struct { + query string + want time.Duration + }{ + {`up`, 0}, + {`x offset 1h`, 0}, + {`x offset -1h`, time.Hour}, + {`x OFFSET - 90m`, 90 * time.Minute}, + {`rate(x[5m] offset -30m) offset -1h`, 90 * time.Minute}, + {`x offset -1h + y offset 2h`, time.Hour}, + {`x @ end()`, 0}, + {`x @ start() offset -1h`, time.Hour}, + {`node_offset_seconds - 1`, 0}, + } + for _, tt := range tests { + t.Run(tt.query, func(t *testing.T) { + got, ok := queryLookahead(tt.query) + if !ok || got != tt.want { + t.Fatalf("queryLookahead = %s, %v; want %s", got, ok, tt.want) + } + }) + } + for _, q := range []string{ + `x @ 1700000000`, + `x @ (end() - 1h)`, + `x{a="a@b"}`, + `x offset -5i`, + `x offset -$__interval`, + `with (f(a) = a offset -1h) f(f(x))`, + } { + if got, ok := queryLookahead(q); ok { + t.Errorf("queryLookahead(%q) = %s, want unknown", q, got) + } + } +} diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index a767e7f9e..c7e7a67ef 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -327,3 +327,39 @@ func TestServerApplicationLookupUnknownLookback(t *testing.T) { t.Fatalf("lookup %s start = %s, want %s", r.path, r.form.Get("start"), want) } } + +// TestServerApplicationLookupCoversNegativeOffset evaluates two hours ago +// a selector shifted an hour forward: the pod lookup must reach the time +// of the samples, or a pod created in between is left out. +func TestServerApplicationLookupCoversNegativeOffset(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + at := testNow.Add(-2 * time.Hour) + q := url.Values{"query": {"container_cpu_usage_seconds_total offset -1h"}, "time": {unix(at)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + r := h.requests()[0] + if want := formatTime(at.Add(time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("end") != want { + t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want) + } +} + +// TestServerApplicationRefusesUnboundedSampleTime refuses a query whose +// samples may lie at a time the lookup cannot be sized for, rather than +// answer with series silently left out. +func TestServerApplicationRefusesUnboundedSampleTime(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{"up @ 1700000000", "up offset -5i"} { + q := url.Values{"query": {query}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for a refused query", n) + } + // The namespace root has no lookup to size and still serves it. + if got := h.status(http.MethodGet, metricsRoot+"api/v1/query?"+url.Values{"query": {"up @ 1700000000"}}.Encode(), "good"); got != http.StatusOK { + t.Fatalf("namespace root answered %d", got) + } +} From 97a55a167bf786809a88d50d65da2f2a2225f55d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:44:23 +0700 Subject: [PATCH 08/41] docs(observability-proxy): put the route review comments on their functions Reviews was inserted between ResourceAttributes and its doc comment, so go doc showed the ResourceAttributes text on Reviews, and that text still called it the single review a route needs, which application roots no longer are. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/route.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/route.go b/internal/observabilityproxy/route.go index 1baff4578..49a7c1770 100644 --- a/internal/observabilityproxy/route.go +++ b/internal/observabilityproxy/route.go @@ -131,8 +131,6 @@ func ParseRoute(escapedPath string) (Route, *HTTPError) { return r, nil } -// ResourceAttributes is the single SubjectAccessReview a route requires. It -// depends on the path only, so a POST query authorizes exactly like a GET. // Reviews lists every access a request on the route needs. An application // root also needs read access to its namespace root: its pod and PVC // branches match by name across the window, so a name another application @@ -148,6 +146,9 @@ func (r Route) Reviews() []authorizationv1.ResourceAttributes { return []authorizationv1.ResourceAttributes{r.ResourceAttributes()} } +// ResourceAttributes is the SubjectAccessReview of the route's own +// resource. It depends on the path only, so a POST query authorizes +// exactly like a GET. func (r Route) ResourceAttributes() authorizationv1.ResourceAttributes { switch r.Kind { case NamespaceLogs: From 1a3517fa81d6ca87f4387ce4e50e831d981d60e7 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:55:50 +0700 Subject: [PATCH 09/41] fix(observability-proxy): fail an application query whose lookup failed When the pod or PVC lookup failed, its branch was dropped and the query still answered 200, without the cAdvisor and volume series of the application, so a store error showed up as an empty graph. That is the same silently incomplete answer that a lookup over its cap is refused for. A failed lookup now fails the request: 504 when the store missed the deadline, 502 otherwise. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 13 ++++++++----- internal/observabilityproxy/server_app_test.go | 9 +++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 88acb4ba4..ff1955dc9 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -95,9 +95,9 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // applicationScope looks the application's pods and PVCs up in store over // the request window, reaching back by the query's own lookback up to the // configured limit and forward to the latest sample a negative offset -// reads. A failed lookup drops its branch, which only narrows the answer; -// a list over its cap, or a query whose latest sample is unknown, is -// refused, because the answer would silently miss series. +// reads. A failed lookup, a list over its cap, or a query whose latest +// sample is unknown fails the request, because a successful answer would +// silently miss the application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root") @@ -115,8 +115,11 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name) names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max) if err != nil { - slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) - continue + slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) + if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusGatewayTimeout { + return nil, herr + } + return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label) } if len(names) > l.max { return nil, errorf(http.StatusUnprocessableEntity, "the application has more than %d %ss in this window; use a shorter range", l.max, l.label) diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index c7e7a67ef..268ea7b8f 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -218,8 +218,9 @@ func TestServerApplicationMetricsBranches(t *testing.T) { want []string }{ {name: "no pods or pvcs", path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, - {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusOK, - want: []string{wantIdentityBranch, `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`}}, + // A failed lookup would leave the application's own series out of + // a successful answer, so it fails the request instead. + {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, @@ -232,8 +233,8 @@ func TestServerApplicationMetricsBranches(t *testing.T) { // series without a pod label. {name: "empty pod name dropped", pods: []string{"", "p"}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, {name: "only empty pod names", pods: []string{""}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, - {name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, - {name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, + {name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, + {name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 66e2b99f0981cd440a6d9b538c29f51fdaacadf0 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:56:32 +0700 Subject: [PATCH 10/41] fix(observability-proxy): reach the aligned end in the app lookup vmselect rounds a range query's end up to a whole step before evaluating it, so the last point can read samples up to one step past the end the client sent. The pod and PVC lookup stopped at that end and could miss a pod created within the last step. The lookup end of a range query now adds one step. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/metricsparams.go | 3 ++- internal/observabilityproxy/server_app_test.go | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index b55f09318..721496fe3 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -173,8 +173,9 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } req.start = queryStart(out.Get("query"), start, step) req.from, req.to = start, end + // vmselect rounds end up to a whole step (AdjustStartEnd). if ahead, ok := queryLookahead(out.Get("query")); ok { - req.last = end.Add(ahead) + req.last = end.Add(step + ahead) } out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 268ea7b8f..395deb18b 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -184,7 +184,8 @@ func TestServerApplicationMetricsScope(t *testing.T) { if want := formatTime(buildMetrics(t, "api/v1/query_range", hostile).start); r.form.Get("start") != want { t.Errorf("%s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } - if r.form.Get("end") != formatTime(testNow) || r.form.Get("limit") != "101" { + // One step past end, which vmselect rounds up to a whole step. + if r.form.Get("end") != formatTime(testNow.Add(time.Minute)) || r.form.Get("limit") != "101" { t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit")) } } @@ -364,3 +365,18 @@ func TestServerApplicationRefusesUnboundedSampleTime(t *testing.T) { t.Fatalf("namespace root answered %d", got) } } + +// TestServerApplicationLookupCoversAlignedEnd: vmselect rounds a range +// query's end up to a whole step, so the lookup reaches one step past it. +func TestServerApplicationLookupCoversAlignedEnd(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + end := testNow.Add(-90 * time.Minute) + q := url.Values{"query": {"up"}, "start": {unix(end.Add(-6 * time.Hour))}, "end": {unix(end)}, "step": {"3600"}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query_range?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + r := h.requests()[0] + if want := formatTime(end.Add(time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("end") != want { + t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want) + } +} From bc823b704e5994301f62d6add1e28971768e7ff9 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:01:57 +0700 Subject: [PATCH 11/41] fix(observability-proxy): answer 504 to a lookup that misses the deadline Only a deadline hit before the lookup's response headers was reported as 504. When the store sent its headers and then stalled, the deadline expired while the body was decoded, and the decode error came back as a generic 502, so a slow store looked like a broken one. The request deadline is now checked whichever step of the lookup it cut, and both cases are covered by a test. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 7 +++-- .../observabilityproxy/server_app_test.go | 28 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index ff1955dc9..80d12cbe2 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -19,6 +19,7 @@ package observabilityproxy import ( "context" "encoding/json" + "errors" "io" "log/slog" "net/http" @@ -116,8 +117,10 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max) if err != nil { slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) - if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusGatewayTimeout { - return nil, herr + // The deadline can also expire while the answer is read, after + // post has handed it over. + if errors.Is(context.Cause(ctx), context.DeadlineExceeded) { + return nil, errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", l.label) } return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label) } diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 395deb18b..53216e575 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -380,3 +380,31 @@ func TestServerApplicationLookupCoversAlignedEnd(t *testing.T) { t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want) } } + +// TestServerApplicationLookupDeadline: a lookup that misses the request +// deadline answers 504, whether the store sent nothing or stalled after +// its headers, and the query itself is not sent. +func TestServerApplicationLookupDeadline(t *testing.T) { + for _, headers := range []bool{false, true} { + h := newPolicyHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/label/pod/values" { + appMetricsUpstream(nil, nil, http.StatusOK)(w, r) + return + } + if headers { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"status":"success","data":[`) + w.(http.Flusher).Flush() + } + <-r.Context().Done() + }, appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusGatewayTimeout { + t.Fatalf("headers sent %v: status = %d, want 504", headers, got) + } + for _, r := range h.requests() { + if !strings.HasPrefix(r.path, "/api/v1/label/") { + t.Fatalf("headers sent %v: the query reached the store after a failed lookup", headers) + } + } + } +} From 84665a42a798d935a1b078d6fbdd3c720c62bffc Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:28:12 +0700 Subject: [PATCH 12/41] fix(observability-proxy): refuse app queries that read past the lookup The pod and PVC lookup never started earlier than --metrics-app-lookback before the window. A query that read further back, such as rate(x[7d]) with the 24h default, or whose lookback is unknown (an @ modifier, WITH, a step-multiple window), answered 200 without the series of pods that existed only before the lookup start. Such queries now answer 422 on the application root, matching how a query whose latest sample cannot be covered is handled; the namespace root has no lookup and still serves them. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 2 +- internal/observabilityproxy/appscope.go | 17 ++++++----- .../observabilityproxy/server_app_test.go | 29 ++++++++++++++----- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 4a99aa4cc..93f4c4553 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -94,7 +94,7 @@ func parseFlags(args []string) (options, error) { fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use") fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return") fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") - fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs at most") + fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs; a query reading further back answers 422") fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 80d12cbe2..835d29b84 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -94,20 +94,21 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { } // applicationScope looks the application's pods and PVCs up in store over -// the request window, reaching back by the query's own lookback up to the -// configured limit and forward to the latest sample a negative offset -// reads. A failed lookup, a list over its cap, or a query whose latest -// sample is unknown fails the request, because a successful answer would -// silently miss the application's series. +// the samples the request reads: back by the query's own lookback, and +// forward to the latest sample a negative offset reads. A query that reads +// further back than the configured lookback, or whose earliest or latest +// sample is unknown, a failed lookup, and a list over its cap all fail the +// request, because a successful answer would silently miss the +// application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root") } lim := s.cfg.Metrics.App - start := req.from.Add(-lim.Lookback) - if req.start.After(start) { - start = req.start + if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { + return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback) } + start := req.start var lists [2][]string for i, l := range []struct { label, metric string diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 53216e575..d3dcb683a 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -316,17 +316,30 @@ func TestServerApplicationNarrowedGrant(t *testing.T) { } } -// An expression whose lookback cannot be bounded looks pods up from the -// configured lookback before the window. -func TestServerApplicationLookupUnknownLookback(t *testing.T) { - h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) - q := url.Values{"query": {"up @ end()"}, "time": {unix(testNow)}} +// TestServerApplicationLookbackBeyondCap: the lookup reaches back at most +// --metrics-app-lookback, so a query that reads further back, or whose +// lookback is unknown, would miss pods that existed only earlier; it is +// refused rather than answered without their series. +func TestServerApplicationLookbackBeyondCap(t *testing.T) { + cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 100, MaxPVCs: 100}}} + h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{"rate(x[7d])", "x[30d] @ end()", "rate(x[5i])"} { + q := url.Values{"query": {query}, "time": {unix(testNow)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for refused queries", n) + } + q := url.Values{"query": {"rate(x[5m])"}, "time": {unix(testNow)}} if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { - t.Fatalf("status = %d", got) + t.Fatalf("rate(x[5m]) answered %d, want 200", got) } r := h.requests()[0] - if want := formatTime(testNow.Add(-24 * time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { - t.Fatalf("lookup %s start = %s, want %s", r.path, r.form.Get("start"), want) + want := formatTime(buildMetrics(t, "api/v1/query", q).start) + if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { + t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } } From 84d5b4ff0aa6e5056d4ea244eb5ba18baff6a0b8 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:28:34 +0700 Subject: [PATCH 13/41] fix(observability-proxy): require a positive application lookback Every query reads some time before its window, at least the implicit rollup window, so with the lookback refusal a zero --metrics-app-lookback would turn every application metrics query into a 422. It is now refused at startup instead. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 4 ++-- cmd/observability-proxy/main_test.go | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 93f4c4553..7fe145229 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -137,8 +137,8 @@ func parseFlags(args []string) (options, error) { return o, errors.New("--metrics-shortterm-retention, --metrics-max-range and --metrics-max-timeout must be positive") case o.metrics.Limits.MaxPoints < 2: return o, errors.New("--metrics-max-points must be at least 2") - case o.metrics.App.Lookback < 0 || o.metrics.App.MaxPods < 1 || o.metrics.App.MaxPVCs < 1: - return o, errors.New("--metrics-app-lookback must not be negative and --metrics-app-max-pods and --metrics-app-max-pvcs must be positive") + case o.metrics.App.Lookback <= 0 || o.metrics.App.MaxPods < 1 || o.metrics.App.MaxPVCs < 1: + return o, errors.New("--metrics-app-lookback, --metrics-app-max-pods and --metrics-app-max-pvcs must be positive") case o.logs.MaxTailResume < 0: return o, errors.New("--logs-max-tail-resume must not be negative") // The proxy's deadline starts before the request reaches vmselect, so diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go index 6b5c60d7d..bf8e032c2 100644 --- a/cmd/observability-proxy/main_test.go +++ b/cmd/observability-proxy/main_test.go @@ -66,7 +66,7 @@ func TestParseFlagsMetricsOverrides(t *testing.T) { o, err := parseFlags([]string{ "--root-tenant=tenant-x", "--metrics-shortterm-store=fast", "--metrics-longterm-store=slow", "--metrics-shortterm-retention=24h", "--metrics-max-range=168h", "--metrics-max-points=500", "--metrics-max-timeout=10s", - "--metrics-app-lookback=0s", "--metrics-app-max-pods=7", "--metrics-app-max-pvcs=3", + "--metrics-app-lookback=2h", "--metrics-app-max-pods=7", "--metrics-app-max-pvcs=3", }) if err != nil { t.Fatal(err) @@ -74,7 +74,7 @@ func TestParseFlagsMetricsOverrides(t *testing.T) { m := o.metrics if m.RootTenant != "tenant-x" || m.ShortTermStore != "fast" || m.LongTermStore != "slow" || m.ShortTermRetention != 24*time.Hour || m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second || - m.App.Lookback != 0 || m.App.MaxPods != 7 || m.App.MaxPVCs != 3 { + m.App.Lookback != 2*time.Hour || m.App.MaxPods != 7 || m.App.MaxPVCs != 3 { t.Errorf("overrides not applied: %+v", m) } } @@ -107,6 +107,9 @@ func TestParseFlagsInvalid(t *testing.T) { {"--metrics-max-points=1"}, {"--metrics-max-timeout=0s"}, {"--metrics-app-lookback=-1h"}, + // Every query reads some time before its window, so a zero lookback + // would refuse every application metrics query. + {"--metrics-app-lookback=0s"}, {"--metrics-app-max-pods=0"}, {"--metrics-app-max-pvcs=0"}, {"--unknown"}, From 1a3e1827ef2104e7f0ae881bd03f284442e4a5d5 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:29:13 +0700 Subject: [PATCH 14/41] fix(observability-proxy): pass a store 422 from the app lookup through vmselect answers 422 when a label lookup exceeds its own series limits. The application root turned that into a 502, which reads as a broken store, while the client could have fixed it by narrowing the window. The 422 and vmselect's message now reach the client. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 9 +++++++++ internal/observabilityproxy/server_app_test.go | 3 +++ 2 files changed, 12 insertions(+) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 835d29b84..eec1f0eb6 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -120,6 +120,9 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) // The deadline can also expire while the answer is read, after // post has handed it over. + if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusUnprocessableEntity { + return nil, herr + } if errors.Is(context.Cause(ctx), context.DeadlineExceeded) { return nil, errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", l.label) } @@ -147,6 +150,12 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, return nil, herr } defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusUnprocessableEntity { + // vmselect's refusal of a lookup over its own limits; the client + // can act on it by narrowing the window. + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<10)) + return nil, errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, strings.TrimSpace(string(msg))) + } if resp.StatusCode != http.StatusOK { return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status) } diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index d3dcb683a..d1d691b00 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -222,6 +222,9 @@ func TestServerApplicationMetricsBranches(t *testing.T) { // A failed lookup would leave the application's own series out of // a successful answer, so it fails the request instead. {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, + // vmselect refuses a lookup over its own series limits with 422; + // the client can act on that by narrowing the window. + {name: "pod lookup over the store's limits", pods: []string{"p"}, podStatus: http.StatusUnprocessableEntity, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, From 86dddb447a81f584f9e017cb4fd9223c9e63137b Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:29:14 +0700 Subject: [PATCH 15/41] docs(observability-proxy): note the store calls behind one app request The per-user rate counts client requests, but an application metrics request makes three store calls, so an operator sizing the rate against store capacity needs to know the multiplier. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 7fe145229..a3fa9b259 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -98,6 +98,9 @@ func parseFlags(args []string) (options, error) { fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") + // Counted per client request: an application metrics request costs the + // store three calls (the pod and PVC lookups and the query) for one + // token, so the store load per user can be three times this rate. fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables") fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user") fs.IntVar(&o.limiter.MaxTailsPerUser, "max-tails-per-user", 5, "open tails per user; 0 disables") From 85fb98dd2cd5029adca903da074ab156877abb2a Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:36:52 +0700 Subject: [PATCH 16/41] fix(observability-proxy): stop refusing ordinary app queries on lookback With the lookback refusal, the 24h default turned common application queries into 422: a day's increase, and month-long panels at the step a dashboard picks for them. The lookback estimate is a loose upper bound, made for the store choice, and counts every "(" as an implicit subquery, so it far exceeds what such queries read. The lookback is only the refusal threshold, since the lookup itself spans the window the query reads, so its default now matches --metrics-max-range. The chart passes it from values, so an operator can tune it. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 6 ++++- cmd/observability-proxy/main_test.go | 2 +- .../observabilityproxy/server_app_test.go | 25 +++++++++++++++++++ .../templates/deployment.yaml | 1 + .../tests/deployment_test.yaml | 7 ++++++ .../system/observability-proxy/values.yaml | 3 +++ 6 files changed, 42 insertions(+), 2 deletions(-) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index a3fa9b259..487b6d04d 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -94,7 +94,11 @@ func parseFlags(args []string) (options, error) { fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use") fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return") fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") - fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs; a query reading further back answers 422") + // Only the refusal threshold: the lookup itself spans the window the + // query reads, and that estimate is a loose upper bound (every "(" + // counts as an implicit subquery), so the default matches + // --metrics-max-range to serve month-long panels at a dashboard step. + fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 30*24*time.Hour, "how far before the window an application metrics query may read; one reading further back, or by an unknown amount, answers 422") fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go index bf8e032c2..616d25c51 100644 --- a/cmd/observability-proxy/main_test.go +++ b/cmd/observability-proxy/main_test.go @@ -40,7 +40,7 @@ func TestParseFlagsDefaults(t *testing.T) { if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second { t.Errorf("metrics limit defaults changed: %+v", m.Limits) } - if m.App.Lookback != 24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 { + if m.App.Lookback != 30*24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 { t.Errorf("application scope defaults changed: %+v", m.App) } if m.RootTenant != o.rootTenant { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index d1d691b00..0b894b8fc 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -424,3 +424,28 @@ func TestServerApplicationLookupDeadline(t *testing.T) { } } } + +// TestServerApplicationDefaultLookbackServesLongPanels: the lookback is +// only the refusal threshold, and the lookback estimate is a loose upper +// bound, so the default must serve a day's increase and a month-long +// panel at the step a dashboard picks for it. +func TestServerApplicationDefaultLookbackServesLongPanels(t *testing.T) { + cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: 30 * 24 * time.Hour, MaxPods: 100, MaxPVCs: 100}}} + h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + month := url.Values{"start": {unix(testNow.Add(-29 * 24 * time.Hour))}, "end": {unix(testNow)}} + for _, tc := range []struct{ path, query, step string }{ + {"api/v1/query", "increase(x[24h])", ""}, + {"api/v1/query_range", "rate(x[5m])", "21600"}, + {"api/v1/query_range", "histogram_quantile(0.99, sum by (le) (rate(x[5m])))", "7200"}, + } { + q := url.Values{"query": {tc.query}} + if tc.step != "" { + q.Set("step", tc.step) + q.Set("start", month.Get("start")) + q.Set("end", month.Get("end")) + } + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tc.path+"?"+q.Encode(), "good"); got != http.StatusOK { + t.Errorf("%s step %s answered %d, want 200", tc.query, tc.step, got) + } + } +} diff --git a/packages/system/observability-proxy/templates/deployment.yaml b/packages/system/observability-proxy/templates/deployment.yaml index d49de140b..5cb32833f 100644 --- a/packages/system/observability-proxy/templates/deployment.yaml +++ b/packages/system/observability-proxy/templates/deployment.yaml @@ -41,6 +41,7 @@ spec: - --metrics-shortterm-store={{ .Values.observabilityProxy.metrics.shorttermStore }} - --metrics-longterm-store={{ .Values.observabilityProxy.metrics.longtermStore }} - --metrics-shortterm-retention={{ .Values.observabilityProxy.metrics.shorttermRetention }} + - --metrics-app-lookback={{ .Values.observabilityProxy.metrics.appLookback }} ports: - name: http containerPort: 8080 diff --git a/packages/system/observability-proxy/tests/deployment_test.yaml b/packages/system/observability-proxy/tests/deployment_test.yaml index a5e948920..0f0104660 100644 --- a/packages/system/observability-proxy/tests/deployment_test.yaml +++ b/packages/system/observability-proxy/tests/deployment_test.yaml @@ -51,6 +51,9 @@ tests: - contains: path: spec.template.spec.containers[0].args content: --metrics-shortterm-retention=72h + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-app-lookback=720h - it: follows renamed metrics stores set: @@ -58,6 +61,7 @@ tests: shorttermStore: fast longtermStore: slow shorttermRetention: 24h + appLookback: 48h asserts: - contains: path: spec.template.spec.containers[0].args @@ -68,6 +72,9 @@ tests: - contains: path: spec.template.spec.containers[0].args content: --metrics-shortterm-retention=24h + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-app-lookback=48h - it: falls back to cozy.local when the platform passes no cluster domain asserts: diff --git a/packages/system/observability-proxy/values.yaml b/packages/system/observability-proxy/values.yaml index 67334a237..3e3ac70fe 100644 --- a/packages/system/observability-proxy/values.yaml +++ b/packages/system/observability-proxy/values.yaml @@ -14,6 +14,9 @@ observabilityProxy: shorttermStore: shortterm longtermStore: longterm shorttermRetention: 72h + # How far before its window an application metrics query may read; + # one reading further back answers 422 on the application root. + appLookback: 720h resources: requests: cpu: 10m From caa4e8d0e3d5ab9f05fce28c96e0c54e4aed35b1 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:37:59 +0700 Subject: [PATCH 17/41] fix(observability-proxy): keep only the series-limit 422 of a lookup vmselect answers 422 to every error that carries no status of its own, its query timeout and storage failures included, so passing every lookup 422 through told the client to narrow a window when the store was failing, and bypassed the 504 for a missed deadline. Only vmselect's series-limit message stays a 422, with the message taken from its JSON answer; its timeout becomes 504 and anything else 502. The request deadline is checked before the store's status. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 35 ++++++++++++---- .../observabilityproxy/server_app_test.go | 41 +++++++++++++++++-- 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index eec1f0eb6..796280eb6 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -120,12 +120,12 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) // The deadline can also expire while the answer is read, after // post has handed it over. - if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusUnprocessableEntity { - return nil, herr - } if errors.Is(context.Cause(ctx), context.DeadlineExceeded) { return nil, errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", l.label) } + if herr, ok := err.(*HTTPError); ok && (herr.Code == http.StatusUnprocessableEntity || herr.Code == http.StatusGatewayTimeout) { + return nil, herr + } return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label) } if len(names) > l.max { @@ -151,10 +151,7 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, } defer func() { _ = resp.Body.Close() }() if resp.StatusCode == http.StatusUnprocessableEntity { - // vmselect's refusal of a lookup over its own limits; the client - // can act on it by narrowing the window. - msg, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<10)) - return nil, errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, strings.TrimSpace(string(msg))) + return nil, lookupRefusal(label, resp.Body) } if resp.StatusCode != http.StatusOK { return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status) @@ -182,3 +179,27 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, } return names, nil } + +// seriesLimitError is the message vmstorage gives when a search matches +// more series than the -search.max* flags allow (lib/storage/index_db.go); +// cluster vmselect wraps it in its own context, hence a substring match. +const seriesLimitError = "the number of matching timeseries exceeds" + +// lookupRefusal reads a vmselect 422. vmselect answers 422 to any error +// without a status of its own (httpserver.SendPrometheusError), storage +// failures and its own timeout included, so only the series limit, which +// a narrower window fixes, stays a 422 for the client. +func lookupRefusal(label string, body io.Reader) *HTTPError { + var answer struct { + Error string `json:"error"` + } + _ = json.NewDecoder(io.LimitReader(body, 64<<10)).Decode(&answer) + switch { + case strings.Contains(answer.Error, seriesLimitError): + return errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, answer.Error) + case strings.Contains(answer.Error, "timeout exceeded"): + return errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", label) + default: + return errorf(http.StatusBadGateway, "the application's %s lookup failed: %s", label, answer.Error) + } +} diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 0b894b8fc..2cd2a0eda 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -222,9 +222,6 @@ func TestServerApplicationMetricsBranches(t *testing.T) { // A failed lookup would leave the application's own series out of // a successful answer, so it fails the request instead. {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, - // vmselect refuses a lookup over its own series limits with 422; - // the client can act on that by narrowing the window. - {name: "pod lookup over the store's limits", pods: []string{"p"}, podStatus: http.StatusUnprocessableEntity, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, @@ -449,3 +446,41 @@ func TestServerApplicationDefaultLookbackServesLongPanels(t *testing.T) { } } } + +// TestServerApplicationLookupStoreErrors: vmselect answers 422 to any error +// without a status of its own (SendPrometheusError), so only its series +// limit, which a narrower window fixes, reaches the client as 422. +func TestServerApplicationLookupStoreErrors(t *testing.T) { + for _, tc := range []struct { + label, message string + want int + }{ + {"pod", "the number of matching timeseries exceeds 300000; either narrow down the search or increase -search.max* command-line flag values", http.StatusUnprocessableEntity}, + {"persistentvolumeclaim", "the number of matching timeseries exceeds 300000; either narrow down the search", http.StatusUnprocessableEntity}, + {"pod", "cannot fetch label values from vmstorage nodes: the number of matching timeseries exceeds 300000", http.StatusUnprocessableEntity}, + {"pod", "timeout exceeded before starting the query processing: 30s", http.StatusGatewayTimeout}, + {"pod", "error when searching for label values: cannot obtain data from vmstorage", http.StatusBadGateway}, + } { + h := newPolicyHarness(t, Config{}, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/label/"+tc.label+"/values" { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnprocessableEntity) + _ = json.NewEncoder(w).Encode(map[string]string{"status": "error", "errorType": "422", "error": tc.message}) + return + } + appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r) + }, appPolicy, nil) + resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good", nil, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != tc.want { + t.Errorf("%s lookup %q answered %d, want %d", tc.label, tc.message, resp.StatusCode, tc.want) + } + if tc.want == http.StatusUnprocessableEntity && !strings.Contains(string(body), "the number of matching timeseries exceeds 300000") { + t.Errorf("the store's limit message did not reach the client: %q", body) + } + if strings.Contains(string(body), `"errorType"`) { + t.Errorf("the store's raw JSON reached the client: %q", body) + } + } +} From 336961d7cd6dc7293caebd670c96b459b4ade6fc Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:38:10 +0700 Subject: [PATCH 18/41] docs(observability-proxy): put the store-call multiplier in the flag help The note that an application metrics request makes three store calls sat in a source comment, where an operator reading --help never sees it. It is now part of the --rate-limit-per-user help text. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 487b6d04d..f5676b6d5 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -102,10 +102,7 @@ func parseFlags(args []string) (options, error) { fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") - // Counted per client request: an application metrics request costs the - // store three calls (the pod and PVC lookups and the query) for one - // token, so the store load per user can be three times this rate. - fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables") + fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained client requests per second per user; an application metrics request makes three store calls (pod lookup, PVC lookup, query); 0 disables") fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user") fs.IntVar(&o.limiter.MaxTailsPerUser, "max-tails-per-user", 5, "open tails per user; 0 disables") fs.IntVar(&o.limiter.MaxTails, "max-tails", 200, "open tails in total; 0 disables") From c9c02b2d52e31f70e2a28a2d24c2929c8d042d74 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:45:13 +0700 Subject: [PATCH 19/41] test(observability-proxy): cover a store 504 on the app lookup The lookup now passes a 504 from the store through, where it used to answer 502, but only the 504 read from a 422 body was tested. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/server_app_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 2cd2a0eda..dcbf1e2a2 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -222,6 +222,7 @@ func TestServerApplicationMetricsBranches(t *testing.T) { // A failed lookup would leave the application's own series out of // a successful answer, so it fails the request instead. {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, + {name: "pod lookup times out at the store", pods: []string{"p"}, podStatus: http.StatusGatewayTimeout, path: "api/v1/query?query=up", wantCode: http.StatusGatewayTimeout}, {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, From 501f0d98f7c82e02169c33219d2772be49c137fa Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:45:13 +0700 Subject: [PATCH 20/41] docs(observability-proxy): drop the claimed link between two flag defaults The comment said the application lookback default matches --metrics-max-range, but nothing ties the two: changing the range flag leaves the lookback where it is. It now gives only the reason for the length. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index f5676b6d5..66cc614f0 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -96,8 +96,8 @@ func parseFlags(args []string) (options, error) { fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") // Only the refusal threshold: the lookup itself spans the window the // query reads, and that estimate is a loose upper bound (every "(" - // counts as an implicit subquery), so the default matches - // --metrics-max-range to serve month-long panels at a dashboard step. + // counts as an implicit subquery), so the default is long enough to + // serve month-long panels at a dashboard step. fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 30*24*time.Hour, "how far before the window an application metrics query may read; one reading further back, or by an unknown amount, answers 422") fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") From 0bb98ced05bfeca07e15edf31c4470d05ae3ecfc Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 02:24:13 +0700 Subject: [PATCH 21/41] fix(observability-proxy): say that every @ is refused on app roots The lookahead scan let @ start() and @ end() through and the 422 named them as allowed, but the lookback scan cannot bound any @, so every @ query was refused anyway, with a message pointing at forms that would not pass. Both scans now treat any @ as unknown and the message says so. The lookback-cap test now exercises the cap with a window just past it rather than an @ query. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 2 +- .../observabilityproxy/metricslookback.go | 20 +++++----------- .../observabilityproxy/metricsparams_test.go | 6 +++-- .../observabilityproxy/server_app_test.go | 24 ++++++++++++++++++- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 796280eb6..e41dd1330 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -102,7 +102,7 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { - return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root") + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ modifier, or an offset that is not a plain duration); use the namespace root") } lim := s.cfg.Metrics.App if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 442e9c8d9..e9e7cb0e9 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -131,22 +131,14 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { // queryLookahead is an upper bound on how far after its evaluation window // a query reads, from the same lexical scan. Only a negative offset reads -// forward; the negative offsets are summed, which covers nesting. An @ -// modifier pins samples to any instant unless it names start() or end(), -// which lie in the window, and WITH can apply one offset several times; -// either makes the bound unknown. +// forward; the negative offsets are summed, which covers nesting. Any @ +// makes the bound unknown, as it does for queryLookback, and so does WITH, +// which can apply one offset several times. func queryLookahead(query string) (time.Duration, bool) { - lower := strings.ToLower(query) - for rest := lower; ; { - at := strings.IndexByte(rest, '@') - if at < 0 { - break - } - rest = strings.TrimLeft(rest[at+1:], " \t\n\r") - if !strings.HasPrefix(rest, "start()") && !strings.HasPrefix(rest, "end()") { - return 0, false - } + if strings.Contains(query, "@") { + return 0, false } + lower := strings.ToLower(query) _, ahead, ok := scanOffsets(lower) if !ok || ahead > 0 && hasKeyword(lower, "with") || ahead > float64(maxMetricsDuration) { return 0, false diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 5c39fdfff..5e0d548b7 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -441,8 +441,6 @@ func TestQueryLookahead(t *testing.T) { {`x OFFSET - 90m`, 90 * time.Minute}, {`rate(x[5m] offset -30m) offset -1h`, 90 * time.Minute}, {`x offset -1h + y offset 2h`, time.Hour}, - {`x @ end()`, 0}, - {`x @ start() offset -1h`, time.Hour}, {`node_offset_seconds - 1`, 0}, } for _, tt := range tests { @@ -455,6 +453,10 @@ func TestQueryLookahead(t *testing.T) { } for _, q := range []string{ `x @ 1700000000`, + // The lookback scan cannot bound any @, so the application root + // refuses every one; the lookahead does not tell them apart. + `x @ end()`, + `x @ start() offset -1h`, `x @ (end() - 1h)`, `x{a="a@b"}`, `x offset -5i`, diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index dcbf1e2a2..8ae395d45 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -324,7 +324,9 @@ func TestServerApplicationNarrowedGrant(t *testing.T) { func TestServerApplicationLookbackBeyondCap(t *testing.T) { cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 100, MaxPVCs: 100}}} h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) - for _, query := range []string{"rate(x[7d])", "x[30d] @ end()", "rate(x[5i])"} { + // max_over_time(x[2h]) reads just past the 1h cap; rate(x[5i]) has an + // unknown lookback. + for _, query := range []string{"rate(x[7d])", "max_over_time(x[2h])", "rate(x[5i])"} { q := url.Values{"query": {query}, "time": {unix(testNow)}} if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity { t.Errorf("%s answered %d, want 422", query, got) @@ -485,3 +487,23 @@ func TestServerApplicationLookupStoreErrors(t *testing.T) { } } } + +// TestServerApplicationRefusesEveryAt: no @ form can be bounded, and the +// answer says so rather than naming forms that would pass. +func TestServerApplicationRefusesEveryAt(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{"up @ end()", "up @ start()", "up @ 1700000000"} { + resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+url.Values{"query": {query}}.Encode(), "good", nil, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, resp.StatusCode) + } + if strings.Contains(string(body), "start() or end()") { + t.Errorf("%s: the refusal names @ forms as allowed: %q", query, body) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for refused queries", n) + } +} From 41e61cb288b5d5a0c2cd250c5effb67601927bfb Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 02:24:54 +0700 Subject: [PATCH 22/41] fix(observability-proxy): keep store error text out of lookup answers A store error other than the series limit was built into the lookup's 502 message, and that text can name vmstorage nodes and addresses. The handler happened to replace the message before it reached the client, but nothing held it there. The text now goes to the log only. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 4 +++- internal/observabilityproxy/appscope_test.go | 13 +++++++++++++ internal/observabilityproxy/server_app_test.go | 5 +++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index e41dd1330..561a92ab8 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -200,6 +200,8 @@ func lookupRefusal(label string, body io.Reader) *HTTPError { case strings.Contains(answer.Error, "timeout exceeded"): return errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", label) default: - return errorf(http.StatusBadGateway, "the application's %s lookup failed: %s", label, answer.Error) + // The text may name vmstorage nodes, so it stays in the log. + slog.Warn("application lookup refused by the store", "label", label, "error", answer.Error) + return errorf(http.StatusBadGateway, "the application's %s lookup failed", label) } } diff --git a/internal/observabilityproxy/appscope_test.go b/internal/observabilityproxy/appscope_test.go index af028b867..c4b8e5d69 100644 --- a/internal/observabilityproxy/appscope_test.go +++ b/internal/observabilityproxy/appscope_test.go @@ -17,10 +17,12 @@ limitations under the License. package observabilityproxy import ( + "net/http" "net/url" "regexp" "slices" "strconv" + "strings" "testing" "time" ) @@ -114,3 +116,14 @@ func TestMetricsRequestScope(t *testing.T) { t.Errorf("range window = [%s, %s]", rng.from, rng.to) } } + +// TestLookupRefusalKeepsStoreErrorsInternal: a store error other than the +// series limit may name vmstorage nodes; it goes to the log, not into a +// message that could reach the client. +func TestLookupRefusalKeepsStoreErrorsInternal(t *testing.T) { + body := strings.NewReader(`{"status":"error","error":"cannot fetch from vmstorage-longterm-0.vmstorage:8401: connection refused"}`) + herr := lookupRefusal("pod", body) + if herr.Code != http.StatusBadGateway || strings.Contains(herr.Message, "vmstorage") { + t.Fatalf("lookupRefusal = %d %q, want 502 without the store's text", herr.Code, herr.Message) + } +} diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 8ae395d45..9a56e9e18 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -485,6 +485,11 @@ func TestServerApplicationLookupStoreErrors(t *testing.T) { if strings.Contains(string(body), `"errorType"`) { t.Errorf("the store's raw JSON reached the client: %q", body) } + // Other store errors may name vmstorage nodes; only the limit + // message, which the client can act on, is passed on. + if tc.want != http.StatusUnprocessableEntity && strings.Contains(string(body), "vmstorage") { + t.Errorf("the store's error text reached the client: %q", body) + } } } From 5fc9e5a0b9fa83f925b62e68783e047b6106869c Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 02:33:22 +0700 Subject: [PATCH 23/41] fix(observability-proxy): pass on only the limit text of a lookup 422 The series-limit refusal of a lookup reached the client with all of cluster vmselect's wrapping, and that context can name vmstorage nodes, the leak the generic 502 had just closed for other errors. The client now gets the message from the limit on. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 5 ++++- internal/observabilityproxy/server_app_test.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 561a92ab8..526bd0b52 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -196,7 +196,10 @@ func lookupRefusal(label string, body io.Reader) *HTTPError { _ = json.NewDecoder(io.LimitReader(body, 64<<10)).Decode(&answer) switch { case strings.Contains(answer.Error, seriesLimitError): - return errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, answer.Error) + // From the limit on: what precedes it is cluster vmselect's context, + // which can name vmstorage nodes. + limit := answer.Error[strings.Index(answer.Error, seriesLimitError):] + return errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, limit) case strings.Contains(answer.Error, "timeout exceeded"): return errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", label) default: diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 9a56e9e18..52cfaa255 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -487,7 +487,7 @@ func TestServerApplicationLookupStoreErrors(t *testing.T) { } // Other store errors may name vmstorage nodes; only the limit // message, which the client can act on, is passed on. - if tc.want != http.StatusUnprocessableEntity && strings.Contains(string(body), "vmstorage") { + if strings.Contains(string(body), "vmstorage") { t.Errorf("the store's error text reached the client: %q", body) } } From 486f13dce690507a12319a63166b1ac85b0ba398 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 02:33:22 +0700 Subject: [PATCH 24/41] docs(observability-proxy): name every query form the app 422 refuses The refusal named only the @ modifier and non-literal offsets, but the same answer covers an @ inside a label value and WITH with a negative offset, so a client could not tell why such a query was refused. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 526bd0b52..6c3a9fb4f 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -102,7 +102,7 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { - return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ modifier, or an offset that is not a plain duration); use the namespace root") + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, WITH with a negative offset, or an offset that is not a plain duration); use the namespace root") } lim := s.cfg.Metrics.App if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { From 5418d6b40e864022c32914e57f6b2084e6ccdc81 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:05:54 +0700 Subject: [PATCH 25/41] fix(observability-proxy): refuse compound offsets on the app root The offset scan read a duration up to its first sign and ignored the rest. metricsql lexes 1h-30m as one duration and DurationValue makes every part after the first negative one subtract, so offset -1h-30m shifts samples 90 minutes forward while the scan covered 60, and offset 1h-2h shifts them an hour forward while the scan counted it as looking back. The pod and PVC lookup then missed pods created in the gap and the query answered 200 without them. A duration followed directly by a minus now makes the scan unknown, which answers 422 on the application root. A plus or a spaced sign is an operator after the offset in metricsql, not part of it, and is still read. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observabilityproxy/metricslookback.go | 6 +++++ .../observabilityproxy/metricsparams_test.go | 13 +++++++++++ .../observabilityproxy/server_app_test.go | 22 +++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index e9e7cb0e9..886d9e43b 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -169,6 +169,12 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { if end < 0 { end = len(arg) } + // metricsql lexes 1h-30m as one duration whose parts after the + // first negative one all subtract (DurationValue), so -1h-30m is + // -90m and 1h-2h is -1h; the scan does not read such compounds. + if end < len(arg) && arg[end] == '-' { + return 0, 0, false + } d, ok := parseQueryDuration(arg[:end]) if !ok { return 0, 0, false diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 5e0d548b7..29802a220 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -442,6 +442,12 @@ func TestQueryLookahead(t *testing.T) { {`rate(x[5m] offset -30m) offset -1h`, 90 * time.Minute}, {`x offset -1h + y offset 2h`, time.Hour}, {`node_offset_seconds - 1`, 0}, + // metricsql lexes a duration as one token, so a sign with spaces, + // or a plus, is an operator after the offset, not part of it. + {`x offset 1h+30m`, 0}, + {`x offset 1h - 30m`, 0}, + {`x offset -1h + 30m`, time.Hour}, + {`x offset -1h30m`, 90 * time.Minute}, } for _, tt := range tests { t.Run(tt.query, func(t *testing.T) { @@ -460,6 +466,13 @@ func TestQueryLookahead(t *testing.T) { `x @ (end() - 1h)`, `x{a="a@b"}`, `x offset -5i`, + // Compound durations with a negative part (metricsql DurationValue): + // -1h-30m is -90m and 1h-2h is -1h, neither of which the scan reads. + `x offset -1h-30m`, + `x offset - 1h-30m`, + `x offset 1h-2h`, + `x offset 1h-30m`, + `rate(x[5m] offset 1h-30m)`, `x offset -$__interval`, `with (f(a) = a offset -1h) f(f(x))`, } { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 52cfaa255..e4f0ca34f 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -512,3 +512,25 @@ func TestServerApplicationRefusesEveryAt(t *testing.T) { t.Fatalf("upstream saw %d requests for refused queries", n) } } + +// TestServerApplicationRefusesForwardReads: queries that read past the +// evaluation time by an amount the scan cannot bound answer 422 and send +// nothing upstream, rather than answer 200 without pods created in that +// time. +func TestServerApplicationRefusesForwardReads(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{ + "container_cpu_usage_seconds_total offset -1h-30m", + "container_cpu_usage_seconds_total offset 1h-2h", + } { + q := url.Values{"query": {query}, "time": {unix(testNow.Add(-2 * time.Hour))}} + resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good", nil, nil) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, resp.StatusCode) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for refused queries", n) + } +} From cb35399ee70f49399c467bb2a4be9117b923255d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:06:04 +0700 Subject: [PATCH 26/41] fix(observability-proxy): refuse rollup_candlestick on the app root vmselect evaluates rollup_candlestick one step ahead of the evaluation time, with no offset in the query: evalRollupFuncWithoutAt adds the step in effect to the window's start and end (app/vmselect/promql/ eval.go in v1.136.0-cluster, unchanged in v1.139.0). The lookahead scan returned zero for it, so the lookup stopped at the evaluation time and pods created during that step were left out of a 200. The step in effect is the subquery's when the call is nested, so rather than size it the application root answers 422. No other function reads ahead: the subquery evaluation past its end in evalRollupFuncWithSubquery is outside the outer rollup's window. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 2 +- internal/observabilityproxy/metricslookback.go | 8 +++++++- internal/observabilityproxy/metricsparams_test.go | 5 +++++ internal/observabilityproxy/server_app_test.go | 1 + 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 6c3a9fb4f..9911abeeb 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -102,7 +102,7 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { - return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, WITH with a negative offset, or an offset that is not a plain duration); use the namespace root") + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, rollup_candlestick, WITH with a negative offset, or an offset that is not a plain duration); use the namespace root") } lim := s.cfg.Metrics.App if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 886d9e43b..379a42be0 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -133,12 +133,18 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { // a query reads, from the same lexical scan. Only a negative offset reads // forward; the negative offsets are summed, which covers nesting. Any @ // makes the bound unknown, as it does for queryLookback, and so does WITH, -// which can apply one offset several times. +// which can apply one offset several times. vmselect also evaluates +// rollup_candlestick one step ahead of the evaluation time +// (evalRollupFuncWithoutAt), with the step of whatever subquery it sits +// in, so it makes the bound unknown too. func queryLookahead(query string) (time.Duration, bool) { if strings.Contains(query, "@") { return 0, false } lower := strings.ToLower(query) + if hasKeyword(lower, "rollup_candlestick") { + return 0, false + } _, ahead, ok := scanOffsets(lower) if !ok || ahead > 0 && hasKeyword(lower, "with") || ahead > float64(maxMetricsDuration) { return 0, false diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 29802a220..d320333fd 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -473,6 +473,11 @@ func TestQueryLookahead(t *testing.T) { `x offset 1h-2h`, `x offset 1h-30m`, `rate(x[5m] offset 1h-30m)`, + // vmselect evaluates rollup_candlestick one step ahead + // (evalRollupFuncWithoutAt), whatever the offset. + `rollup_candlestick(x)`, + `ROLLUP_CANDLESTICK(x[5m])`, + `sum(rollup_candlestick(x[1h:5m]))`, `x offset -$__interval`, `with (f(a) = a offset -1h) f(f(x))`, } { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index e4f0ca34f..3ede9f239 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -522,6 +522,7 @@ func TestServerApplicationRefusesForwardReads(t *testing.T) { for _, query := range []string{ "container_cpu_usage_seconds_total offset -1h-30m", "container_cpu_usage_seconds_total offset 1h-2h", + "rollup_candlestick(container_cpu_usage_seconds_total)", } { q := url.Values{"query": {query}, "time": {unix(testNow.Add(-2 * time.Hour))}} resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good", nil, nil) From 84f19e14d88b7a6567ad6b98f67120128e71465b Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:11:42 +0700 Subject: [PATCH 27/41] fix(observability-proxy): reach the subquery steps in the app lookup vmselect evaluates a subquery up to its end plus one subquery step, rounded up to the step (evalRollupFuncWithSubquery in app/vmselect/promql/eval.go), and range_* transforms (setLastValues) and keep_next_value carry those later values back onto earlier points. A series that starts after the evaluation time, such as a pod created within that reach, can therefore appear in the answer, which the earlier claim that only rollup_candlestick reads ahead missed. The lookup stopped at the evaluation time and left such a pod out of a 200. The lookahead now adds two steps per subquery level, counting every "(" as a possible implicit subquery and using the largest step seen, as the lookback already does. Overreaching only widens the lookup over the application's own pods. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observabilityproxy/metricslookback.go | 52 +++++++++++++++---- internal/observabilityproxy/metricsparams.go | 4 +- .../observabilityproxy/metricsparams_test.go | 22 ++++++-- .../observabilityproxy/server_app_test.go | 22 +++++++- 4 files changed, 84 insertions(+), 16 deletions(-) diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 379a42be0..ffff274a2 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -130,14 +130,19 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { } // queryLookahead is an upper bound on how far after its evaluation window -// a query reads, from the same lexical scan. Only a negative offset reads -// forward; the negative offsets are summed, which covers nesting. Any @ -// makes the bound unknown, as it does for queryLookback, and so does WITH, -// which can apply one offset several times. vmselect also evaluates -// rollup_candlestick one step ahead of the evaluation time -// (evalRollupFuncWithoutAt), with the step of whatever subquery it sits -// in, so it makes the bound unknown too. -func queryLookahead(query string) (time.Duration, bool) { +// a query reads, from the same lexical scan, with step the query's step. +// A negative offset reads forward; the negative offsets are summed, which +// covers nesting. vmselect also evaluates a subquery up to its end plus +// its step, rounded up to the step (evalRollupFuncWithSubquery), and +// range_* transforms and keep_next_value carry those later values back +// onto earlier points, so each subquery level adds two steps; as for the +// lookback, every "(" counts as a possible implicit subquery and each +// term uses the largest step seen. Any @ makes the bound unknown, as it +// does for queryLookback, and so does WITH, which can apply one offset +// several times. vmselect also evaluates rollup_candlestick one step ahead +// of the evaluation time (evalRollupFuncWithoutAt), so it makes the bound +// unknown too. +func queryLookahead(query string, step time.Duration) (time.Duration, bool) { if strings.Contains(query, "@") { return 0, false } @@ -146,7 +151,36 @@ func queryLookahead(query string) (time.Duration, bool) { return 0, false } _, ahead, ok := scanOffsets(lower) - if !ok || ahead > 0 && hasKeyword(lower, "with") || ahead > float64(maxMetricsDuration) { + if !ok || ahead > 0 && hasKeyword(lower, "with") { + return 0, false + } + subqueries := 0 + maxStep := float64(step) + for rest := query; ; { + open := strings.IndexByte(rest, '[') + if open < 0 { + break + } + closing := strings.IndexByte(rest[open:], ']') + if closing < 0 { + return 0, false + } + _, subStep, isSubquery := strings.Cut(rest[open+1:open+closing], ":") + rest = rest[open+closing+1:] + if !isSubquery { + continue + } + subqueries++ + if strings.TrimSpace(subStep) != "" { + d, ok := parseQueryDuration(subStep) + if !ok { + return 0, false + } + maxStep = max(maxStep, d) + } + } + ahead += 2 * float64(subqueries+strings.Count(query, "(")) * maxStep + if ahead > float64(maxMetricsDuration) { return 0, false } return time.Duration(ahead), true diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index 721496fe3..63be3c497 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -146,7 +146,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step) req.from, req.to = at, at - if ahead, ok := queryLookahead(out.Get("query")); ok { + if ahead, ok := queryLookahead(out.Get("query"), step); ok { req.last = at.Add(ahead) } return req, nil @@ -174,7 +174,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti req.start = queryStart(out.Get("query"), start, step) req.from, req.to = start, end // vmselect rounds end up to a whole step (AdjustStartEnd). - if ahead, ok := queryLookahead(out.Get("query")); ok { + if ahead, ok := queryLookahead(out.Get("query"), step); ok { req.last = end.Add(step + ahead) } out.Set("start", formatTime(start)) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index d320333fd..95ee047d8 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -439,7 +439,9 @@ func TestQueryLookahead(t *testing.T) { {`x offset 1h`, 0}, {`x offset -1h`, time.Hour}, {`x OFFSET - 90m`, 90 * time.Minute}, - {`rate(x[5m] offset -30m) offset -1h`, 90 * time.Minute}, + // Each "(" may be an implicit subquery, which vmselect evaluates + // up to two steps past its end; here the step is 1m. + {`rate(x[5m] offset -30m) offset -1h`, 92 * time.Minute}, {`x offset -1h + y offset 2h`, time.Hour}, {`node_offset_seconds - 1`, 0}, // metricsql lexes a duration as one token, so a sign with spaces, @@ -451,7 +453,7 @@ func TestQueryLookahead(t *testing.T) { } for _, tt := range tests { t.Run(tt.query, func(t *testing.T) { - got, ok := queryLookahead(tt.query) + got, ok := queryLookahead(tt.query, time.Minute) if !ok || got != tt.want { t.Fatalf("queryLookahead = %s, %v; want %s", got, ok, tt.want) } @@ -481,8 +483,22 @@ func TestQueryLookahead(t *testing.T) { `x offset -$__interval`, `with (f(a) = a offset -1h) f(f(x))`, } { - if got, ok := queryLookahead(q); ok { + if got, ok := queryLookahead(q, time.Minute); ok { t.Errorf("queryLookahead(%q) = %s, want unknown", q, got) } } } + +// TestQueryLookaheadSubquery: vmselect evaluates a subquery up to its end +// plus one subquery step, rounded up to the step (evalRollupFuncWithSubquery), +// and range_last and the other range_* transforms copy the last of those +// values back onto earlier points, so a series that starts after the +// evaluation time can reach the answer. +func TestQueryLookaheadSubquery(t *testing.T) { + for _, q := range []string{`max_over_time(range_last(x)[1h:1h])`, `max_over_time(keep_next_value(x)[1h:1h])`} { + got, ok := queryLookahead(q, time.Minute) + if !ok || got < 2*time.Hour { + t.Errorf("queryLookahead(%q) = %s, %v; want at least two subquery steps", q, got, ok) + } + } +} diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 3ede9f239..2d0d9b56e 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -184,8 +184,9 @@ func TestServerApplicationMetricsScope(t *testing.T) { if want := formatTime(buildMetrics(t, "api/v1/query_range", hostile).start); r.form.Get("start") != want { t.Errorf("%s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } - // One step past end, which vmselect rounds up to a whole step. - if r.form.Get("end") != formatTime(testNow.Add(time.Minute)) || r.form.Get("limit") != "101" { + // One step past end, which vmselect rounds up to a whole step, and + // two steps for each "(" that may be an implicit subquery. + if r.form.Get("end") != formatTime(testNow.Add(5*time.Minute)) || r.form.Get("limit") != "101" { t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit")) } } @@ -535,3 +536,20 @@ func TestServerApplicationRefusesForwardReads(t *testing.T) { t.Fatalf("upstream saw %d requests for refused queries", n) } } + +// TestServerApplicationLookupCoversSubquery: a pod created during the two +// subquery steps past the evaluation time can show up through range_last, +// so the lookup reaches that far. +func TestServerApplicationLookupCoversSubquery(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + at := testNow.Add(-6 * time.Hour) + q := url.Values{"query": {"max_over_time(range_last(container_cpu_usage_seconds_total)[1h:1h])"}, "time": {unix(at)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + r := h.requests()[0] + end, err := time.Parse(time.RFC3339, r.form.Get("end")) + if err != nil || r.path != "/api/v1/label/pod/values" || end.Before(at.Add(2*time.Hour)) { + t.Fatalf("lookup %s end = %s, want at least %s", r.path, r.form.Get("end"), formatTime(at.Add(2*time.Hour))) + } +} From 56fc64575a4bcc58d5cf5d66732dcdb190040188 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:11:54 +0700 Subject: [PATCH 28/41] fix(observability-proxy): read a minus after an offset as an operator The compound-offset refusal took any minus right after an offset's duration for a compound, so x offset 1h-1 or x offset 5m-y answered 422. metricsql continues a duration token only with a minus followed by a duration part (scanSingleDuration), so only that form is refused now; anything else is the subtraction it is. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/metricslookback.go | 7 ++++++- internal/observabilityproxy/metricsparams_test.go | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index ffff274a2..16f1f4cff 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -49,6 +49,10 @@ var ( querySeconds = regexp.MustCompile(`^\d+(?:\.\d+)?$`) ) +// negativeDurationPart is what metricsql's scanSingleDuration accepts +// after a minus inside one duration token. +var negativeDurationPart = regexp.MustCompile(`^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d|w|y|i)|\$__interval)`) + var queryDurationUnits = map[string]time.Duration{ "ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour, "d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour, @@ -212,7 +216,8 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { // metricsql lexes 1h-30m as one duration whose parts after the // first negative one all subtract (DurationValue), so -1h-30m is // -90m and 1h-2h is -1h; the scan does not read such compounds. - if end < len(arg) && arg[end] == '-' { + // A minus not followed by a duration part is an operator. + if end < len(arg) && arg[end] == '-' && negativeDurationPart.MatchString(arg[end+1:]) { return 0, 0, false } d, ok := parseQueryDuration(arg[:end]) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 95ee047d8..26db536cf 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -450,6 +450,9 @@ func TestQueryLookahead(t *testing.T) { {`x offset 1h - 30m`, 0}, {`x offset -1h + 30m`, time.Hour}, {`x offset -1h30m`, 90 * time.Minute}, + // A minus not followed by a duration is an operator. + {`x offset 1h-1`, 0}, + {`x offset 5m-y`, 0}, } for _, tt := range tests { t.Run(tt.query, func(t *testing.T) { From b876368a3fd057debcb8147fa96a1cb00d594522 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:16:24 +0700 Subject: [PATCH 29/41] fix(observability-proxy): refuse any minus right after an offset again Telling a compound duration from a subtraction by a duration-part pattern missed forms metricsql accepts after the minus: offset 5m-1.h is 55 minutes forward and offset 5m-.5h is 25, yet both read as five minutes back, and the lookup silently missed pods created in between. A pattern that followed scanSingleDuration exactly would still be a second copy of the grammar to keep in step, so any minus right after an offset's duration is refused again; offset 1h-1 and offset 5m-y now answer 422 as well, and metricsql cannot parse the latter anyway. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/metricslookback.go | 10 ++++------ internal/observabilityproxy/metricsparams_test.go | 10 +++++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 16f1f4cff..9f01936b7 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -49,10 +49,6 @@ var ( querySeconds = regexp.MustCompile(`^\d+(?:\.\d+)?$`) ) -// negativeDurationPart is what metricsql's scanSingleDuration accepts -// after a minus inside one duration token. -var negativeDurationPart = regexp.MustCompile(`^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d|w|y|i)|\$__interval)`) - var queryDurationUnits = map[string]time.Duration{ "ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour, "d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour, @@ -216,8 +212,10 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { // metricsql lexes 1h-30m as one duration whose parts after the // first negative one all subtract (DurationValue), so -1h-30m is // -90m and 1h-2h is -1h; the scan does not read such compounds. - // A minus not followed by a duration part is an operator. - if end < len(arg) && arg[end] == '-' && negativeDurationPart.MatchString(arg[end+1:]) { + // metricsql continues a duration after a minus in more forms than + // the scan can tell from a subtraction (-1.h and -.5h among them), + // so any minus right after the duration is refused. + if end < len(arg) && arg[end] == '-' { return 0, 0, false } d, ok := parseQueryDuration(arg[:end]) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 26db536cf..15067ed6c 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -450,9 +450,6 @@ func TestQueryLookahead(t *testing.T) { {`x offset 1h - 30m`, 0}, {`x offset -1h + 30m`, time.Hour}, {`x offset -1h30m`, 90 * time.Minute}, - // A minus not followed by a duration is an operator. - {`x offset 1h-1`, 0}, - {`x offset 5m-y`, 0}, } for _, tt := range tests { t.Run(tt.query, func(t *testing.T) { @@ -477,6 +474,12 @@ func TestQueryLookahead(t *testing.T) { `x offset - 1h-30m`, `x offset 1h-2h`, `x offset 1h-30m`, + // metricsql continues a duration with -1.h or -.5h too, so any minus + // right after the duration is refused rather than told apart. + `x offset 5m-1.h`, + `x offset 5m-.5h`, + `x offset 1h-1`, + `x offset 5m-y`, `rate(x[5m] offset 1h-30m)`, // vmselect evaluates rollup_candlestick one step ahead // (evalRollupFuncWithoutAt), whatever the offset. @@ -498,6 +501,7 @@ func TestQueryLookahead(t *testing.T) { // values back onto earlier points, so a series that starts after the // evaluation time can reach the answer. func TestQueryLookaheadSubquery(t *testing.T) { + for _, q := range []string{`max_over_time(range_last(x)[1h:1h])`, `max_over_time(keep_next_value(x)[1h:1h])`} { got, ok := queryLookahead(q, time.Minute) if !ok || got < 2*time.Hour { From d164790946a4b3a557c1bc1204ca6b4f8b78e57c Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:16:31 +0700 Subject: [PATCH 30/41] test(observability-proxy): pin the subquery lookahead exactly The subquery sizing was tested only as "at least two hours", so which step drives it, the fallback to the query step for an empty subquery step, and its refusals (an unreadable step, an unclosed bracket, a sum past the duration range) were not held by any test. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observabilityproxy/metricsparams_test.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 15067ed6c..75b88d7f3 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -480,6 +480,12 @@ func TestQueryLookahead(t *testing.T) { `x offset 5m-.5h`, `x offset 1h-1`, `x offset 5m-y`, + // A subquery step the scan cannot read, an unclosed bracket, and a + // sum past the duration range. + `rate(x[1h:5i])`, + `rate(x[1h:$__interval])`, + `rate(x[1h`, + `max_over_time(x[1h:100y])`, `rate(x[5m] offset 1h-30m)`, // vmselect evaluates rollup_candlestick one step ahead // (evalRollupFuncWithoutAt), whatever the offset. @@ -501,7 +507,16 @@ func TestQueryLookahead(t *testing.T) { // values back onto earlier points, so a series that starts after the // evaluation time can reach the answer. func TestQueryLookaheadSubquery(t *testing.T) { - + // One explicit subquery and one "(": two levels of two steps, at the + // subquery's step, or at the query's step when the subquery has none. + for q, want := range map[string]time.Duration{ + `max_over_time(x[1h:1h])`: 4 * time.Hour, + `max_over_time(x[1h:])`: 4 * time.Minute, + } { + if got, ok := queryLookahead(q, time.Minute); !ok || got != want { + t.Errorf("queryLookahead(%q) = %s, %v; want %s", q, got, ok, want) + } + } for _, q := range []string{`max_over_time(range_last(x)[1h:1h])`, `max_over_time(keep_next_value(x)[1h:1h])`} { got, ok := queryLookahead(q, time.Minute) if !ok || got < 2*time.Hour { From 054fac6ab0fa4c3e618080744d168c3a57f4bcc9 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:16:59 +0700 Subject: [PATCH 31/41] docs(observability-proxy): name subquery steps in the app lookup texts Since the lookup reaches subquery steps past the evaluation time, the applicationScope comment still described only negative offsets, and a query refused for a subquery step the scan cannot read got a 422 that listed every other cause but that one. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 5 +++-- internal/observabilityproxy/server_app_test.go | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 9911abeeb..4918c9588 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -95,14 +95,15 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // applicationScope looks the application's pods and PVCs up in store over // the samples the request reads: back by the query's own lookback, and -// forward to the latest sample a negative offset reads. A query that reads +// forward to the latest sample a negative offset or a subquery's +// evaluation past its end reads. A query that reads // further back than the configured lookback, or whose earliest or latest // sample is unknown, a failed lookup, and a list over its cap all fail the // request, because a successful answer would silently miss the // application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { - return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, rollup_candlestick, WITH with a negative offset, or an offset that is not a plain duration); use the namespace root") + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, rollup_candlestick, WITH with a negative offset, an offset that is not a plain duration, or a subquery step that is not one); use the namespace root") } lim := s.cfg.Metrics.App if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 2d0d9b56e..c451d35af 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -553,3 +553,15 @@ func TestServerApplicationLookupCoversSubquery(t *testing.T) { t.Fatalf("lookup %s end = %s, want at least %s", r.path, r.form.Get("end"), formatTime(at.Add(2*time.Hour))) } } + +// TestServerApplicationNamesUnreadableSubqueryStep: the refusal names the +// cause when a subquery step is what the scan cannot read. +func TestServerApplicationNamesUnreadableSubqueryStep(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+url.Values{"query": {"max_over_time(x[1h:5i])"}}.Encode(), "good", nil, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnprocessableEntity || !strings.Contains(string(body), "subquery step") { + t.Fatalf("answered %d %q, want 422 naming the subquery step", resp.StatusCode, body) + } +} From 8e491d15ddeb9dad0b82a9b5940342bcf2d666c5 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:21:16 +0700 Subject: [PATCH 32/41] fix(observability-proxy): accept only a token end after an offset metricsql also reads a trailing $__interval as part of an offset's duration: offset -1h$__interval is -(1h + step). The scan stopped at the "$" and read one hour, so the lookup missed pods created in the last step and the query answered 200 without them. Refusing a minus alone left every other continuation open. Only what certainly ends the duration token is now accepted after it: whitespace, a closing bracket or separator, or a binary operator other than minus; anything else answers 422 on the application root. The grammar comment no longer claims upper-case offset units are refused, as offsets are scanned lower-cased like metricsql reads them. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observabilityproxy/metricslookback.go | 19 +++++++++++++------ .../observabilityproxy/metricsparams_test.go | 4 ++++ .../observabilityproxy/server_app_test.go | 1 + 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 9f01936b7..55e78a0c2 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -41,14 +41,20 @@ const latencyOffset = 30 * time.Second // queryDurationPart is the subset of the MetricsQL duration grammar // (scanSingleDuration in metricsql) the scan understands. Step multiples -// (5i, $__interval), negative parts and upper-case units are left out on -// purpose: anything else makes the bound unknown. +// (5i, $__interval) and negative parts are left out on purpose: anything +// else makes the bound unknown. Offsets are scanned lower-cased, as +// metricsql reads units in any case; windows and steps in brackets are +// not, so an upper-case unit there is unknown. var ( queryDurationPart = regexp.MustCompile(`(\d+(?:\.\d+)?)(ms|s|m|h|d|w|y)`) queryDuration = regexp.MustCompile(`^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d|w|y))+$`) querySeconds = regexp.MustCompile(`^\d+(?:\.\d+)?$`) ) +// offsetTerminators may follow an offset's duration: whitespace, a +// closing bracket or separator, or a binary operator other than minus. +const offsetTerminators = " \t\n\r)]},+*/^%<>=!" + var queryDurationUnits = map[string]time.Duration{ "ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour, "d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour, @@ -212,10 +218,11 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { // metricsql lexes 1h-30m as one duration whose parts after the // first negative one all subtract (DurationValue), so -1h-30m is // -90m and 1h-2h is -1h; the scan does not read such compounds. - // metricsql continues a duration after a minus in more forms than - // the scan can tell from a subtraction (-1.h and -.5h among them), - // so any minus right after the duration is refused. - if end < len(arg) && arg[end] == '-' { + // metricsql continues a duration token in more forms than the scan + // reads: after a minus (-30m, -1.h, -.5h) and with a trailing + // $__interval, all summed by DurationValue. Only what certainly + // ends the token is accepted after the duration. + if end < len(arg) && !strings.ContainsRune(offsetTerminators, rune(arg[end])) { return 0, 0, false } d, ok := parseQueryDuration(arg[:end]) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 75b88d7f3..0fbbcd7a9 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -480,6 +480,10 @@ func TestQueryLookahead(t *testing.T) { `x offset 5m-.5h`, `x offset 1h-1`, `x offset 5m-y`, + // metricsql also reads a trailing $__interval as part of the + // duration: -1h$__interval is -(1h + step). + `x offset -1h$__interval`, + `rate(x[5m]) offset -1h$__interval`, // A subquery step the scan cannot read, an unclosed bracket, and a // sum past the duration range. `rate(x[1h:5i])`, diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index c451d35af..8b3a4e587 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -523,6 +523,7 @@ func TestServerApplicationRefusesForwardReads(t *testing.T) { for _, query := range []string{ "container_cpu_usage_seconds_total offset -1h-30m", "container_cpu_usage_seconds_total offset 1h-2h", + "container_cpu_usage_seconds_total offset -1h$__interval", "rollup_candlestick(container_cpu_usage_seconds_total)", } { q := url.Values{"query": {query}, "time": {unix(testNow.Add(-2 * time.Hour))}} From 3198de893ad0869995111f8879a62a3155e14225 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:37:04 +0700 Subject: [PATCH 33/41] fix(observability-proxy): refuse escapes outside strings on the app root metricsql unescapes identifiers (unescapeIdent in parser.go), so rollup\_candlestick(x + y) is rollup_candlestick to vmselect while the lexical scan saw another name: the lookup ended ten minutes past the evaluation time and vmselect read nearly thirteen. Any escaped keyword the scan looks for, offset included, had the same hole. A backslash outside a string literal now makes the scan unknown, which answers 422 on the application root; escapes inside label values are unaffected. With rollup_candlestick refused, two steps per "(" stay enough: a subquery evaluates at most one step past its end plus the alignment of that end to the step (evalRollupFuncWithSubquery), and the one step a candlestick over a non-selector would add on top cannot occur. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 2 +- .../observabilityproxy/metricslookback.go | 32 ++++++++++++++++--- .../observabilityproxy/metricsparams_test.go | 10 ++++++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 4918c9588..2f0f23751 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -103,7 +103,7 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { - return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, rollup_candlestick, WITH with a negative offset, an offset that is not a plain duration, or a subquery step that is not one); use the namespace root") + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, a backslash outside a string, rollup_candlestick, WITH with a negative offset, an offset that is not a plain duration, or a subquery step that is not one); use the namespace root") } lim := s.cfg.Metrics.App if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 55e78a0c2..7f936307f 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -145,11 +145,12 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { // lookback, every "(" counts as a possible implicit subquery and each // term uses the largest step seen. Any @ makes the bound unknown, as it // does for queryLookback, and so does WITH, which can apply one offset -// several times. vmselect also evaluates rollup_candlestick one step ahead -// of the evaluation time (evalRollupFuncWithoutAt), so it makes the bound -// unknown too. +// several times, and a backslash outside a string, which can spell an +// identifier the scan looks for differently. vmselect also evaluates +// rollup_candlestick one step ahead of the evaluation time +// (evalRollupFuncWithoutAt), so it makes the bound unknown too. func queryLookahead(query string, step time.Duration) (time.Duration, bool) { - if strings.Contains(query, "@") { + if strings.Contains(query, "@") || escapeOutsideString(query) { return 0, false } lower := strings.ToLower(query) @@ -237,6 +238,29 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { } } +// escapeOutsideString reports a backslash outside a string literal. +// metricsql unescapes identifiers (unescapeIdent), so rollup\_candlestick +// or offs\et name what the lexical scan looks for without its spelling. +// Inside double or single quotes a backslash escapes the next character; +// inside backquotes it is literal. +func escapeOutsideString(query string) bool { + var quote byte + for i := 0; i < len(query); i++ { + c := query[i] + switch { + case quote == 0 && c == '\\': + return true + case quote == 0 && (c == '"' || c == '\'' || c == '`'): + quote = c + case quote != 0 && quote != '`' && c == '\\': + i++ + case quote != 0 && c == quote: + quote = 0 + } + } + return false +} + // isKeyword reports whether lower[start:end] stands on its own rather than // inside an identifier. func isKeyword(lower string, start, end int) bool { diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 0fbbcd7a9..cf452a74f 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -444,6 +444,9 @@ func TestQueryLookahead(t *testing.T) { {`rate(x[5m] offset -30m) offset -1h`, 92 * time.Minute}, {`x offset -1h + y offset 2h`, time.Hour}, {`node_offset_seconds - 1`, 0}, + // An escape inside a string literal is only part of a label value. + {`x{a=~"a\\.b"}`, 0}, + {`x{a="\"rollup\\_candlestick\""}`, 0}, // metricsql lexes a duration as one token, so a sign with spaces, // or a plus, is an operator after the offset, not part of it. {`x offset 1h+30m`, 0}, @@ -491,6 +494,13 @@ func TestQueryLookahead(t *testing.T) { `rate(x[1h`, `max_over_time(x[1h:100y])`, `rate(x[5m] offset 1h-30m)`, + // metricsql unescapes identifiers (unescapeIdent), so an escaped + // rollup_candlestick, or any escape outside a string, could hide + // what the scan looks for. + `rollup\_candlestick(x + y)`, + `rollup_candle\stick(x)`, + `rollup\x5fcandlestick(x)`, + `x offs\et -1h`, // vmselect evaluates rollup_candlestick one step ahead // (evalRollupFuncWithoutAt), whatever the offset. `rollup_candlestick(x)`, From a54760fd9cc145b79c6c901011fe5cb6857a530a Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:37:53 +0700 Subject: [PATCH 34/41] fix(observability-proxy): start the instant app lookup a latency early vmselect evaluates an instant query whose time is within 30s of its own clock at that clock minus 30s (queryOffset in prometheus.go), not at time minus 30s. For a time slightly in the future the evaluation starts earlier than the proxy assumed: time=now+29s with a 1s step read from six minutes before, while the lookup started at five minutes 32 seconds, and pods present only in that gap were missing from a 200. The evaluation instant is always within (time-60s, time], so the lookup of an instant query now starts one more latency offset early. That needs no assumption about the proxy's clock matching vmselect's. The store choice on the namespace root is left as it is. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 8 +++++++ .../observabilityproxy/server_app_test.go | 24 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 2f0f23751..0fe8b05e9 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -110,6 +110,14 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback) } start := req.start + // vmselect evaluates an instant query whose time is within 30s of its + // own clock at that clock minus 30s (queryOffset in prometheus.go), so + // the evaluation instant lies anywhere in (time-60s, time], while + // req.start assumes time-30s. One more latency offset covers it without + // trusting the proxy's clock to match vmselect's. + if req.path == "api/v1/query" { + start = start.Add(-latencyOffset) + } var lists [2][]string for i, l := range []struct { label, metric string diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 8b3a4e587..39b6ac60f 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -341,7 +341,8 @@ func TestServerApplicationLookbackBeyondCap(t *testing.T) { t.Fatalf("rate(x[5m]) answered %d, want 200", got) } r := h.requests()[0] - want := formatTime(buildMetrics(t, "api/v1/query", q).start) + // An instant query may be evaluated up to one more latency offset early. + want := formatTime(buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset)) if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } @@ -566,3 +567,24 @@ func TestServerApplicationNamesUnreadableSubqueryStep(t *testing.T) { t.Fatalf("answered %d %q, want 422 naming the subquery step", resp.StatusCode, body) } } + +// TestServerApplicationLookupCoversLatencyOffset: vmselect evaluates an +// instant query whose time is within 30s of its own clock at its clock +// minus 30s (prometheus.go, queryOffset), so for a time slightly in the +// future it reads from earlier than time minus 30s. The lookup starts +// early enough for any evaluation instant in (time-60s, time]. +func TestServerApplicationLookupCoversLatencyOffset(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + at := testNow.Add(29 * time.Second) + q := url.Values{"query": {"container_cpu_usage_seconds_total"}, "time": {at.Format(time.RFC3339)}, "step": {"1s"}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + // The earliest read of the query evaluated at the earliest instant. + want := buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset) + r := h.requests()[0] + start, err := time.Parse(time.RFC3339, r.form.Get("start")) + if err != nil || start.After(want) { + t.Fatalf("lookup start = %s, want at most %s", r.form.Get("start"), formatTime(want)) + } +} From cf45a23bf3c79772928e606176133857ce800bd1 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:44:21 +0700 Subject: [PATCH 35/41] fix(observability-proxy): refuse comments outside strings on the app root The escape check tracked string literals but not comments: in metricsql a "#" outside a string comments out the rest of the line, so a quote inside a comment made the check treat the next line's code as string content, and rollup\_candlestick there was accepted again. The bracket and offset scans do not understand comments either. A "#" outside a string now makes the lookahead unknown as well, which answers 422 on the application root, rather than teaching every scan about comments; API clients do not send them. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 2 +- .../observabilityproxy/metricslookback.go | 23 +++++++++++-------- .../observabilityproxy/metricsparams_test.go | 5 ++++ 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 0fe8b05e9..53f8d5582 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -103,7 +103,7 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { - return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, a backslash outside a string, rollup_candlestick, WITH with a negative offset, an offset that is not a plain duration, or a subquery step that is not one); use the namespace root") + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, a backslash or a comment outside a string, rollup_candlestick, WITH with a negative offset, an offset that is not a plain duration, or a subquery step that is not one); use the namespace root") } lim := s.cfg.Metrics.App if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 7f936307f..b478f582b 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -145,12 +145,12 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { // lookback, every "(" counts as a possible implicit subquery and each // term uses the largest step seen. Any @ makes the bound unknown, as it // does for queryLookback, and so does WITH, which can apply one offset -// several times, and a backslash outside a string, which can spell an -// identifier the scan looks for differently. vmselect also evaluates +// several times, and a backslash or a comment outside a string, which can +// hide what the scan looks for. vmselect also evaluates // rollup_candlestick one step ahead of the evaluation time // (evalRollupFuncWithoutAt), so it makes the bound unknown too. func queryLookahead(query string, step time.Duration) (time.Duration, bool) { - if strings.Contains(query, "@") || escapeOutsideString(query) { + if strings.Contains(query, "@") || unscannableOutsideString(query) { return 0, false } lower := strings.ToLower(query) @@ -238,17 +238,20 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { } } -// escapeOutsideString reports a backslash outside a string literal. -// metricsql unescapes identifiers (unescapeIdent), so rollup\_candlestick -// or offs\et name what the lexical scan looks for without its spelling. -// Inside double or single quotes a backslash escapes the next character; -// inside backquotes it is literal. -func escapeOutsideString(query string) bool { +// unscannableOutsideString reports a backslash or a "#" outside a string +// literal. metricsql unescapes identifiers (unescapeIdent), so +// rollup\_candlestick or offs\et name what the lexical scan looks for +// without its spelling; "#" starts a comment to the end of the line, and +// quotes or brackets inside one would mislead the string tracking here and +// the bracket and offset scans. Inside double or single quotes a backslash +// escapes the next character; inside backquotes it is literal, and a +// backquote escaped there makes metricsql refuse the string anyway. +func unscannableOutsideString(query string) bool { var quote byte for i := 0; i < len(query); i++ { c := query[i] switch { - case quote == 0 && c == '\\': + case quote == 0 && (c == '\\' || c == '#'): return true case quote == 0 && (c == '"' || c == '\'' || c == '`'): quote = c diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index cf452a74f..e201a6fa8 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -501,6 +501,11 @@ func TestQueryLookahead(t *testing.T) { `rollup_candle\stick(x)`, `rollup\x5fcandlestick(x)`, `x offs\et -1h`, + // A comment runs to the end of the line, so quotes in it would + // confuse the string tracking and brackets in it the scans. + "x # '\n+ rollup\\_candlestick(y) # '", + "x # \"\n+ rollup\\_candlestick(y) # \"", + "rate(x[5m]) # comment", // vmselect evaluates rollup_candlestick one step ahead // (evalRollupFuncWithoutAt), whatever the offset. `rollup_candlestick(x)`, From ac5a3c88c418b60216b0c6b93441f26b4229a98d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:45:21 +0700 Subject: [PATCH 36/41] fix(observability-proxy): put the latency window in the instant start The second latency offset was added to the application lookup only, so the other readers of the instant query's earliest read still assumed time minus 30s: the store choice could send a query to the short-term store up to 30 seconds too late near its retention edge, the lookback cap compared against a start the lookup no longer used, and the field no longer meant the earliest instant read. The earliest read of an instant query now starts from time minus two latency offsets, the earliest instant vmselect can evaluate it at, and the lookup uses it as is. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 8 -------- internal/observabilityproxy/metricsparams.go | 6 +++++- .../observabilityproxy/metricsparams_test.go | 18 +++++++++++++++++- internal/observabilityproxy/server_app_test.go | 6 +++--- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 53f8d5582..e6df8ba02 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -110,14 +110,6 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback) } start := req.start - // vmselect evaluates an instant query whose time is within 30s of its - // own clock at that clock minus 30s (queryOffset in prometheus.go), so - // the evaluation instant lies anywhere in (time-60s, time], while - // req.start assumes time-30s. One more latency offset covers it without - // trusting the proxy's clock to match vmselect's. - if req.path == "api/v1/query" { - start = start.Add(-latencyOffset) - } var lists [2][]string for i, l := range []struct { label, metric string diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index 63be3c497..f02620b3c 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -144,7 +144,11 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } out.Set("step", formatSeconds(step)) } - req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step) + // vmselect evaluates an instant query whose time is within the + // latency offset of its own clock at that clock minus the offset + // (queryOffset in prometheus.go), so the evaluation instant lies in + // (time-2*offset, time]; the earlier end needs no clock to agree. + req.start = queryStart(out.Get("query"), at.Add(-2*latencyOffset), step) req.from, req.to = at, at if ahead, ok := queryLookahead(out.Get("query"), step); ok { req.last = at.Add(ahead) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index e201a6fa8..34d36f7e5 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -227,7 +227,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) { if req.params.Get("end") != "2026-11-04T12:00:00.000Z" || req.params.Get("start") != "2026-11-04T11:55:00.000Z" { t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end")) } - if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-30*time.Second)) { + if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-2*latencyOffset)) { t.Errorf("instant query without time: params %v, start %v", q.params, q.start) } } @@ -543,3 +543,19 @@ func TestQueryLookaheadSubquery(t *testing.T) { } } } + +// TestBuildMetricsParamsInstantStartCoversLatencyOffset: vmselect evaluates +// an instant query whose time is within 30s of its clock at that clock +// minus 30s, anywhere in (time-60s, time], so the earliest read the store +// choice and the lookup rely on starts from time-60s. +func TestBuildMetricsParamsInstantStartCoversLatencyOffset(t *testing.T) { + at := testNow.Add(29 * time.Second) + req := buildMetrics(t, "api/v1/query", url.Values{"query": {"x"}, "time": {at.Format(time.RFC3339)}, "step": {"1s"}}) + lookback, ok := queryLookback("x", time.Second) + if !ok { + t.Fatal("lookback of x unknown") + } + if want := at.Add(-2*latencyOffset - lookback); req.start.After(want) { + t.Fatalf("start = %s, want at most %s", req.start, want) + } +} diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 39b6ac60f..e5c0d3710 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -341,8 +341,7 @@ func TestServerApplicationLookbackBeyondCap(t *testing.T) { t.Fatalf("rate(x[5m]) answered %d, want 200", got) } r := h.requests()[0] - // An instant query may be evaluated up to one more latency offset early. - want := formatTime(buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset)) + want := formatTime(buildMetrics(t, "api/v1/query", q).start) if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } @@ -581,7 +580,8 @@ func TestServerApplicationLookupCoversLatencyOffset(t *testing.T) { t.Fatalf("status = %d", got) } // The earliest read of the query evaluated at the earliest instant. - want := buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset) + lookback, _ := queryLookback("container_cpu_usage_seconds_total", time.Second) + want := at.Add(-2*latencyOffset - lookback) r := h.requests()[0] start, err := time.Parse(time.RFC3339, r.form.Get("start")) if err != nil || start.After(want) { From 79ea8561de6f419d52830d3822a331d04c77e7c3 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 07:00:58 +0700 Subject: [PATCH 37/41] fix(observability-proxy): end a backquoted string only at an unescaped quote The string tracking treated a backslash inside backquotes as literal, but metricsql's scanString lets a backslash escape the closing quote in all three quote kinds, and a quoted label name keeps that escape ({`\``=""} parses). The proxy ended such a string early, read the rest of the query as string content, and accepted an escaped rollup_candlestick after it with the lookup ending too soon. Inside any quote a backslash now skips the next character, as scanString does, and the comment no longer claims metricsql refuses an escaped backquote. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observabilityproxy/metricslookback.go | 9 +++++---- .../observabilityproxy/metricsparams_test.go | 5 +++++ .../observabilityproxy/server_app_test.go | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index b478f582b..3571afc4c 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -243,9 +243,10 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { // rollup\_candlestick or offs\et name what the lexical scan looks for // without its spelling; "#" starts a comment to the end of the line, and // quotes or brackets inside one would mislead the string tracking here and -// the bracket and offset scans. Inside double or single quotes a backslash -// escapes the next character; inside backquotes it is literal, and a -// backquote escaped there makes metricsql refuse the string anyway. +// the bracket and offset scans. Inside any of the three quotes a backslash +// skips the next character, as metricsql's scanString does: an escaped +// closing quote does not end the string, backquotes included (a quoted +// label name keeps such an escape). func unscannableOutsideString(query string) bool { var quote byte for i := 0; i < len(query); i++ { @@ -255,7 +256,7 @@ func unscannableOutsideString(query string) bool { return true case quote == 0 && (c == '"' || c == '\'' || c == '`'): quote = c - case quote != 0 && quote != '`' && c == '\\': + case quote != 0 && c == '\\': i++ case quote != 0 && c == quote: quote = 0 diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 34d36f7e5..2cc22a4ae 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -506,6 +506,11 @@ func TestQueryLookahead(t *testing.T) { "x # '\n+ rollup\\_candlestick(y) # '", "x # \"\n+ rollup\\_candlestick(y) # \"", "rate(x[5m]) # comment", + // metricsql lets a backslash escape the closing quote in backquotes + // too (scanString), and quoted label names keep such escapes, so + // a backquoted string must not end at an escaped backquote. + "container_cpu_usage_seconds_total{`\\``=\"\"} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)", + "{`\\`a`} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)", // vmselect evaluates rollup_candlestick one step ahead // (evalRollupFuncWithoutAt), whatever the offset. `rollup_candlestick(x)`, diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index e5c0d3710..ade81ca04 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -588,3 +588,22 @@ func TestServerApplicationLookupCoversLatencyOffset(t *testing.T) { t.Fatalf("lookup start = %s, want at most %s", r.form.Get("start"), formatTime(want)) } } + +// TestServerApplicationRefusesBackquoteEscapeTail: an escaped backquote +// inside a quoted label name does not end the string in metricsql, so the +// escaped rollup_candlestick after it must still be refused. +func TestServerApplicationRefusesBackquoteEscapeTail(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{ + "container_cpu_usage_seconds_total{`\\``=\"\"} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)", + "{`\\`a`} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)", + } { + q := url.Values{"query": {query}, "time": {unix(testNow.Add(-6 * time.Hour))}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for refused queries", n) + } +} From e8e0cede61222e884441b7203094c18c13133736 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 18:09:46 +0700 Subject: [PATCH 38/41] fix(observability-proxy): forward the default instant query time An instant query without time had its application lookup sized at the proxy's clock, but the time was not forwarded, so vmselect evaluated at its own later clock. With lookups taking most of the request deadline, the evaluation lands well past the end of the lookup and leaves out pods first sampled in between. Forward the instant the lookup was sized for. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/metricsparams.go | 5 ++++- .../observabilityproxy/metricsparams_test.go | 2 +- .../observabilityproxy/server_app_test.go | 21 +++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index f02620b3c..c40bc058f 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -134,8 +134,11 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti return req, herr } at = t - out.Set("time", formatTime(t)) } + // The default is forwarded too: the application lookup is sized + // for at, and vmselect's own clock reads later, by as long as the + // lookups took. + out.Set("time", formatTime(at)) step := instantDefaultStep if v := in.Get("step"); v != "" { var herr *HTTPError diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 2cc22a4ae..ba14a9781 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -227,7 +227,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) { if req.params.Get("end") != "2026-11-04T12:00:00.000Z" || req.params.Get("start") != "2026-11-04T11:55:00.000Z" { t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end")) } - if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-2*latencyOffset)) { + if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Get("time") != "2026-11-04T12:00:00.000Z" || !q.start.Equal(testNow.Add(-15*time.Minute-2*latencyOffset)) { t.Errorf("instant query without time: params %v, start %v", q.params, q.start) } } diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index ade81ca04..7afaa3618 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -363,6 +363,27 @@ func TestServerApplicationLookupCoversNegativeOffset(t *testing.T) { } } +// TestServerApplicationForwardsDefaultTime: without time the lookup is +// sized at the proxy's clock, so that instant is forwarded. Otherwise +// vmselect evaluates at its own later clock, past the end of the lookup by +// however long the lookups took, and leaves out pods first sampled since. +func TestServerApplicationForwardsDefaultTime(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + q := url.Values{"query": {"container_cpu_usage_seconds_total"}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + reqs := h.requests() + lookup, final := reqs[0], reqs[len(reqs)-1] + want := formatTime(testNow) + if lookup.path != "/api/v1/label/pod/values" || lookup.form.Get("end") != want { + t.Fatalf("lookup %s end = %s, want %s", lookup.path, lookup.form.Get("end"), want) + } + if final.path != "/api/v1/query" || final.form.Get("time") != want { + t.Fatalf("query %s time = %q, want the lookup's instant %s", final.path, final.form.Get("time"), want) + } +} + // TestServerApplicationRefusesUnboundedSampleTime refuses a query whose // samples may lie at a time the lookup cannot be sized for, rather than // answer with series silently left out. From 19dbee15fc7cb2aef22fd36151061d18cb26a207 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 18:15:09 +0700 Subject: [PATCH 39/41] docs(observability-proxy): explain why both application roots need the namespace root The reason given for the second access review held only for the metrics root. The log root needs it too, because an ownerless pod keeps the lineage labels its creator set and can claim any application. The tenant ClusterRole comment also named only the namespace roots, though the application roots now depend on the same rule. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/route.go | 12 ++++++++---- internal/observabilityproxy/server_app_test.go | 7 ++++--- .../templates/tenant-clusterroles.yaml | 6 +++++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/internal/observabilityproxy/route.go b/internal/observabilityproxy/route.go index 49a7c1770..bc12c6276 100644 --- a/internal/observabilityproxy/route.go +++ b/internal/observabilityproxy/route.go @@ -132,10 +132,14 @@ func ParseRoute(escapedPath string) (Route, *HTTPError) { } // Reviews lists every access a request on the route needs. An application -// root also needs read access to its namespace root: its pod and PVC -// branches match by name across the window, so a name another application -// held earlier reaches that application's series, and a grant narrowed by -// resourceNames would read more than it names. +// root also needs read access to its namespace root, because its scope +// narrows within the namespace but does not isolate: on both roots the +// identity comes from lineage labels that the lineage webhook keeps as set +// on an object it finds no owner for, so anyone who can create a pod there +// can label it as the application; on the metrics root the pod and PVC +// branches also match by name across the window, so a name another +// application held earlier reaches that application's series. A grant +// narrowed by resourceNames would read more than it names. func (r Route) Reviews() []authorizationv1.ResourceAttributes { switch r.Kind { case ApplicationLog: diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 7afaa3618..461cbc16d 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -282,9 +282,10 @@ func TestServerApplicationMetricsBranches(t *testing.T) { // A grant narrowed to application "a" by resourceNames must not reach "b" // through any endpoint of either root. Without read access to the -// namespace roots it does not reach "a" either: the pod and PVC branches -// match by name, and a name another application used earlier in the -// window would pull its series in. +// namespace roots it does not reach "a" either: an ownerless pod keeps the +// lineage labels its creator set, so it can claim to be "a" on both roots, +// and on the metrics root the pod and PVC branches match by name, so a name +// another application used earlier in the window would pull its series in. func TestServerApplicationNarrowedGrant(t *testing.T) { h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"a-0"}, nil, http.StatusOK), appPolicy, nil) paths := []string{ diff --git a/packages/system/observability-proxy/templates/tenant-clusterroles.yaml b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml index a2bf94f3c..ff5f3b987 100644 --- a/packages/system/observability-proxy/templates/tenant-clusterroles.yaml +++ b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml @@ -1,7 +1,11 @@ # Aggregated into cozy-tenant-view (and consequently use, admin, super-admin). # No API server serves cozystack.io logs or metrics: the rule exists only for # the SubjectAccessReview the observability proxy asks before it answers a -# read under /apis/cozystack.io/v1alpha1/namespaces//{logs,metrics}/. +# read under /apis/cozystack.io/v1alpha1/namespaces//{logs,metrics}/, +# and before a read under an application root, +# /apis/apps.cozystack.io/v1alpha1/namespaces////{log,metrics}/, +# which needs the namespace root too. Narrowing this role also cuts off the +# application roots. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: From 3a5eb4fbf200005c88da0e3c68da23d1de6679a0 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 10 Nov 2026 03:40:26 +0700 Subject: [PATCH 40/41] test(observability-proxy): cover stalled reviews on the application roots Application roots take two access reviews before any body is read; a POST to either must answer 503 within the request timeout when kube-apiserver stalls on them. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/server_deadline_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/observabilityproxy/server_deadline_test.go b/internal/observabilityproxy/server_deadline_test.go index 6b367229c..5723365b3 100644 --- a/internal/observabilityproxy/server_deadline_test.go +++ b/internal/observabilityproxy/server_deadline_test.go @@ -428,6 +428,8 @@ func TestServerStalledReviewAnswersInTime(t *testing.T) { {"logs subject access review", logsRoot + "query", false}, {"metrics token review", metricsRoot + "api/v1/query", true}, {"metrics subject access review", metricsRoot + "api/v1/query", false}, + {"application logs subject access review", appRoot + "db/log/select/logsql/query", false}, + {"application metrics subject access review", appRoot + "db/metrics/api/v1/query", false}, } { t.Run(tc.name, func(t *testing.T) { h := newHarness(t, Config{RequestTimeout: 200 * time.Millisecond}, nil) From 6fe3e5f6ff831ab87b4af11f14db0e583716e5cb Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 10 Nov 2026 03:46:57 +0700 Subject: [PATCH 41/41] fix(observability-proxy): refuse a partial application lookup A cluster vmselect answers label values with isPartial when a storage node does not respond, and the list it returns misses that node's names. Scoping the query to it would drop those pods' series without a word, which the lookup exists to prevent, so a partial list now fails the request as any other failed lookup does. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 10 ++++- .../observabilityproxy/server_app_test.go | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index e6df8ba02..659b55052 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -158,8 +158,9 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status) } var body struct { - Status string `json:"status"` - Data []string `json:"data"` + Status string `json:"status"` + IsPartial bool `json:"isPartial"` + Data []string `json:"data"` } if err := json.NewDecoder(io.LimitReader(resp.Body, maxLookupBytes)).Decode(&body); err != nil { return nil, err @@ -170,8 +171,13 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, if len(body.Data) > limit { // Over the cap before dropping empty values: the store cut the // list at limit+1, so a shorter filtered list could hide names. + // A partial list over the cap settles it too. return body.Data, nil } + // A storage node that did not answer leaves names out of the list. + if body.IsPartial { + return nil, errorf(http.StatusBadGateway, "metrics store answered a partial %s list", label) + } names := body.Data[:0] for _, n := range body.Data { if n != "" { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 461cbc16d..056e1a555 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -516,6 +516,43 @@ func TestServerApplicationLookupStoreErrors(t *testing.T) { } } +// TestServerApplicationRefusesPartialLookup: vmselect answers a label +// values lookup with isPartial when a vmstorage node is down, and a short +// pod list would drop the missing pods' series without a word. +func TestServerApplicationRefusesPartialLookup(t *testing.T) { + for _, label := range []string{"pod", "persistentvolumeclaim"} { + h := newPolicyHarness(t, Config{}, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/label/"+label+"/values" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"status":"success","isPartial":true,"data":["db-0"]}`) + return + } + appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r) + }, appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusBadGateway { + t.Errorf("partial %s lookup answered %d, want 502", label, got) + } + for _, req := range h.requests() { + if req.path == "/api/v1/query" { + t.Errorf("partial %s lookup still sent the query", label) + } + } + } + // A partial list already over the cap settles it: the full one is longer. + cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}} + h := newPolicyHarness(t, cfg, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/label/pod/values" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"status":"success","isPartial":true,"data":["db-0","db-1"]}`) + return + } + appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r) + }, appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusUnprocessableEntity { + t.Errorf("partial pod list over the cap answered %d, want 422", got) + } +} + // TestServerApplicationRefusesEveryAt: no @ form can be bounded, and the // answer says so rather than naming forms that would pass. func TestServerApplicationRefusesEveryAt(t *testing.T) {