Files
cozystack/internal/observabilityproxy/server_app_test.go
T
ZverGuy 6fe3e5f6ff fix(observability-proxy): refuse a partial application lookup
A cluster vmselect answers label values with isPartial when a storage
node does not respond, and the list it returns misses that node's
names. Scoping the query to it would drop those pods' series without a
word, which the lookup exists to prevent, so a partial list now fails
the request as any other failed lookup does.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
2026-11-10 04:03:24 +07:00

669 lines
32 KiB
Go

/*
Copyright 2026 The Cozystack Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package observabilityproxy
import (
"encoding/json"
"io"
"net/http"
"net/url"
"slices"
"strings"
"testing"
"time"
authorizationv1 "k8s.io/api/authorization/v1"
)
const appRoot = "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/"
// appPolicy lets alice read every postgres in tenant-a and the namespace
// roots, as the view role does. bob and carol may read only postgres "a",
// as a grant narrowed with resourceNames does; carol may also read the
// namespace roots.
func appPolicy(user string, a *authorizationv1.ResourceAttributes) bool {
if a.Namespace != "tenant-a" {
return false
}
if a.Verb == "list" && a.Group == "cozystack.io" && (a.Resource == "logs" || a.Resource == "metrics") && a.Name == "" && a.Subresource == "" {
return user == "alice" || user == "carol"
}
if a.Verb != "get" || a.Group != "apps.cozystack.io" || a.Resource != "postgreses" ||
(a.Subresource != "log" && a.Subresource != "metrics") {
return false
}
return user == "alice" || a.Name == "a"
}
func TestServerApplicationLogsScope(t *testing.T) {
h := newPolicyHarness(t, Config{}, nil, appPolicy, nil)
hostile := url.Values{
"query": {"error"},
"field": {"level"},
"extra_filters": {`{kubernetes_namespace_name="tenant-b"}`},
"extra_stream_filters": {`{apps_cozystack_io_application_name=~".*"}`},
"AccountID": {"9"},
}
headers := map[string]string{"AccountID": "9", "ProjectID": "9"}
endpoints := []string{"query", "hits", "stats_query_range", "tail", "field_values", "stream_field_values", "facets"}
for _, ep := range endpoints {
resp := h.do(http.MethodPost, appRoot+"db.main/log/select/logsql/"+ep+"?extra_stream_filters=x", "good", hostile, headers)
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("%s answered %d", ep, resp.StatusCode)
}
}
want := authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "log", Name: "db.main", Namespace: "tenant-a"}
if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want {
t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want)
}
reqs := h.requests()
if len(reqs) != len(endpoints) {
t.Fatalf("upstream saw %d requests, want %d", len(reqs), len(endpoints))
}
// Two values: vlselect ANDs every extra_stream_filters value into the
// query and each of its subqueries.
wantFilters := []string{wantScope, `{apps_cozystack_io_application_kind="Postgres",apps_cozystack_io_application_name="db.main"}`}
for i, r := range reqs {
if r.path != "/select/logsql/"+endpoints[i] {
t.Errorf("upstream path = %s, want %s", r.path, endpoints[i])
}
if got := r.form["extra_stream_filters"]; !slices.Equal(got, wantFilters) {
t.Errorf("%s: extra_stream_filters = %q, want %q", endpoints[i], got, wantFilters)
}
if r.form.Has("extra_filters") || r.form.Has("AccountID") || r.headers.Get("AccountID") != "0" || r.headers.Get("ProjectID") != "0" {
t.Errorf("%s: client scope reached the store: %v %v", endpoints[i], r.form, r.headers)
}
}
}
func TestServerApplicationKinds(t *testing.T) {
tests := []struct {
name string
kinds KindLookup
path string
want int
}{
{"unknown plural logs", nil, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/mysqls/a/log/select/logsql/query", http.StatusNotFound},
{"unknown plural metrics", nil, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/mysqls/a/metrics/api/v1/query?query=up", http.StatusNotFound},
{"not synced logs", newTestKinds(t, false), appRoot + "a/log/select/logsql/query", http.StatusServiceUnavailable},
{"not synced metrics", newTestKinds(t, false), appRoot + "a/metrics/api/v1/query?query=up", http.StatusServiceUnavailable},
{"ambiguous plural", newTestKinds(t, true, appDefinition("x", "Postgres", "postgreses"), appDefinition("y", "Pg", "postgreses")), appRoot + "a/log/select/logsql/query", http.StatusServiceUnavailable},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
h := newPolicyHarness(t, Config{}, nil, func(string, *authorizationv1.ResourceAttributes) bool { return true }, tt.kinds)
if got := h.status(http.MethodGet, tt.path, "good"); got != tt.want {
t.Fatalf("status = %d, want %d", got, tt.want)
}
if n := len(h.requests()); n != 0 {
t.Fatalf("upstream saw %d requests", n)
}
})
}
// The kind is looked up only for an authorized caller.
h := newPolicyHarness(t, Config{}, nil, appPolicy, newTestKinds(t, false))
if got := h.status(http.MethodGet, appRoot+"b/log/select/logsql/query", "bob-token"); got != http.StatusForbidden {
t.Fatalf("denied caller answered %d, want 403", got)
}
}
const wantPodLookup = `kube_pod_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}`
// appMetricsUpstream answers the pod and PVC lookups with the given names,
// or with status when it is not 200, and every other path like vmselect.
func appMetricsUpstream(pods, pvcs []string, podStatus int) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
names := map[string][]string{"/api/v1/label/pod/values": pods, "/api/v1/label/persistentvolumeclaim/values": pvcs}
list, lookup := names[r.URL.Path]
if !lookup {
promUpstream(w, r)
return
}
if r.URL.Path == "/api/v1/label/pod/values" && podStatus != http.StatusOK {
http.Error(w, "unavailable", podStatus)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{"status": "success", "data": list})
}
}
func TestServerApplicationMetricsScope(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"db-main-0", "db.main-1"}, []string{"data-db-main-0"}, http.StatusOK), appPolicy, nil)
hostile := url.Values{
"query": {"sum(rate(container_cpu_usage_seconds_total[5m]))"},
"start": {unix(testNow.Add(-time.Hour))},
"end": {unix(testNow)},
"step": {"60"},
"extra_filters": {`{namespace=~".+"}`},
"extra_filters[]": {`{namespace="tenant-a"}`},
"extra_label": {"apps_cozystack_io_application_name=b"},
}
resp := h.do(http.MethodPost, appRoot+"db.main/metrics/api/v1/query_range", "good", hostile, nil)
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("answered %d %q", resp.StatusCode, body)
}
want := authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "metrics", Name: "db.main", Namespace: "tenant-a"}
if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want {
t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want)
}
reqs := h.requests()
if len(reqs) != 3 {
t.Fatalf("upstream saw %d requests, want two lookups and the query", len(reqs))
}
lookups := map[string]string{
"/api/v1/label/pod/values": wantPodLookup,
"/api/v1/label/persistentvolumeclaim/values": `kube_persistentvolumeclaim_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}`,
}
for _, r := range reqs[:2] {
if got := r.form["match[]"]; len(got) != 1 || got[0] != lookups[r.path] {
t.Errorf("%s match[] = %q", r.path, got)
}
if got := r.form["extra_filters[]"]; !slices.Equal(got, []string{wantMetricsScope}) {
t.Errorf("%s extra_filters[] = %q", r.path, got)
}
// The query's own lookback is known and shorter than the 24h cap.
if want := formatTime(buildMetrics(t, "api/v1/query_range", hostile).start); r.form.Get("start") != want {
t.Errorf("%s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want)
}
// One step past end, which vmselect rounds up to a whole step, and
// two steps for each "(" that may be an implicit subquery.
if r.form.Get("end") != formatTime(testNow.Add(5*time.Minute)) || r.form.Get("limit") != "101" {
t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit"))
}
}
main := reqs[2]
wantBranches := []string{
wantIdentityBranch,
`{namespace="tenant-a",pod=~"^(db-main-0|db\\.main-1)$"}`,
`{namespace="tenant-a",persistentvolumeclaim=~"^(data-db-main-0)$"}`,
}
if main.path != "/api/v1/query_range" || !slices.Equal(main.form["extra_filters[]"], wantBranches) {
t.Fatalf("query %s extra_filters[] = %q, want %q", main.path, main.form["extra_filters[]"], wantBranches)
}
if main.form.Has("extra_filters") || main.form.Has("extra_label") {
t.Fatalf("client scope reached the store: %v", main.form)
}
if !slices.Equal(h.stores, []string{"shortterm", "shortterm", "shortterm"}) {
t.Fatalf("stores = %q, want the lookups on the query's store", h.stores)
}
}
func TestServerApplicationMetricsBranches(t *testing.T) {
podBranch := `{namespace="tenant-a",pod=~"^(p)$"}`
tests := []struct {
name string
pods, pvcs []string
podStatus int
podBody string
cfg Config
path string
wantCode int
want []string
}{
{name: "no pods or pvcs", path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}},
// A failed lookup would leave the application's own series out of
// a successful answer, so it fails the request instead.
{name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway},
{name: "pod lookup times out at the store", pods: []string{"p"}, podStatus: http.StatusGatewayTimeout, path: "api/v1/query?query=up", wantCode: http.StatusGatewayTimeout},
{name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}},
path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity},
{name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}},
path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity},
{name: "pvcs over the cap", pods: []string{"p"}, pvcs: []string{"v", "w"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}},
path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity},
{name: "labels get the branches as match[]", pods: []string{"p"}, path: "api/v1/labels", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}},
{name: "series needs match[]", path: "api/v1/series", wantCode: http.StatusBadRequest},
// An empty name would make pod=~"^(|p)$", which matches every
// series without a pod label.
{name: "empty pod name dropped", pods: []string{"", "p"}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}},
{name: "only empty pod names", pods: []string{""}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}},
{name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway},
{name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.podStatus == 0 {
tt.podStatus = http.StatusOK
}
upstream := appMetricsUpstream(tt.pods, tt.pvcs, tt.podStatus)
if tt.podBody != "" {
upstream = func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/v1/label/pod/values" {
_, _ = io.WriteString(w, tt.podBody)
return
}
appMetricsUpstream(nil, nil, http.StatusOK)(w, r)
}
}
h := newPolicyHarness(t, tt.cfg, upstream, appPolicy, nil)
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tt.path, "good"); got != tt.wantCode {
t.Fatalf("status = %d, want %d", got, tt.wantCode)
}
var main *upstreamRequest
for _, r := range h.requests() {
if !strings.HasPrefix(r.path, "/api/v1/label/pod/") && !strings.HasPrefix(r.path, "/api/v1/label/persistentvolumeclaim/") {
main = &r
}
}
if tt.want == nil {
if main != nil {
t.Fatalf("query reached the store: %s %v", main.path, main.form)
}
return
}
if main == nil || !slices.Equal(main.form["extra_filters[]"], tt.want) {
t.Fatalf("extra_filters[] = %v, want %q", main, tt.want)
}
if strings.HasSuffix(tt.path, "labels") && !slices.Equal(main.form["match[]"], tt.want) {
t.Fatalf("match[] = %q, want the branches", main.form["match[]"])
}
})
}
}
// A grant narrowed to application "a" by resourceNames must not reach "b"
// through any endpoint of either root. Without read access to the
// namespace roots it does not reach "a" either: an ownerless pod keeps the
// lineage labels its creator set, so it can claim to be "a" on both roots,
// and on the metrics root the pod and PVC branches match by name, so a name
// another application used earlier in the window would pull its series in.
func TestServerApplicationNarrowedGrant(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"a-0"}, nil, http.StatusOK), appPolicy, nil)
paths := []string{
"metrics/api/v1/query?query=up",
"metrics/api/v1/query_range?query=up&start=" + unix(testNow.Add(-time.Hour)) + "&end=" + unix(testNow) + "&step=60",
"metrics/api/v1/series?match[]=up",
"metrics/api/v1/labels",
"metrics/api/v1/label/pod/values",
"log/select/logsql/query",
"log/select/logsql/hits",
"log/select/logsql/stats_query_range?query=" + url.QueryEscape("* | stats count()"),
"log/select/logsql/tail",
"log/select/logsql/field_values?field=level",
"log/select/logsql/stream_field_values?field=level",
"log/select/logsql/facets",
}
for _, p := range paths {
for _, denied := range []struct{ app, token string }{{"b", "bob-token"}, {"b", "carol-token"}, {"a", "bob-token"}} {
if got := h.status(http.MethodGet, appRoot+denied.app+"/"+p, denied.token); got != http.StatusForbidden {
t.Errorf("%s on %s/%s answered %d, want 403", denied.token, denied.app, p, got)
}
}
}
if n := len(h.requests()); n != 0 {
t.Fatalf("upstream saw %d requests for a denied application", n)
}
for _, p := range []string{"metrics/api/v1/labels", "log/select/logsql/query"} {
if got := h.status(http.MethodGet, appRoot+"a/"+p, "carol-token"); got != http.StatusOK {
t.Errorf("a/%s answered %d, want 200", p, got)
}
}
}
// TestServerApplicationLookbackBeyondCap: the lookup reaches back at most
// --metrics-app-lookback, so a query that reads further back, or whose
// lookback is unknown, would miss pods that existed only earlier; it is
// refused rather than answered without their series.
func TestServerApplicationLookbackBeyondCap(t *testing.T) {
cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 100, MaxPVCs: 100}}}
h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
// max_over_time(x[2h]) reads just past the 1h cap; rate(x[5i]) has an
// unknown lookback.
for _, query := range []string{"rate(x[7d])", "max_over_time(x[2h])", "rate(x[5i])"} {
q := url.Values{"query": {query}, "time": {unix(testNow)}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity {
t.Errorf("%s answered %d, want 422", query, got)
}
}
if n := len(h.requests()); n != 0 {
t.Fatalf("upstream saw %d requests for refused queries", n)
}
q := url.Values{"query": {"rate(x[5m])"}, "time": {unix(testNow)}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK {
t.Fatalf("rate(x[5m]) answered %d, want 200", got)
}
r := h.requests()[0]
want := formatTime(buildMetrics(t, "api/v1/query", q).start)
if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want {
t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want)
}
}
// TestServerApplicationLookupCoversNegativeOffset evaluates two hours ago
// a selector shifted an hour forward: the pod lookup must reach the time
// of the samples, or a pod created in between is left out.
func TestServerApplicationLookupCoversNegativeOffset(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
at := testNow.Add(-2 * time.Hour)
q := url.Values{"query": {"container_cpu_usage_seconds_total offset -1h"}, "time": {unix(at)}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK {
t.Fatalf("status = %d", got)
}
r := h.requests()[0]
if want := formatTime(at.Add(time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("end") != want {
t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want)
}
}
// TestServerApplicationForwardsDefaultTime: without time the lookup is
// sized at the proxy's clock, so that instant is forwarded. Otherwise
// vmselect evaluates at its own later clock, past the end of the lookup by
// however long the lookups took, and leaves out pods first sampled since.
func TestServerApplicationForwardsDefaultTime(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
q := url.Values{"query": {"container_cpu_usage_seconds_total"}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK {
t.Fatalf("status = %d", got)
}
reqs := h.requests()
lookup, final := reqs[0], reqs[len(reqs)-1]
want := formatTime(testNow)
if lookup.path != "/api/v1/label/pod/values" || lookup.form.Get("end") != want {
t.Fatalf("lookup %s end = %s, want %s", lookup.path, lookup.form.Get("end"), want)
}
if final.path != "/api/v1/query" || final.form.Get("time") != want {
t.Fatalf("query %s time = %q, want the lookup's instant %s", final.path, final.form.Get("time"), want)
}
}
// TestServerApplicationRefusesUnboundedSampleTime refuses a query whose
// samples may lie at a time the lookup cannot be sized for, rather than
// answer with series silently left out.
func TestServerApplicationRefusesUnboundedSampleTime(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
for _, query := range []string{"up @ 1700000000", "up offset -5i"} {
q := url.Values{"query": {query}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity {
t.Errorf("%s answered %d, want 422", query, got)
}
}
if n := len(h.requests()); n != 0 {
t.Fatalf("upstream saw %d requests for a refused query", n)
}
// The namespace root has no lookup to size and still serves it.
if got := h.status(http.MethodGet, metricsRoot+"api/v1/query?"+url.Values{"query": {"up @ 1700000000"}}.Encode(), "good"); got != http.StatusOK {
t.Fatalf("namespace root answered %d", got)
}
}
// TestServerApplicationLookupCoversAlignedEnd: vmselect rounds a range
// query's end up to a whole step, so the lookup reaches one step past it.
func TestServerApplicationLookupCoversAlignedEnd(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
end := testNow.Add(-90 * time.Minute)
q := url.Values{"query": {"up"}, "start": {unix(end.Add(-6 * time.Hour))}, "end": {unix(end)}, "step": {"3600"}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query_range?"+q.Encode(), "good"); got != http.StatusOK {
t.Fatalf("status = %d", got)
}
r := h.requests()[0]
if want := formatTime(end.Add(time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("end") != want {
t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want)
}
}
// TestServerApplicationLookupDeadline: a lookup that misses the request
// deadline answers 504, whether the store sent nothing or stalled after
// its headers, and the query itself is not sent.
func TestServerApplicationLookupDeadline(t *testing.T) {
for _, headers := range []bool{false, true} {
h := newPolicyHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/label/pod/values" {
appMetricsUpstream(nil, nil, http.StatusOK)(w, r)
return
}
if headers {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"status":"success","data":[`)
w.(http.Flusher).Flush()
}
<-r.Context().Done()
}, appPolicy, nil)
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusGatewayTimeout {
t.Fatalf("headers sent %v: status = %d, want 504", headers, got)
}
for _, r := range h.requests() {
if !strings.HasPrefix(r.path, "/api/v1/label/") {
t.Fatalf("headers sent %v: the query reached the store after a failed lookup", headers)
}
}
}
}
// TestServerApplicationDefaultLookbackServesLongPanels: the lookback is
// only the refusal threshold, and the lookback estimate is a loose upper
// bound, so the default must serve a day's increase and a month-long
// panel at the step a dashboard picks for it.
func TestServerApplicationDefaultLookbackServesLongPanels(t *testing.T) {
cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: 30 * 24 * time.Hour, MaxPods: 100, MaxPVCs: 100}}}
h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
month := url.Values{"start": {unix(testNow.Add(-29 * 24 * time.Hour))}, "end": {unix(testNow)}}
for _, tc := range []struct{ path, query, step string }{
{"api/v1/query", "increase(x[24h])", ""},
{"api/v1/query_range", "rate(x[5m])", "21600"},
{"api/v1/query_range", "histogram_quantile(0.99, sum by (le) (rate(x[5m])))", "7200"},
} {
q := url.Values{"query": {tc.query}}
if tc.step != "" {
q.Set("step", tc.step)
q.Set("start", month.Get("start"))
q.Set("end", month.Get("end"))
}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tc.path+"?"+q.Encode(), "good"); got != http.StatusOK {
t.Errorf("%s step %s answered %d, want 200", tc.query, tc.step, got)
}
}
}
// TestServerApplicationLookupStoreErrors: vmselect answers 422 to any error
// without a status of its own (SendPrometheusError), so only its series
// limit, which a narrower window fixes, reaches the client as 422.
func TestServerApplicationLookupStoreErrors(t *testing.T) {
for _, tc := range []struct {
label, message string
want int
}{
{"pod", "the number of matching timeseries exceeds 300000; either narrow down the search or increase -search.max* command-line flag values", http.StatusUnprocessableEntity},
{"persistentvolumeclaim", "the number of matching timeseries exceeds 300000; either narrow down the search", http.StatusUnprocessableEntity},
{"pod", "cannot fetch label values from vmstorage nodes: the number of matching timeseries exceeds 300000", http.StatusUnprocessableEntity},
{"pod", "timeout exceeded before starting the query processing: 30s", http.StatusGatewayTimeout},
{"pod", "error when searching for label values: cannot obtain data from vmstorage", http.StatusBadGateway},
} {
h := newPolicyHarness(t, Config{}, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/v1/label/"+tc.label+"/values" {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnprocessableEntity)
_ = json.NewEncoder(w).Encode(map[string]string{"status": "error", "errorType": "422", "error": tc.message})
return
}
appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r)
}, appPolicy, nil)
resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good", nil, nil)
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if resp.StatusCode != tc.want {
t.Errorf("%s lookup %q answered %d, want %d", tc.label, tc.message, resp.StatusCode, tc.want)
}
if tc.want == http.StatusUnprocessableEntity && !strings.Contains(string(body), "the number of matching timeseries exceeds 300000") {
t.Errorf("the store's limit message did not reach the client: %q", body)
}
if strings.Contains(string(body), `"errorType"`) {
t.Errorf("the store's raw JSON reached the client: %q", body)
}
// Other store errors may name vmstorage nodes; only the limit
// message, which the client can act on, is passed on.
if strings.Contains(string(body), "vmstorage") {
t.Errorf("the store's error text reached the client: %q", body)
}
}
}
// TestServerApplicationRefusesPartialLookup: vmselect answers a label
// values lookup with isPartial when a vmstorage node is down, and a short
// pod list would drop the missing pods' series without a word.
func TestServerApplicationRefusesPartialLookup(t *testing.T) {
for _, label := range []string{"pod", "persistentvolumeclaim"} {
h := newPolicyHarness(t, Config{}, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/v1/label/"+label+"/values" {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"status":"success","isPartial":true,"data":["db-0"]}`)
return
}
appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r)
}, appPolicy, nil)
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusBadGateway {
t.Errorf("partial %s lookup answered %d, want 502", label, got)
}
for _, req := range h.requests() {
if req.path == "/api/v1/query" {
t.Errorf("partial %s lookup still sent the query", label)
}
}
}
// A partial list already over the cap settles it: the full one is longer.
cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}
h := newPolicyHarness(t, cfg, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/v1/label/pod/values" {
w.Header().Set("Content-Type", "application/json")
_, _ = io.WriteString(w, `{"status":"success","isPartial":true,"data":["db-0","db-1"]}`)
return
}
appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r)
}, appPolicy, nil)
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusUnprocessableEntity {
t.Errorf("partial pod list over the cap answered %d, want 422", got)
}
}
// TestServerApplicationRefusesEveryAt: no @ form can be bounded, and the
// answer says so rather than naming forms that would pass.
func TestServerApplicationRefusesEveryAt(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
for _, query := range []string{"up @ end()", "up @ start()", "up @ 1700000000"} {
resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+url.Values{"query": {query}}.Encode(), "good", nil, nil)
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if resp.StatusCode != http.StatusUnprocessableEntity {
t.Errorf("%s answered %d, want 422", query, resp.StatusCode)
}
if strings.Contains(string(body), "start() or end()") {
t.Errorf("%s: the refusal names @ forms as allowed: %q", query, body)
}
}
if n := len(h.requests()); n != 0 {
t.Fatalf("upstream saw %d requests for refused queries", n)
}
}
// TestServerApplicationRefusesForwardReads: queries that read past the
// evaluation time by an amount the scan cannot bound answer 422 and send
// nothing upstream, rather than answer 200 without pods created in that
// time.
func TestServerApplicationRefusesForwardReads(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
for _, query := range []string{
"container_cpu_usage_seconds_total offset -1h-30m",
"container_cpu_usage_seconds_total offset 1h-2h",
"container_cpu_usage_seconds_total offset -1h$__interval",
"rollup_candlestick(container_cpu_usage_seconds_total)",
} {
q := url.Values{"query": {query}, "time": {unix(testNow.Add(-2 * time.Hour))}}
resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good", nil, nil)
_ = resp.Body.Close()
if resp.StatusCode != http.StatusUnprocessableEntity {
t.Errorf("%s answered %d, want 422", query, resp.StatusCode)
}
}
if n := len(h.requests()); n != 0 {
t.Fatalf("upstream saw %d requests for refused queries", n)
}
}
// TestServerApplicationLookupCoversSubquery: a pod created during the two
// subquery steps past the evaluation time can show up through range_last,
// so the lookup reaches that far.
func TestServerApplicationLookupCoversSubquery(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
at := testNow.Add(-6 * time.Hour)
q := url.Values{"query": {"max_over_time(range_last(container_cpu_usage_seconds_total)[1h:1h])"}, "time": {unix(at)}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK {
t.Fatalf("status = %d", got)
}
r := h.requests()[0]
end, err := time.Parse(time.RFC3339, r.form.Get("end"))
if err != nil || r.path != "/api/v1/label/pod/values" || end.Before(at.Add(2*time.Hour)) {
t.Fatalf("lookup %s end = %s, want at least %s", r.path, r.form.Get("end"), formatTime(at.Add(2*time.Hour)))
}
}
// TestServerApplicationNamesUnreadableSubqueryStep: the refusal names the
// cause when a subquery step is what the scan cannot read.
func TestServerApplicationNamesUnreadableSubqueryStep(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+url.Values{"query": {"max_over_time(x[1h:5i])"}}.Encode(), "good", nil, nil)
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if resp.StatusCode != http.StatusUnprocessableEntity || !strings.Contains(string(body), "subquery step") {
t.Fatalf("answered %d %q, want 422 naming the subquery step", resp.StatusCode, body)
}
}
// TestServerApplicationLookupCoversLatencyOffset: vmselect evaluates an
// instant query whose time is within 30s of its own clock at its clock
// minus 30s (prometheus.go, queryOffset), so for a time slightly in the
// future it reads from earlier than time minus 30s. The lookup starts
// early enough for any evaluation instant in (time-60s, time].
func TestServerApplicationLookupCoversLatencyOffset(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
at := testNow.Add(29 * time.Second)
q := url.Values{"query": {"container_cpu_usage_seconds_total"}, "time": {at.Format(time.RFC3339)}, "step": {"1s"}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK {
t.Fatalf("status = %d", got)
}
// The earliest read of the query evaluated at the earliest instant.
lookback, _ := queryLookback("container_cpu_usage_seconds_total", time.Second)
want := at.Add(-2*latencyOffset - lookback)
r := h.requests()[0]
start, err := time.Parse(time.RFC3339, r.form.Get("start"))
if err != nil || start.After(want) {
t.Fatalf("lookup start = %s, want at most %s", r.form.Get("start"), formatTime(want))
}
}
// TestServerApplicationRefusesBackquoteEscapeTail: an escaped backquote
// inside a quoted label name does not end the string in metricsql, so the
// escaped rollup_candlestick after it must still be refused.
func TestServerApplicationRefusesBackquoteEscapeTail(t *testing.T) {
h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil)
for _, query := range []string{
"container_cpu_usage_seconds_total{`\\``=\"\"} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)",
"{`\\`a`} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)",
} {
q := url.Values{"query": {query}, "time": {unix(testNow.Add(-6 * time.Hour))}}
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity {
t.Errorf("%s answered %d, want 422", query, got)
}
}
if n := len(h.requests()); n != 0 {
t.Fatalf("upstream saw %d requests for refused queries", n)
}
}