mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
feat(observability-proxy): serve the application metrics root
Graphs of one application need its series without the caller knowing which pods and volumes it ran on. Exporter series carry the identity the scrape class stamps, but cAdvisor and kubelet volume series do not, so the scope ORs three branches: the identity, and the application's pods and PVCs as kube-state-metrics labelled them over the window. Each branch also carries the namespace, so the scope is never wider than the namespace root. The lookup reaches back by the query's own lookback, bounded by a flag, so pods replaced inside an expression's range still count. A failed lookup only drops its branch. A list over its cap answers 422 instead of being truncated, which would hide series without saying so. Assisted-by: LLM Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
2f600d5354
commit
dcdf0adab3
8 files changed
+492
-8
No files matched your search
@@ -94,6 +94,9 @@ func parseFlags(args []string) (options, error) {
|
||||
fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use")
|
||||
fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return")
|
||||
fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set")
|
||||
fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs at most")
|
||||
fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422")
|
||||
fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422")
|
||||
fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables")
|
||||
fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables")
|
||||
fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user")
|
||||
@@ -134,6 +137,8 @@ func parseFlags(args []string) (options, error) {
|
||||
return o, errors.New("--metrics-shortterm-retention, --metrics-max-range and --metrics-max-timeout must be positive")
|
||||
case o.metrics.Limits.MaxPoints < 2:
|
||||
return o, errors.New("--metrics-max-points must be at least 2")
|
||||
case o.metrics.App.Lookback < 0 || o.metrics.App.MaxPods < 1 || o.metrics.App.MaxPVCs < 1:
|
||||
return o, errors.New("--metrics-app-lookback must not be negative and --metrics-app-max-pods and --metrics-app-max-pvcs must be positive")
|
||||
case o.logs.MaxTailResume < 0:
|
||||
return o, errors.New("--logs-max-tail-resume must not be negative")
|
||||
// The proxy's deadline starts before the request reaches vmselect, so
|
||||
|
||||
@@ -40,6 +40,9 @@ func TestParseFlagsDefaults(t *testing.T) {
|
||||
if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second {
|
||||
t.Errorf("metrics limit defaults changed: %+v", m.Limits)
|
||||
}
|
||||
if m.App.Lookback != 24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 {
|
||||
t.Errorf("application scope defaults changed: %+v", m.App)
|
||||
}
|
||||
if m.RootTenant != o.rootTenant {
|
||||
t.Errorf("metrics root tenant = %q, want --root-tenant", m.RootTenant)
|
||||
}
|
||||
@@ -63,13 +66,15 @@ func TestParseFlagsMetricsOverrides(t *testing.T) {
|
||||
o, err := parseFlags([]string{
|
||||
"--root-tenant=tenant-x", "--metrics-shortterm-store=fast", "--metrics-longterm-store=slow",
|
||||
"--metrics-shortterm-retention=24h", "--metrics-max-range=168h", "--metrics-max-points=500", "--metrics-max-timeout=10s",
|
||||
"--metrics-app-lookback=0s", "--metrics-app-max-pods=7", "--metrics-app-max-pvcs=3",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := o.metrics
|
||||
if m.RootTenant != "tenant-x" || m.ShortTermStore != "fast" || m.LongTermStore != "slow" || m.ShortTermRetention != 24*time.Hour ||
|
||||
m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second {
|
||||
m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second ||
|
||||
m.App.Lookback != 0 || m.App.MaxPods != 7 || m.App.MaxPVCs != 3 {
|
||||
t.Errorf("overrides not applied: %+v", m)
|
||||
}
|
||||
}
|
||||
@@ -101,6 +106,9 @@ func TestParseFlagsInvalid(t *testing.T) {
|
||||
{"--metrics-max-range=0s"},
|
||||
{"--metrics-max-points=1"},
|
||||
{"--metrics-max-timeout=0s"},
|
||||
{"--metrics-app-lookback=-1h"},
|
||||
{"--metrics-app-max-pods=0"},
|
||||
{"--metrics-app-max-pvcs=0"},
|
||||
{"--unknown"},
|
||||
// The proxy would cut the request while vmselect is still within
|
||||
// the timeout it was given.
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package observabilityproxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AppScopeLimits bound the pod and PVC lookup of the application metrics
|
||||
// root.
|
||||
type AppScopeLimits struct {
|
||||
// Lookback is how far before the window the lookup reaches at most, so
|
||||
// pods gone before the window but inside an expression's range count.
|
||||
Lookback time.Duration
|
||||
MaxPods int
|
||||
MaxPVCs int
|
||||
}
|
||||
|
||||
// maxLookupBytes bounds a lookup answer; the caps keep a valid one far
|
||||
// below it.
|
||||
const maxLookupBytes = 8 << 20
|
||||
|
||||
// The lineage labels as the scrape class writes them on series, and as
|
||||
// kube-state-metrics writes them on kube_pod_labels and
|
||||
// kube_persistentvolumeclaim_labels.
|
||||
const (
|
||||
seriesKindLabel = "apps_cozystack_io_application_kind"
|
||||
seriesNameLabel = "apps_cozystack_io_application_name"
|
||||
ksmKindLabel = "label_" + seriesKindLabel
|
||||
ksmNameLabel = "label_" + seriesNameLabel
|
||||
)
|
||||
|
||||
func identitySelector(ns, kindLabel, nameLabel, kind, name string) string {
|
||||
return "{namespace=" + strconv.Quote(ns) + "," + kindLabel + "=" + strconv.Quote(kind) + "," + nameLabel + "=" + strconv.Quote(name) + "}"
|
||||
}
|
||||
|
||||
// nameBranch matches label against any of names within ns. With no names
|
||||
// it is "": a branch of pod=~"^()$" would match every series without a pod
|
||||
// label.
|
||||
func nameBranch(ns, label string, names []string) string {
|
||||
if len(names) == 0 {
|
||||
return ""
|
||||
}
|
||||
quoted := make([]string, len(names))
|
||||
for i, n := range names {
|
||||
quoted[i] = regexp.QuoteMeta(n)
|
||||
}
|
||||
return "{namespace=" + strconv.Quote(ns) + "," + label + "=~" + strconv.Quote("^("+strings.Join(quoted, "|")+")$") + "}"
|
||||
}
|
||||
|
||||
// applicationSelectors is the application scope: series stamped with the
|
||||
// application's identity, plus series of its pods and PVCs for those not
|
||||
// stamped (cAdvisor, kubelet volume stats). Every branch carries the
|
||||
// namespace, so their OR stays inside it.
|
||||
//
|
||||
// Within the namespace, both the identity and the pod and PVC lists come
|
||||
// from lineage labels on pods and PVCs. They separate applications for a
|
||||
// grant narrowed by resourceNames only while whoever creates a pod or PVC
|
||||
// cannot choose those labels; the lineage webhook keeps them as set on an
|
||||
// object it finds no owner for, and the logs scope rests on the same.
|
||||
func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string {
|
||||
out := []string{identitySelector(ns, seriesKindLabel, seriesNameLabel, kind, name)}
|
||||
for _, b := range []string{nameBranch(ns, "pod", pods), nameBranch(ns, "persistentvolumeclaim", pvcs)} {
|
||||
if b != "" {
|
||||
out = append(out, b)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// applicationScope looks the application's pods and PVCs up in store over
|
||||
// the request window, reaching back by the query's own lookback up to the
|
||||
// configured limit. A failed lookup drops its branch, which only narrows
|
||||
// the answer; a list over its cap is refused, because truncating it would
|
||||
// silently drop series.
|
||||
func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) {
|
||||
lim := s.cfg.Metrics.App
|
||||
start := req.from.Add(-lim.Lookback)
|
||||
if req.start.After(start) {
|
||||
start = req.start
|
||||
}
|
||||
var lists [2][]string
|
||||
for i, l := range []struct {
|
||||
label, metric string
|
||||
max int
|
||||
}{{"pod", "kube_pod_labels", lim.MaxPods}, {"persistentvolumeclaim", "kube_persistentvolumeclaim_labels", lim.MaxPVCs}} {
|
||||
match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name)
|
||||
names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.to, l.max)
|
||||
if err != nil {
|
||||
slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err)
|
||||
continue
|
||||
}
|
||||
if len(names) > l.max {
|
||||
return nil, errorf(http.StatusUnprocessableEntity, "the application has more than %d %ss in this window; use a shorter range", l.max, l.label)
|
||||
}
|
||||
lists[i] = names
|
||||
}
|
||||
return applicationSelectors(ns, kind, name, lists[0], lists[1]), nil
|
||||
}
|
||||
|
||||
// lookupNames asks for at most max+1 values, enough to tell a list over
|
||||
// the cap without reading all of it.
|
||||
func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, match, ns string, start, end time.Time, max int) ([]string, error) {
|
||||
params := url.Values{}
|
||||
params.Set("match[]", match)
|
||||
params.Set("extra_filters[]", namespaceSelector(ns))
|
||||
params.Set("start", formatTime(start))
|
||||
params.Set("end", formatTime(end))
|
||||
params.Set("limit", strconv.Itoa(max+1))
|
||||
resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/api/v1/label/"+label+"/values", params, "metrics store", nil)
|
||||
if herr != nil {
|
||||
return nil, herr
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status)
|
||||
}
|
||||
var body struct {
|
||||
Status string `json:"status"`
|
||||
Data []string `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxLookupBytes)).Decode(&body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if body.Status != "success" {
|
||||
return nil, errorf(http.StatusBadGateway, "metrics store answered status %q", body.Status)
|
||||
}
|
||||
names := body.Data[:0]
|
||||
for _, n := range body.Data {
|
||||
if n != "" {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package observabilityproxy
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const wantIdentityBranch = `{namespace="tenant-a",apps_cozystack_io_application_kind="Postgres",apps_cozystack_io_application_name="db.main"}`
|
||||
|
||||
func TestApplicationSelectors(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pods, pvcs []string
|
||||
want []string
|
||||
}{
|
||||
{"identity only", nil, nil, []string{wantIdentityBranch}},
|
||||
{"pods", []string{"db-main-1", "db.main-2"}, nil, []string{
|
||||
wantIdentityBranch,
|
||||
`{namespace="tenant-a",pod=~"^(db-main-1|db\\.main-2)$"}`,
|
||||
}},
|
||||
{"pvcs", nil, []string{"data-db-main-1"}, []string{
|
||||
wantIdentityBranch,
|
||||
`{namespace="tenant-a",persistentvolumeclaim=~"^(data-db-main-1)$"}`,
|
||||
}},
|
||||
{"all three", []string{"p"}, []string{"v"}, []string{
|
||||
wantIdentityBranch,
|
||||
`{namespace="tenant-a",pod=~"^(p)$"}`,
|
||||
`{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`,
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := applicationSelectors("tenant-a", "Postgres", "db.main", tt.pods, tt.pvcs); !slices.Equal(got, tt.want) {
|
||||
t.Fatalf("got %q\nwant %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// DNS-1123 keeps most metacharacters out of names, but the names come
|
||||
// back from the store, so the encoder must not rely on that.
|
||||
func TestNameBranchEscapes(t *testing.T) {
|
||||
names := []string{`a.b`, `x|.*`, `)|(.*`, `q"} or {namespace=~".+`, `back\slash`, `$^+?[]{}`}
|
||||
sel := nameBranch("tenant-a", "pod", names)
|
||||
m := regexp.MustCompile(`^\{namespace="tenant-a",pod=~(".*")\}$`).FindStringSubmatch(sel)
|
||||
if m == nil {
|
||||
t.Fatalf("selector %s is not one namespace and one pod matcher", sel)
|
||||
}
|
||||
pattern, err := strconv.Unquote(m[1])
|
||||
if err != nil {
|
||||
t.Fatalf("pod matcher %s does not unquote: %v", m[1], err)
|
||||
}
|
||||
re := regexp.MustCompile(pattern)
|
||||
for _, n := range names {
|
||||
if !re.MatchString(n) {
|
||||
t.Errorf("%q does not match its own branch", n)
|
||||
}
|
||||
}
|
||||
for _, other := range []string{"axb", "x", "anything", "tenant-b", "", `back\\slash`} {
|
||||
if re.MatchString(other) {
|
||||
t.Errorf("branch matches %q", other)
|
||||
}
|
||||
}
|
||||
if nameBranch("tenant-a", "pod", nil) != "" {
|
||||
t.Error("an empty list produced a branch, which would match series without the label")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsRequestScope(t *testing.T) {
|
||||
branches := []string{"{a}", "{b}"}
|
||||
labels := buildMetrics(t, "api/v1/labels", url.Values{})
|
||||
labels.setScope(branches)
|
||||
if got := labels.params["extra_filters[]"]; !slices.Equal(got, branches) {
|
||||
t.Errorf("extra_filters[] = %q", got)
|
||||
}
|
||||
if got := labels.params["match[]"]; !slices.Equal(got, branches) {
|
||||
t.Errorf("default match[] = %q, want the branches", got)
|
||||
}
|
||||
client := buildMetrics(t, "api/v1/labels", url.Values{"match[]": {"up"}})
|
||||
client.setScope(branches)
|
||||
if got := client.params["match[]"]; !slices.Equal(got, []string{"up"}) {
|
||||
t.Errorf("client match[] replaced: %q", got)
|
||||
}
|
||||
query := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}})
|
||||
query.setScope(branches)
|
||||
if query.params.Has("match[]") || !slices.Equal(query.params["extra_filters[]"], branches) {
|
||||
t.Errorf("query params = %v", query.params)
|
||||
}
|
||||
if !query.from.Equal(testNow) || !query.to.Equal(testNow) {
|
||||
t.Errorf("instant window = [%s, %s], want now", query.from, query.to)
|
||||
}
|
||||
rng := buildMetrics(t, "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-time.Hour))}, "end": {unix(testNow)}, "step": {"60"}})
|
||||
if !rng.from.Equal(testNow.Add(-time.Hour)) || !rng.to.Equal(testNow) {
|
||||
t.Errorf("range window = [%s, %s]", rng.from, rng.to)
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,7 @@ type MetricsConfig struct {
|
||||
LongTermStore string
|
||||
ShortTermRetention time.Duration
|
||||
Limits MetricsLimits
|
||||
App AppScopeLimits
|
||||
// UpstreamURL maps a store name to its Prometheus API base URL, without
|
||||
// a trailing slash; nil means the in-cluster vmselect Service.
|
||||
UpstreamURL func(store string) string
|
||||
@@ -77,6 +78,20 @@ type metricsRequest struct {
|
||||
// start is the earliest instant the request reads, for the store choice;
|
||||
// zero when that is unknown, which picks the long-term store.
|
||||
start time.Time
|
||||
// from and to are the window the query is evaluated over, without the
|
||||
// lookback of its expression.
|
||||
from, to time.Time
|
||||
// scopedMatch is set when match[] holds the scope, not client matchers.
|
||||
scopedMatch bool
|
||||
}
|
||||
|
||||
// setScope replaces the namespace scope with the given selectors, which
|
||||
// vmselect ORs.
|
||||
func (req *metricsRequest) setScope(selectors []string) {
|
||||
req.params["extra_filters[]"] = selectors
|
||||
if req.scopedMatch {
|
||||
req.params["match[]"] = selectors
|
||||
}
|
||||
}
|
||||
|
||||
// namespaceSelector quotes ns the way MetricsQL unquotes a label value
|
||||
@@ -127,6 +142,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
|
||||
out.Set("step", formatSeconds(step))
|
||||
}
|
||||
req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step)
|
||||
req.from, req.to = at, at
|
||||
return req, nil
|
||||
|
||||
case "api/v1/query_range":
|
||||
@@ -150,6 +166,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
|
||||
return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints)
|
||||
}
|
||||
req.start = queryStart(out.Get("query"), start, step)
|
||||
req.from, req.to = start, end
|
||||
out.Set("start", formatTime(start))
|
||||
out.Set("end", formatTime(end))
|
||||
out.Set("step", formatSeconds(step))
|
||||
@@ -172,7 +189,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
|
||||
if herr != nil {
|
||||
return req, herr
|
||||
}
|
||||
req.start = start
|
||||
req.start, req.from, req.to = start, start, end
|
||||
out.Set("start", formatTime(start))
|
||||
out.Set("end", formatTime(end))
|
||||
|
||||
@@ -194,6 +211,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
|
||||
// extra_filters[] on the label APIs unless a match[] is present
|
||||
// (getCommonParamsInternal), so the scope is repeated here.
|
||||
out.Set("match[]", namespaceSelector(namespace))
|
||||
req.scopedMatch = true
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
@@ -193,11 +193,8 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
|
||||
return herr
|
||||
}
|
||||
}
|
||||
switch route.Kind {
|
||||
case NamespaceMetrics:
|
||||
return s.serveMetrics(w, r, route, user)
|
||||
case ApplicationMetrics:
|
||||
return errorf(http.StatusNotFound, "not found")
|
||||
if route.Kind == NamespaceMetrics || route.Kind == ApplicationMetrics {
|
||||
return s.serveMetrics(w, r, route, user, kind)
|
||||
}
|
||||
endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/")
|
||||
if !ok {
|
||||
@@ -288,7 +285,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError {
|
||||
// serveMetrics scopes to the namespace, or with kind set to the application
|
||||
// route.Name of that kind within it.
|
||||
func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user, kind string) *HTTPError {
|
||||
release, ok := s.limits.acquire(user)
|
||||
if !ok {
|
||||
return tooManyConcurrent(w)
|
||||
@@ -310,6 +309,13 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout
|
||||
defer cancel()
|
||||
}
|
||||
store := s.cfg.Metrics.store(req.start, now)
|
||||
if kind != "" {
|
||||
selectors, herr := s.applicationScope(ctx, r, store, req, route.Namespace, kind, route.Name)
|
||||
if herr != nil {
|
||||
return herr
|
||||
}
|
||||
req.setScope(selectors)
|
||||
}
|
||||
resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/"+req.path, req.params, "metrics store", nil)
|
||||
if herr != nil {
|
||||
return herr
|
||||
|
||||
@@ -17,11 +17,14 @@ limitations under the License.
|
||||
package observabilityproxy
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
authorizationv1 "k8s.io/api/authorization/v1"
|
||||
)
|
||||
@@ -112,3 +115,170 @@ func TestServerApplicationKinds(t *testing.T) {
|
||||
t.Fatalf("denied caller answered %d, want 403", got)
|
||||
}
|
||||
}
|
||||
|
||||
const wantPodLookup = `kube_pod_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}`
|
||||
|
||||
// appMetricsUpstream answers the pod and PVC lookups with the given names,
|
||||
// or with status when it is not 200, and every other path like vmselect.
|
||||
func appMetricsUpstream(pods, pvcs []string, podStatus int) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
names := map[string][]string{"/api/v1/label/pod/values": pods, "/api/v1/label/persistentvolumeclaim/values": pvcs}
|
||||
list, lookup := names[r.URL.Path]
|
||||
if !lookup {
|
||||
promUpstream(w, r)
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/api/v1/label/pod/values" && podStatus != http.StatusOK {
|
||||
http.Error(w, "unavailable", podStatus)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"status": "success", "data": list})
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerApplicationMetricsScope(t *testing.T) {
|
||||
h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"db-main-0", "db.main-1"}, []string{"data-db-main-0"}, http.StatusOK), appPolicy, nil)
|
||||
hostile := url.Values{
|
||||
"query": {"sum(rate(container_cpu_usage_seconds_total[5m]))"},
|
||||
"start": {unix(testNow.Add(-time.Hour))},
|
||||
"end": {unix(testNow)},
|
||||
"step": {"60"},
|
||||
"extra_filters": {`{namespace=~".+"}`},
|
||||
"extra_filters[]": {`{namespace="tenant-a"}`},
|
||||
"extra_label": {"apps_cozystack_io_application_name=b"},
|
||||
}
|
||||
resp := h.do(http.MethodPost, appRoot+"db.main/metrics/api/v1/query_range", "good", hostile, nil)
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("answered %d %q", resp.StatusCode, body)
|
||||
}
|
||||
want := authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "metrics", Name: "db.main", Namespace: "tenant-a"}
|
||||
if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want {
|
||||
t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want)
|
||||
}
|
||||
reqs := h.requests()
|
||||
if len(reqs) != 3 {
|
||||
t.Fatalf("upstream saw %d requests, want two lookups and the query", len(reqs))
|
||||
}
|
||||
lookups := map[string]string{
|
||||
"/api/v1/label/pod/values": wantPodLookup,
|
||||
"/api/v1/label/persistentvolumeclaim/values": `kube_persistentvolumeclaim_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}`,
|
||||
}
|
||||
for _, r := range reqs[:2] {
|
||||
if got := r.form["match[]"]; len(got) != 1 || got[0] != lookups[r.path] {
|
||||
t.Errorf("%s match[] = %q", r.path, got)
|
||||
}
|
||||
if got := r.form["extra_filters[]"]; !slices.Equal(got, []string{wantMetricsScope}) {
|
||||
t.Errorf("%s extra_filters[] = %q", r.path, got)
|
||||
}
|
||||
start, herr := parseMetricsTime("start", r.form.Get("start"))
|
||||
if herr != nil || start.After(testNow.Add(-time.Hour)) || start.Before(testNow.Add(-25*time.Hour)) {
|
||||
t.Errorf("%s start = %s, want within the lookback before the window", r.path, r.form.Get("start"))
|
||||
}
|
||||
if r.form.Get("end") != formatTime(testNow) || r.form.Get("limit") != "101" {
|
||||
t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit"))
|
||||
}
|
||||
}
|
||||
main := reqs[2]
|
||||
wantBranches := []string{
|
||||
wantIdentityBranch,
|
||||
`{namespace="tenant-a",pod=~"^(db-main-0|db\\.main-1)$"}`,
|
||||
`{namespace="tenant-a",persistentvolumeclaim=~"^(data-db-main-0)$"}`,
|
||||
}
|
||||
if main.path != "/api/v1/query_range" || !slices.Equal(main.form["extra_filters[]"], wantBranches) {
|
||||
t.Fatalf("query %s extra_filters[] = %q, want %q", main.path, main.form["extra_filters[]"], wantBranches)
|
||||
}
|
||||
if main.form.Has("extra_filters") || main.form.Has("extra_label") {
|
||||
t.Fatalf("client scope reached the store: %v", main.form)
|
||||
}
|
||||
if !slices.Equal(h.stores, []string{"shortterm", "shortterm", "shortterm"}) {
|
||||
t.Fatalf("stores = %q, want the lookups on the query's store", h.stores)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerApplicationMetricsBranches(t *testing.T) {
|
||||
podBranch := `{namespace="tenant-a",pod=~"^(p)$"}`
|
||||
tests := []struct {
|
||||
name string
|
||||
pods, pvcs []string
|
||||
podStatus int
|
||||
cfg Config
|
||||
path string
|
||||
wantCode int
|
||||
want []string
|
||||
}{
|
||||
{name: "no pods or pvcs", path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}},
|
||||
{name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusOK,
|
||||
want: []string{wantIdentityBranch, `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`}},
|
||||
{name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}},
|
||||
path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity},
|
||||
{name: "pvcs over the cap", pods: []string{"p"}, pvcs: []string{"v", "w"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}},
|
||||
path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity},
|
||||
{name: "labels get the branches as match[]", pods: []string{"p"}, path: "api/v1/labels", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}},
|
||||
{name: "series needs match[]", path: "api/v1/series", wantCode: http.StatusBadRequest},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.podStatus == 0 {
|
||||
tt.podStatus = http.StatusOK
|
||||
}
|
||||
h := newPolicyHarness(t, tt.cfg, appMetricsUpstream(tt.pods, tt.pvcs, tt.podStatus), appPolicy, nil)
|
||||
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tt.path, "good"); got != tt.wantCode {
|
||||
t.Fatalf("status = %d, want %d", got, tt.wantCode)
|
||||
}
|
||||
var main *upstreamRequest
|
||||
for _, r := range h.requests() {
|
||||
if !strings.HasPrefix(r.path, "/api/v1/label/pod/") && !strings.HasPrefix(r.path, "/api/v1/label/persistentvolumeclaim/") {
|
||||
main = &r
|
||||
}
|
||||
}
|
||||
if tt.want == nil {
|
||||
if main != nil {
|
||||
t.Fatalf("query reached the store: %s %v", main.path, main.form)
|
||||
}
|
||||
return
|
||||
}
|
||||
if main == nil || !slices.Equal(main.form["extra_filters[]"], tt.want) {
|
||||
t.Fatalf("extra_filters[] = %v, want %q", main, tt.want)
|
||||
}
|
||||
if strings.HasSuffix(tt.path, "labels") && !slices.Equal(main.form["match[]"], tt.want) {
|
||||
t.Fatalf("match[] = %q, want the branches", main.form["match[]"])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A grant narrowed to application "a" by resourceNames must not reach "b"
|
||||
// through any endpoint of either root.
|
||||
func TestServerApplicationNarrowedGrant(t *testing.T) {
|
||||
h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"a-0"}, nil, http.StatusOK), appPolicy, nil)
|
||||
paths := []string{
|
||||
"metrics/api/v1/query?query=up",
|
||||
"metrics/api/v1/query_range?query=up&start=" + unix(testNow.Add(-time.Hour)) + "&end=" + unix(testNow) + "&step=60",
|
||||
"metrics/api/v1/series?match[]=up",
|
||||
"metrics/api/v1/labels",
|
||||
"metrics/api/v1/label/pod/values",
|
||||
"log/select/logsql/query",
|
||||
"log/select/logsql/hits",
|
||||
"log/select/logsql/stats_query_range?query=" + url.QueryEscape("* | stats count()"),
|
||||
"log/select/logsql/tail",
|
||||
"log/select/logsql/field_values?field=level",
|
||||
"log/select/logsql/stream_field_values?field=level",
|
||||
"log/select/logsql/facets",
|
||||
}
|
||||
for _, p := range paths {
|
||||
if got := h.status(http.MethodGet, appRoot+"b/"+p, "bob-token"); got != http.StatusForbidden {
|
||||
t.Errorf("b/%s answered %d, want 403", p, got)
|
||||
}
|
||||
}
|
||||
if n := len(h.requests()); n != 0 {
|
||||
t.Fatalf("upstream saw %d requests for a denied application", n)
|
||||
}
|
||||
for _, p := range []string{"metrics/api/v1/labels", "log/select/logsql/query"} {
|
||||
if got := h.status(http.MethodGet, appRoot+"a/"+p, "bob-token"); got != http.StatusOK {
|
||||
t.Errorf("a/%s answered %d, want 200", p, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -96,6 +96,9 @@ func newPolicyHarness(t *testing.T, cfg Config, handler http.HandlerFunc, allow
|
||||
if cfg.Metrics.Limits == (MetricsLimits{}) {
|
||||
cfg.Metrics.Limits = testMetricsLimits()
|
||||
}
|
||||
if cfg.Metrics.App == (AppScopeLimits{}) {
|
||||
cfg.Metrics.App = AppScopeLimits{Lookback: 24 * time.Hour, MaxPods: 100, MaxPVCs: 100}
|
||||
}
|
||||
if cfg.Logs == (LogsLimits{}) {
|
||||
cfg.Logs = testLimits()
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user