mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(observability-proxy): refuse a partial application lookup
A cluster vmselect answers label values with isPartial when a storage node does not respond, and the list it returns misses that node's names. Scoping the query to it would drop those pods' series without a word, which the lookup exists to prevent, so a partial list now fails the request as any other failed lookup does. Assisted-by: LLM Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
3a5eb4fbf2
commit
6fe3e5f6ff
2 files changed
+45
-2
No files matched your search
@@ -158,8 +158,9 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label,
|
||||
return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status)
|
||||
}
|
||||
var body struct {
|
||||
Status string `json:"status"`
|
||||
Data []string `json:"data"`
|
||||
Status string `json:"status"`
|
||||
IsPartial bool `json:"isPartial"`
|
||||
Data []string `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, maxLookupBytes)).Decode(&body); err != nil {
|
||||
return nil, err
|
||||
@@ -170,8 +171,13 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label,
|
||||
if len(body.Data) > limit {
|
||||
// Over the cap before dropping empty values: the store cut the
|
||||
// list at limit+1, so a shorter filtered list could hide names.
|
||||
// A partial list over the cap settles it too.
|
||||
return body.Data, nil
|
||||
}
|
||||
// A storage node that did not answer leaves names out of the list.
|
||||
if body.IsPartial {
|
||||
return nil, errorf(http.StatusBadGateway, "metrics store answered a partial %s list", label)
|
||||
}
|
||||
names := body.Data[:0]
|
||||
for _, n := range body.Data {
|
||||
if n != "" {
|
||||
|
||||
@@ -516,6 +516,43 @@ func TestServerApplicationLookupStoreErrors(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestServerApplicationRefusesPartialLookup: vmselect answers a label
|
||||
// values lookup with isPartial when a vmstorage node is down, and a short
|
||||
// pod list would drop the missing pods' series without a word.
|
||||
func TestServerApplicationRefusesPartialLookup(t *testing.T) {
|
||||
for _, label := range []string{"pod", "persistentvolumeclaim"} {
|
||||
h := newPolicyHarness(t, Config{}, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/v1/label/"+label+"/values" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{"status":"success","isPartial":true,"data":["db-0"]}`)
|
||||
return
|
||||
}
|
||||
appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r)
|
||||
}, appPolicy, nil)
|
||||
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusBadGateway {
|
||||
t.Errorf("partial %s lookup answered %d, want 502", label, got)
|
||||
}
|
||||
for _, req := range h.requests() {
|
||||
if req.path == "/api/v1/query" {
|
||||
t.Errorf("partial %s lookup still sent the query", label)
|
||||
}
|
||||
}
|
||||
}
|
||||
// A partial list already over the cap settles it: the full one is longer.
|
||||
cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}
|
||||
h := newPolicyHarness(t, cfg, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/v1/label/pod/values" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{"status":"success","isPartial":true,"data":["db-0","db-1"]}`)
|
||||
return
|
||||
}
|
||||
appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r)
|
||||
}, appPolicy, nil)
|
||||
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusUnprocessableEntity {
|
||||
t.Errorf("partial pod list over the cap answered %d, want 422", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestServerApplicationRefusesEveryAt: no @ form can be bounded, and the
|
||||
// answer says so rather than naming forms that would pass.
|
||||
func TestServerApplicationRefusesEveryAt(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user