mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(observability-proxy): keep only the series-limit 422 of a lookup
vmselect answers 422 to every error that carries no status of its own, its query timeout and storage failures included, so passing every lookup 422 through told the client to narrow a window when the store was failing, and bypassed the 504 for a missed deadline. Only vmselect's series-limit message stays a 422, with the message taken from its JSON answer; its timeout becomes 504 and anything else 502. The request deadline is checked before the store's status. Assisted-by: LLM Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
85fb98dd2c
commit
caa4e8d0e3
2 files changed
+66
-10
No files matched your search
@@ -120,12 +120,12 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st
|
||||
slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err)
|
||||
// The deadline can also expire while the answer is read, after
|
||||
// post has handed it over.
|
||||
if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusUnprocessableEntity {
|
||||
return nil, herr
|
||||
}
|
||||
if errors.Is(context.Cause(ctx), context.DeadlineExceeded) {
|
||||
return nil, errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", l.label)
|
||||
}
|
||||
if herr, ok := err.(*HTTPError); ok && (herr.Code == http.StatusUnprocessableEntity || herr.Code == http.StatusGatewayTimeout) {
|
||||
return nil, herr
|
||||
}
|
||||
return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label)
|
||||
}
|
||||
if len(names) > l.max {
|
||||
@@ -151,10 +151,7 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label,
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode == http.StatusUnprocessableEntity {
|
||||
// vmselect's refusal of a lookup over its own limits; the client
|
||||
// can act on it by narrowing the window.
|
||||
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<10))
|
||||
return nil, errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, strings.TrimSpace(string(msg)))
|
||||
return nil, lookupRefusal(label, resp.Body)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status)
|
||||
@@ -182,3 +179,27 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label,
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// seriesLimitError is the message vmstorage gives when a search matches
|
||||
// more series than the -search.max* flags allow (lib/storage/index_db.go);
|
||||
// cluster vmselect wraps it in its own context, hence a substring match.
|
||||
const seriesLimitError = "the number of matching timeseries exceeds"
|
||||
|
||||
// lookupRefusal reads a vmselect 422. vmselect answers 422 to any error
|
||||
// without a status of its own (httpserver.SendPrometheusError), storage
|
||||
// failures and its own timeout included, so only the series limit, which
|
||||
// a narrower window fixes, stays a 422 for the client.
|
||||
func lookupRefusal(label string, body io.Reader) *HTTPError {
|
||||
var answer struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
_ = json.NewDecoder(io.LimitReader(body, 64<<10)).Decode(&answer)
|
||||
switch {
|
||||
case strings.Contains(answer.Error, seriesLimitError):
|
||||
return errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, answer.Error)
|
||||
case strings.Contains(answer.Error, "timeout exceeded"):
|
||||
return errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", label)
|
||||
default:
|
||||
return errorf(http.StatusBadGateway, "the application's %s lookup failed: %s", label, answer.Error)
|
||||
}
|
||||
}
|
||||
@@ -222,9 +222,6 @@ func TestServerApplicationMetricsBranches(t *testing.T) {
|
||||
// A failed lookup would leave the application's own series out of
|
||||
// a successful answer, so it fails the request instead.
|
||||
{name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway},
|
||||
// vmselect refuses a lookup over its own series limits with 422;
|
||||
// the client can act on that by narrowing the window.
|
||||
{name: "pod lookup over the store's limits", pods: []string{"p"}, podStatus: http.StatusUnprocessableEntity, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity},
|
||||
{name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}},
|
||||
path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity},
|
||||
{name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}},
|
||||
@@ -449,3 +446,41 @@ func TestServerApplicationDefaultLookbackServesLongPanels(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestServerApplicationLookupStoreErrors: vmselect answers 422 to any error
|
||||
// without a status of its own (SendPrometheusError), so only its series
|
||||
// limit, which a narrower window fixes, reaches the client as 422.
|
||||
func TestServerApplicationLookupStoreErrors(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
label, message string
|
||||
want int
|
||||
}{
|
||||
{"pod", "the number of matching timeseries exceeds 300000; either narrow down the search or increase -search.max* command-line flag values", http.StatusUnprocessableEntity},
|
||||
{"persistentvolumeclaim", "the number of matching timeseries exceeds 300000; either narrow down the search", http.StatusUnprocessableEntity},
|
||||
{"pod", "cannot fetch label values from vmstorage nodes: the number of matching timeseries exceeds 300000", http.StatusUnprocessableEntity},
|
||||
{"pod", "timeout exceeded before starting the query processing: 30s", http.StatusGatewayTimeout},
|
||||
{"pod", "error when searching for label values: cannot obtain data from vmstorage", http.StatusBadGateway},
|
||||
} {
|
||||
h := newPolicyHarness(t, Config{}, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/v1/label/"+tc.label+"/values" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusUnprocessableEntity)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"status": "error", "errorType": "422", "error": tc.message})
|
||||
return
|
||||
}
|
||||
appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r)
|
||||
}, appPolicy, nil)
|
||||
resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good", nil, nil)
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != tc.want {
|
||||
t.Errorf("%s lookup %q answered %d, want %d", tc.label, tc.message, resp.StatusCode, tc.want)
|
||||
}
|
||||
if tc.want == http.StatusUnprocessableEntity && !strings.Contains(string(body), "the number of matching timeseries exceeds 300000") {
|
||||
t.Errorf("the store's limit message did not reach the client: %q", body)
|
||||
}
|
||||
if strings.Contains(string(body), `"errorType"`) {
|
||||
t.Errorf("the store's raw JSON reached the client: %q", body)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user