mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(observability-proxy): fail an application query whose lookup failed
When the pod or PVC lookup failed, its branch was dropped and the query still answered 200, without the cAdvisor and volume series of the application, so a store error showed up as an empty graph. That is the same silently incomplete answer that a lookup over its cap is refused for. A failed lookup now fails the request: 504 when the store missed the deadline, 502 otherwise. Assisted-by: LLM Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
97a55a167b
commit
1a3517fa81
2 files changed
+13
-9
No files matched your search
@@ -95,9 +95,9 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string {
|
||||
// applicationScope looks the application's pods and PVCs up in store over
|
||||
// the request window, reaching back by the query's own lookback up to the
|
||||
// configured limit and forward to the latest sample a negative offset
|
||||
// reads. A failed lookup drops its branch, which only narrows the answer;
|
||||
// a list over its cap, or a query whose latest sample is unknown, is
|
||||
// refused, because the answer would silently miss series.
|
||||
// reads. A failed lookup, a list over its cap, or a query whose latest
|
||||
// sample is unknown fails the request, because a successful answer would
|
||||
// silently miss the application's series.
|
||||
func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) {
|
||||
if req.last.IsZero() {
|
||||
return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root")
|
||||
@@ -115,8 +115,11 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st
|
||||
match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name)
|
||||
names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max)
|
||||
if err != nil {
|
||||
slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err)
|
||||
continue
|
||||
slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err)
|
||||
if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusGatewayTimeout {
|
||||
return nil, herr
|
||||
}
|
||||
return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label)
|
||||
}
|
||||
if len(names) > l.max {
|
||||
return nil, errorf(http.StatusUnprocessableEntity, "the application has more than %d %ss in this window; use a shorter range", l.max, l.label)
|
||||
|
||||
@@ -218,8 +218,9 @@ func TestServerApplicationMetricsBranches(t *testing.T) {
|
||||
want []string
|
||||
}{
|
||||
{name: "no pods or pvcs", path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}},
|
||||
{name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusOK,
|
||||
want: []string{wantIdentityBranch, `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`}},
|
||||
// A failed lookup would leave the application's own series out of
|
||||
// a successful answer, so it fails the request instead.
|
||||
{name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway},
|
||||
{name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}},
|
||||
path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity},
|
||||
{name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}},
|
||||
@@ -232,8 +233,8 @@ func TestServerApplicationMetricsBranches(t *testing.T) {
|
||||
// series without a pod label.
|
||||
{name: "empty pod name dropped", pods: []string{"", "p"}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}},
|
||||
{name: "only empty pod names", pods: []string{""}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}},
|
||||
{name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}},
|
||||
{name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}},
|
||||
{name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway},
|
||||
{name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user