mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(observability-proxy): answer 504 to a lookup that misses the deadline
Only a deadline hit before the lookup's response headers was reported as 504. When the store sent its headers and then stalled, the deadline expired while the body was decoded, and the decode error came back as a generic 502, so a slow store looked like a broken one. The request deadline is now checked whichever step of the lookup it cut, and both cases are covered by a test. Assisted-by: LLM Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
66e2b99f09
commit
bc823b704e
2 files changed
+33
-2
No files matched your search
@@ -19,6 +19,7 @@ package observabilityproxy
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -116,8 +117,10 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st
|
||||
names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max)
|
||||
if err != nil {
|
||||
slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err)
|
||||
if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusGatewayTimeout {
|
||||
return nil, herr
|
||||
// The deadline can also expire while the answer is read, after
|
||||
// post has handed it over.
|
||||
if errors.Is(context.Cause(ctx), context.DeadlineExceeded) {
|
||||
return nil, errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", l.label)
|
||||
}
|
||||
return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label)
|
||||
}
|
||||
|
||||
@@ -380,3 +380,31 @@ func TestServerApplicationLookupCoversAlignedEnd(t *testing.T) {
|
||||
t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestServerApplicationLookupDeadline: a lookup that misses the request
|
||||
// deadline answers 504, whether the store sent nothing or stalled after
|
||||
// its headers, and the query itself is not sent.
|
||||
func TestServerApplicationLookupDeadline(t *testing.T) {
|
||||
for _, headers := range []bool{false, true} {
|
||||
h := newPolicyHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/v1/label/pod/values" {
|
||||
appMetricsUpstream(nil, nil, http.StatusOK)(w, r)
|
||||
return
|
||||
}
|
||||
if headers {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{"status":"success","data":[`)
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
<-r.Context().Done()
|
||||
}, appPolicy, nil)
|
||||
if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusGatewayTimeout {
|
||||
t.Fatalf("headers sent %v: status = %d, want 504", headers, got)
|
||||
}
|
||||
for _, r := range h.requests() {
|
||||
if !strings.HasPrefix(r.path, "/api/v1/label/") {
|
||||
t.Fatalf("headers sent %v: the query reached the store after a failed lookup", headers)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user