fix(observability-proxy): bound the metrics request lifetime

The metrics root parsed its form before taking a concurrency slot,
under no read deadline, and wrote its answer under no write deadline,
so a client that sent its body slowly or stopped reading held the
handler outside every limit. A store answer that broke off after the
status line still ended as a clean 200.

The metrics handler now follows the logs root: the slot is taken
before the body is read under the request timeout, the answer must be
taken by the request deadline, and a failed copy cuts the connection.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy committed 2026-11-10 03:56:18 +07:00
1 parent 1ead35a37a
commit 0d3a0487e4
2 files changed
+70 -9

No files matched your search

+14 -9
View File
@@ -280,20 +280,19 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
}
func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError {
r.Body = http.MaxBytesReader(w, r.Body, maxFormBytes)
if err := r.ParseForm(); err != nil {
return errorf(http.StatusBadRequest, "invalid form: %v", err)
release, ok := s.limits.acquire(user)
if !ok {
return errorf(http.StatusTooManyRequests, "too many concurrent requests")
}
defer release()
if herr := s.readForm(w, r); herr != nil {
return herr
}
now := s.now()
req, herr := BuildMetricsParams(route.Endpoint, r.Form, route.Namespace, now, s.cfg.Metrics.Limits)
if herr != nil {
return herr
}
release, ok := s.limits.acquire(user)
if !ok {
return errorf(http.StatusTooManyRequests, "too many concurrent requests")
}
defer release()
ctx := r.Context()
if s.cfg.RequestTimeout > 0 {
@@ -310,9 +309,15 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout
if ct := resp.Header.Get("Content-Type"); ct != "" {
w.Header().Set("Content-Type", ct)
}
// As for a bounded logs answer: the client must take it by the request
// deadline, and a break-off cuts the connection.
if deadline, ok := ctx.Deadline(); ok {
_ = http.NewResponseController(w).SetWriteDeadline(deadline)
}
w.WriteHeader(resp.StatusCode)
if _, err := io.Copy(w, resp.Body); err != nil && !errors.Is(err, context.Canceled) {
if _, err := io.Copy(w, resp.Body); err != nil {
slog.Debug("copying upstream response", "error", err)
return errAborted
}
return nil
}
@@ -444,3 +444,59 @@ func TestServerStalledReviewAnswersInTime(t *testing.T) {
})
}
}
// TestServerMetricsRequestLifetime repeats the body, write and break-off
// checks on the metrics root, which has its own handler.
func TestServerMetricsRequestLifetime(t *testing.T) {
query := metricsRoot + "api/v1/query?query=up"
t.Run("slow body", func(t *testing.T) {
h := newHarness(t, Config{
RequestTimeout: 200 * time.Millisecond,
Limiter: LimiterConfig{MaxConcurrentPerUser: 1},
}, promUpstream)
pipe := servePipe(t, h.server)
conn := pipe.send(t, "POST "+metricsRoot+"api/v1/query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n"+
"Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 100\r\n\r\nquery=")
eventually(t, time.Second, func() bool {
return h.status(http.MethodGet, query, "good") == http.StatusTooManyRequests
}, "a metrics request with an unfinished body does not hold a concurrency slot")
if resp := readResponse(t, conn); resp.StatusCode != http.StatusRequestTimeout {
t.Fatalf("unfinished body answered %d, want 408", resp.StatusCode)
}
})
t.Run("stalled client", func(t *testing.T) {
// Larger than the server's write buffer, so the handler itself
// blocks on the client.
big := `{"status":"success","data":"` + strings.Repeat("x", 64<<10) + `"}`
h := newHarness(t, Config{
RequestTimeout: 200 * time.Millisecond,
Limiter: LimiterConfig{MaxConcurrentPerUser: 1},
}, func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, big) })
pipe := servePipe(t, h.server)
pipe.send(t, "GET "+query+" HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n")
eventually(t, time.Second, func() bool {
return h.status(http.MethodGet, query, "good") == http.StatusTooManyRequests
}, "the stalled metrics request never held its slot")
eventually(t, 2*time.Second, func() bool {
return h.status(http.MethodGet, query, "good") == http.StatusOK
}, "a metrics client that stopped reading keeps its concurrency slot")
})
t.Run("truncated answer", func(t *testing.T) {
h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", "100")
_, _ = io.WriteString(w, `{"status":"success",`)
})
// Nothing has been flushed yet when the copy fails, so the
// connection may close before the status line.
req, _ := http.NewRequest(http.MethodGet, h.proxy.URL+query, nil)
req.Header.Set("Authorization", "Bearer good")
resp, err := http.DefaultClient.Do(req)
if err != nil {
return
}
defer func() { _ = resp.Body.Close() }()
if _, err := io.ReadAll(resp.Body); err == nil {
t.Fatal("a truncated store answer reached the client as a complete response")
}
})
}