mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
fix(observability-proxy): bound the metrics request lifetime
The metrics root parsed its form before taking a concurrency slot, under no read deadline, and wrote its answer under no write deadline, so a client that sent its body slowly or stopped reading held the handler outside every limit. A store answer that broke off after the status line still ended as a clean 200. The metrics handler now follows the logs root: the slot is taken before the body is read under the request timeout, the answer must be taken by the request deadline, and a failed copy cuts the connection. Assisted-by: LLM Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
1ead35a37a
commit
0d3a0487e4
2 files changed
+70
-9
No files matched your search
@@ -280,20 +280,19 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
|
||||
}
|
||||
|
||||
func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxFormBytes)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
return errorf(http.StatusBadRequest, "invalid form: %v", err)
|
||||
release, ok := s.limits.acquire(user)
|
||||
if !ok {
|
||||
return errorf(http.StatusTooManyRequests, "too many concurrent requests")
|
||||
}
|
||||
defer release()
|
||||
if herr := s.readForm(w, r); herr != nil {
|
||||
return herr
|
||||
}
|
||||
now := s.now()
|
||||
req, herr := BuildMetricsParams(route.Endpoint, r.Form, route.Namespace, now, s.cfg.Metrics.Limits)
|
||||
if herr != nil {
|
||||
return herr
|
||||
}
|
||||
release, ok := s.limits.acquire(user)
|
||||
if !ok {
|
||||
return errorf(http.StatusTooManyRequests, "too many concurrent requests")
|
||||
}
|
||||
defer release()
|
||||
|
||||
ctx := r.Context()
|
||||
if s.cfg.RequestTimeout > 0 {
|
||||
@@ -310,9 +309,15 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout
|
||||
if ct := resp.Header.Get("Content-Type"); ct != "" {
|
||||
w.Header().Set("Content-Type", ct)
|
||||
}
|
||||
// As for a bounded logs answer: the client must take it by the request
|
||||
// deadline, and a break-off cuts the connection.
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
_ = http.NewResponseController(w).SetWriteDeadline(deadline)
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
if _, err := io.Copy(w, resp.Body); err != nil && !errors.Is(err, context.Canceled) {
|
||||
if _, err := io.Copy(w, resp.Body); err != nil {
|
||||
slog.Debug("copying upstream response", "error", err)
|
||||
return errAborted
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -444,3 +444,59 @@ func TestServerStalledReviewAnswersInTime(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestServerMetricsRequestLifetime repeats the body, write and break-off
|
||||
// checks on the metrics root, which has its own handler.
|
||||
func TestServerMetricsRequestLifetime(t *testing.T) {
|
||||
query := metricsRoot + "api/v1/query?query=up"
|
||||
t.Run("slow body", func(t *testing.T) {
|
||||
h := newHarness(t, Config{
|
||||
RequestTimeout: 200 * time.Millisecond,
|
||||
Limiter: LimiterConfig{MaxConcurrentPerUser: 1},
|
||||
}, promUpstream)
|
||||
pipe := servePipe(t, h.server)
|
||||
conn := pipe.send(t, "POST "+metricsRoot+"api/v1/query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n"+
|
||||
"Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 100\r\n\r\nquery=")
|
||||
eventually(t, time.Second, func() bool {
|
||||
return h.status(http.MethodGet, query, "good") == http.StatusTooManyRequests
|
||||
}, "a metrics request with an unfinished body does not hold a concurrency slot")
|
||||
if resp := readResponse(t, conn); resp.StatusCode != http.StatusRequestTimeout {
|
||||
t.Fatalf("unfinished body answered %d, want 408", resp.StatusCode)
|
||||
}
|
||||
})
|
||||
t.Run("stalled client", func(t *testing.T) {
|
||||
// Larger than the server's write buffer, so the handler itself
|
||||
// blocks on the client.
|
||||
big := `{"status":"success","data":"` + strings.Repeat("x", 64<<10) + `"}`
|
||||
h := newHarness(t, Config{
|
||||
RequestTimeout: 200 * time.Millisecond,
|
||||
Limiter: LimiterConfig{MaxConcurrentPerUser: 1},
|
||||
}, func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, big) })
|
||||
pipe := servePipe(t, h.server)
|
||||
pipe.send(t, "GET "+query+" HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n")
|
||||
eventually(t, time.Second, func() bool {
|
||||
return h.status(http.MethodGet, query, "good") == http.StatusTooManyRequests
|
||||
}, "the stalled metrics request never held its slot")
|
||||
eventually(t, 2*time.Second, func() bool {
|
||||
return h.status(http.MethodGet, query, "good") == http.StatusOK
|
||||
}, "a metrics client that stopped reading keeps its concurrency slot")
|
||||
})
|
||||
t.Run("truncated answer", func(t *testing.T) {
|
||||
h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Length", "100")
|
||||
_, _ = io.WriteString(w, `{"status":"success",`)
|
||||
})
|
||||
// Nothing has been flushed yet when the copy fails, so the
|
||||
// connection may close before the status line.
|
||||
req, _ := http.NewRequest(http.MethodGet, h.proxy.URL+query, nil)
|
||||
req.Header.Set("Authorization", "Bearer good")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if _, err := io.ReadAll(resp.Body); err == nil {
|
||||
t.Fatal("a truncated store answer reached the client as a complete response")
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user