diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index 4760f7421..50b6ac89e 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -280,20 +280,19 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT } func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError { - r.Body = http.MaxBytesReader(w, r.Body, maxFormBytes) - if err := r.ParseForm(); err != nil { - return errorf(http.StatusBadRequest, "invalid form: %v", err) + release, ok := s.limits.acquire(user) + if !ok { + return errorf(http.StatusTooManyRequests, "too many concurrent requests") + } + defer release() + if herr := s.readForm(w, r); herr != nil { + return herr } now := s.now() req, herr := BuildMetricsParams(route.Endpoint, r.Form, route.Namespace, now, s.cfg.Metrics.Limits) if herr != nil { return herr } - release, ok := s.limits.acquire(user) - if !ok { - return errorf(http.StatusTooManyRequests, "too many concurrent requests") - } - defer release() ctx := r.Context() if s.cfg.RequestTimeout > 0 { @@ -310,9 +309,15 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout if ct := resp.Header.Get("Content-Type"); ct != "" { w.Header().Set("Content-Type", ct) } + // As for a bounded logs answer: the client must take it by the request + // deadline, and a break-off cuts the connection. + if deadline, ok := ctx.Deadline(); ok { + _ = http.NewResponseController(w).SetWriteDeadline(deadline) + } w.WriteHeader(resp.StatusCode) - if _, err := io.Copy(w, resp.Body); err != nil && !errors.Is(err, context.Canceled) { + if _, err := io.Copy(w, resp.Body); err != nil { slog.Debug("copying upstream response", "error", err) + return errAborted } return nil } diff --git a/internal/observabilityproxy/server_deadline_test.go b/internal/observabilityproxy/server_deadline_test.go index 7415b46fa..03d03f218 100644 --- a/internal/observabilityproxy/server_deadline_test.go +++ b/internal/observabilityproxy/server_deadline_test.go @@ -444,3 +444,59 @@ func TestServerStalledReviewAnswersInTime(t *testing.T) { }) } } + +// TestServerMetricsRequestLifetime repeats the body, write and break-off +// checks on the metrics root, which has its own handler. +func TestServerMetricsRequestLifetime(t *testing.T) { + query := metricsRoot + "api/v1/query?query=up" + t.Run("slow body", func(t *testing.T) { + h := newHarness(t, Config{ + RequestTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1}, + }, promUpstream) + pipe := servePipe(t, h.server) + conn := pipe.send(t, "POST "+metricsRoot+"api/v1/query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n"+ + "Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 100\r\n\r\nquery=") + eventually(t, time.Second, func() bool { + return h.status(http.MethodGet, query, "good") == http.StatusTooManyRequests + }, "a metrics request with an unfinished body does not hold a concurrency slot") + if resp := readResponse(t, conn); resp.StatusCode != http.StatusRequestTimeout { + t.Fatalf("unfinished body answered %d, want 408", resp.StatusCode) + } + }) + t.Run("stalled client", func(t *testing.T) { + // Larger than the server's write buffer, so the handler itself + // blocks on the client. + big := `{"status":"success","data":"` + strings.Repeat("x", 64<<10) + `"}` + h := newHarness(t, Config{ + RequestTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1}, + }, func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, big) }) + pipe := servePipe(t, h.server) + pipe.send(t, "GET "+query+" HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n") + eventually(t, time.Second, func() bool { + return h.status(http.MethodGet, query, "good") == http.StatusTooManyRequests + }, "the stalled metrics request never held its slot") + eventually(t, 2*time.Second, func() bool { + return h.status(http.MethodGet, query, "good") == http.StatusOK + }, "a metrics client that stopped reading keeps its concurrency slot") + }) + t.Run("truncated answer", func(t *testing.T) { + h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "100") + _, _ = io.WriteString(w, `{"status":"success",`) + }) + // Nothing has been flushed yet when the copy fails, so the + // connection may close before the status line. + req, _ := http.NewRequest(http.MethodGet, h.proxy.URL+query, nil) + req.Header.Set("Authorization", "Bearer good") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return + } + defer func() { _ = resp.Body.Close() }() + if _, err := io.ReadAll(resp.Body); err == nil { + t.Fatal("a truncated store answer reached the client as a complete response") + } + }) +}