fix(observability-proxy): keep the vmselect timeout below the request's

An equal vmselect timeout still lost the race: the proxy's deadline
starts before the request reaches vmselect, so the proxy cut first
and answered 504 for a store within its time. The vmselect timeout
must now be strictly below a set request timeout, the flag help says
so, and tests pin the accepted side too, including no request timeout.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy committed 2026-11-10 03:56:18 +07:00
1 parent 0c5d1cd93f
commit 3e2472280e
2 files changed
+20 -5

No files matched your search

+6 -5
View File
@@ -91,7 +91,7 @@ func parseFlags(args []string) (options, error) {
fs.DurationVar(&o.metrics.ShortTermRetention, "metrics-shortterm-retention", 72*time.Hour, "retention of the short-term store; older windows are read from the long-term one")
fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use")
fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return")
fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped")
fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set")
fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables")
fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables")
fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user")
@@ -134,10 +134,11 @@ func parseFlags(args []string) (options, error) {
return o, errors.New("--metrics-max-points must be at least 2")
case o.logs.MaxTailResume < 0:
return o, errors.New("--logs-max-tail-resume must not be negative")
// Past the request timeout the proxy cuts a query vmselect is still
// allowed to run, and answers 504 for a store that is within its time.
case o.requestTimeout > 0 && o.metrics.Limits.MaxTimeout > o.requestTimeout:
return o, errors.New("--metrics-max-timeout must not exceed --request-timeout")
// The proxy's deadline starts before the request reaches vmselect, so
// at an equal or longer timeout it cuts a query vmselect is still
// allowed to run and answers 504 for a store that is within its time.
case o.requestTimeout > 0 && o.metrics.Limits.MaxTimeout >= o.requestTimeout:
return o, errors.New("--metrics-max-timeout must be below --request-timeout")
case o.requestTimeout < 0 || o.tailIdleTimeout < 0 || o.authCacheTTL < 0 || o.shutdownTimeout < 0:
return o, errors.New("timeouts and the auth cache TTL must not be negative")
// NaN passes every comparison and +Inf never empties a bucket; either
+14
View File
@@ -106,6 +106,7 @@ func TestParseFlagsInvalid(t *testing.T) {
// The proxy would cut the request while vmselect is still within
// the timeout it was given.
{"--request-timeout=10s", "--metrics-max-timeout=30s"},
{"--request-timeout=30s", "--metrics-max-timeout=30s"},
{"unexpected", "--root-tenant=tenant-x"},
} {
if _, err := parseFlags(args); err == nil {
@@ -113,3 +114,16 @@ func TestParseFlagsInvalid(t *testing.T) {
}
}
}
// The vmselect timeout must stay strictly below a request timeout, and is
// free when the request has none.
func TestParseFlagsTimeoutPairs(t *testing.T) {
for _, args := range [][]string{
{"--request-timeout=31s", "--metrics-max-timeout=30s"},
{"--request-timeout=0", "--metrics-max-timeout=10m"},
} {
if _, err := parseFlags(args); err != nil {
t.Errorf("parseFlags(%v) refused: %v", args, err)
}
}
}