From 65944fcb79171fdf542192cb243fcd685231214d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 02:23:09 +0700 Subject: [PATCH] fix(observability-proxy): send metrics times vmselect cannot misread Times went to vmselect as "%d.%03d" unix seconds. vmselect reads a number without "-" as a unix time whose unit it guesses from its size, so a time in the first 24.8 days of 1970 arrived as a thousand times as many seconds: a window the proxy checked as 8.8 days and 30 points reached vmselect as 2001..2025 and 30000 points, and the label and series APIs, which have no points check, scanned the whole retention. Times past 2038 were read as microseconds. Times now go out as RFC3339 with milliseconds, which vmselect parses as a date and never as a guessed unit, and times outside 1970..2262, which vmselect cannot hold as nanoseconds, are refused. A test repeats vmselect's parsing of the exact strings the proxy sends. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/metricsparams.go | 24 +++- .../observabilityproxy/metricsparams_test.go | 15 +-- .../observabilityproxy/vmselect_parse_test.go | 117 ++++++++++++++++++ 3 files changed, 140 insertions(+), 16 deletions(-) create mode 100644 internal/observabilityproxy/vmselect_parse_test.go diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index 54c46f527..8beb347d0 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -18,6 +18,7 @@ package observabilityproxy import ( "fmt" + "math" "net/http" "net/url" "regexp" @@ -280,18 +281,33 @@ func parseMetricsTime(key, v string) (time.Time, *HTTPError) { if ok { nanos, _ = strconv.ParseInt((frac + "000000000")[:9], 10, 64) } - return time.UnixMilli(time.Unix(secs, nanos).UnixMilli()).UTC(), nil + return metricsTimeInRange(key, v, time.Unix(secs, nanos)) } t, err := time.Parse(time.RFC3339Nano, v) - if err != nil || t.Before(time.Unix(0, 0)) { + if err != nil { return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q", key, v) } + return metricsTimeInRange(key, v, t) +} + +// maxMetricsTime is the last instant vmselect can read: it takes an +// RFC3339 time through time.UnixNano (timeutil.ParseTimeAt). +var maxMetricsTime = time.Unix(0, math.MaxInt64) + +func metricsTimeInRange(key, v string, t time.Time) (time.Time, *HTTPError) { + if t.Before(time.Unix(0, 0)) || !t.Before(maxMetricsTime) { + return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q: outside 1970..2262", key, v) + } return time.UnixMilli(t.UnixMilli()).UTC(), nil } +// formatTime writes RFC3339 with milliseconds. vmselect reads a number +// without "-" as a unix time whose unit it guesses from its size +// (timeutil.getUnixTimestampNanoseconds): "1000000.000" is taken as +// 1000000000 seconds, and anything past 2038 in milliseconds as +// microseconds. A dated form never reaches that guess. func formatTime(t time.Time) string { - ms := t.UnixMilli() - return fmt.Sprintf("%d.%03d", ms/1000, ms%1000) + return t.UTC().Format("2006-01-02T15:04:05.000Z07:00") } // formatSeconds writes a whole-millisecond duration so that vmselect reads diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index a4957a19a..a4a435567 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -17,7 +17,6 @@ limitations under the License. package observabilityproxy import ( - "math" "net/http" "net/url" "strconv" @@ -198,7 +197,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) { "query": {"up"}, "start": {"2026-11-04T11:00:00.25Z"}, "end": {unix(testNow)}, "step": {"1m30s"}, }) want := url.Values{ - "query": {"up"}, "start": {"1793790000.250"}, "end": {"1793793600.000"}, "step": {"90"}, + "query": {"up"}, "start": {"2026-11-04T11:00:00.250Z"}, "end": {"2026-11-04T12:00:00.000Z"}, "step": {"90"}, "timeout": {"30"}, "extra_filters[]": {wantMetricsScope}, } if req.params.Encode() != want.Encode() { @@ -225,7 +224,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) { } req = buildMetrics(t, "api/v1/labels", url.Values{}) - if req.params.Get("end") != "1793793600.000" || req.params.Get("start") != "1793793300.000" { + if req.params.Get("end") != "2026-11-04T12:00:00.000Z" || req.params.Get("start") != "2026-11-04T11:55:00.000Z" { t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end")) } if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-30*time.Second)) { @@ -422,15 +421,7 @@ func TestBuildMetricsParamsPointsAsVmselectCounts(t *testing.T) { } req := buildMetrics(t, "api/v1/query_range", rng(11000)) stepMs := vmselectMillis(t, req.params.Get("step")) - // Times, unlike durations, are rounded (timeutil.ParseTimeMsec). - timeMs := func(v string) int64 { - secs, err := strconv.ParseFloat(v, 64) - if err != nil { - t.Fatal(err) - } - return int64(math.Round(secs * 1e3)) - } - startMs, endMs := timeMs(req.params.Get("start")), timeMs(req.params.Get("end")) + startMs, endMs := vmselectTimeMillis(t, req.params.Get("start")), vmselectTimeMillis(t, req.params.Get("end")) if points := (endMs-startMs)/stepMs + 1; stepMs != 1001 || points != 11000 { t.Fatalf("vmselect reads step %dms and %d points, want 1001ms and 11000", stepMs, points) } diff --git a/internal/observabilityproxy/vmselect_parse_test.go b/internal/observabilityproxy/vmselect_parse_test.go new file mode 100644 index 000000000..6d75bbf7f --- /dev/null +++ b/internal/observabilityproxy/vmselect_parse_test.go @@ -0,0 +1,117 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "math" + "net/url" + "strconv" + "strings" + "testing" + "time" +) + +// vmselectTimeMillis repeats how vmselect v1.136.0-cluster reads a time +// argument (httputil.GetTime -> timeutil.ParseTimeMsec -> ParseTimeAt in +// lib/timeutil/time.go) for the forms a client or the proxy may send: a +// value without "-" is a number whose unit vmselect guesses from its size +// (getUnixTimestampNanoseconds), anything else with a zone is RFC3339 read +// through time.UnixNano. Relative and partial dates are left out; the +// proxy refuses them. +func vmselectTimeMillis(t *testing.T, s string) int64 { + t.Helper() + if strings.Contains(s, "-") { + ts, err := time.Parse(time.RFC3339, s) + if err != nil { + t.Fatalf("vmselect cannot parse %q: %v", s, err) + } + return int64(math.Round(float64(ts.UnixNano()) / 1e6)) + } + intPart, frac, _ := strings.Cut(s, ".") + n, err := strconv.ParseInt(intPart+frac, 10, 64) + if err != nil { + t.Fatalf("vmselect cannot parse %q: %v", s, err) + } + for e := len(frac); e%3 != 0; e++ { + n *= 10 + } + var nanos int64 + switch { + case n < 1<<31: + nanos = n * 1e9 + case n < 1e3*(1<<31): + nanos = n * 1e6 + case n < 1e6*(1<<31): + nanos = n * 1e3 + default: + nanos = n + } + return int64(math.Round(float64(nanos) / 1e6)) +} + +// TestVmselectReadsProxyTimes sends times across the range the proxy +// accepts, in both client forms, and checks that vmselect reads the +// forwarded value as the same instant the proxy validated. A number such +// as "1000000.000" is seconds to the proxy but milliseconds-as-seconds +// guesswork to vmselect, so the proxy must not forward one. +func TestVmselectReadsProxyTimes(t *testing.T) { + for _, client := range []string{ + "0", "1000000", "1762400", "2147483", "2147484", "1000000.5", + "1793793600", "1793793600.123", "2147483647", "2147483648", "9223372035", + "1970-01-01T00:00:00Z", "1970-01-12T13:46:40.001Z", "2026-11-04T12:00:00.123Z", "2262-01-01T00:00:00Z", + } { + want, herr := parseMetricsTime("time", client) + if herr != nil { + t.Fatalf("%q refused: %v", client, herr) + } + req := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}, "time": {client}}) + sent := req.params.Get("time") + if got := vmselectTimeMillis(t, sent); got != want.UnixMilli() { + t.Errorf("client time %q forwarded as %q reaches vmselect as %dms, want %dms", client, sent, got, want.UnixMilli()) + } + } +} + +// TestVmselectReadsSecondsWithFraction is the window that passed the proxy as +// 8.8 days and 30 points while vmselect, reading "1000000.000" as a +// thousand times that many seconds, saw 2001..2025 and 30000 points. +func TestVmselectReadsSecondsWithFraction(t *testing.T) { + in := url.Values{"query": {"up"}, "start": {"1000000"}, "end": {"1762400"}, "step": {"25414"}} + req := buildMetrics(t, "api/v1/query_range", in) + start, end := vmselectTimeMillis(t, req.params.Get("start")), vmselectTimeMillis(t, req.params.Get("end")) + step := vmselectMillis(t, req.params.Get("step")) + if start != 1000000*1000 || end != 1762400*1000 { + t.Fatalf("vmselect reads %dms..%dms, want 1000000s..1762400s", start, end) + } + if points := (end-start)/step + 1; points != 30 { + t.Fatalf("vmselect counts %d points, want the 30 the proxy checked", points) + } + labels := buildMetrics(t, "api/v1/labels", url.Values{"start": {"1000000"}, "end": {"1000300"}}) + if got := vmselectTimeMillis(t, labels.params.Get("start")); got != 1000000*1000 { + t.Fatalf("label API start reaches vmselect as %dms, want 1000000s", got) + } +} + +func TestParseMetricsTimeRange(t *testing.T) { + // vmselect reads an RFC3339 time through time.UnixNano, which is + // undefined from 2262-04-11 on. + for _, v := range []string{"9223372037", "9999999999", "2262-04-12T00:00:00Z", "9999-01-01T00:00:00Z"} { + if _, herr := parseMetricsTime("time", v); herr == nil { + t.Errorf("%q accepted, want refused", v) + } + } +}