From 311cb1c90b751bcc24cf962e68676a6997680c4d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:10:02 +0700 Subject: [PATCH] fix(dashboard): skip tail dedup for rows without a stream ID A pipe such as `delete _stream_id` leaves rows from different streams that share a timestamp with the same dedup key, so a resumed tail dropped a record it had never shown. Only rows that still carry both _stream_id and _time are deduplicated now, as the accepted logs design specifies. Assisted-by: LLM Signed-off-by: ZverGuy --- .../console/src/lib/observability/logs.test.ts | 18 ++++++++++++++++++ .../apps/console/src/lib/observability/logs.ts | 15 ++++++++++----- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts index 62d1ac249..278da6890 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts @@ -63,6 +63,24 @@ describe("mergeTail", () => { expect(out).toHaveLength(2) }) + it("never deduplicates lines a pipe stripped of _stream_id, even on a resume", () => { + const t = "2026-01-01T00:00:02Z" + const buffer = [{ _time: t, _msg: "from stream 1" }] + + const out = mergeTail(buffer, [{ _time: t, _msg: "from stream 2" }], 10, t) + + expect(out.map((x) => x._msg)).toEqual(["from stream 1", "from stream 2"]) + }) + + it("deduplicates only the complete rows of a mixed resume", () => { + const t = "2026-01-01T00:00:02Z" + const buffer = [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }] + + const out = mergeTail(buffer, [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }], 10, t) + + expect(out.map((x) => x._msg)).toEqual(["seen", "no stream", "no time", "no stream", "no time"]) + }) + it("does not deduplicate outside a resume", () => { const out = mergeTail([e("t1", "a")], [e("t1", "a")], 10) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts index eb24a5f96..b1005a1cd 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts @@ -58,16 +58,21 @@ export function compareTime(a: string, b: string): number { return x < y ? -1 : x > y ? 1 : 0 } -const entryKey = (e: LogEntry) => `${e._stream_id ?? ""}\u0000${e._time}` +// Null when a pipe removed either identity field: such a row cannot be told from another stream's. +const entryKey = (e: LogEntry) => + e._stream_id !== undefined && e._time !== undefined ? `${e._stream_id}\u0000${e._time}` : null // (_stream_id, _time) is not unique in VictoriaLogs, so dedup applies only to -// lines a resumed tail replays at or before the resume point, and never to -// lines a pipe stripped of _time. +// lines a resumed tail replays at or before the resume point, and only to +// lines that still carry both fields. export function mergeTail(buffer: LogEntry[], incoming: LogEntry[], cap: number, resumedFrom?: string): LogEntry[] { let fresh = incoming if (resumedFrom) { - const seen = new Set(buffer.map(entryKey)) - fresh = incoming.filter((e) => e._time === undefined || compareTime(e._time, resumedFrom) > 0 || !seen.has(entryKey(e))) + const seen = new Set(buffer.map(entryKey).filter((k) => k !== null)) + fresh = incoming.filter((e) => { + const key = entryKey(e) + return key === null || compareTime(e._time ?? "", resumedFrom) > 0 || !seen.has(key) + }) } const merged = buffer.concat(fresh) return merged.length > cap ? merged.slice(merged.length - cap) : merged