fix(dashboard): skip tail dedup for rows without a stream ID

A pipe such as `delete _stream_id` leaves rows from different streams
that share a timestamp with the same dedup key, so a resumed tail
dropped a record it had never shown. Only rows that still carry both
_stream_id and _time are deduplicated now, as the accepted logs design
specifies.

Assisted-by: LLM
Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
ZverGuy committed 2026-11-05 06:10:02 +07:00
1 parent bfa85e57bf
commit 311cb1c90b
2 files changed
+28 -5

No files matched your search

@@ -63,6 +63,24 @@ describe("mergeTail", () => {
expect(out).toHaveLength(2)
})
it("never deduplicates lines a pipe stripped of _stream_id, even on a resume", () => {
const t = "2026-01-01T00:00:02Z"
const buffer = [{ _time: t, _msg: "from stream 1" }]
const out = mergeTail(buffer, [{ _time: t, _msg: "from stream 2" }], 10, t)
expect(out.map((x) => x._msg)).toEqual(["from stream 1", "from stream 2"])
})
it("deduplicates only the complete rows of a mixed resume", () => {
const t = "2026-01-01T00:00:02Z"
const buffer = [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }]
const out = mergeTail(buffer, [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }], 10, t)
expect(out.map((x) => x._msg)).toEqual(["seen", "no stream", "no time", "no stream", "no time"])
})
it("does not deduplicate outside a resume", () => {
const out = mergeTail([e("t1", "a")], [e("t1", "a")], 10)
@@ -58,16 +58,21 @@ export function compareTime(a: string, b: string): number {
return x < y ? -1 : x > y ? 1 : 0
}
const entryKey = (e: LogEntry) => `${e._stream_id ?? ""}\u0000${e._time}`
// Null when a pipe removed either identity field: such a row cannot be told from another stream's.
const entryKey = (e: LogEntry) =>
e._stream_id !== undefined && e._time !== undefined ? `${e._stream_id}\u0000${e._time}` : null
// (_stream_id, _time) is not unique in VictoriaLogs, so dedup applies only to
// lines a resumed tail replays at or before the resume point, and never to
// lines a pipe stripped of _time.
// lines a resumed tail replays at or before the resume point, and only to
// lines that still carry both fields.
export function mergeTail(buffer: LogEntry[], incoming: LogEntry[], cap: number, resumedFrom?: string): LogEntry[] {
let fresh = incoming
if (resumedFrom) {
const seen = new Set(buffer.map(entryKey))
fresh = incoming.filter((e) => e._time === undefined || compareTime(e._time, resumedFrom) > 0 || !seen.has(entryKey(e)))
const seen = new Set(buffer.map(entryKey).filter((k) => k !== null))
fresh = incoming.filter((e) => {
const key = entryKey(e)
return key === null || compareTime(e._time ?? "", resumedFrom) > 0 || !seen.has(key)
})
}
const merged = buffer.concat(fresh)
return merged.length > cap ? merged.slice(merged.length - cap) : merged