mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-22 12:43:32 +09:00
fix(dashboard): skip tail dedup for rows without a stream ID
A pipe such as `delete _stream_id` leaves rows from different streams that share a timestamp with the same dedup key, so a resumed tail dropped a record it had never shown. Only rows that still carry both _stream_id and _time are deduplicated now, as the accepted logs design specifies. Assisted-by: LLM Signed-off-by: ZverGuy <maximbel2003@gmail.com>
This commit is contained in:
1 parent
bfa85e57bf
commit
311cb1c90b
2 files changed
+28
-5
No files matched your search
+18
@@ -63,6 +63,24 @@ describe("mergeTail", () => {
|
||||
expect(out).toHaveLength(2)
|
||||
})
|
||||
|
||||
it("never deduplicates lines a pipe stripped of _stream_id, even on a resume", () => {
|
||||
const t = "2026-01-01T00:00:02Z"
|
||||
const buffer = [{ _time: t, _msg: "from stream 1" }]
|
||||
|
||||
const out = mergeTail(buffer, [{ _time: t, _msg: "from stream 2" }], 10, t)
|
||||
|
||||
expect(out.map((x) => x._msg)).toEqual(["from stream 1", "from stream 2"])
|
||||
})
|
||||
|
||||
it("deduplicates only the complete rows of a mixed resume", () => {
|
||||
const t = "2026-01-01T00:00:02Z"
|
||||
const buffer = [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }]
|
||||
|
||||
const out = mergeTail(buffer, [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }], 10, t)
|
||||
|
||||
expect(out.map((x) => x._msg)).toEqual(["seen", "no stream", "no time", "no stream", "no time"])
|
||||
})
|
||||
|
||||
it("does not deduplicate outside a resume", () => {
|
||||
const out = mergeTail([e("t1", "a")], [e("t1", "a")], 10)
|
||||
|
||||
|
||||
+10
-5
@@ -58,16 +58,21 @@ export function compareTime(a: string, b: string): number {
|
||||
return x < y ? -1 : x > y ? 1 : 0
|
||||
}
|
||||
|
||||
const entryKey = (e: LogEntry) => `${e._stream_id ?? ""}\u0000${e._time}`
|
||||
// Null when a pipe removed either identity field: such a row cannot be told from another stream's.
|
||||
const entryKey = (e: LogEntry) =>
|
||||
e._stream_id !== undefined && e._time !== undefined ? `${e._stream_id}\u0000${e._time}` : null
|
||||
|
||||
// (_stream_id, _time) is not unique in VictoriaLogs, so dedup applies only to
|
||||
// lines a resumed tail replays at or before the resume point, and never to
|
||||
// lines a pipe stripped of _time.
|
||||
// lines a resumed tail replays at or before the resume point, and only to
|
||||
// lines that still carry both fields.
|
||||
export function mergeTail(buffer: LogEntry[], incoming: LogEntry[], cap: number, resumedFrom?: string): LogEntry[] {
|
||||
let fresh = incoming
|
||||
if (resumedFrom) {
|
||||
const seen = new Set(buffer.map(entryKey))
|
||||
fresh = incoming.filter((e) => e._time === undefined || compareTime(e._time, resumedFrom) > 0 || !seen.has(entryKey(e)))
|
||||
const seen = new Set(buffer.map(entryKey).filter((k) => k !== null))
|
||||
fresh = incoming.filter((e) => {
|
||||
const key = entryKey(e)
|
||||
return key === null || compareTime(e._time ?? "", resumedFrom) > 0 || !seen.has(key)
|
||||
})
|
||||
}
|
||||
const merged = buffer.concat(fresh)
|
||||
return merged.length > cap ? merged.slice(merged.length - cap) : merged
|
||||
|
||||
Reference in new issue
Block a user