From e5f17148368f6187fd36c7fbd0348ed033d98630 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 04:12:02 +0700 Subject: [PATCH] feat(monitoring): keep application identity on container log lines A reader that scopes logs to one application needs the owning application on every line, and the lineage webhook already puts it on the pod as apps.cozystack.io/application.{group,kind,name}. Fluent Bit fetched those labels and then dropped them with the rest of the kubernetes map, so the store could only be scoped by namespace. Keep the three labels as apps_cozystack_io_application_{group,kind,name}, the names the metrics side uses, and make them stream fields on every container-log output so the scope is an indexed stream filter. A pod without the labels produces no such fields, and VictoriaLogs builds the stream from the fields present, so existing streams keep their identity; a labelled pod starts a new stream once. Assisted-by: LLM Signed-off-by: ZverGuy --- .../fluent-bit_application_identity_test.yaml | 67 +++++++++++++++++++ packages/system/monitoring-agents/values.yaml | 20 +++++- 2 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 packages/system/monitoring-agents/tests/fluent-bit_application_identity_test.yaml diff --git a/packages/system/monitoring-agents/tests/fluent-bit_application_identity_test.yaml b/packages/system/monitoring-agents/tests/fluent-bit_application_identity_test.yaml new file mode 100644 index 000000000..d1815eb1b --- /dev/null +++ b/packages/system/monitoring-agents/tests/fluent-bit_application_identity_test.yaml @@ -0,0 +1,67 @@ +suite: fluent-bit keeps the application identity of a container log line + +# The lineage webhook labels a pod with apps.cozystack.io/application.{group, +# kind,name}. A container log line keeps them as top-level +# apps_cozystack_io_application_{group,kind,name}, the names the metrics side +# uses, and they are stream fields so a reader can scope to one application +# with extra_stream_filters. The kubernetes map, labels included, is otherwise +# dropped by Remove_wildcard kubernetes_, so the three keys have to be lifted +# and renamed out of the kubernetes_ prefix before that filter runs. +templates: + - charts/fluent-bit/templates/configmap.yaml +release: + name: monitoring-agents + namespace: cozy-monitoring +set: + global: + target: tenant-root +kubernetesProvider: + scheme: + "v1/Namespace": + gvr: + version: v1 + resource: namespaces + namespaced: false + objects: + - kind: Namespace + apiVersion: v1 + metadata: + name: tenant-a + labels: + namespace.cozystack.io/monitoring: tenant-a +tests: + - it: lifts the pod labels after the kubernetes map is lifted + asserts: + - matchRegex: + path: data["fluent-bit.conf"] + pattern: '(?s)Operation lift\n\s+Nested_under kubernetes\n.*Name nest\n\s+Match kube\.\*\n\s+Operation lift\n\s+Nested_under kubernetes_labels\n\s+Add_prefix kubernetes_labels_\n' + + - it: renames the three lineage labels out of the kubernetes_ prefix before Remove_wildcard + asserts: + - matchRegex: + path: data["fluent-bit.conf"] + pattern: '(?s)Nested_under kubernetes_labels\n.*Rename kubernetes_labels_apps\.cozystack\.io/application\.group apps_cozystack_io_application_group\n\s+Rename kubernetes_labels_apps\.cozystack\.io/application\.kind apps_cozystack_io_application_kind\n\s+Rename kubernetes_labels_apps\.cozystack\.io/application\.name apps_cozystack_io_application_name\n.*Remove_wildcard kubernetes_\n' + + - it: makes the lineage keys stream fields on the default container-log output + asserts: + - matchRegex: + path: data["fluent-bit.conf"] + pattern: 'Match kube\.\*\n\s+Host vlinsert-generic\.tenant-root\.svc\n\s+port 9481\n\s+compress gzip\n\s+uri /insert/jsonline\?_stream_fields=log_source,stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name,apps_cozystack_io_application_group,apps_cozystack_io_application_kind,apps_cozystack_io_application_name&' + + - it: makes the lineage keys stream fields on a routed tenant's container-log output + asserts: + - matchRegex: + path: data["fluent-bit.conf"] + pattern: 'Match tenant\.tenant-a\.\*\n\s+Host vlinsert-generic\.tenant-a\.svc\n\s+port 9481\n\s+compress gzip\n\s+uri /insert/jsonline\?_stream_fields=log_source,stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name,apps_cozystack_io_application_group,apps_cozystack_io_application_kind,apps_cozystack_io_application_name&' + + - it: leaves the events and audit outputs without lineage stream fields + asserts: + - matchRegex: + path: data["fluent-bit.conf"] + pattern: 'uri /insert/jsonline\?_stream_fields=log_source,reason,metadata_namespace,metadata_name&' + - matchRegex: + path: data["fluent-bit.conf"] + pattern: 'uri /insert/jsonline\?_stream_fields=log_source,stage,user_username,verb,requestUri&' + - notMatchRegex: + path: data["fluent-bit.conf"] + pattern: 'Match (events\.\*|audit\.\*|\*)\n[^\[]*Nested_under kubernetes_labels' diff --git a/packages/system/monitoring-agents/values.yaml b/packages/system/monitoring-agents/values.yaml index a9c22cb14..186828f70 100644 --- a/packages/system/monitoring-agents/values.yaml +++ b/packages/system/monitoring-agents/values.yaml @@ -405,7 +405,7 @@ fluent-bit: Host vlinsert-generic.{{ .Values.global.target }}.svc port 9481 compress gzip - uri /insert/jsonline?_stream_fields=log_source,stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name&_msg_field=log&_time_field=date + uri /insert/jsonline?_stream_fields=log_source,stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name,apps_cozystack_io_application_group,apps_cozystack_io_application_kind,apps_cozystack_io_application_name&_msg_field=log&_time_field=date format json_lines json_date_format iso8601 header AccountID 0 @@ -439,7 +439,7 @@ fluent-bit: Host vlinsert-generic.{{ $target }}.svc port 9481 compress gzip - uri /insert/jsonline?_stream_fields=log_source,stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name&_msg_field=log&_time_field=date + uri /insert/jsonline?_stream_fields=log_source,stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name,apps_cozystack_io_application_group,apps_cozystack_io_application_kind,apps_cozystack_io_application_name&_msg_field=log&_time_field=date format json_lines json_date_format iso8601 header AccountID 0 @@ -461,6 +461,19 @@ fluent-bit: Operation lift Nested_under kubernetes Add_prefix kubernetes_ + {{- /* + The pod's lineage labels name the application it belongs to. lift + cannot pick keys, so every label comes up; all but these three keep + the kubernetes_ prefix and fall to Remove_wildcard below. An absent + label stays absent, and VictoriaLogs builds _stream from the stream + fields that are present, so an unlabelled pod keeps its stream. + */}} + [FILTER] + Name nest + Match kube.* + Operation lift + Nested_under kubernetes_labels + Add_prefix kubernetes_labels_ [FILTER] Name modify Match kube.* @@ -501,6 +514,9 @@ fluent-bit: Rename kubernetes_pod_name keep_kubernetes_pod_name Rename kubernetes_namespace_name keep_kubernetes_namespace_name Rename kubernetes_container_name keep_kubernetes_container_name + Rename kubernetes_labels_apps.cozystack.io/application.group apps_cozystack_io_application_group + Rename kubernetes_labels_apps.cozystack.io/application.kind apps_cozystack_io_application_kind + Rename kubernetes_labels_apps.cozystack.io/application.name apps_cozystack_io_application_name [FILTER] Name modify Match kube.*