mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-09 14:02:56 +09:00
Aleksei Sviridkin
45a273eba5
The fluent-bit DaemonSet crash-loops on a loaded install, timing out its HelmRelease and cascading into velero and backupstrategy-controller DependencyNotReady failures. The subchart default liveness/readiness probes (initialDelaySeconds 0, timeoutSeconds 1, failureThreshold 3) target the HTTP server on :2020, but that server does not bind until AFTER the kubernetes filter's 'testing connectivity with API server' step. Under a concurrent install the kube-API can be unreachable for ~10s, delaying the :2020 bind, while the single-threaded HTTP server can miss a 1s timeout when the engine is busy. The probes then fire into a connection-refused port and liveness-kill fluent-bit into a restart loop. Give both probes load headroom (initialDelaySeconds 15, timeoutSeconds 5, failureThreshold 6) so the API-gated late bind and busy-engine blips are tolerated. The httpGet target is unchanged (path / on port http = :2020). Add a helm-unittest pinning the relaxed timing and the preserved target. Assisted-By: Claude <noreply@anthropic.com> Signed-off-by: Aleksei Sviridkin <f@lex.la>
448 lines
16 KiB
YAML
448 lines
16 KiB
YAML
global:
|
|
target: cozy-monitoring
|
|
|
|
kube-state-metrics:
|
|
extraArgs:
|
|
- --metric-labels-allowlist=pods=[*],deployments=[*]
|
|
rbac:
|
|
extraRules:
|
|
- apiGroups:
|
|
- source.toolkit.fluxcd.io
|
|
- kustomize.toolkit.fluxcd.io
|
|
- helm.toolkit.fluxcd.io
|
|
- notification.toolkit.fluxcd.io
|
|
- image.toolkit.fluxcd.io
|
|
resources:
|
|
- gitrepositories
|
|
- buckets
|
|
- helmrepositories
|
|
- helmcharts
|
|
- ocirepositories
|
|
- kustomizations
|
|
- helmreleases
|
|
- alerts
|
|
- providers
|
|
- receivers
|
|
- imagerepositories
|
|
- imagepolicies
|
|
- imageupdateautomations
|
|
verbs: [ "list", "watch" ]
|
|
customResourceState:
|
|
enabled: true
|
|
config:
|
|
spec:
|
|
resources:
|
|
- groupVersionKind:
|
|
group: kustomize.toolkit.fluxcd.io
|
|
version: v1
|
|
kind: Kustomization
|
|
metricNamePrefix: gotk
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux Kustomization resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [ metadata, name ]
|
|
labelsFromPath:
|
|
exported_namespace: [ metadata, namespace ]
|
|
ready: [ status, conditions, "[type=Ready]", status ]
|
|
suspended: [ spec, suspend ]
|
|
revision: [ status, lastAppliedRevision ]
|
|
source_name: [ spec, sourceRef, name ]
|
|
- groupVersionKind:
|
|
group: helm.toolkit.fluxcd.io
|
|
version: v2
|
|
kind: HelmRelease
|
|
metricNamePrefix: gotk
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux HelmRelease resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [ metadata, name ]
|
|
labelsFromPath:
|
|
exported_namespace: [ metadata, namespace ]
|
|
ready: [ status, conditions, "[type=Ready]", status ]
|
|
suspended: [ spec, suspend ]
|
|
revision: [ status, history, "0", chartVersion ]
|
|
chart_name: [ status, history, "0", chartName ]
|
|
chart_app_version: [ status, history, "0", appVersion ]
|
|
chart_ref_name: [ spec, chartRef, name ]
|
|
chart_source_name: [ spec, chart, spec, sourceRef, name ]
|
|
- groupVersionKind:
|
|
group: source.toolkit.fluxcd.io
|
|
version: v1
|
|
kind: GitRepository
|
|
metricNamePrefix: gotk
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux GitRepository resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [ metadata, name ]
|
|
labelsFromPath:
|
|
exported_namespace: [ metadata, namespace ]
|
|
ready: [ status, conditions, "[type=Ready]", status ]
|
|
suspended: [ spec, suspend ]
|
|
revision: [ status, artifact, revision ]
|
|
url: [ spec, url ]
|
|
- groupVersionKind:
|
|
group: source.toolkit.fluxcd.io
|
|
version: v1beta2
|
|
kind: Bucket
|
|
metricNamePrefix: gotk
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux Bucket resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [ metadata, name ]
|
|
labelsFromPath:
|
|
exported_namespace: [ metadata, namespace ]
|
|
ready: [ status, conditions, "[type=Ready]", status ]
|
|
suspended: [ spec, suspend ]
|
|
revision: [ status, artifact, revision ]
|
|
endpoint: [ spec, endpoint ]
|
|
bucket_name: [ spec, bucketName ]
|
|
- groupVersionKind:
|
|
group: source.toolkit.fluxcd.io
|
|
version: v1
|
|
kind: HelmRepository
|
|
metricNamePrefix: gotk
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux HelmRepository resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [ metadata, name ]
|
|
labelsFromPath:
|
|
exported_namespace: [ metadata, namespace ]
|
|
ready: [ status, conditions, "[type=Ready]", status ]
|
|
suspended: [ spec, suspend ]
|
|
revision: [ status, artifact, revision ]
|
|
url: [ spec, url ]
|
|
- groupVersionKind:
|
|
group: source.toolkit.fluxcd.io
|
|
version: v1
|
|
kind: HelmChart
|
|
metricNamePrefix: gotk
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux HelmChart resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [ metadata, name ]
|
|
labelsFromPath:
|
|
exported_namespace: [ metadata, namespace ]
|
|
ready: [ status, conditions, "[type=Ready]", status ]
|
|
suspended: [ spec, suspend ]
|
|
revision: [ status, artifact, revision ]
|
|
chart_name: [ spec, chart ]
|
|
chart_version: [ spec, version ]
|
|
- groupVersionKind:
|
|
group: source.toolkit.fluxcd.io
|
|
version: v1beta2
|
|
kind: OCIRepository
|
|
metricNamePrefix: gotk
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux OCIRepository resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [ metadata, name ]
|
|
labelsFromPath:
|
|
exported_namespace: [ metadata, namespace ]
|
|
ready: [ status, conditions, "[type=Ready]", status ]
|
|
suspended: [ spec, suspend ]
|
|
revision: [ status, artifact, revision ]
|
|
url: [ spec, url ]
|
|
- groupVersionKind:
|
|
group: notification.toolkit.fluxcd.io
|
|
version: v1beta3
|
|
kind: Alert
|
|
metricNamePrefix: gotk
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux Alert resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [ metadata, name ]
|
|
labelsFromPath:
|
|
exported_namespace: [ metadata, namespace ]
|
|
suspended: [ spec, suspend ]
|
|
- groupVersionKind:
|
|
group: notification.toolkit.fluxcd.io
|
|
version: v1beta3
|
|
kind: Provider
|
|
metricNamePrefix: gotk
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux Provider resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [ metadata, name ]
|
|
labelsFromPath:
|
|
exported_namespace: [ metadata, namespace ]
|
|
suspended: [ spec, suspend ]
|
|
- groupVersionKind:
|
|
group: notification.toolkit.fluxcd.io
|
|
version: v1
|
|
kind: Receiver
|
|
metricNamePrefix: gotk
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux Receiver resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [ metadata, name ]
|
|
labelsFromPath:
|
|
exported_namespace: [ metadata, namespace ]
|
|
ready: [ status, conditions, "[type=Ready]", status ]
|
|
suspended: [ spec, suspend ]
|
|
webhook_path: [ status, webhookPath ]
|
|
- groupVersionKind:
|
|
group: image.toolkit.fluxcd.io
|
|
version: v1beta2
|
|
kind: ImageRepository
|
|
metricNamePrefix: gotk
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux ImageRepository resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [ metadata, name ]
|
|
labelsFromPath:
|
|
exported_namespace: [ metadata, namespace ]
|
|
ready: [ status, conditions, "[type=Ready]", status ]
|
|
suspended: [ spec, suspend ]
|
|
image: [ spec, image ]
|
|
- groupVersionKind:
|
|
group: image.toolkit.fluxcd.io
|
|
version: v1beta2
|
|
kind: ImagePolicy
|
|
metricNamePrefix: gotk
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux ImagePolicy resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [ metadata, name ]
|
|
labelsFromPath:
|
|
exported_namespace: [ metadata, namespace ]
|
|
ready: [ status, conditions, "[type=Ready]", status ]
|
|
suspended: [ spec, suspend ]
|
|
source_name: [ spec, imageRepositoryRef, name ]
|
|
- groupVersionKind:
|
|
group: image.toolkit.fluxcd.io
|
|
version: v1beta2
|
|
kind: ImageUpdateAutomation
|
|
metricNamePrefix: gotk
|
|
metrics:
|
|
- name: "resource_info"
|
|
help: "The current state of a Flux ImageUpdateAutomation resource."
|
|
each:
|
|
type: Info
|
|
info:
|
|
labelsFromPath:
|
|
name: [ metadata, name ]
|
|
labelsFromPath:
|
|
exported_namespace: [ metadata, namespace ]
|
|
ready: [ status, conditions, "[type=Ready]", status ]
|
|
suspended: [ spec, suspend ]
|
|
source_name: [ spec, sourceRef, name ]
|
|
|
|
vmagent:
|
|
externalLabels:
|
|
cluster: cozystack
|
|
remoteWrite:
|
|
urls:
|
|
- http://vminsert-shortterm.{{ .Values.global.target }}.svc:8480/insert/0/prometheus
|
|
- http://vminsert-longterm.{{ .Values.global.target }}.svc:8480/insert/0/prometheus
|
|
extraArgs: {}
|
|
|
|
fluent-bit:
|
|
rbac:
|
|
eventsAccess: true
|
|
# Relax probe timing to survive a loaded install. Fluent-bit's HTTP server on
|
|
# :2020 (the probe target) does not bind until AFTER the kubernetes filter's
|
|
# "testing connectivity with API server" step; under concurrent install the
|
|
# kube-API can be unreachable for ~10s, delaying the :2020 bind, and the
|
|
# single-threaded HTTP server can miss a 1s timeout while the engine is busy.
|
|
# The subchart defaults (initialDelaySeconds 0, timeoutSeconds 1,
|
|
# failureThreshold 3) then liveness-kill fluent-bit into a crash loop. httpGet
|
|
# stays on path / port http (:2020) — same target as the subchart defaults.
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /
|
|
port: http
|
|
initialDelaySeconds: 15
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
failureThreshold: 6
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /
|
|
port: http
|
|
timeoutSeconds: 5
|
|
failureThreshold: 6
|
|
daemonSetVolumes:
|
|
- name: varlog
|
|
hostPath:
|
|
path: /var/log
|
|
- name: varlibdockercontainers
|
|
hostPath:
|
|
path: /var/lib/docker/containers
|
|
daemonSetVolumeMounts:
|
|
- name: varlog
|
|
mountPath: /var/log
|
|
- name: varlibdockercontainers
|
|
mountPath: /var/lib/docker/containers
|
|
readOnly: true
|
|
config:
|
|
inputs: |
|
|
[INPUT]
|
|
Name tail
|
|
Path /var/log/containers/*.log
|
|
multiline.parser docker, cri
|
|
Tag kube.*
|
|
Mem_Buf_Limit 5MB
|
|
Skip_Long_Lines On
|
|
[INPUT]
|
|
Name kubernetes_events
|
|
Tag events.*
|
|
Kube_url https://kubernetes.default.svc
|
|
[INPUT]
|
|
Name tail
|
|
Alias audit
|
|
Path /var/log/audit/kube/*.log
|
|
Parser audit
|
|
Tag audit.*
|
|
customParsers: |
|
|
[PARSER]
|
|
Name docker_no_time
|
|
Format json
|
|
Time_Keep Off
|
|
Time_Key time
|
|
Time_Format %Y-%m-%dT%H:%M:%S.%L
|
|
[PARSER]
|
|
Name audit
|
|
Format json
|
|
Time_Key requestReceivedTimestamp
|
|
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
|
|
[PARSER]
|
|
Name containerd
|
|
Format regex
|
|
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<event>.*)$
|
|
Time_Key time
|
|
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
|
|
outputs: |
|
|
[OUTPUT]
|
|
Name http
|
|
Match kube.*
|
|
Host vlinsert-generic.{{ .Values.global.target }}.svc
|
|
port 9481
|
|
compress gzip
|
|
uri /insert/jsonline?_stream_fields=log_source,stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name&_msg_field=log&_time_field=date
|
|
format json_lines
|
|
json_date_format iso8601
|
|
header AccountID 0
|
|
header ProjectID 0
|
|
[OUTPUT]
|
|
Name http
|
|
Match events.*
|
|
Host vlinsert-generic.{{ .Values.global.target }}.svc
|
|
port 9481
|
|
compress gzip
|
|
uri /insert/jsonline?_stream_fields=log_source,reason,meatdata_namespace,metadata_name&_msg_field=message&_time_field=date
|
|
format json_lines
|
|
json_date_format iso8601
|
|
header AccountID 0
|
|
header ProjectID 0
|
|
[OUTPUT]
|
|
Name http
|
|
Match audit.*
|
|
Host vlinsert-generic.{{ .Values.global.target }}.svc
|
|
port 9481
|
|
compress gzip
|
|
uri /insert/jsonline?_stream_fields=log_source,stage,user_username,verb,requestUri&_msg_field=requestURI&_time_field=date
|
|
format json_lines
|
|
json_date_format iso8601
|
|
header AccountID 0
|
|
header ProjectID 0
|
|
filters: |
|
|
[FILTER]
|
|
Name kubernetes
|
|
Match kube.*
|
|
Merge_Log On
|
|
Keep_Log On
|
|
K8S-Logging.Parser On
|
|
K8S-Logging.Exclude On
|
|
[FILTER]
|
|
Name nest
|
|
Match kube.*
|
|
Wildcard pod_name
|
|
Operation lift
|
|
Nested_under kubernetes
|
|
Add_prefix kubernetes_
|
|
[FILTER]
|
|
Name modify
|
|
Match kube.*
|
|
Add log_source container_log
|
|
[FILTER]
|
|
Name nest
|
|
Match events.*
|
|
Wildcard metadata.*
|
|
Operation lift
|
|
Nested_under metadata
|
|
Add_prefix metadata_
|
|
[FILTER]
|
|
Name nest
|
|
Match audit.*
|
|
Wildcard user.*
|
|
Operation lift
|
|
Nested_under user
|
|
Add_prefix user_
|
|
[FILTER]
|
|
Name modify
|
|
Match events.*
|
|
Add log_source kube_events
|
|
[FILTER]
|
|
Name modify
|
|
Match audit.*
|
|
Add log_source audit_log
|
|
[FILTER]
|
|
Name modify
|
|
Match *
|
|
Add tenant {{ .Values.global.target }}
|
|
[FILTER]
|
|
Name modify
|
|
Match *
|
|
Add cluster root-cluster
|
|
scrapeRules:
|
|
etcd:
|
|
enabled: false
|