From 89028b455ef2fbf4772672c2fe6342150efc3200 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 18 Oct 2026 23:16:45 +0300
Subject: [PATCH 001/114] fix(monitoring): route InfoInhibitor to the blackhole
receiver
InfoInhibitor exists to be the source side of an Alertmanager inhibit
rule and, as its own description says, to go to a null receiver. The
platform Alertmanager config has no inhibit rule, and the rule has
carried severity major since it was imported alongside Alerta, so
Alerta shows it one level below critical. Alerta's Telegram and Slack
plugins forward it by default, and a notify_email label inherited from
the info alert gets it emailed too.
It fires whenever a rule labelled severity="info" is active with no
warning or critical alert in the same namespace and cluster. The
platform's own rules no longer use info, but rules it does not write
still do. CDI generates CDIStorageProfilesIncomplete, KubeVirt
generates KubeVirtVMGuestMemoryAvailableLow, and operators add their
own. Each of them turns into a major InfoInhibitor in Alerta.
none is the severity upstream kube-prometheus gives this rule, and the
platform config already sends severity="none" to the blackhole ahead
of every route that delivers. The severity contract refuses none
because such an alert is never delivered, so InfoInhibitor is named as
the one alert for which that is the point. Tests hold both ends, so
neither the label nor the route order can drift back to delivering it.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
hack/alert-severity-contract.bats | 89 +++++++++++++++++++
.../alerts/general.rules.yaml | 2 +-
.../tests/alertmanager_blackhole_test.yaml | 45 ++++++++++
3 files changed, 135 insertions(+), 1 deletion(-)
create mode 100644 packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
diff --git a/hack/alert-severity-contract.bats b/hack/alert-severity-contract.bats
index e1315eb96..44f197000 100644
--- a/hack/alert-severity-contract.bats
+++ b/hack/alert-severity-contract.bats
@@ -9,6 +9,10 @@ load test_helper
# scope: they are re-vendored by `make update`, so a bad severity there is
# fixed upstream or in patches/, not by editing the label.
ALERT_SEVERITY_ALLOWED=" security critical major minor warning indeterminate informational normal ok cleared debug trace unknown "
+# The alerts that exist only to be dropped, the one place `none` is right.
+# InfoInhibitor is the source side of an Alertmanager inhibit rule, which the
+# platform config does not have, so delivering it at any severity is noise.
+ALERT_SEVERITY_BLACKHOLED=" InfoInhibitor "
helm_actions_stripped() {
awk '
@@ -104,6 +108,14 @@ alert_severity_contract() {
break
fi
done
+ if [ "$severity" = "none" ]; then
+ for blackholed in $ALERT_SEVERITY_BLACKHOLED; do
+ if [ "$blackholed" = "$alert" ]; then
+ accepted=1
+ break
+ fi
+ done
+ fi
if [ "$accepted" -eq 0 ]; then
echo "$file: $alert -> '$severity'" >&2
bad=$((bad + 1))
@@ -128,6 +140,83 @@ alert_severity_contract() {
alert_severity_contract
}
+@test "InfoInhibitor is labelled for the blackhole route" {
+ rows=$(mktemp) || return 1
+ if ! alert_severity_rows > "$rows"; then
+ rm -f "$rows"
+ return 1
+ fi
+ tab=$(printf '\t')
+ found=$(grep -c "${tab}InfoInhibitor${tab}" "$rows" || true)
+ delivered=$(grep "${tab}InfoInhibitor${tab}" "$rows" | grep -vc "${tab}none\$" || true)
+ rm -f "$rows"
+ [ "$found" -gt 0 ]
+ [ "$delivered" -eq 0 ]
+}
+
+@test "severity none passes only on an alert that exists to be dropped" {
+ fixture=$(mktemp -d)
+ mkdir -p "$fixture/packages/system/example/alerts"
+ printf '%s\n' \
+ 'apiVersion: operator.victoriametrics.com/v1beta1' \
+ 'kind: VMRule' \
+ 'spec:' \
+ ' groups:' \
+ ' - name: general' \
+ ' rules:' \
+ ' - alert: InfoInhibitor' \
+ ' labels:' \
+ ' severity: none' \
+ > "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
+ git -C "$fixture" init -q
+ git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
+
+ inhibitor_succeeded=0
+ ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || inhibitor_succeeded=$?
+
+ printf '%s\n' \
+ 'apiVersion: operator.victoriametrics.com/v1beta1' \
+ 'kind: VMRule' \
+ 'spec:' \
+ ' groups:' \
+ ' - name: silent' \
+ ' rules:' \
+ ' - alert: SilentAlert' \
+ ' labels:' \
+ ' severity: none' \
+ > "$fixture/packages/system/example/alerts/01-silent.yaml"
+ git -C "$fixture" add packages/system/example/alerts/01-silent.yaml
+
+ silent_succeeded=0
+ ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || silent_succeeded=$?
+ rm -rf "$fixture"
+ [ "$inhibitor_succeeded" -eq 0 ]
+ [ "$silent_succeeded" -ne 0 ]
+}
+
+@test "an alert that exists to be dropped still fails on a refused severity" {
+ fixture=$(mktemp -d)
+ mkdir -p "$fixture/packages/system/example/alerts"
+ printf '%s\n' \
+ 'apiVersion: operator.victoriametrics.com/v1beta1' \
+ 'kind: VMRule' \
+ 'spec:' \
+ ' groups:' \
+ ' - name: general' \
+ ' rules:' \
+ ' - alert: InfoInhibitor' \
+ ' labels:' \
+ ' severity: info' \
+ > "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
+ git -C "$fixture" init -q
+ git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
+
+ contract_succeeded=0
+ ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$?
+ rm -rf "$fixture"
+ [ "$contract_succeeded" -ne 0 ]
+}
+
@test "a parser failure cannot leave the contract green" {
fixture=$(mktemp -d)
mkdir -p "$fixture/packages/system/example/alerts"
diff --git a/packages/system/monitoring-agents/alerts/general.rules.yaml b/packages/system/monitoring-agents/alerts/general.rules.yaml
index 924870c40..45dd6100f 100644
--- a/packages/system/monitoring-agents/alerts/general.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/general.rules.yaml
@@ -52,6 +52,6 @@ spec:
!= "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} ==
1
labels:
- severity: major
+ severity: none
exported_instance: global
service: general.rules
diff --git a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
new file mode 100644
index 000000000..e987bd405
--- /dev/null
+++ b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
@@ -0,0 +1,45 @@
+suite: Alertmanager blackhole route
+templates:
+ - templates/alerta/alerta.yaml
+
+release:
+ name: monitoring
+ namespace: tenant-root
+
+set:
+ _cluster:
+ root-host: example.org
+ _namespace:
+ host: tenant-root.example.org
+ ingress: tenant-root
+
+# Rules labelled severity="none", InfoInhibitor among them, exist to be
+# dropped, and Alerta answers that severity with a 500. Only the heartbeat
+# route may come ahead of the blackhole, so no delivering route claims them.
+tests:
+ - it: drops severity none ahead of every delivering route
+ asserts:
+ - documentSelector:
+ path: metadata.labels.app
+ value: alertmanager
+ matchRegex:
+ path: stringData["alertmanager.yaml"]
+ pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'
+
+ - it: keeps severity none out of email even when its alert name is listed
+ set:
+ alerta:
+ alerts:
+ email:
+ to: "oncall@example.org"
+ from: "alerts@example.org"
+ smarthost: "smtp.example.org:465"
+ alertnames:
+ - InfoInhibitor
+ asserts:
+ - documentSelector:
+ path: metadata.labels.app
+ value: alertmanager
+ matchRegex:
+ path: stringData["alertmanager.yaml"]
+ pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'
From 8d82348e85fb9588ad213d763ece81f7b88e84a8 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Mon, 19 Oct 2026 16:04:58 +0300
Subject: [PATCH 002/114] fix(vm-disk): refuse a disk smaller than the golden
image it clones
CDI checks a clone against its source before it creates the claim. When
the golden image's PVC is in Block mode and requests more than the disk,
it records CloneValidationFailed and leaves the DataVolume pending for
good. The DataVolume spec is immutable, so raising storage on the VMDisk
afterwards changes nothing, and the resize hook finds no claim to patch.
Every image in vm-default-images is 20Gi on replicated, where CDI picks
Block mode for the LINSTOR driver, while the chart defaults storage to
5Gi, so a disk created from an image at the default size never comes up.
The chart now refuses such a disk at render time and names both sizes,
so the user sets storage to the golden's size before any DataVolume
exists. It compares what validateClone compares in CDI v1.66.1, the
golden PVC's request against the disk's request raised to the target
StorageProfile's minimumSupportedPvcSize. A Filesystem golden is not
compared, because CDI sizes that clone from the image inside it or with
a filesystem overhead, and neither is a golden whose PVC does not exist
yet. An existing disk is exempt, as for the other golden checks.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
packages/apps/vm-disk/templates/dv.yaml | 20 ++
.../vm-disk/tests/dv_golden_size_test.yaml | 252 ++++++++++++++++++
2 files changed, 272 insertions(+)
create mode 100644 packages/apps/vm-disk/tests/dv_golden_size_test.yaml
diff --git a/packages/apps/vm-disk/templates/dv.yaml b/packages/apps/vm-disk/templates/dv.yaml
index 09831fc79..9bec73f5f 100644
--- a/packages/apps/vm-disk/templates/dv.yaml
+++ b/packages/apps/vm-disk/templates/dv.yaml
@@ -146,6 +146,26 @@
{{- fail (printf "source.image %q: golden image %q lives in StorageClass %q, and a clone into StorageClass %q of the same provisioner gets no host-assisted fallback from CDI. LINSTOR creates the clone next to the source data, and StorageClass %q does not allow remote volume access from every node, so the clone may never bind. Set storageClass to %q, or ask the platform administrator to set cloneStrategyOverride to copy in the kubevirt-cdi package." .Values.source.image.name $desiredSource.pvc.name $goldenSC $targetSC $targetSC $goldenSC) }}
{{- end }}
{{- end }}
+ {{- /* CDI v1.66.1 refuses to clone a Block source PVC into a smaller
+ request before it creates the claim (validateClone in
+ pkg/controller/datavolume/pvc-clone-controller.go), and the
+ DataVolume then waits forever. It raises the request to the target
+ StorageProfile's minimumSupportedPvcSize before comparing. A
+ Filesystem source is not compared, as CDI sizes that clone from
+ the image inside it or with a filesystem overhead the chart does
+ not reproduce. */}}
+ {{- $goldenSize := dig "spec" "resources" "requests" "storage" "" ($goldenPVC | default dict) | toString }}
+ {{- if and .Values.storage $goldenSize (eq (dig "spec" "volumeMode" "" ($goldenPVC | default dict) | toString) "Block") }}
+ {{- $goldenBytes := include "cozy-lib.resources.toFloat" $goldenSize | float64 }}
+ {{- $diskBytes := include "cozy-lib.resources.toFloat" (.Values.storage | toString) | float64 }}
+ {{- if and (lt $diskBytes $goldenBytes) $targetSC }}
+ {{- $minimum := dig "metadata" "annotations" "cdi.kubevirt.io/minimumSupportedPvcSize" "" (lookup "cdi.kubevirt.io/v1beta1" "StorageProfile" "" $targetSC | default dict) | toString }}
+ {{- $diskBytes = maxf $diskBytes (include "cozy-lib.resources.toFloat" $minimum | float64) }}
+ {{- end }}
+ {{- if lt $diskBytes $goldenBytes }}
+ {{- fail (printf "storage %s is smaller than golden image %q (%s), so CDI refuses to clone it and the disk would stay pending forever. Set storage to %s or more." (.Values.storage | toString) $desiredSource.pvc.name $goldenSize $goldenSize) }}
+ {{- end }}
+ {{- end }}
{{- end }}
{{- else if hasKey .Values.source "http" }}
{{- $desiredSource = dict "http" (dict "url" (required "A valid .Values.source.http.url entry required!" .Values.source.http.url)) }}
diff --git a/packages/apps/vm-disk/tests/dv_golden_size_test.yaml b/packages/apps/vm-disk/tests/dv_golden_size_test.yaml
new file mode 100644
index 000000000..6a68874c7
--- /dev/null
+++ b/packages/apps/vm-disk/tests/dv_golden_size_test.yaml
@@ -0,0 +1,252 @@
+suite: vm-disk refuses a disk smaller than the golden image it clones
+release:
+ name: test-vm-disk
+ namespace: tenant-test
+templates:
+ - templates/dv.yaml
+tests:
+ - it: refuses a new disk at the chart default size from a larger Block golden
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: &scheme
+ "cdi.kubevirt.io/v1beta1/DataVolume":
+ gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: datavolumes}
+ namespaced: true
+ "cdi.kubevirt.io/v1beta1/StorageProfile":
+ gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: storageprofiles}
+ namespaced: false
+ "v1/Namespace":
+ gvr: {group: "", version: v1, resource: namespaces}
+ namespaced: false
+ "v1/PersistentVolumeClaim":
+ gvr: {group: "", version: v1, resource: persistentvolumeclaims}
+ namespaced: true
+ "storage.k8s.io/v1/StorageClass":
+ gvr: {group: storage.k8s.io, version: v1, resource: storageclasses}
+ namespaced: false
+ objects:
+ - &namespace
+ kind: Namespace
+ apiVersion: v1
+ metadata: {name: tenant-test}
+ - &goldenPVC
+ kind: PersistentVolumeClaim
+ apiVersion: v1
+ metadata: {name: vm-default-images-ubuntu, namespace: cozy-public}
+ spec:
+ storageClassName: replicated
+ volumeMode: Block
+ resources: {requests: {storage: 20Gi}}
+ asserts:
+ - failedTemplate:
+ errorPattern: 'storage 5Gi is smaller than golden image "vm-default-images-ubuntu" \(20Gi\), so CDI refuses to clone it and the disk would stay pending forever\. Set storage to 20Gi or more\.'
+
+ - it: clones into a disk the size of the golden
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 20Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - equal:
+ path: spec.storage.resources.requests.storage
+ value: 20Gi
+
+ - it: clones into a disk larger than the golden
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 25Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - equal:
+ path: spec.storage.resources.requests.storage
+ value: 25Gi
+
+ - it: compares bytes, so 21G is smaller than a 20Gi golden
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 21G
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - failedTemplate:
+ errorPattern: 'storage 21G is smaller than golden image "vm-default-images-ubuntu" \(20Gi\)'
+
+ - it: compares bytes, so 20480Mi is the size of a 20Gi golden
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 20480Mi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - equal:
+ path: spec.storage.resources.requests.storage
+ value: 20480Mi
+
+ - it: clones when the target StorageProfile's minimum PVC size covers the golden
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ - kind: StorageProfile
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata:
+ name: replicated
+ annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 20Gi}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: refuses when the target StorageProfile's minimum PVC size is still below the golden
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ - kind: StorageProfile
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata:
+ name: replicated
+ annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 10Gi}
+ asserts:
+ - failedTemplate:
+ errorPattern: 'storage 5Gi is smaller than golden image "vm-default-images-ubuntu" \(20Gi\)'
+
+ - it: reads the minimum PVC size of the default class for an empty storageClass
+ set:
+ source: {image: {name: ubuntu}}
+ storageClass: ""
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ - kind: StorageClass
+ apiVersion: storage.k8s.io/v1
+ metadata:
+ name: replicated
+ annotations: {storageclass.kubernetes.io/is-default-class: "true"}
+ provisioner: linstor.csi.linbit.com
+ - kind: StorageProfile
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata:
+ name: replicated
+ annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 20Gi}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: does not compare sizes for a Filesystem golden
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - kind: PersistentVolumeClaim
+ apiVersion: v1
+ metadata: {name: vm-default-images-ubuntu, namespace: cozy-public}
+ spec:
+ storageClassName: replicated
+ volumeMode: Filesystem
+ resources: {requests: {storage: 20Gi}}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: renders without a cluster to look the golden up in
+ set:
+ source: {image: {name: ubuntu}}
+ asserts:
+ - equal:
+ path: spec.storage.resources.requests.storage
+ value: 5Gi
+
+ - it: clones a golden whose PVC does not exist yet
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - kind: DataVolume
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata: {name: vm-default-images-ubuntu, namespace: cozy-public}
+ status: {phase: ImportScheduled}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: leaves a disk without a size to CDI, which takes the golden's
+ set:
+ source: {image: {name: ubuntu}}
+ storage: null
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: leaves an existing disk alone
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 25Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ - kind: DataVolume
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata:
+ name: test-vm-disk
+ namespace: tenant-test
+ annotations:
+ vm-disk.cozystack.io/source: '{"image":{"name":"ubuntu"}}'
+ vm-disk.cozystack.io/storage-class: replicated
+ spec:
+ source: {pvc: {name: vm-default-images-ubuntu, namespace: cozy-public}}
+ storage: {resources: {requests: {storage: 10Gi}}, storageClassName: replicated}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: does not compare sizes for an http source
+ set:
+ source: {http: {url: "https://example.com/disk.img"}}
+ storage: 1Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: does not compare sizes for an upload source
+ set:
+ source: {upload: {}}
+ storage: 1Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - notFailedTemplate: {}
From f1f30b3071254e3640292b4eb483b04f5d40d77a Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Tue, 20 Oct 2026 12:05:51 +0300
Subject: [PATCH 003/114] fix(etcd): require an even spread of members over
nodes by default
The default spread of the etcd members was ScheduleAnyway, which only
adds to the score of a node, and other scores can outweigh it. A new
etcd on a three-node cluster got its first two members on one node.
On a node-local StorageClass a member cannot leave its node, so losing
that node takes two of the three members and the quorum.
DoNotSchedule with maxSkew 1 still lets an etcd with more members than
nodes start, because the skew is counted between the nodes that exist.
One node takes all three members and two nodes take two and one.
nodeTaintsPolicy is Ignore, so a cordoned node still counts and a
member waits for it instead of landing next to another member.
The operator copies the constraints into an EtcdMember when it creates
it and builds the Pod from that copy, so members that exist keep the
old setting and only members created later get the new one.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../extra/etcd/templates/etcd-cluster.yaml | 3 +-
.../etcd/tests/topology-spread_test.yaml | 46 ++++++++++++++-----
2 files changed, 37 insertions(+), 12 deletions(-)
diff --git a/packages/extra/etcd/templates/etcd-cluster.yaml b/packages/extra/etcd/templates/etcd-cluster.yaml
index 2824ce371..67dd34bd9 100644
--- a/packages/extra/etcd/templates/etcd-cluster.yaml
+++ b/packages/extra/etcd/templates/etcd-cluster.yaml
@@ -69,7 +69,8 @@ spec:
topologySpreadConstraints:
- maxSkew: 1
topologyKey: "kubernetes.io/hostname"
- whenUnsatisfiable: ScheduleAnyway
+ whenUnsatisfiable: DoNotSchedule
+ nodeTaintsPolicy: Ignore
labelSelector:
matchLabels:
app.kubernetes.io/instance: etcd
diff --git a/packages/extra/etcd/tests/topology-spread_test.yaml b/packages/extra/etcd/tests/topology-spread_test.yaml
index 4b96fc310..11bb92243 100644
--- a/packages/extra/etcd/tests/topology-spread_test.yaml
+++ b/packages/extra/etcd/tests/topology-spread_test.yaml
@@ -8,7 +8,7 @@ templates:
- templates/etcd-cluster.yaml
tests:
- - it: uses a soft per-host spread by default
+ - it: uses a strict per-host spread by default
set:
_cluster: {}
documentSelector:
@@ -16,17 +16,41 @@ tests:
value: EtcdCluster
asserts:
- equal:
- path: spec.topologySpreadConstraints[0].maxSkew
- value: 1
+ path: spec.topologySpreadConstraints[0]
+ value:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: DoNotSchedule
+ nodeTaintsPolicy: Ignore
+ labelSelector:
+ matchLabels:
+ app.kubernetes.io/instance: etcd
+ - lengthEqual:
+ path: spec.topologySpreadConstraints
+ count: 1
+
+ - it: renders a soft cluster-wide spread policy as written
+ set:
+ _cluster:
+ scheduling:
+ globalAppTopologySpreadConstraints: |
+ topologySpreadConstraints:
+ - maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+ documentSelector:
+ path: kind
+ value: EtcdCluster
+ asserts:
- equal:
- path: spec.topologySpreadConstraints[0].topologyKey
- value: kubernetes.io/hostname
- - equal:
- path: spec.topologySpreadConstraints[0].whenUnsatisfiable
- value: ScheduleAnyway
- - equal:
- path: spec.topologySpreadConstraints[0].labelSelector.matchLabels["app.kubernetes.io/instance"]
- value: etcd
+ path: spec.topologySpreadConstraints[0]
+ value:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+ labelSelector:
+ matchLabels:
+ app.kubernetes.io/instance: etcd
- lengthEqual:
path: spec.topologySpreadConstraints
count: 1
From 9f6fdd77c6ca964e0adcc387e50993487f2e594e Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sat, 24 Oct 2026 15:52:37 +0300
Subject: [PATCH 004/114] docs(etcd): update the affinity notes for the hard
member spread
The affinity description and the comment above the affinity block
explained the soft default by a member that would stay Pending on its
node-local volume. Now that the spread is DoNotSchedule by default, the
spread keeps such a member Pending as well, once its node holds another
member and some other node holds none. Both texts now explain the soft
default by what is left, that a required rule forbids two members on
one node even when every node has one, and state the node-local case
for both rules.
The description also says that the operator copies affinity into each
EtcdMember when it creates it, so a recreated Pod keeps the rule its
member was created with. README, values.schema.json and the
ApplicationDefinition are regenerated from it.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
packages/extra/etcd/README.md | 20 +++++++++----------
.../extra/etcd/templates/etcd-cluster.yaml | 14 ++++++++-----
packages/extra/etcd/values.schema.json | 2 +-
packages/extra/etcd/values.yaml | 2 +-
packages/system/etcd-rd/cozyrds/etcd.yaml | 2 +-
5 files changed, 22 insertions(+), 18 deletions(-)
diff --git a/packages/extra/etcd/README.md b/packages/extra/etcd/README.md
index e96c63832..91341ff32 100644
--- a/packages/extra/etcd/README.md
+++ b/packages/extra/etcd/README.md
@@ -34,14 +34,14 @@ coordinates).
### Common parameters
-| Name | Description | Type | Value |
-| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------- |
-| `version` | etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image. | `string` | `3.6.11` |
-| `size` | Persistent Volume size. | `quantity` | `4Gi` |
-| `storageClass` | StorageClass used to store the data. | `string` | `""` |
-| `replicas` | Number of etcd replicas. | `int` | `3` |
-| `resources` | Resource configuration for etcd. | `object` | `{}` |
-| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` |
-| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` |
-| `affinity` | Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively. | `object` | `{}` |
+| Name | Description | Type | Value |
+| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------- |
+| `version` | etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image. | `string` | `3.6.11` |
+| `size` | Persistent Volume size. | `quantity` | `4Gi` |
+| `storageClass` | StorageClass used to store the data. | `string` | `""` |
+| `replicas` | Number of etcd replicas. | `int` | `3` |
+| `resources` | Resource configuration for etcd. | `object` | `{}` |
+| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` |
+| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` |
+| `affinity` | Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones. | `object` | `{}` |
diff --git a/packages/extra/etcd/templates/etcd-cluster.yaml b/packages/extra/etcd/templates/etcd-cluster.yaml
index 67dd34bd9..3676f53cb 100644
--- a/packages/extra/etcd/templates/etcd-cluster.yaml
+++ b/packages/extra/etcd/templates/etcd-cluster.yaml
@@ -79,11 +79,15 @@ spec:
affinity:
{{- toYaml . | nindent 4 }}
{{- else }}
- # Soft anti-affinity: spread etcd members across nodes on a best-effort
- # basis so a single node failure is unlikely to take out more than one
- # member. Preferred (not required) because a hard rule would leave a member
- # Pending when replicas exceed schedulable nodes, or when node-local storage
- # pins a member to a node. Override via .Values.affinity for a hard rule.
+ # Soft anti-affinity, a preference on top of the spread constraint above,
+ # which by default is the hard rule that keeps two members off one node
+ # while another node has none. Preferred, not required, because a required
+ # rule also forbids two members on one node when every node already has
+ # one, so an etcd with more members than schedulable nodes could not start
+ # all of them. With node-local storage a recreated member Pod can only
+ # return to its own node, and the default spread keeps it Pending there
+ # while that node holds another member and some other node holds none.
+ # Override via .Values.affinity for a hard rule.
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
diff --git a/packages/extra/etcd/values.schema.json b/packages/extra/etcd/values.schema.json
index b438ca9d7..c33219f4f 100644
--- a/packages/extra/etcd/values.schema.json
+++ b/packages/extra/etcd/values.schema.json
@@ -67,7 +67,7 @@
}
},
"affinity": {
- "description": "Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.",
+ "description": "Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.",
"type": "object",
"default": {},
"x-kubernetes-preserve-unknown-fields": true
diff --git a/packages/extra/etcd/values.yaml b/packages/extra/etcd/values.yaml
index 61b994d7c..c5be94909 100644
--- a/packages/extra/etcd/values.yaml
+++ b/packages/extra/etcd/values.yaml
@@ -23,5 +23,5 @@ resources:
cpu: 1000m
memory: 512Mi
-## @param {object} [affinity] - Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.
+## @param {object} [affinity] - Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.
affinity: {}
diff --git a/packages/system/etcd-rd/cozyrds/etcd.yaml b/packages/system/etcd-rd/cozyrds/etcd.yaml
index 90ed8a80a..1ce88e8ae 100644
--- a/packages/system/etcd-rd/cozyrds/etcd.yaml
+++ b/packages/system/etcd-rd/cozyrds/etcd.yaml
@@ -8,7 +8,7 @@ spec:
plural: etcds
singular: etcd
openAPISchema: |-
- {"title":"Chart Values","type":"object","properties":{"version":{"description":"etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image.","type":"string","default":"3.6.11"},"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"affinity":{"description":"Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}
+ {"title":"Chart Values","type":"object","properties":{"version":{"description":"etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image.","type":"string","default":"3.6.11"},"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"affinity":{"description":"Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}
release:
prefix: ""
labels:
From 00c11c7a745db9b57d419631c3c88b7ac8c274cb Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:50 +0300
Subject: [PATCH 005/114] test(monitoring): check the listed alert still
reaches email after the blackhole
The case lists InfoInhibitor in email.alertnames but only asserted the
route order up to the blackhole, so it would still pass if the email
route for a listed name stopped being rendered.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../monitoring/tests/alertmanager_blackhole_test.yaml | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
index e987bd405..d4e366180 100644
--- a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
+++ b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
@@ -43,3 +43,9 @@ tests:
matchRegex:
path: stringData["alertmanager.yaml"]
pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'
+ - documentSelector:
+ path: metadata.labels.app
+ value: alertmanager
+ matchRegex:
+ path: stringData["alertmanager.yaml"]
+ pattern: '(?s)\n - receiver: blackhole\n matchers:\n - .severity="none".\n.*\n - receiver: email\n group_wait: 1m\n group_interval: 5m\n repeat_interval: .12h.\n matchers:\n - .alertname=~"\^\(InfoInhibitor\)\$".\n'
From 7bda2bdbc07dd941724e5ea12e51e8c808313b1b Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:30:14 +0300
Subject: [PATCH 006/114] fix(monitoring): select the vmalert Service the
operator creates
The VMServiceScrape selected app.kubernetes.io/instance=vmalert, while the
chart names the VMAlert vmalert- and the
operator puts that name into the instance label of its Service. The scrape
matched nothing, so the metrics of vmalert itself were never collected.
The instance stays in the selector so the scrape keeps pointing at the one
VMAlert this release renders.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../templates/vm/vmalert-scrape.yaml | 4 +-
.../monitoring/tests/vmalert_scrape_test.yaml | 38 +++++++++++++++++++
2 files changed, 41 insertions(+), 1 deletion(-)
create mode 100644 packages/system/monitoring/tests/vmalert_scrape_test.yaml
diff --git a/packages/system/monitoring/templates/vm/vmalert-scrape.yaml b/packages/system/monitoring/templates/vm/vmalert-scrape.yaml
index ee406a275..b467d5564 100644
--- a/packages/system/monitoring/templates/vm/vmalert-scrape.yaml
+++ b/packages/system/monitoring/templates/vm/vmalert-scrape.yaml
@@ -1,3 +1,4 @@
+{{- with first .Values.metricsStorages }}
---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
@@ -13,5 +14,6 @@ spec:
selector:
matchLabels:
app.kubernetes.io/component: monitoring
- app.kubernetes.io/instance: vmalert
+ app.kubernetes.io/instance: vmalert-{{ .name }}
app.kubernetes.io/name: vmalert
+{{- end }}
diff --git a/packages/system/monitoring/tests/vmalert_scrape_test.yaml b/packages/system/monitoring/tests/vmalert_scrape_test.yaml
new file mode 100644
index 000000000..1dfb9ec3e
--- /dev/null
+++ b/packages/system/monitoring/tests/vmalert_scrape_test.yaml
@@ -0,0 +1,38 @@
+suite: vmalert metrics are scraped
+templates:
+ - templates/vm/vmalert-scrape.yaml
+
+release:
+ name: monitoring
+ namespace: tenant-root
+
+set:
+ _cluster:
+ root-host: example.org
+ _namespace:
+ host: tenant-root.example.org
+ ingress: tenant-root
+ metricsStorages:
+ - name: shortterm
+ retentionPeriod: "3d"
+ deduplicationInterval: "15s"
+ - name: longterm
+ retentionPeriod: "14d"
+ deduplicationInterval: "5m"
+
+tests:
+ - it: selects the Service of the VMAlert the chart renders
+ asserts:
+ - equal:
+ path: spec.selector.matchLabels
+ value:
+ app.kubernetes.io/component: monitoring
+ app.kubernetes.io/instance: vmalert-shortterm
+ app.kubernetes.io/name: vmalert
+
+ - it: renders nothing without metrics storages, like the VMAlert itself
+ set:
+ metricsStorages: []
+ asserts:
+ - hasDocuments:
+ count: 0
From 34766ff0b50af0801f2c94c1435dd744211bbc23 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 17:06:52 +0300
Subject: [PATCH 007/114] fix(vertical-pod-autoscaler): run two admission
controllers spread over nodes
With one admission controller and failurePolicy Ignore, a pod created
while that controller is being rescheduled is admitted without its VPA
recommendation, and with updateMode Initial it keeps the defaults for
the rest of its life. On a v1.6.5 cluster a vmstorage pod created during
a node drain got the operator defaults, a 1500Mi limit against a 3.4Gi
recommendation, and was OOMKilled weeks later.
A second replica on another node and a PodDisruptionBudget keep one
controller serving through a drain. The controller still deletes and
recreates its webhook configuration on start, so a restart leaves only
that short gap instead of the whole reschedule.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../tests/vpa_test.yaml | 29 +++++++++++++++++++
.../vertical-pod-autoscaler/values.yaml | 12 ++++++++
2 files changed, 41 insertions(+)
diff --git a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
index 6b8c82675..1d2011330 100644
--- a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
+++ b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
@@ -242,3 +242,32 @@ tests:
count: 0
+
+ - it: runs two admission controllers spread over nodes
+ template: charts/vertical-pod-autoscaler/templates/admission-controller/deployment.yaml
+ asserts:
+ - equal:
+ path: spec.replicas
+ value: 2
+ - equal:
+ path: spec.template.spec.topologySpreadConstraints[0].topologyKey
+ value: kubernetes.io/hostname
+ - equal:
+ path: spec.template.spec.topologySpreadConstraints[0].labelSelector.matchLabels
+ value:
+ app.kubernetes.io/name: vertical-pod-autoscaler
+ app.kubernetes.io/instance: vpa
+ app.kubernetes.io/component: admission-controller
+
+ - it: keeps one admission controller up through a voluntary disruption
+ template: charts/vertical-pod-autoscaler/templates/admission-controller/pdb.yaml
+ asserts:
+ - equal:
+ path: spec.minAvailable
+ value: 1
+ - equal:
+ path: spec.selector.matchLabels
+ value:
+ app.kubernetes.io/name: vertical-pod-autoscaler
+ app.kubernetes.io/instance: vpa
+ app.kubernetes.io/component: admission-controller
diff --git a/packages/system/vertical-pod-autoscaler/values.yaml b/packages/system/vertical-pod-autoscaler/values.yaml
index 46678903e..456b99912 100644
--- a/packages/system/vertical-pod-autoscaler/values.yaml
+++ b/packages/system/vertical-pod-autoscaler/values.yaml
@@ -37,6 +37,18 @@ vertical-pod-autoscaler:
prometheus-cadvisor-job-name: cadvisor
admissionController:
+ replicaCount: 2
+ pdb:
+ create: true
+ topologySpreadConstraints:
+ - maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+ labelSelector:
+ matchLabels:
+ app.kubernetes.io/name: vertical-pod-autoscaler
+ app.kubernetes.io/instance: "{{ .Release.Name }}"
+ app.kubernetes.io/component: admission-controller
resources:
limits:
memory: 256Mi
From 61289c721011af0ca4a61eb278ae2664ae7cd8e7 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 17:09:16 +0300
Subject: [PATCH 008/114] fix(etcd): size the defrag job for the client it runs
The hourly defrag pod runs the etcd-defrag client, which only sends
defragment calls to the members, yet it requested 200m CPU and 256Mi
from the tenant quota. On a sandbox tenant whose quota was mostly taken
by its own etcd and SeaweedFS, the job could not create a pod and the
defrag never ran. The client peaked at 5.1 MiB working set over a day
on a v1.6.5 cluster.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
packages/extra/etcd/templates/etcd-defrag.yaml | 8 ++++----
packages/extra/etcd/tests/defrag_test.yaml | 13 +++++++++++++
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/packages/extra/etcd/templates/etcd-defrag.yaml b/packages/extra/etcd/templates/etcd-defrag.yaml
index 52cc63518..c1eefa0db 100644
--- a/packages/extra/etcd/templates/etcd-defrag.yaml
+++ b/packages/extra/etcd/templates/etcd-defrag.yaml
@@ -29,11 +29,11 @@ spec:
drop: ["ALL"]
resources:
requests:
- cpu: 200m
- memory: 256Mi
+ cpu: 10m
+ memory: 32Mi
limits:
- cpu: 500m
- memory: 512Mi
+ cpu: 100m
+ memory: 64Mi
args:
# The v1alpha2 operator names the headless Service after the
# cluster ({{ $.Release.Name }}) and does not use StatefulSet
diff --git a/packages/extra/etcd/tests/defrag_test.yaml b/packages/extra/etcd/tests/defrag_test.yaml
index 8d74e1a6b..b3acc3401 100644
--- a/packages/extra/etcd/tests/defrag_test.yaml
+++ b/packages/extra/etcd/tests/defrag_test.yaml
@@ -41,3 +41,16 @@ tests:
documentSelector:
path: kind
value: EtcdCluster
+
+ - it: asks the tenant quota only for what the defrag client uses
+ template: templates/etcd-defrag.yaml
+ asserts:
+ - equal:
+ path: spec.jobTemplate.spec.template.spec.containers[0].resources
+ value:
+ requests:
+ cpu: 10m
+ memory: 32Mi
+ limits:
+ cpu: 100m
+ memory: 64Mi
From a7ec05f9eb759e0c1025cf4bf95631df338a85c0 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 17:15:04 +0300
Subject: [PATCH 009/114] fix(monitoring-agents): make a silent Watchdog an
alarm in Alerta
Alerta turns the Watchdog into a heartbeat and stores its severity in the
heartbeat attributes. The alerta heartbeats --alert job of the alerta-web
image takes the severity of a stale heartbeat from those attributes ahead
of its HEARTBEAT_SEVERITY default, so with severity ok the HeartbeatFail
it raises is a normal alert and nothing says the alerting pipeline
stopped. Without a timeout label the heartbeat also took a day to go
stale, while Alertmanager refreshes it every minute.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../alerts/general.rules.yaml | 3 +-
.../tests/watchdog_heartbeat_test.yaml | 29 +++++++++++++++++++
2 files changed, 31 insertions(+), 1 deletion(-)
create mode 100644 packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml
diff --git a/packages/system/monitoring-agents/alerts/general.rules.yaml b/packages/system/monitoring-agents/alerts/general.rules.yaml
index 924870c40..0ac83fe72 100644
--- a/packages/system/monitoring-agents/alerts/general.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/general.rules.yaml
@@ -33,7 +33,8 @@ spec:
is working properly.
expr: vector(1)
labels:
- severity: ok
+ severity: major
+ timeout: "300"
exported_instance: global
service: general.rules
event: Heartbeat
diff --git a/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml b/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml
new file mode 100644
index 000000000..fe630a80c
--- /dev/null
+++ b/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml
@@ -0,0 +1,29 @@
+suite: Watchdog heartbeat raises an alarm in Alerta when it stops
+templates:
+ - templates/alerts.yaml
+
+release:
+ name: monitoring-agents
+ namespace: cozy-monitoring
+
+tests:
+ - it: gives the heartbeat a severity Alerta reports as an alarm
+ documentSelector:
+ path: metadata.name
+ value: alerts-general.rules
+ asserts:
+ - equal:
+ path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.severity
+ value: major
+
+ - it: lets the heartbeat go stale within minutes, not a day
+ documentSelector:
+ path: metadata.name
+ value: alerts-general.rules
+ asserts:
+ - equal:
+ path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.timeout
+ value: "300"
+ - equal:
+ path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.event
+ value: Heartbeat
From 70e1e8b1454db65222a0b3c0952650fa69239fdc Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 20:33:46 +0300
Subject: [PATCH 010/114] test(vertical-pod-autoscaler): check maxSkew and
whenUnsatisfiable of the spread
The test only looked at the topology key and the label selector, so a
change to maxSkew or to whenUnsatisfiable went unnoticed.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
index 1d2011330..0ff338b48 100644
--- a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
+++ b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
@@ -252,6 +252,12 @@ tests:
- equal:
path: spec.template.spec.topologySpreadConstraints[0].topologyKey
value: kubernetes.io/hostname
+ - equal:
+ path: spec.template.spec.topologySpreadConstraints[0].maxSkew
+ value: 1
+ - equal:
+ path: spec.template.spec.topologySpreadConstraints[0].whenUnsatisfiable
+ value: ScheduleAnyway
- equal:
path: spec.template.spec.topologySpreadConstraints[0].labelSelector.matchLabels
value:
From e96ceedd7ebd6a8e138e8b363ef36a2ffbf7e20f Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:25:58 +0700
Subject: [PATCH 011/114] feat(dashboard): add a console client for the
observability proxy
The observability proxy serves metrics and logs under Kubernetes-shaped
paths but answers in native Prometheus JSON and VictoriaLogs NDJSON, so
the typed K8sClient methods cannot read it. A raw fetch on the client
keeps the same credentials and 401 handling the rest of the console
uses, and the proxy module maps each documented status to its own
error kind so a cluster without the proxy (404) can be told apart from
a denied request or a store that is down.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../__tests__/k8s-client/client.raw.test.ts | 47 ++++
.../src/lib/observability/proxy.test.ts | 179 +++++++++++++++
.../console/src/lib/observability/proxy.ts | 203 ++++++++++++++++++
.../console/packages/k8s-client/src/client.ts | 18 ++
4 files changed, 447 insertions(+)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
diff --git a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
new file mode 100644
index 000000000..ab6c056f5
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
@@ -0,0 +1,47 @@
+import { describe, it, expect, vi, afterEach } from "vitest"
+import { K8sClient } from "@cozystack/k8s-client"
+
+afterEach(() => {
+ vi.unstubAllGlobals()
+ vi.restoreAllMocks()
+})
+
+describe("K8sClient.raw", () => {
+ it("returns the response untouched, including a non-2xx one", async () => {
+ const res = new Response("not found", { status: 404 })
+ vi.stubGlobal("fetch", vi.fn(async () => res))
+ const client = new K8sClient()
+
+ await expect(client.raw("/x")).resolves.toBe(res)
+ })
+
+ it("prefixes the base URL and attaches the bearer token", async () => {
+ const fetchMock = vi.fn(async () => new Response("{}"))
+ vi.stubGlobal("fetch", fetchMock)
+ const client = new K8sClient({ baseUrl: "/base", getToken: async () => "t0k" })
+
+ await client.raw("/apis/x", { headers: { Accept: "application/json" } })
+
+ const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
+ expect(url).toBe("/base/apis/x")
+ expect(init.headers).toEqual({ Accept: "application/json", Authorization: "Bearer t0k" })
+ })
+
+ it("routes a 401 through the client's unauthorized handler", async () => {
+ const onUnauthorized = vi.fn()
+ vi.stubGlobal("fetch", vi.fn(async () => new Response("", { status: 401 })))
+ const client = new K8sClient({ onUnauthorized })
+
+ const res = await client.raw("/x")
+
+ expect(res.status).toBe(401)
+ expect(onUnauthorized).toHaveBeenCalledTimes(1)
+ })
+
+ it("does not swallow a network failure", async () => {
+ vi.stubGlobal("fetch", vi.fn(async () => { throw new TypeError("offline") }))
+ const client = new K8sClient()
+
+ await expect(client.raw("/x")).rejects.toThrow("offline")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
new file mode 100644
index 000000000..f06835ff7
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
@@ -0,0 +1,179 @@
+import { describe, it, expect, vi } from "vitest"
+import { K8sClient } from "@cozystack/k8s-client"
+import {
+ ProxyError,
+ errorKindForStatus,
+ fieldValues,
+ logsRoot,
+ metricsRoot,
+ queryLogs,
+ queryRange,
+ tailLogs,
+} from "./proxy.ts"
+
+const app = { namespace: "tenant-a", plural: "postgreses", name: "db" }
+const ns = { namespace: "tenant-a" }
+
+function clientReturning(...responses: Response[]) {
+ const client = new K8sClient({ onUnauthorized: () => {} })
+ const raw = vi.spyOn(client, "raw")
+ for (const r of responses) raw.mockResolvedValueOnce(r)
+ return { client, raw }
+}
+
+function calledUrl(raw: ReturnType["raw"], call = 0): URL {
+ return new URL(raw.mock.calls[call][0], "http://x")
+}
+
+describe("roots", () => {
+ it("builds the application roots under apps.cozystack.io", () => {
+ expect(metricsRoot(app)).toBe("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/")
+ expect(logsRoot(app)).toBe("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/")
+ })
+
+ it("builds the namespace roots under cozystack.io", () => {
+ expect(metricsRoot(ns)).toBe("/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/")
+ expect(logsRoot(ns)).toBe("/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/")
+ })
+})
+
+describe("errorKindForStatus", () => {
+ it.each([
+ [400, "bad-request"],
+ [401, "unauthorized"],
+ [403, "forbidden"],
+ [404, "not-available"],
+ [422, "limit"],
+ [429, "rate-limited"],
+ [502, "store-unreachable"],
+ [503, "unavailable"],
+ [504, "timeout"],
+ [500, "unknown"],
+ ])("maps %i to %s", (status, kind) => {
+ expect(errorKindForStatus(status)).toBe(kind)
+ })
+})
+
+describe("queryRange", () => {
+ it("GETs query_range under the root and returns the matrix", async () => {
+ const body = { status: "success", data: { resultType: "matrix", result: [{ metric: { pod: "p" }, values: [[1, "2"]] }] } }
+ const { client, raw } = clientReturning(Response.json(body))
+
+ const result = await queryRange(client, metricsRoot(app), { query: "up", start: 10, end: 20, step: 5 })
+
+ const url = calledUrl(raw)
+ expect(url.pathname).toBe(`${metricsRoot(app)}api/v1/query_range`)
+ expect(Object.fromEntries(url.searchParams)).toEqual({ query: "up", start: "10", end: "20", step: "5" })
+ expect(result).toEqual(body.data.result)
+ })
+
+ it("throws a ProxyError carrying the Prometheus error text", async () => {
+ const { client } = clientReturning(
+ Response.json({ status: "error", error: "range too long" }, { status: 422 }),
+ )
+
+ const err = await queryRange(client, metricsRoot(app), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
+
+ expect(err).toBeInstanceOf(ProxyError)
+ expect(err).toMatchObject({ kind: "limit", status: 422, message: "range too long" })
+ })
+
+ it("reports a fetch failure as a network error", async () => {
+ const client = new K8sClient()
+ vi.spyOn(client, "raw").mockRejectedValue(new TypeError("offline"))
+
+ const err = await queryRange(client, metricsRoot(app), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
+
+ expect(err).toMatchObject({ kind: "network" })
+ })
+})
+
+describe("queryLogs", () => {
+ it("parses NDJSON lines and sorts them by _time ascending", async () => {
+ const ndjson = [
+ JSON.stringify({ _time: "2026-11-04T10:00:02Z", _msg: "b" }),
+ "",
+ JSON.stringify({ _time: "2026-11-04T10:00:01Z", _msg: "a" }),
+ ].join("\n")
+ const { client, raw } = clientReturning(new Response(ndjson))
+
+ const entries = await queryLogs(client, logsRoot(ns), { query: "*", start: "s", end: "e", limit: 100 })
+
+ expect(calledUrl(raw).pathname).toBe(`${logsRoot(ns)}select/logsql/query`)
+ expect(Object.fromEntries(calledUrl(raw).searchParams)).toEqual({ query: "*", start: "s", end: "e", limit: "100" })
+ expect(entries.map((e) => e._msg)).toEqual(["a", "b"])
+ })
+
+ it("keeps a plain-text error body as the message", async () => {
+ const { client } = clientReturning(new Response("cannot parse query", { status: 400 }))
+
+ const err = await queryLogs(client, logsRoot(ns), { query: "(", start: "s", end: "e", limit: 1 }).catch((e) => e)
+
+ expect(err).toMatchObject({ kind: "bad-request", message: "cannot parse query" })
+ })
+})
+
+describe("fieldValues", () => {
+ it("uses stream_field_values for stream fields and returns the values", async () => {
+ const { client, raw } = clientReturning(
+ Response.json({ values: [{ value: "pod-b", hits: 1 }, { value: "pod-a", hits: 9 }] }),
+ )
+
+ const values = await fieldValues(client, logsRoot(app), {
+ field: "kubernetes_pod_name",
+ stream: true,
+ query: "*",
+ start: "s",
+ end: "e",
+ })
+
+ expect(calledUrl(raw).pathname).toBe(`${logsRoot(app)}select/logsql/stream_field_values`)
+ expect(calledUrl(raw).searchParams.get("field")).toBe("kubernetes_pod_name")
+ expect(values).toEqual(["pod-a", "pod-b"])
+ })
+
+ it("uses field_values for other fields", async () => {
+ const { client, raw } = clientReturning(Response.json({ values: [] }))
+
+ await fieldValues(client, logsRoot(app), { field: "level", stream: false, query: "*", start: "s", end: "e" })
+
+ expect(calledUrl(raw).pathname).toBe(`${logsRoot(app)}select/logsql/field_values`)
+ })
+})
+
+describe("tailLogs", () => {
+ it("delivers every line as it arrives, including one split across chunks", async () => {
+ const enc = new TextEncoder()
+ const chunks = ['{"_time":"t1","_msg":"a"}\n{"_ti', 'me":"t2","_msg":"b"}\n']
+ const body = new ReadableStream({
+ start(c) {
+ for (const ch of chunks) c.enqueue(enc.encode(ch))
+ c.close()
+ },
+ })
+ const { client, raw } = clientReturning(new Response(body))
+ const seen: string[] = []
+
+ await tailLogs(client, logsRoot(ns), { query: "*", since: "t0" }, (e) => seen.push(e._msg))
+
+ expect(calledUrl(raw).pathname).toBe(`${logsRoot(ns)}select/logsql/tail`)
+ expect(calledUrl(raw).searchParams.get("since")).toBe("t0")
+ expect(seen).toEqual(["a", "b"])
+ })
+
+ it("omits since on the first connect", async () => {
+ const { client, raw } = clientReturning(new Response(""))
+
+ await tailLogs(client, logsRoot(ns), { query: "*" }, () => {})
+
+ expect(calledUrl(raw).searchParams.has("since")).toBe(false)
+ })
+
+ it("throws a ProxyError when the tail is refused", async () => {
+ const { client } = clientReturning(new Response("", { status: 403 }))
+
+ const err = await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}).catch((e) => e)
+
+ expect(err).toMatchObject({ kind: "forbidden" })
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
new file mode 100644
index 000000000..a257825c9
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -0,0 +1,203 @@
+import type { K8sClient } from "@cozystack/k8s-client"
+
+export type ObservabilityScope =
+ | { namespace: string }
+ | { namespace: string; plural: string; name: string }
+
+function appBase(scope: ObservabilityScope): string | null {
+ if (!("plural" in scope)) return null
+ return `/apis/apps.cozystack.io/v1alpha1/namespaces/${scope.namespace}/${scope.plural}/${scope.name}`
+}
+
+export function metricsRoot(scope: ObservabilityScope): string {
+ const app = appBase(scope)
+ return app ? `${app}/metrics/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/metrics/`
+}
+
+export function logsRoot(scope: ObservabilityScope): string {
+ const app = appBase(scope)
+ return app ? `${app}/log/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/logs/`
+}
+
+export type ProxyErrorKind =
+ | "bad-request"
+ | "unauthorized"
+ | "forbidden"
+ | "not-available"
+ | "limit"
+ | "rate-limited"
+ | "store-unreachable"
+ | "unavailable"
+ | "timeout"
+ | "network"
+ | "unknown"
+
+const KIND_BY_STATUS: Record = {
+ 400: "bad-request",
+ 401: "unauthorized",
+ 403: "forbidden",
+ 404: "not-available",
+ 422: "limit",
+ 429: "rate-limited",
+ 502: "store-unreachable",
+ 503: "unavailable",
+ 504: "timeout",
+}
+
+export function errorKindForStatus(status: number): ProxyErrorKind {
+ return KIND_BY_STATUS[status] ?? "unknown"
+}
+
+export class ProxyError extends Error {
+ kind: ProxyErrorKind
+ status: number
+
+ constructor(kind: ProxyErrorKind, status: number, message: string) {
+ super(message)
+ this.name = "ProxyError"
+ this.kind = kind
+ this.status = status
+ }
+}
+
+// Prometheus answers {"error": "..."}, VictoriaLogs and nginx answer plain text.
+function errorMessage(text: string, status: number): string {
+ try {
+ const parsed: unknown = JSON.parse(text)
+ if (parsed && typeof parsed === "object" && "error" in parsed && typeof parsed.error === "string") {
+ return parsed.error
+ }
+ } catch {
+ // not JSON
+ }
+ const trimmed = text.trim()
+ return trimmed && trimmed.length < 300 && !trimmed.startsWith("<") ? trimmed : `Server returned ${status}`
+}
+
+async function proxyGet(
+ client: K8sClient,
+ path: string,
+ params: Record,
+ signal?: AbortSignal,
+): Promise {
+ const search = new URLSearchParams()
+ for (const [k, v] of Object.entries(params)) if (v !== undefined) search.set(k, String(v))
+ let res: Response
+ try {
+ res = await client.raw(`${path}?${search}`, { signal })
+ } catch (err) {
+ if (signal?.aborted) throw err
+ throw new ProxyError("network", 0, "Unable to connect to the server")
+ }
+ if (!res.ok) {
+ const text = await res.text().catch(() => "")
+ throw new ProxyError(errorKindForStatus(res.status), res.status, errorMessage(text, res.status))
+ }
+ return res
+}
+
+export interface PromSeries {
+ metric: Record
+ values: [number, string][]
+}
+
+export interface RangeParams {
+ query: string
+ start: number
+ end: number
+ step: number
+}
+
+export async function queryRange(
+ client: K8sClient,
+ root: string,
+ params: RangeParams,
+ signal?: AbortSignal,
+): Promise {
+ const res = await proxyGet(client, `${root}api/v1/query_range`, { ...params }, signal)
+ const body = (await res.json()) as { data?: { result?: PromSeries[] } }
+ return body.data?.result ?? []
+}
+
+export interface LogEntry {
+ _time: string
+ _msg: string
+ _stream_id?: string
+ [field: string]: string | undefined
+}
+
+function parseLine(line: string): LogEntry | null {
+ if (!line.trim()) return null
+ try {
+ const value: unknown = JSON.parse(line)
+ if (value && typeof value === "object" && "_time" in value) return value as LogEntry
+ } catch {
+ // a torn or non-JSON line carries nothing to show
+ }
+ return null
+}
+
+export interface LogsQueryParams {
+ query: string
+ start: string
+ end: string
+ limit: number
+}
+
+export async function queryLogs(
+ client: K8sClient,
+ root: string,
+ params: LogsQueryParams,
+ signal?: AbortSignal,
+): Promise {
+ const res = await proxyGet(client, `${root}select/logsql/query`, { ...params }, signal)
+ const entries = (await res.text()).split("\n").map(parseLine).filter((e) => e !== null)
+ return entries.sort((a, b) => (a._time < b._time ? -1 : a._time > b._time ? 1 : 0))
+}
+
+export interface FieldValuesParams {
+ field: string
+ stream: boolean
+ query: string
+ start: string
+ end: string
+}
+
+export async function fieldValues(
+ client: K8sClient,
+ root: string,
+ { stream, ...params }: FieldValuesParams,
+ signal?: AbortSignal,
+): Promise {
+ const endpoint = stream ? "stream_field_values" : "field_values"
+ const res = await proxyGet(client, `${root}select/logsql/${endpoint}`, { ...params }, signal)
+ const body = (await res.json()) as { values?: { value: string }[] }
+ return (body.values ?? []).map((v) => v.value).sort()
+}
+
+export async function tailLogs(
+ client: K8sClient,
+ root: string,
+ params: { query: string; since?: string },
+ onEntry: (entry: LogEntry) => void,
+ signal?: AbortSignal,
+): Promise {
+ const res = await proxyGet(client, `${root}select/logsql/tail`, params, signal)
+ if (!res.body) return
+ const reader = res.body.getReader()
+ const decoder = new TextDecoder()
+ let buffer = ""
+ for (;;) {
+ const { done, value } = await reader.read()
+ if (done) break
+ buffer += decoder.decode(value, { stream: true })
+ const lines = buffer.split("\n")
+ buffer = lines.pop() ?? ""
+ for (const line of lines) {
+ const entry = parseLine(line)
+ if (entry) onEntry(entry)
+ }
+ }
+ const last = parseLine(buffer)
+ if (last) onEntry(last)
+}
diff --git a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
index 12f50e0fd..04c528760 100644
--- a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
+++ b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
@@ -241,6 +241,24 @@ export class K8sClient {
})
}
+ /**
+ * Fetch a same-origin path with the client's credentials and 401 handling,
+ * leaving status and body to the caller. For endpoints that do not speak
+ * Kubernetes JSON, such as the observability proxy's streamed NDJSON.
+ */
+ async raw(path: string, init?: RequestInit): Promise {
+ const headers: Record = {
+ ...(init?.headers as Record),
+ }
+ if (this.getToken) {
+ headers["Authorization"] = `Bearer ${await this.getToken()}`
+ }
+ const res = await fetch(`${this.baseUrl}${path}`, { ...init, headers })
+ if (res.status === 401) this.handleUnauthorized()
+ else if (res.ok) this.unauthorizedHandled = false
+ return res
+ }
+
getApiGroups(): Promise {
return this.request("/apis")
}
From 063e56ede57a481b2db223a37641e601a2d51a05 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:30:03 +0700
Subject: [PATCH 012/114] feat(dashboard): show resource consumption on
application pages
Tenants had no way to see what their applications consume. A Metrics
tab on every application detail page (except VM disks, which run no
containers) charts CPU, memory and network for a selectable range
through the observability proxy's application root, which injects the
scope server-side, so the queries stay kind-agnostic and no per-kind
UI is needed.
Charts use uPlot: about 25 KB gzipped with no dependencies, built for
time series, and loaded only with the Metrics tab so other pages do
not pay for it. A cluster without the proxy answers 404 and the tab
says metrics are not available there instead of showing an error.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../images/console/apps/console/package.json | 3 +-
.../observability/MetricsView.test.tsx | 88 +++++++++++++
.../components/observability/MetricsView.tsx | 72 +++++++++++
.../observability/ProxyNotice.test.tsx | 43 +++++++
.../components/observability/ProxyNotice.tsx | 49 +++++++
.../components/observability/RangePicker.tsx | 30 +++++
.../observability/TimeSeriesChart.test.tsx | 46 +++++++
.../observability/TimeSeriesChart.tsx | 51 ++++++++
.../src/lib/observability/metrics.test.ts | 107 ++++++++++++++++
.../console/src/lib/observability/metrics.ts | 121 ++++++++++++++++++
.../src/lib/observability/proxy.test.ts | 10 ++
.../console/src/lib/observability/proxy.ts | 8 ++
.../detail/ApplicationDetailPage.test.tsx | 28 ++++
.../routes/detail/ApplicationDetailPage.tsx | 12 ++
.../console/src/routes/detail/MetricsTab.tsx | 15 +++
.../dashboard/images/console/pnpm-lock.yaml | 8 ++
16 files changed, 690 insertions(+), 1 deletion(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/package.json b/packages/system/dashboard/images/console/apps/console/package.json
index fcddb7023..254195f28 100644
--- a/packages/system/dashboard/images/console/apps/console/package.json
+++ b/packages/system/dashboard/images/console/apps/console/package.json
@@ -31,7 +31,8 @@
"react": "^19.2.4",
"react-dom": "^19.2.4",
"react-router": "^7.6.1",
- "simple-icons": "^16.22.0"
+ "simple-icons": "^16.22.0",
+ "uplot": "^1.6.32"
},
"devDependencies": {
"@tailwindcss/vite": "^4.2.2",
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
new file mode 100644
index 000000000..f09e15201
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
@@ -0,0 +1,88 @@
+import { describe, it, expect, vi } from "vitest"
+import { screen, waitFor } from "@testing-library/react"
+import userEvent from "@testing-library/user-event"
+import { K8sClient } from "@cozystack/k8s-client"
+import { renderWithK8sProvider } from "../../test-utils/render.tsx"
+import type { ChartData } from "../../lib/observability/metrics.ts"
+
+vi.mock("./TimeSeriesChart.tsx", () => ({
+ TimeSeriesChart: ({ data }: { data: ChartData }) => (
+ {data.series.map((s) => s.label).join(",")}
+ ),
+}))
+
+const { MetricsView } = await import("./MetricsView.tsx")
+
+const scope = { namespace: "tenant-a", plural: "postgreses", name: "db" }
+
+function matrix(pod: string) {
+ return Response.json({
+ status: "success",
+ data: { resultType: "matrix", result: [{ metric: { pod }, values: [[1, "1"]] }] },
+ })
+}
+
+function clientWith(handler: (url: URL) => Response) {
+ const client = new K8sClient()
+ const raw = vi.spyOn(client, "raw").mockImplementation(async (path) => handler(new URL(path, "http://x")))
+ return { client, raw }
+}
+
+describe("MetricsView", () => {
+ it("draws CPU, memory and network from the application root", async () => {
+ const { client, raw } = clientWith(() => matrix("db-0"))
+
+ renderWithK8sProvider( , { client })
+
+ await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3))
+ expect(screen.getByText("CPU (cores)")).toBeInTheDocument()
+ expect(screen.getByText("Memory (working set)")).toBeInTheDocument()
+ expect(screen.getByText("Network")).toBeInTheDocument()
+ for (const [path] of raw.mock.calls) {
+ expect(path.startsWith("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query_range?")).toBe(true)
+ }
+ })
+
+ it("requeries with a coarser step when a longer range is picked", async () => {
+ const { client, raw } = clientWith(() => matrix("db-0"))
+ renderWithK8sProvider( , { client })
+ await waitFor(() => expect(raw).toHaveBeenCalled())
+
+ await userEvent.click(screen.getByRole("button", { name: "7d" }))
+
+ await waitFor(() =>
+ expect(raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("step") === "2520")).toBe(true),
+ )
+ })
+
+ it("shows one quiet notice instead of three errors when the proxy is absent", async () => {
+ const { client } = clientWith(() => new Response("404 page not found", { status: 404 }))
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("Metrics are not available on this cluster.")).toBeInTheDocument()
+ expect(screen.getAllByRole("status")).toHaveLength(1)
+ expect(screen.queryByText("CPU (cores)")).toBeNull()
+ })
+
+ it("keeps the other panels when one query hits a limit", async () => {
+ const { client } = clientWith((url) =>
+ url.searchParams.get("query")?.includes("memory")
+ ? Response.json({ status: "error", error: "too many series" }, { status: 422 })
+ : matrix("db-0"),
+ )
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("too many series. Choose a shorter range.")).toBeInTheDocument()
+ await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(2))
+ })
+
+ it("says so when a panel has no data", async () => {
+ const { client } = clientWith(() => Response.json({ status: "success", data: { resultType: "matrix", result: [] } }))
+
+ renderWithK8sProvider( , { client })
+
+ await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(3))
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
new file mode 100644
index 000000000..35c8e8417
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
@@ -0,0 +1,72 @@
+import { useState } from "react"
+import { useQueries } from "@tanstack/react-query"
+import { useK8sClient } from "@cozystack/k8s-client"
+import { Section, Spinner } from "@cozystack/ui"
+import {
+ METRIC_PANELS,
+ RANGES,
+ rangeWindow,
+ toChartData,
+ type RangeKey,
+} from "../../lib/observability/metrics.ts"
+import { isScopeWide, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts"
+import { ProxyNotice } from "./ProxyNotice.tsx"
+import { TimeSeriesChart } from "./TimeSeriesChart.tsx"
+import { RangePicker } from "./RangePicker.tsx"
+
+const REFRESH_MS = 60_000
+
+export function MetricsView({ scope }: { scope: ObservabilityScope }) {
+ const client = useK8sClient()
+ const root = metricsRoot(scope)
+ const [range, setRange] = useState("1h")
+
+ const results = useQueries({
+ queries: METRIC_PANELS.map((panel) => ({
+ queryKey: ["observability", "metrics", root, panel.id, range],
+ queryFn: async ({ signal }: { signal: AbortSignal }) => {
+ const window = rangeWindow(range, Math.floor(Date.now() / 1000))
+ const series = await Promise.all(
+ panel.queries(window).map(async (q) => ({
+ label: q.label,
+ series: await queryRange(client, root, { query: q.expr, ...window }, signal),
+ })),
+ )
+ return toChartData(series)
+ },
+ retry: false,
+ refetchInterval: REFRESH_MS,
+ })),
+ })
+
+ const wide = results.find((r) => isScopeWide(r.error))?.error
+
+ return (
+
+
+
Resource consumption
+
+
+ {wide ? (
+
+ ) : (
+ METRIC_PANELS.map((panel, i) => {
+ const { data, error, isLoading } = results[i]
+ return (
+
+ {error ? (
+
+ ) : isLoading || !data ? (
+
+ ) : data.series.length === 0 ? (
+ No data for this range.
+ ) : (
+
+ )}
+
+ )
+ })
+ )}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
new file mode 100644
index 000000000..fd001bcae
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
@@ -0,0 +1,43 @@
+import { describe, it, expect } from "vitest"
+import { render, screen } from "@testing-library/react"
+import { ProxyError } from "../../lib/observability/proxy.ts"
+import { ProxyNotice } from "./ProxyNotice.tsx"
+
+describe("ProxyNotice", () => {
+ it("renders a 404 as not available on this cluster, not as an error", () => {
+ render( )
+
+ expect(screen.getByRole("status")).toHaveTextContent("Metrics are not available on this cluster.")
+ expect(screen.queryByRole("alert")).toBeNull()
+ })
+
+ it("renders a 502 as a quiet notice about the store", () => {
+ render( )
+
+ expect(screen.getByRole("status")).toHaveTextContent("The logs store is unreachable right now.")
+ })
+
+ it.each([
+ ["forbidden", 403, "You do not have permission to read metrics here."],
+ ["limit", 422, "too many points. Choose a shorter range."],
+ ["rate-limited", 429, "Too many requests. Try again in a moment."],
+ ["timeout", 504, "The store did not answer in time. Choose a shorter range or a narrower filter."],
+ ["unavailable", 503, "The service is temporarily unavailable. Try again in a moment."],
+ ] as const)("explains %s", (kind, status, text) => {
+ render( )
+
+ expect(screen.getByRole("alert")).toHaveTextContent(text)
+ })
+
+ it("shows the server message for a rejected query", () => {
+ render( )
+
+ expect(screen.getByRole("alert")).toHaveTextContent("cannot parse query")
+ })
+
+ it("falls back to the message of an unexpected error", () => {
+ render( )
+
+ expect(screen.getByRole("alert")).toHaveTextContent("boom")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
new file mode 100644
index 000000000..a60b831b0
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
@@ -0,0 +1,49 @@
+import { ProxyError } from "../../lib/observability/proxy.ts"
+
+interface ProxyNoticeProps {
+ what: "Metrics" | "Logs"
+ error: unknown
+}
+
+function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: boolean; text: string } {
+ const lower = what.toLowerCase()
+ if (!(error instanceof ProxyError)) {
+ return { quiet: false, text: error instanceof Error ? error.message : String(error) }
+ }
+ switch (error.kind) {
+ case "not-available":
+ return { quiet: true, text: `${what} are not available on this cluster.` }
+ case "store-unreachable":
+ return { quiet: true, text: `The ${lower} store is unreachable right now.` }
+ case "unauthorized":
+ return { quiet: false, text: "Your session has expired. Sign in again." }
+ case "forbidden":
+ return { quiet: false, text: `You do not have permission to read ${lower} here.` }
+ case "limit":
+ return { quiet: false, text: `${error.message}. Choose a shorter range.` }
+ case "rate-limited":
+ return { quiet: false, text: "Too many requests. Try again in a moment." }
+ case "timeout":
+ return { quiet: false, text: "The store did not answer in time. Choose a shorter range or a narrower filter." }
+ case "unavailable":
+ return { quiet: false, text: "The service is temporarily unavailable. Try again in a moment." }
+ default:
+ return { quiet: false, text: error.message }
+ }
+}
+
+export function ProxyNotice({ what, error }: ProxyNoticeProps) {
+ const { quiet, text } = describe(what, error)
+ if (quiet) {
+ return (
+
+ {text}
+
+ )
+ }
+ return (
+
+ {text}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
new file mode 100644
index 000000000..22030c706
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
@@ -0,0 +1,30 @@
+import { cn } from "@cozystack/ui"
+
+export function RangePicker({
+ options,
+ value,
+ onChange,
+}: {
+ options: readonly K[]
+ value: K
+ onChange: (value: K) => void
+}) {
+ return (
+
+ {options.map((key) => (
+ onChange(key)}
+ className={cn(
+ "rounded px-2.5 py-1 text-xs",
+ value === key ? "bg-slate-900 text-white" : "text-slate-600 hover:bg-slate-100",
+ )}
+ >
+ {key}
+
+ ))}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
new file mode 100644
index 000000000..57cf6fb99
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
@@ -0,0 +1,46 @@
+import { describe, it, expect, vi, beforeEach } from "vitest"
+import { render } from "@testing-library/react"
+
+const created: { opts: { series: { label?: string }[] }; data: unknown[]; destroy: ReturnType }[] = []
+
+vi.mock("uplot", () => ({
+ default: class {
+ destroy = vi.fn()
+ constructor(opts: { series: { label?: string }[] }, data: unknown[]) {
+ created.push({ opts, data, destroy: this.destroy })
+ }
+ setSize() {}
+ },
+}))
+
+const { TimeSeriesChart } = await import("./TimeSeriesChart.tsx")
+
+beforeEach(() => {
+ created.length = 0
+})
+
+describe("TimeSeriesChart", () => {
+ const data = {
+ timestamps: [1, 2],
+ series: [
+ { label: "db-0", values: [1, 2] },
+ { label: "db-1", values: [null, 3] },
+ ],
+ }
+
+ it("hands uPlot the timestamps first and one labelled series per input", () => {
+ render( )
+
+ expect(created).toHaveLength(1)
+ expect(created[0].data).toEqual([[1, 2], [1, 2], [null, 3]])
+ expect(created[0].opts.series.slice(1).map((s) => s.label)).toEqual(["db-0", "db-1"])
+ })
+
+ it("destroys the chart on unmount", () => {
+ const { unmount } = render( )
+
+ unmount()
+
+ expect(created[0].destroy).toHaveBeenCalled()
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
new file mode 100644
index 000000000..6b596e17f
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
@@ -0,0 +1,51 @@
+import { useEffect, useRef } from "react"
+import uPlot from "uplot"
+import "uplot/dist/uPlot.min.css"
+import type { ChartData } from "../../lib/observability/metrics.ts"
+
+const COLORS = ["#2563eb", "#16a34a", "#d97706", "#9333ea", "#dc2626", "#0891b2", "#4b5563", "#db2777"]
+const HEIGHT = 200
+
+interface TimeSeriesChartProps {
+ data: ChartData
+ format: (value: number) => string
+}
+
+export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) {
+ const ref = useRef(null)
+
+ useEffect(() => {
+ const el = ref.current
+ if (!el) return
+ const fmt = (v: number | null) => (v === null ? "–" : format(v))
+ const chart = new uPlot(
+ {
+ width: el.clientWidth || 600,
+ height: HEIGHT,
+ scales: { x: { time: true } },
+ legend: { live: true },
+ series: [
+ {},
+ ...data.series.map((s, i) => ({
+ label: s.label,
+ stroke: COLORS[i % COLORS.length],
+ width: 1.5,
+ value: (_: uPlot, v: number | null) => fmt(v),
+ })),
+ ],
+ axes: [{}, { size: 70, values: (_: uPlot, splits: number[]) => splits.map(format) }],
+ },
+ [data.timestamps, ...data.series.map((s) => s.values)],
+ el,
+ )
+ if (typeof ResizeObserver === "undefined") return () => chart.destroy()
+ const observer = new ResizeObserver(() => chart.setSize({ width: el.clientWidth, height: HEIGHT }))
+ observer.observe(el)
+ return () => {
+ observer.disconnect()
+ chart.destroy()
+ }
+ }, [data, format])
+
+ return
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts
new file mode 100644
index 000000000..0803940bf
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts
@@ -0,0 +1,107 @@
+import { describe, it, expect } from "vitest"
+import {
+ METRIC_PANELS,
+ formatBytes,
+ formatCores,
+ rangeWindow,
+ toChartData,
+} from "./metrics.ts"
+
+describe("rangeWindow", () => {
+ it("ends now and spans the range in about 240 points", () => {
+ const w = rangeWindow("6h", 1_000_000)
+
+ expect(w.end).toBe(1_000_000)
+ expect(w.start).toBe(1_000_000 - 6 * 3600)
+ expect(w.step).toBe(90)
+ })
+
+ it("never steps finer than 15s", () => {
+ expect(rangeWindow("1h", 0).step).toBe(15)
+ })
+})
+
+describe("METRIC_PANELS", () => {
+ const window = { start: 0, end: 3600, step: 15 }
+
+ it("rates CPU over at least five minutes so sparse scrapes still draw", () => {
+ const [query] = METRIC_PANELS.find((p) => p.id === "cpu")!.queries(window)
+
+ expect(query.expr).toBe(
+ 'sum by (pod) (rate(container_cpu_usage_seconds_total{container!="",container!="POD"}[300s]))',
+ )
+ })
+
+ it("widens the rate window with the step on long ranges", () => {
+ const [query] = METRIC_PANELS.find((p) => p.id === "cpu")!.queries({ start: 0, end: 1, step: 2520 })
+
+ expect(query.expr).toContain("[5040s]")
+ })
+
+ it("reads memory as working set per pod", () => {
+ const [query] = METRIC_PANELS.find((p) => p.id === "memory")!.queries(window)
+
+ expect(query.expr).toBe('sum by (pod) (container_memory_working_set_bytes{container!="",container!="POD"})')
+ })
+
+ it("draws network receive and transmit as two named series", () => {
+ const queries = METRIC_PANELS.find((p) => p.id === "network")!.queries(window)
+
+ expect(queries.map((q) => q.label)).toEqual(["receive", "transmit"])
+ expect(queries[0].expr).toBe("sum(rate(container_network_receive_bytes_total[300s]))")
+ expect(queries[1].expr).toBe("sum(rate(container_network_transmit_bytes_total[300s]))")
+ })
+
+ it("names no application kind in any query", () => {
+ for (const panel of METRIC_PANELS) {
+ for (const q of panel.queries(window)) expect(q.expr).not.toMatch(/apps_cozystack_io|namespace=/)
+ }
+ })
+})
+
+describe("toChartData", () => {
+ it("aligns series on the union of timestamps and leaves gaps as null", () => {
+ const data = toChartData([
+ { label: "a", series: [{ metric: {}, values: [[10, "1"], [20, "2"]] }] },
+ { label: "b", series: [{ metric: {}, values: [[20, "5"], [30, "6"]] }] },
+ ])
+
+ expect(data.timestamps).toEqual([10, 20, 30])
+ expect(data.series).toEqual([
+ { label: "a", values: [1, 2, null] },
+ { label: "b", values: [null, 5, 6] },
+ ])
+ })
+
+ it("labels each series by its pod, falling back to the query label", () => {
+ const data = toChartData([
+ {
+ label: "cpu",
+ series: [
+ { metric: { pod: "db-1" }, values: [[1, "0.5"]] },
+ { metric: {}, values: [[1, "0.1"]] },
+ ],
+ },
+ ])
+
+ expect(data.series.map((s) => s.label)).toEqual(["db-1", "cpu"])
+ })
+
+ it("drops NaN samples instead of drawing them as zero", () => {
+ const data = toChartData([{ label: "x", series: [{ metric: {}, values: [[1, "NaN"]] }] }])
+
+ expect(data.series[0].values).toEqual([null])
+ })
+})
+
+describe("formatters", () => {
+ it("formats CPU in millicores below one core", () => {
+ expect(formatCores(0.25)).toBe("250m")
+ expect(formatCores(2)).toBe("2")
+ })
+
+ it("formats bytes with binary units", () => {
+ expect(formatBytes(512)).toBe("512 B")
+ expect(formatBytes(3 * 1024 * 1024)).toBe("3.0 MiB")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
new file mode 100644
index 000000000..b23a51e27
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
@@ -0,0 +1,121 @@
+import type { PromSeries } from "./proxy.ts"
+
+export const RANGES = {
+ "1h": 3600,
+ "6h": 6 * 3600,
+ "24h": 24 * 3600,
+ "7d": 7 * 24 * 3600,
+} as const
+
+export type RangeKey = keyof typeof RANGES
+
+export interface RangeWindow {
+ start: number
+ end: number
+ step: number
+}
+
+const POINTS = 240
+const MIN_STEP = 15
+
+export function rangeWindow(range: RangeKey, nowSeconds: number): RangeWindow {
+ const span = RANGES[range]
+ return { start: nowSeconds - span, end: nowSeconds, step: Math.max(MIN_STEP, Math.round(span / POINTS)) }
+}
+
+// A rate needs several scrapes inside its window, and on long ranges the window
+// must also cover the step or samples between points are skipped.
+function rateWindow(step: number): string {
+ return `${Math.max(300, step * 2)}s`
+}
+
+const CONTAINERS = 'container!="",container!="POD"'
+
+export interface MetricQuery {
+ label: string
+ expr: string
+}
+
+export interface MetricPanel {
+ id: "cpu" | "memory" | "network"
+ title: string
+ format: (value: number) => string
+ queries: (window: RangeWindow) => MetricQuery[]
+}
+
+export function formatCores(value: number): string {
+ if (value === 0) return "0"
+ if (Math.abs(value) < 1) return `${Math.round(value * 1000)}m`
+ return `${Number(value.toFixed(2))}`
+}
+
+const BYTE_UNITS = ["B", "KiB", "MiB", "GiB", "TiB"]
+
+export function formatBytes(value: number): string {
+ let v = value
+ let unit = 0
+ while (Math.abs(v) >= 1024 && unit < BYTE_UNITS.length - 1) {
+ v /= 1024
+ unit++
+ }
+ return unit === 0 ? `${Math.round(v)} B` : `${v.toFixed(1)} ${BYTE_UNITS[unit]}`
+}
+
+// The proxy injects the namespace or application scope, so these stay kind-agnostic.
+export const METRIC_PANELS: MetricPanel[] = [
+ {
+ id: "cpu",
+ title: "CPU (cores)",
+ format: formatCores,
+ queries: ({ step }) => [
+ {
+ label: "cpu",
+ expr: `sum by (pod) (rate(container_cpu_usage_seconds_total{${CONTAINERS}}[${rateWindow(step)}]))`,
+ },
+ ],
+ },
+ {
+ id: "memory",
+ title: "Memory (working set)",
+ format: formatBytes,
+ queries: () => [
+ { label: "memory", expr: `sum by (pod) (container_memory_working_set_bytes{${CONTAINERS}})` },
+ ],
+ },
+ {
+ id: "network",
+ title: "Network",
+ format: (v) => `${formatBytes(v)}/s`,
+ queries: ({ step }) => [
+ { label: "receive", expr: `sum(rate(container_network_receive_bytes_total[${rateWindow(step)}]))` },
+ { label: "transmit", expr: `sum(rate(container_network_transmit_bytes_total[${rateWindow(step)}]))` },
+ ],
+ },
+]
+
+export interface ChartSeries {
+ label: string
+ values: (number | null)[]
+}
+
+export interface ChartData {
+ timestamps: number[]
+ series: ChartSeries[]
+}
+
+export function toChartData(results: { label: string; series: PromSeries[] }[]): ChartData {
+ const flat = results.flatMap(({ label, series }) =>
+ series.map((s) => ({ label: s.metric.pod ?? label, points: new Map(s.values.map(([t, v]) => [t, Number(v)])) })),
+ )
+ const timestamps = [...new Set(flat.flatMap((s) => [...s.points.keys()]))].sort((a, b) => a - b)
+ return {
+ timestamps,
+ series: flat.map(({ label, points }) => ({
+ label,
+ values: timestamps.map((t) => {
+ const v = points.get(t)
+ return v === undefined || Number.isNaN(v) ? null : v
+ }),
+ })),
+ }
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
index f06835ff7..f8cf44656 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
@@ -4,6 +4,7 @@ import {
ProxyError,
errorKindForStatus,
fieldValues,
+ isScopeWide,
logsRoot,
metricsRoot,
queryLogs,
@@ -177,3 +178,12 @@ describe("tailLogs", () => {
expect(err).toMatchObject({ kind: "forbidden" })
})
})
+
+describe("isScopeWide", () => {
+ it("is true for answers that do not depend on the query", () => {
+ expect(isScopeWide(new ProxyError("not-available", 404, ""))).toBe(true)
+ expect(isScopeWide(new ProxyError("forbidden", 403, ""))).toBe(true)
+ expect(isScopeWide(new ProxyError("limit", 422, ""))).toBe(false)
+ expect(isScopeWide(new Error("x"))).toBe(false)
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
index a257825c9..40b451b0b 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -60,6 +60,14 @@ export class ProxyError extends Error {
}
}
+// These answers hold for every query in the scope, so one notice replaces the whole view.
+export function isScopeWide(error: unknown): boolean {
+ return (
+ error instanceof ProxyError &&
+ (error.kind === "not-available" || error.kind === "forbidden" || error.kind === "unauthorized")
+ )
+}
+
// Prometheus answers {"error": "..."}, VictoriaLogs and nginx answer plain text.
function errorMessage(text: string, status: number): string {
try {
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx
index a015a98bf..40b923f62 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx
@@ -59,6 +59,7 @@ vi.mock("./EventsTab.tsx", () => ({ EventsTab: () => null }))
vi.mock("./VncTab.tsx", () => ({ VncTab: () => null }))
vi.mock("./SerialTab.tsx", () => ({ SerialTab: () => null }))
vi.mock("./VMPowerControls.tsx", () => ({ VMPowerControls: () => null }))
+vi.mock("./MetricsTab.tsx", () => ({ MetricsTab: () => null }))
const { ApplicationDetailPage } = await import("./ApplicationDetailPage.tsx")
@@ -111,11 +112,38 @@ describe("ApplicationDetailPage tabs for a virtual machine", () => {
"VNC",
"Workloads",
"Services",
+ "Metrics",
"Events",
])
})
})
+describe("ApplicationDetailPage observability tabs", () => {
+ function showKind(kind: string, plural: string) {
+ h.appKind = kind
+ h.appPlural = plural
+ h.get = {
+ data: { kind, metadata: { name: "demo", namespace: "tenant-test" } },
+ isLoading: false,
+ error: undefined,
+ }
+ h.configMaps = []
+ h.configMapsError = undefined
+ h.tabLabels = []
+ renderPage()
+ }
+
+ it("offers Metrics on any application kind, ahead of Events", () => {
+ showKind("Postgres", "postgreses")
+ expect(h.tabLabels.slice(-3)).toEqual(["Secrets", "Metrics", "Events"])
+ })
+
+ it("leaves Metrics off a VM disk, which runs no containers", () => {
+ showKind("VMDisk", "vmdisks")
+ expect(h.tabLabels).not.toContain("Metrics")
+ })
+})
+
describe("ApplicationDetailPage configuration", () => {
it("offers ConfigMaps declared by the application's resource map", () => {
h.appKind = "FoundationDB"
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
index 55fa684b8..4520244d5 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
@@ -37,6 +37,8 @@ import { VncTab } from "./VncTab.tsx"
// fetched only when a VM's console is opened. Every dashboard page reaches this
// file, and most of them never touch a VM.
const SerialTab = lazy(() => import("./SerialTab.tsx").then((m) => ({ default: m.SerialTab })))
+// Same for the charting library behind the Metrics tab.
+const MetricsTab = lazy(() => import("./MetricsTab.tsx").then((m) => ({ default: m.MetricsTab })))
import { VMPowerControls } from "./VMPowerControls.tsx"
import { useResourceBasePath } from "../../lib/portal.ts"
import { useResourcePresence } from "./use-resource-presence.ts"
@@ -143,6 +145,7 @@ export function ApplicationDetailPage() {
{ to: `${base}/vnc`, label: "VNC", end: false },
{ to: `${base}/workloads`, label: "Workloads", end: false },
{ to: `${base}/services`, label: "Services", end: false },
+ { to: `${base}/metrics`, label: "Metrics", end: false },
{ to: `${base}/events`, label: "Events", end: false },
)
} else {
@@ -163,6 +166,7 @@ export function ApplicationDetailPage() {
}
tabs.push(
{ to: `${base}/secrets`, label: "Secrets", end: false },
+ { to: `${base}/metrics`, label: "Metrics", end: false },
{ to: `${base}/events`, label: "Events", end: false },
)
}
@@ -262,6 +266,14 @@ export function ApplicationDetailPage() {
}
/>
} />
+ }>
+
+
+ }
+ />
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx
new file mode 100644
index 000000000..a928e633d
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx
@@ -0,0 +1,15 @@
+import type { ApplicationDefinition, ApplicationInstance } from "@cozystack/types"
+import { useTenantContext } from "../../lib/tenant-context.tsx"
+import { MetricsView } from "../../components/observability/MetricsView.tsx"
+
+interface MetricsTabProps {
+ ad: ApplicationDefinition
+ instance: ApplicationInstance
+}
+
+export function MetricsTab({ ad, instance }: MetricsTabProps) {
+ const { tenantNamespace } = useTenantContext()
+ const plural = ad.spec?.application.plural
+ if (!tenantNamespace || !plural) return null
+ return
+}
diff --git a/packages/system/dashboard/images/console/pnpm-lock.yaml b/packages/system/dashboard/images/console/pnpm-lock.yaml
index 9d90b0c5c..4661c444a 100644
--- a/packages/system/dashboard/images/console/pnpm-lock.yaml
+++ b/packages/system/dashboard/images/console/pnpm-lock.yaml
@@ -92,6 +92,9 @@ importers:
simple-icons:
specifier: ^16.22.0
version: 16.22.0
+ uplot:
+ specifier: ^1.6.32
+ version: 1.6.32
devDependencies:
'@tailwindcss/vite':
specifier: ^4.2.2
@@ -1773,6 +1776,9 @@ packages:
peerDependencies:
browserslist: '>= 4.21.0'
+ uplot@1.6.32:
+ resolution: {integrity: sha512-KIMVnG68zvu5XXUbC4LQEPnhwOxBuLyW1AHtpm6IKTXImkbLgkMy+jabjLgSLMasNuGGzQm/ep3tOkyTxpiQIw==}
+
uri-js@4.4.1:
resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
@@ -3405,6 +3411,8 @@ snapshots:
escalade: 3.2.0
picocolors: 1.1.1
+ uplot@1.6.32: {}
+
uri-js@4.4.1:
dependencies:
punycode: 2.3.1
From b5a0ab8de6db7e68a6f4396f01becd8cb98b1dff Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:34:01 +0700
Subject: [PATCH 013/114] feat(dashboard): add a log viewer for the
observability proxy
Tenants could not read their applications' logs without Grafana. The
viewer reads VictoriaLogs' native responses through a proxy root: a
time-ranged LogsQL query, pod and container pickers fed by the stream
field values of the same scope, and a follow mode over the streamed
tail.
The picked pod and container wrap the filter part of the query rather
than being appended to it, because AND binds tighter than OR in LogsQL
and the user's pipes must still run last. A dropped tail resumes from
the last line's _time; the proxy may replay lines around that point,
so the client drops replayed lines it already holds by stream and
time, which VictoriaLogs does not guarantee unique and is therefore
applied only at the resume boundary. Follow keeps the newest 1000
lines so a chatty stream cannot grow the page without bound.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/LogsView.test.tsx | 111 +++++++++++++
.../src/components/observability/LogsView.tsx | 152 ++++++++++++++++++
.../src/lib/observability/logs.test.ts | 65 ++++++++
.../console/src/lib/observability/logs.ts | 73 +++++++++
.../console/src/lib/observability/proxy.ts | 3 +-
.../lib/observability/use-log-tail.test.tsx | 65 ++++++++
.../src/lib/observability/use-log-tail.ts | 95 +++++++++++
7 files changed, 563 insertions(+), 1 deletion(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
new file mode 100644
index 000000000..693adf70a
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -0,0 +1,111 @@
+import { describe, it, expect, vi } from "vitest"
+import { screen, waitFor } from "@testing-library/react"
+import userEvent from "@testing-library/user-event"
+import { K8sClient } from "@cozystack/k8s-client"
+import { renderWithK8sProvider } from "../../test-utils/render.tsx"
+import { LogsView } from "./LogsView.tsx"
+
+const app = { namespace: "tenant-a", plural: "postgreses", name: "db" }
+const ROOT = "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/"
+
+const line = (time: string, msg: string, pod = "db-0") =>
+ JSON.stringify({ _time: time, _msg: msg, _stream_id: "s", kubernetes_pod_name: pod, kubernetes_container_name: "postgres" })
+
+function clientWith(handler: (endpoint: string, params: URLSearchParams) => Response | Promise) {
+ const client = new K8sClient()
+ const raw = vi.spyOn(client, "raw").mockImplementation(async (path) => {
+ const url = new URL(path, "http://x")
+ return handler(url.pathname, url.searchParams)
+ })
+ return { client, raw }
+}
+
+const values = (...v: string[]) => Response.json({ values: v.map((value) => ({ value, hits: 1 })) })
+
+function defaultHandler(endpoint: string, params?: URLSearchParams): Response | Promise {
+ if (endpoint.endsWith("stream_field_values")) {
+ return params?.get("field") === "kubernetes_pod_name" ? values("db-0", "db-1") : values("postgres")
+ }
+ if (endpoint.endsWith("query")) return new Response(`${line("2026-11-04T10:00:01Z", "ready to accept connections")}\n`)
+ return new Promise(() => {})
+}
+
+describe("LogsView", () => {
+ it("shows recent lines from the application root", async () => {
+ const { client, raw } = clientWith(defaultHandler)
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
+ const call = raw.mock.calls.find(([p]) => p.includes("select/logsql/query?"))!
+ expect(call[0].startsWith(`${ROOT}select/logsql/query?`)).toBe(true)
+ const params = new URL(call[0], "http://x").searchParams
+ expect(params.get("query")).toBe("*")
+ expect(params.get("limit")).toBe("500")
+ })
+
+ it("narrows to the picked pod with a stream filter", async () => {
+ const { client, raw } = clientWith(defaultHandler)
+ renderWithK8sProvider( , { client })
+ await screen.findByRole("option", { name: "db-1" })
+
+ await userEvent.selectOptions(screen.getByLabelText("Pod"), "db-1")
+
+ await waitFor(() =>
+ expect(
+ raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("query") === '{kubernetes_pod_name="db-1"} (*)'),
+ ).toBe(true),
+ )
+ })
+
+ it("applies the LogsQL filter on submit", async () => {
+ const { client, raw } = clientWith(defaultHandler)
+ renderWithK8sProvider( , { client })
+
+ await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}")
+
+ await waitFor(() =>
+ expect(raw.mock.calls.some(([p]) => p.includes("select/logsql/query?") && p.includes("query=error"))).toBe(true),
+ )
+ })
+
+ it("follows the live tail when Follow is pressed", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("tail") ? new Response(`${line("2026-11-04T10:00:05Z", "checkpoint complete")}\n`) : defaultHandler(endpoint, params),
+ )
+ renderWithK8sProvider( , { client })
+
+ await userEvent.click(screen.getByRole("button", { name: "Follow" }))
+
+ expect(await screen.findByText("checkpoint complete")).toBeInTheDocument()
+ expect(screen.getByRole("button", { name: "Follow" })).toHaveAttribute("aria-pressed", "true")
+ })
+
+ it("renders a cluster without the proxy as not available", async () => {
+ const { client } = clientWith(() => new Response("404 page not found", { status: 404 }))
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("Logs are not available on this cluster.")).toBeInTheDocument()
+ expect(screen.queryByLabelText("LogsQL filter")).toBeNull()
+ })
+
+ it("keeps the controls and explains a rejected query", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query") ? new Response("cannot parse query", { status: 400 }) : defaultHandler(endpoint, params),
+ )
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("cannot parse query")).toBeInTheDocument()
+ expect(screen.getByLabelText("LogsQL filter")).toBeInTheDocument()
+ })
+
+ it("names the source pod and container of each line", async () => {
+ const { client } = clientWith(defaultHandler)
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("db-0/postgres")).toBeInTheDocument()
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
new file mode 100644
index 000000000..03235cda3
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -0,0 +1,152 @@
+import { useEffect, useRef, useState } from "react"
+import { useQuery } from "@tanstack/react-query"
+import { useK8sClient } from "@cozystack/k8s-client"
+import { Button, Spinner } from "@cozystack/ui"
+import {
+ CONTAINER_FIELD,
+ LOG_RANGES,
+ POD_FIELD,
+ buildLogsQuery,
+ type LogRangeKey,
+} from "../../lib/observability/logs.ts"
+import {
+ fieldValues,
+ isScopeWide,
+ logsRoot,
+ queryLogs,
+ type LogEntry,
+ type ObservabilityScope,
+} from "../../lib/observability/proxy.ts"
+import { useLogTail } from "../../lib/observability/use-log-tail.ts"
+import { ProxyNotice } from "./ProxyNotice.tsx"
+import { RangePicker } from "./RangePicker.tsx"
+
+const HISTORY_LIMIT = 500
+const FOLLOW_CAP = 1000
+
+function windowFor(range: LogRangeKey) {
+ const end = Date.now()
+ return { start: new Date(end - LOG_RANGES[range] * 1000).toISOString(), end: new Date(end).toISOString() }
+}
+
+function Picker({ label, value, options, onChange }: { label: string; value: string; options: string[]; onChange: (v: string) => void }) {
+ return (
+
+ {label}
+ onChange(e.target.value)}
+ className="rounded-md border border-slate-200 bg-white px-2 py-1 text-xs text-slate-900"
+ >
+ All
+ {options.map((o) => (
+
+ {o}
+
+ ))}
+
+
+ )
+}
+
+function LogLines({ entries, follow }: { entries: LogEntry[]; follow: boolean }) {
+ const ref = useRef(null)
+ useEffect(() => {
+ if (follow && ref.current) ref.current.scrollTop = ref.current.scrollHeight
+ }, [entries, follow])
+ if (entries.length === 0) return No log lines for this range and filter.
+ return (
+
+ {entries.map((e, i) => (
+
+ {new Date(e._time).toLocaleTimeString()} {" "}
+ {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`} {" "}
+ {e._msg}
+
+ ))}
+
+ )
+}
+
+export function LogsView({ scope }: { scope: ObservabilityScope }) {
+ const client = useK8sClient()
+ const root = logsRoot(scope)
+ const [range, setRange] = useState("1h")
+ const [draft, setDraft] = useState("")
+ const [filter, setFilter] = useState("")
+ const [pod, setPod] = useState("")
+ const [container, setContainer] = useState("")
+ const [follow, setFollow] = useState(false)
+ const query = buildLogsQuery({ filter, pod, container })
+
+ const pickerQuery = (field: string, narrowTo?: string) => ({
+ queryKey: ["observability", "logs-fields", root, field, range, narrowTo ?? ""],
+ queryFn: ({ signal }: { signal: AbortSignal }) =>
+ fieldValues(client, root, {
+ field,
+ stream: true,
+ query: buildLogsQuery({ filter: "", pod: narrowTo }),
+ ...windowFor(range),
+ }, signal),
+ retry: false,
+ })
+ const pods = useQuery(pickerQuery(POD_FIELD))
+ const containers = useQuery(pickerQuery(CONTAINER_FIELD, pod || undefined))
+
+ const history = useQuery({
+ queryKey: ["observability", "logs", root, query, range],
+ queryFn: ({ signal }) => queryLogs(client, root, { query, limit: HISTORY_LIMIT, ...windowFor(range) }, signal),
+ enabled: !follow,
+ retry: false,
+ })
+ const tail = useLogTail(root, query, follow, { cap: FOLLOW_CAP })
+
+ const wide = [pods.error, history.error, tail.error].find(isScopeWide)
+ if (wide) {
+ return (
+
+ )
+ }
+
+ const error = follow ? tail.error : history.error
+ const entries = follow ? tail.entries : (history.data ?? [])
+
+ return (
+
+
+
+
{ setPod(v); setContainer("") }} />
+
+ {!follow && }
+ setFollow(!follow)}>
+ Follow
+
+
+ {follow && (
+
+ {tail.state === "live" ? "Live" : tail.state === "reconnecting" ? "Reconnecting…" : "Connecting…"}
+ {` · newest ${FOLLOW_CAP} lines kept`}
+
+ )}
+ {error ?
: null}
+ {!follow && history.isLoading ?
:
}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts
new file mode 100644
index 000000000..25d26a23a
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts
@@ -0,0 +1,65 @@
+import { describe, it, expect } from "vitest"
+import { buildLogsQuery, compareTime, mergeTail } from "./logs.ts"
+
+describe("buildLogsQuery", () => {
+ it("passes the filter through when nothing is picked", () => {
+ expect(buildLogsQuery({ filter: "error or warn" })).toBe("error or warn")
+ expect(buildLogsQuery({ filter: " " })).toBe("*")
+ })
+
+ it("ANDs the picked pod and container as a stream filter around the whole filter", () => {
+ expect(buildLogsQuery({ filter: "error or warn", pod: "db-0", container: "postgres" })).toBe(
+ '{kubernetes_pod_name="db-0",kubernetes_container_name="postgres"} (error or warn)',
+ )
+ })
+
+ it("keeps the user's pipes after the scoped filter", () => {
+ expect(buildLogsQuery({ filter: 'a or "x|y" | fields _msg', pod: "db-0" })).toBe(
+ '{kubernetes_pod_name="db-0"} (a or "x|y") | fields _msg',
+ )
+ })
+
+ it("escapes quotes in picked values", () => {
+ expect(buildLogsQuery({ filter: "", pod: 'a"b' })).toBe('{kubernetes_pod_name="a\\"b"} (*)')
+ })
+})
+
+describe("compareTime", () => {
+ it("orders RFC3339 times with differing fraction lengths", () => {
+ expect(compareTime("2026-11-04T10:00:01Z", "2026-11-04T10:00:01.1Z")).toBeLessThan(0)
+ expect(compareTime("2026-11-04T10:00:01.5Z", "2026-11-04T10:00:01.123Z")).toBeGreaterThan(0)
+ expect(compareTime("2026-11-04T10:00:01.10Z", "2026-11-04T10:00:01.1Z")).toBe(0)
+ })
+})
+
+const e = (time: string, msg: string, stream = "s1") => ({ _time: time, _msg: msg, _stream_id: stream })
+
+describe("mergeTail", () => {
+ it("appends and keeps only the newest lines over the cap", () => {
+ const out = mergeTail([e("t1", "a"), e("t2", "b")], [e("t3", "c")], 2)
+
+ expect(out.map((x) => x._msg)).toEqual(["b", "c"])
+ })
+
+ it("drops replayed lines at or before the resume point", () => {
+ const buffer = [e("2026-01-01T00:00:01Z", "a"), e("2026-01-01T00:00:02Z", "b")]
+
+ const out = mergeTail(buffer, [e("2026-01-01T00:00:02Z", "b"), e("2026-01-01T00:00:03Z", "c")], 10, "2026-01-01T00:00:02Z")
+
+ expect(out.map((x) => x._msg)).toEqual(["a", "b", "c"])
+ })
+
+ it("keeps a line at the resume point from another stream", () => {
+ const buffer = [e("2026-01-01T00:00:02Z", "b", "s1")]
+
+ const out = mergeTail(buffer, [e("2026-01-01T00:00:02Z", "other", "s2")], 10, "2026-01-01T00:00:02Z")
+
+ expect(out).toHaveLength(2)
+ })
+
+ it("does not deduplicate outside a resume", () => {
+ const out = mergeTail([e("t1", "a")], [e("t1", "a")], 10)
+
+ expect(out).toHaveLength(2)
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts
new file mode 100644
index 000000000..a6255e541
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts
@@ -0,0 +1,73 @@
+import type { LogEntry } from "./proxy.ts"
+
+export const LOG_RANGES = {
+ "15m": 15 * 60,
+ "1h": 3600,
+ "6h": 6 * 3600,
+ "24h": 24 * 3600,
+} as const
+
+export type LogRangeKey = keyof typeof LOG_RANGES
+
+export const POD_FIELD = "kubernetes_pod_name"
+export const CONTAINER_FIELD = "kubernetes_container_name"
+
+function quote(value: string): string {
+ return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`
+}
+
+function splitAtFirstPipe(query: string): [string, string] {
+ let quoteChar = ""
+ for (let i = 0; i < query.length; i++) {
+ const c = query[i]
+ if (quoteChar) {
+ if (c === "\\") i++
+ else if (c === quoteChar) quoteChar = ""
+ } else if (c === '"' || c === "'" || c === "`") {
+ quoteChar = c
+ } else if (c === "|") {
+ return [query.slice(0, i), query.slice(i + 1)]
+ }
+ }
+ return [query, ""]
+}
+
+// The picker narrows the filter part only: AND binds tighter than OR in
+// LogsQL, and the user's pipes must still run after the narrowed filter.
+export function buildLogsQuery({ filter, pod, container }: { filter: string; pod?: string; container?: string }): string {
+ const trimmed = filter.trim() || "*"
+ const matchers = [
+ pod ? `${POD_FIELD}=${quote(pod)}` : "",
+ container ? `${CONTAINER_FIELD}=${quote(container)}` : "",
+ ].filter(Boolean)
+ if (matchers.length === 0) return trimmed
+ const [head, pipes] = splitAtFirstPipe(trimmed)
+ const scoped = `{${matchers.join(",")}} (${head.trim() || "*"})`
+ return pipes.trim() ? `${scoped} | ${pipes.trim()}` : scoped
+}
+
+// VictoriaLogs trims trailing zeros from the fraction, so the strings do not sort lexically.
+function padFraction(time: string): string {
+ const m = /^([^.Z]+)(?:\.(\d+))?Z$/.exec(time)
+ return m ? `${m[1]}.${(m[2] ?? "").padEnd(9, "0")}Z` : time
+}
+
+export function compareTime(a: string, b: string): number {
+ const x = padFraction(a)
+ const y = padFraction(b)
+ return x < y ? -1 : x > y ? 1 : 0
+}
+
+const entryKey = (e: LogEntry) => `${e._stream_id ?? ""}\u0000${e._time}`
+
+// (_stream_id, _time) is not unique in VictoriaLogs, so dedup applies only to
+// lines a resumed tail replays at or before the resume point.
+export function mergeTail(buffer: LogEntry[], incoming: LogEntry[], cap: number, resumedFrom?: string): LogEntry[] {
+ let fresh = incoming
+ if (resumedFrom) {
+ const seen = new Set(buffer.map(entryKey))
+ fresh = incoming.filter((e) => compareTime(e._time, resumedFrom) > 0 || !seen.has(entryKey(e)))
+ }
+ const merged = buffer.concat(fresh)
+ return merged.length > cap ? merged.slice(merged.length - cap) : merged
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
index 40b451b0b..345d49815 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -1,4 +1,5 @@
import type { K8sClient } from "@cozystack/k8s-client"
+import { compareTime } from "./logs.ts"
export type ObservabilityScope =
| { namespace: string }
@@ -160,7 +161,7 @@ export async function queryLogs(
): Promise {
const res = await proxyGet(client, `${root}select/logsql/query`, { ...params }, signal)
const entries = (await res.text()).split("\n").map(parseLine).filter((e) => e !== null)
- return entries.sort((a, b) => (a._time < b._time ? -1 : a._time > b._time ? 1 : 0))
+ return entries.sort((a, b) => compareTime(a._time, b._time))
}
export interface FieldValuesParams {
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
new file mode 100644
index 000000000..5c420314a
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
@@ -0,0 +1,65 @@
+import { describe, it, expect, vi } from "vitest"
+import { renderHook, waitFor } from "@testing-library/react"
+import type { ReactNode } from "react"
+import { K8sClient, K8sProvider } from "@cozystack/k8s-client"
+import { useLogTail } from "./use-log-tail.ts"
+
+const line = (time: string, msg: string) => JSON.stringify({ _time: time, _msg: msg, _stream_id: "s" })
+
+function ndjson(...lines: string[]): Response {
+ return new Response(lines.map((l) => `${l}\n`).join(""))
+}
+
+function setup(...responses: Response[]) {
+ const client = new K8sClient()
+ const raw = vi.spyOn(client, "raw")
+ for (const r of responses) raw.mockResolvedValueOnce(r)
+ raw.mockImplementation(() => new Promise(() => {}))
+ const wrapper = ({ children }: { children: ReactNode }) => {children}
+ return { raw, wrapper }
+}
+
+const T1 = "2026-11-04T10:00:01Z"
+const T2 = "2026-11-04T10:00:02Z"
+const T3 = "2026-11-04T10:00:03Z"
+
+describe("useLogTail", () => {
+ it("streams lines and resumes after a disconnect from the last _time without duplicates", async () => {
+ const { raw, wrapper } = setup(ndjson(line(T1, "a"), line(T2, "b")), ndjson(line(T2, "b"), line(T3, "c")))
+
+ const { result } = renderHook(() => useLogTail("/root/", "*", true, { cap: 100, retryDelayMs: 1 }), { wrapper })
+
+ await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a", "b", "c"]))
+ expect(new URL(raw.mock.calls[0][0], "http://x").searchParams.has("since")).toBe(false)
+ expect(new URL(raw.mock.calls[1][0], "http://x").searchParams.get("since")).toBe(T2)
+ })
+
+ it("stops and reports an error the proxy will repeat on every retry", async () => {
+ const { raw, wrapper } = setup(new Response("", { status: 403 }))
+
+ const { result } = renderHook(() => useLogTail("/root/", "*", true, { cap: 100, retryDelayMs: 1 }), { wrapper })
+
+ await waitFor(() => expect(result.current.state).toBe("stopped"))
+ expect(result.current.error).toMatchObject({ kind: "forbidden" })
+ expect(raw).toHaveBeenCalledTimes(1)
+ })
+
+ it("does not connect while disabled", () => {
+ const { raw, wrapper } = setup()
+
+ renderHook(() => useLogTail("/root/", "*", false), { wrapper })
+
+ expect(raw).not.toHaveBeenCalled()
+ })
+
+ it("aborts the stream on unmount", async () => {
+ const { raw, wrapper } = setup()
+ const { unmount } = renderHook(() => useLogTail("/root/", "*", true), { wrapper })
+ await waitFor(() => expect(raw).toHaveBeenCalled())
+
+ unmount()
+
+ const init = raw.mock.calls[0][1] as RequestInit
+ expect(init.signal?.aborted).toBe(true)
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
new file mode 100644
index 000000000..cb9222f48
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
@@ -0,0 +1,95 @@
+import { useEffect, useState } from "react"
+import { useK8sClient } from "@cozystack/k8s-client"
+import { mergeTail } from "./logs.ts"
+import { ProxyError, tailLogs, type LogEntry } from "./proxy.ts"
+
+export type TailState = "connecting" | "live" | "reconnecting" | "stopped"
+
+interface TailSnapshot {
+ key: string
+ entries: LogEntry[]
+ state: TailState
+ error: unknown
+}
+
+const FLUSH_MS = 100
+const MAX_RETRY_MS = 30_000
+const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forbidden", "not-available", "limit"])
+
+const sleep = (ms: number, signal: AbortSignal) =>
+ new Promise((resolve) => {
+ const t = setTimeout(resolve, ms)
+ signal.addEventListener("abort", () => {
+ clearTimeout(t)
+ resolve()
+ })
+ })
+
+export function useLogTail(
+ root: string,
+ query: string,
+ enabled: boolean,
+ { cap = 1000, retryDelayMs = 2000 }: { cap?: number; retryDelayMs?: number } = {},
+): Omit {
+ const client = useK8sClient()
+ const key = `${root}\n${query}\n${enabled}`
+ const [snap, setSnap] = useState({ key, entries: [], state: "connecting", error: null })
+
+ useEffect(() => {
+ if (!enabled) return
+ const controller = new AbortController()
+ const { signal } = controller
+ let pending: LogEntry[] = []
+ let timer: ReturnType | undefined
+ let resumedFrom: string | undefined
+ let lastTime: string | undefined
+
+ const update = (patch: Partial, batch: LogEntry[] = [], from = resumedFrom) =>
+ setSnap((prev) => {
+ const base = prev.key === key ? prev : { key, entries: [], state: "connecting" as const, error: null }
+ return { ...base, ...patch, entries: batch.length ? mergeTail(base.entries, batch, cap, from) : base.entries }
+ })
+ const flush = () => {
+ clearTimeout(timer)
+ timer = undefined
+ if (pending.length === 0 || signal.aborted) return
+ const batch = pending
+ pending = []
+ update({ state: "live", error: null }, batch)
+ }
+
+ const run = async () => {
+ for (let attempt = 0; !signal.aborted; attempt++) {
+ resumedFrom = lastTime
+ try {
+ await tailLogs(client, root, { query, since: lastTime }, (entry) => {
+ attempt = 0
+ lastTime = entry._time
+ pending.push(entry)
+ timer ??= setTimeout(flush, FLUSH_MS)
+ }, signal)
+ flush()
+ } catch (err) {
+ flush()
+ if (signal.aborted) return
+ if (err instanceof ProxyError && FATAL.has(err.kind)) {
+ update({ state: "stopped", error: err })
+ return
+ }
+ update({ error: err })
+ }
+ if (signal.aborted) return
+ update({ state: "reconnecting" })
+ await sleep(Math.min(retryDelayMs * 2 ** attempt, MAX_RETRY_MS), signal)
+ }
+ }
+ void run()
+ return () => {
+ controller.abort()
+ clearTimeout(timer)
+ }
+ }, [client, root, query, enabled, key, cap, retryDelayMs])
+
+ const current = snap.key === key ? snap : { key, entries: [], state: "connecting" as const, error: null }
+ return { entries: current.entries, state: current.state, error: current.error }
+}
From f9ed2afad749646bb5c1de0a56b4e8d17e773bc1 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:34:01 +0700
Subject: [PATCH 014/114] feat(dashboard): show logs on application pages and
per tenant
A Logs tab on application detail pages reads the application root of
the observability proxy, and a Logs page under a new Observability
sidebar group reads the tenant namespace root, so lines from before
the application identity labels existed stay reachable there.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../apps/console/src/routes/ConsolePage.tsx | 2 ++
.../apps/console/src/routes/LogsPage.test.tsx | 20 +++++++++++++++++++
.../apps/console/src/routes/LogsPage.tsx | 15 ++++++++++++++
.../detail/ApplicationDetailPage.test.tsx | 9 ++++++---
.../routes/detail/ApplicationDetailPage.tsx | 4 ++++
.../console/src/routes/detail/LogsTab.tsx | 15 ++++++++++++++
.../src/routes/sidebar-sections.test.tsx | 10 ++++++++++
.../console/src/routes/sidebar-sections.tsx | 8 +++++++-
8 files changed, 79 insertions(+), 4 deletions(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx
index 37caae92f..5e204e84f 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx
@@ -15,6 +15,7 @@ import { MigrationSourceEditPage } from "./MigrationSourceEditPage.tsx"
import { MigrationTaskCreatePage } from "./MigrationTaskCreatePage.tsx"
import { MigrationTaskDetailPage } from "./MigrationTaskDetailPage.tsx"
import { ApplicationOrderPage } from "./ApplicationOrderPage.tsx"
+import { LogsPage } from "./LogsPage.tsx"
export function ConsolePage() {
return (
@@ -96,6 +97,7 @@ export function ConsolePage() {
path="migration/vmimporttasks/:name"
element={ }
/>
+ } />
} />
} />
} />
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx
new file mode 100644
index 000000000..1f4263cf1
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx
@@ -0,0 +1,20 @@
+import { describe, it, expect, vi } from "vitest"
+import { render, screen } from "@testing-library/react"
+
+vi.mock("../lib/tenant-context.tsx", () => ({
+ useTenantContext: () => ({ tenantNamespace: "tenant-a" }),
+}))
+vi.mock("../components/observability/LogsView.tsx", () => ({
+ LogsView: ({ scope }: { scope: object }) => {JSON.stringify(scope)} ,
+}))
+
+const { LogsPage } = await import("./LogsPage.tsx")
+
+describe("LogsPage", () => {
+ it("reads logs from the selected tenant's namespace root", () => {
+ render( )
+
+ expect(screen.getByRole("heading", { name: "Logs" })).toBeInTheDocument()
+ expect(JSON.parse(screen.getByTestId("scope").textContent ?? "")).toEqual({ namespace: "tenant-a" })
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx
new file mode 100644
index 000000000..7612dfbb2
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx
@@ -0,0 +1,15 @@
+import { useTenantContext } from "../lib/tenant-context.tsx"
+import { LogsView } from "../components/observability/LogsView.tsx"
+
+export function LogsPage() {
+ const { tenantNamespace } = useTenantContext()
+ return (
+
+
+
Logs
+
Every application in {tenantNamespace ?? "this tenant"}
+
+ {tenantNamespace &&
}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx
index 40b923f62..a0c3a6b84 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx
@@ -60,6 +60,7 @@ vi.mock("./VncTab.tsx", () => ({ VncTab: () => null }))
vi.mock("./SerialTab.tsx", () => ({ SerialTab: () => null }))
vi.mock("./VMPowerControls.tsx", () => ({ VMPowerControls: () => null }))
vi.mock("./MetricsTab.tsx", () => ({ MetricsTab: () => null }))
+vi.mock("./LogsTab.tsx", () => ({ LogsTab: () => null }))
const { ApplicationDetailPage } = await import("./ApplicationDetailPage.tsx")
@@ -113,6 +114,7 @@ describe("ApplicationDetailPage tabs for a virtual machine", () => {
"Workloads",
"Services",
"Metrics",
+ "Logs",
"Events",
])
})
@@ -133,14 +135,15 @@ describe("ApplicationDetailPage observability tabs", () => {
renderPage()
}
- it("offers Metrics on any application kind, ahead of Events", () => {
+ it("offers Metrics and Logs on any application kind, ahead of Events", () => {
showKind("Postgres", "postgreses")
- expect(h.tabLabels.slice(-3)).toEqual(["Secrets", "Metrics", "Events"])
+ expect(h.tabLabels.slice(-4)).toEqual(["Secrets", "Metrics", "Logs", "Events"])
})
- it("leaves Metrics off a VM disk, which runs no containers", () => {
+ it("leaves Metrics and Logs off a VM disk, which runs no containers", () => {
showKind("VMDisk", "vmdisks")
expect(h.tabLabels).not.toContain("Metrics")
+ expect(h.tabLabels).not.toContain("Logs")
})
})
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
index 4520244d5..ca499aac1 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
@@ -44,6 +44,7 @@ import { useResourceBasePath } from "../../lib/portal.ts"
import { useResourcePresence } from "./use-resource-presence.ts"
import { useApplicationConfigMaps } from "./use-app-configmaps.ts"
import { ConfigMapsTab } from "./ConfigMapsTab.tsx"
+import { LogsTab } from "./LogsTab.tsx"
export function ApplicationDetailPage() {
const { plural, name } = useParams<{ plural: string; name: string }>()
@@ -146,6 +147,7 @@ export function ApplicationDetailPage() {
{ to: `${base}/workloads`, label: "Workloads", end: false },
{ to: `${base}/services`, label: "Services", end: false },
{ to: `${base}/metrics`, label: "Metrics", end: false },
+ { to: `${base}/logs`, label: "Logs", end: false },
{ to: `${base}/events`, label: "Events", end: false },
)
} else {
@@ -167,6 +169,7 @@ export function ApplicationDetailPage() {
tabs.push(
{ to: `${base}/secrets`, label: "Secrets", end: false },
{ to: `${base}/metrics`, label: "Metrics", end: false },
+ { to: `${base}/logs`, label: "Logs", end: false },
{ to: `${base}/events`, label: "Events", end: false },
)
}
@@ -274,6 +277,7 @@ export function ApplicationDetailPage() {
}
/>
+ } />
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx
new file mode 100644
index 000000000..4ac5f878d
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx
@@ -0,0 +1,15 @@
+import type { ApplicationDefinition, ApplicationInstance } from "@cozystack/types"
+import { useTenantContext } from "../../lib/tenant-context.tsx"
+import { LogsView } from "../../components/observability/LogsView.tsx"
+
+interface LogsTabProps {
+ ad: ApplicationDefinition
+ instance: ApplicationInstance
+}
+
+export function LogsTab({ ad, instance }: LogsTabProps) {
+ const { tenantNamespace } = useTenantContext()
+ const plural = ad.spec?.application.plural
+ if (!tenantNamespace || !plural) return null
+ return
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx
index efb69dc7b..da40081bc 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx
@@ -110,6 +110,16 @@ describe("useConsoleSidebarSections — admin areas moved out", () => {
expect(findItem(result.current, "Imports")?.to).toBe("/console/migration/vmimporttasks")
expect(result.current.some((s) => s.title === "Migration")).toBe(true)
})
+
+ it("offers the tenant's namespace logs in an Observability group", async () => {
+ const client = makeClient({ nodes: true, backupclasses: true })
+ const { result } = renderHook(() => useConsoleSidebarSections(), {
+ wrapper: makeWrapper(client),
+ })
+ await waitFor(() => expect(result.current.length).toBeGreaterThan(0))
+ expect(findItem(result.current, "Logs")?.to).toBe("/console/logs")
+ expect(result.current.find((s) => s.title === "Observability")?.items).toHaveLength(1)
+ })
})
function appDef(name: string, kind: string, category: string, module = false) {
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx
index d5d100813..4cec3a94c 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx
@@ -11,6 +11,7 @@ import {
Layers,
Network,
Package,
+ ScrollText,
Plug,
Server,
ToyBrick,
@@ -118,7 +119,12 @@ export function useConsoleSidebarSections(): SidebarSection[] {
],
}
- return [...categorySections, backupsSection, migrationSection]
+ const observabilitySection: SidebarSection = {
+ title: "Observability",
+ items: [{ label: "Logs", to: "/console/logs", icon: ScrollText }],
+ }
+
+ return [...categorySections, backupsSection, migrationSection, observabilitySection]
}, [grouped])
}
From c5b57bffd173ea5e4bb371edb75a13a60f549d96 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:39:27 +0700
Subject: [PATCH 015/114] fix(dashboard): open log history at the newest line
A historical query lists lines oldest first, so the viewer opened at
the top of a 500-line window while the line a user looks for is
usually the latest. The chart's value axis also clipped wide labels
such as "476.8 MiB".
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../console/src/components/observability/LogsView.tsx | 8 ++++----
.../src/components/observability/TimeSeriesChart.tsx | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index 03235cda3..707b037b9 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -50,11 +50,11 @@ function Picker({ label, value, options, onChange }: { label: string; value: str
)
}
-function LogLines({ entries, follow }: { entries: LogEntry[]; follow: boolean }) {
+function LogLines({ entries }: { entries: LogEntry[] }) {
const ref = useRef(null)
useEffect(() => {
- if (follow && ref.current) ref.current.scrollTop = ref.current.scrollHeight
- }, [entries, follow])
+ if (ref.current) ref.current.scrollTop = ref.current.scrollHeight
+ }, [entries])
if (entries.length === 0) return No log lines for this range and filter.
return (
@@ -146,7 +146,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
)}
{error ? : null}
- {!follow && history.isLoading ? : }
+ {!follow && history.isLoading ? : }
)
}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
index 6b596e17f..683df2dff 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
@@ -33,7 +33,7 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) {
value: (_: uPlot, v: number | null) => fmt(v),
})),
],
- axes: [{}, { size: 70, values: (_: uPlot, splits: number[]) => splits.map(format) }],
+ axes: [{}, { size: 84, values: (_: uPlot, splits: number[]) => splits.map(format) }],
},
[data.timestamps, ...data.series.map((s) => s.values)],
el,
From 97dcf4c4f02d78dc2dcd4ec862cdaa77631de4f3 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:45:42 +0700
Subject: [PATCH 016/114] fix(dashboard): treat an apiserver answer on a proxy
path as no proxy
Until the console nginx routes the proxy roots, those paths reach
kube-apiserver, which RBAC-denies the namespace root with 403 for a
tenant, so the Logs page told every tenant they lacked permission. The
proxy itself answers in plain text, so a Kubernetes Status body is a
reliable sign that the path did not reach it.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../components/observability/LogsView.test.tsx | 10 ++++++++++
.../console/src/lib/observability/proxy.test.ts | 17 +++++++++++++++++
.../apps/console/src/lib/observability/proxy.ts | 14 +++++++++++++-
3 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 693adf70a..fae3f6a4b 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -90,6 +90,16 @@ describe("LogsView", () => {
expect(screen.queryByLabelText("LogsQL filter")).toBeNull()
})
+ it("renders the tenant logs page as not available when kube-apiserver answers instead of the proxy", async () => {
+ const { client } = clientWith(() =>
+ Response.json({ kind: "Status", apiVersion: "v1", status: "Failure", reason: "Forbidden", code: 403 }, { status: 403 }),
+ )
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("Logs are not available on this cluster.")).toBeInTheDocument()
+ })
+
it("keeps the controls and explains a rejected query", async () => {
const { client } = clientWith((endpoint, params) =>
endpoint.endsWith("query") ? new Response("cannot parse query", { status: 400 }) : defaultHandler(endpoint, params),
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
index f8cf44656..69a8c7c8c 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
@@ -79,6 +79,23 @@ describe("queryRange", () => {
expect(err).toMatchObject({ kind: "limit", status: 422, message: "range too long" })
})
+ it("treats a Kubernetes Status 403 as a cluster without the proxy route", async () => {
+ const status = { kind: "Status", apiVersion: "v1", status: "Failure", reason: "Forbidden", code: 403, message: "logs.cozystack.io \"select\" is forbidden" }
+ const { client } = clientReturning(Response.json(status, { status: 403 }))
+
+ const err = await queryRange(client, metricsRoot(ns), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
+
+ expect(err).toMatchObject({ kind: "not-available", status: 403 })
+ })
+
+ it("keeps the proxy's own plain-text 403 as forbidden", async () => {
+ const { client } = clientReturning(new Response("forbidden\n", { status: 403 }))
+
+ const err = await queryRange(client, metricsRoot(ns), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
+
+ expect(err).toMatchObject({ kind: "forbidden" })
+ })
+
it("reports a fetch failure as a network error", async () => {
const client = new K8sClient()
vi.spyOn(client, "raw").mockRejectedValue(new TypeError("offline"))
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
index 345d49815..c9fce3d4f 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -69,6 +69,17 @@ export function isScopeWide(error: unknown): boolean {
)
}
+// The proxy answers in plain text; a Kubernetes Status body means nginx sent
+// the path to kube-apiserver, which RBAC-denies or 404s a route it does not serve.
+function isKubernetesStatus(text: string): boolean {
+ try {
+ const parsed: unknown = JSON.parse(text)
+ return !!parsed && typeof parsed === "object" && "kind" in parsed && parsed.kind === "Status"
+ } catch {
+ return false
+ }
+}
+
// Prometheus answers {"error": "..."}, VictoriaLogs and nginx answer plain text.
function errorMessage(text: string, status: number): string {
try {
@@ -100,7 +111,8 @@ async function proxyGet(
}
if (!res.ok) {
const text = await res.text().catch(() => "")
- throw new ProxyError(errorKindForStatus(res.status), res.status, errorMessage(text, res.status))
+ const kind = isKubernetesStatus(text) ? "not-available" : errorKindForStatus(res.status)
+ throw new ProxyError(kind, res.status, errorMessage(text, res.status))
}
return res
}
From 4b44971d125cce2c2fd06457c68f23cf1645ac87 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:47:13 +0700
Subject: [PATCH 017/114] fix(dashboard): start each log follow run with its
own buffer
Turning Follow off and on again reused the previous run's buffer, so
new lines were appended after a silent gap and, on a quick toggle,
duplicated. A follow run is now identified by a session value, so the
buffer starts empty each time.
A resume now uses the newest _time seen rather than the last line
received, so an out-of-order line cannot move since backwards, and
the backoff sleep no longer leaves an abort listener per retry.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../lib/observability/use-log-tail.test.tsx | 31 ++++++++++++++++---
.../src/lib/observability/use-log-tail.ts | 23 ++++++++------
2 files changed, 41 insertions(+), 13 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
index 5c420314a..46df47970 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
@@ -27,7 +27,7 @@ describe("useLogTail", () => {
it("streams lines and resumes after a disconnect from the last _time without duplicates", async () => {
const { raw, wrapper } = setup(ndjson(line(T1, "a"), line(T2, "b")), ndjson(line(T2, "b"), line(T3, "c")))
- const { result } = renderHook(() => useLogTail("/root/", "*", true, { cap: 100, retryDelayMs: 1 }), { wrapper })
+ const { result } = renderHook(() => useLogTail("/root/", "*", 1, { cap: 100, retryDelayMs: 1 }), { wrapper })
await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a", "b", "c"]))
expect(new URL(raw.mock.calls[0][0], "http://x").searchParams.has("since")).toBe(false)
@@ -37,24 +37,47 @@ describe("useLogTail", () => {
it("stops and reports an error the proxy will repeat on every retry", async () => {
const { raw, wrapper } = setup(new Response("", { status: 403 }))
- const { result } = renderHook(() => useLogTail("/root/", "*", true, { cap: 100, retryDelayMs: 1 }), { wrapper })
+ const { result } = renderHook(() => useLogTail("/root/", "*", 1, { cap: 100, retryDelayMs: 1 }), { wrapper })
await waitFor(() => expect(result.current.state).toBe("stopped"))
expect(result.current.error).toMatchObject({ kind: "forbidden" })
expect(raw).toHaveBeenCalledTimes(1)
})
+ it("starts a new follow session with an empty buffer", async () => {
+ const { wrapper } = setup(ndjson(line(T1, "first")), ndjson(line(T3, "second")))
+ const { result, rerender } = renderHook(
+ ({ session }: { session: number | null }) => useLogTail("/root/", "*", session, { retryDelayMs: 60_000 }),
+ { wrapper, initialProps: { session: 1 as number | null } },
+ )
+ await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["first"]))
+
+ rerender({ session: null })
+ rerender({ session: 2 })
+
+ await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["second"]))
+ })
+
+ it("resumes from the newest line seen, not the last one received", async () => {
+ const { raw, wrapper } = setup(ndjson(line(T3, "c"), line(T1, "a")), ndjson())
+
+ renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper })
+
+ await waitFor(() => expect(raw.mock.calls.length).toBeGreaterThan(1))
+ expect(new URL(raw.mock.calls[1][0], "http://x").searchParams.get("since")).toBe(T3)
+ })
+
it("does not connect while disabled", () => {
const { raw, wrapper } = setup()
- renderHook(() => useLogTail("/root/", "*", false), { wrapper })
+ renderHook(() => useLogTail("/root/", "*", null), { wrapper })
expect(raw).not.toHaveBeenCalled()
})
it("aborts the stream on unmount", async () => {
const { raw, wrapper } = setup()
- const { unmount } = renderHook(() => useLogTail("/root/", "*", true), { wrapper })
+ const { unmount } = renderHook(() => useLogTail("/root/", "*", 1), { wrapper })
await waitFor(() => expect(raw).toHaveBeenCalled())
unmount()
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
index cb9222f48..ddc835821 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
@@ -1,6 +1,6 @@
import { useEffect, useState } from "react"
import { useK8sClient } from "@cozystack/k8s-client"
-import { mergeTail } from "./logs.ts"
+import { compareTime, mergeTail } from "./logs.ts"
import { ProxyError, tailLogs, type LogEntry } from "./proxy.ts"
export type TailState = "connecting" | "live" | "reconnecting" | "stopped"
@@ -18,25 +18,30 @@ const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forb
const sleep = (ms: number, signal: AbortSignal) =>
new Promise((resolve) => {
- const t = setTimeout(resolve, ms)
- signal.addEventListener("abort", () => {
+ const onAbort = () => {
clearTimeout(t)
resolve()
- })
+ }
+ const t = setTimeout(() => {
+ signal.removeEventListener("abort", onAbort)
+ resolve()
+ }, ms)
+ signal.addEventListener("abort", onAbort, { once: true })
})
+// Each non-null session is one follow run with its own buffer; null stops following.
export function useLogTail(
root: string,
query: string,
- enabled: boolean,
+ session: number | null,
{ cap = 1000, retryDelayMs = 2000 }: { cap?: number; retryDelayMs?: number } = {},
): Omit {
const client = useK8sClient()
- const key = `${root}\n${query}\n${enabled}`
+ const key = `${root}\n${query}\n${session}`
const [snap, setSnap] = useState({ key, entries: [], state: "connecting", error: null })
useEffect(() => {
- if (!enabled) return
+ if (session === null) return
const controller = new AbortController()
const { signal } = controller
let pending: LogEntry[] = []
@@ -64,7 +69,7 @@ export function useLogTail(
try {
await tailLogs(client, root, { query, since: lastTime }, (entry) => {
attempt = 0
- lastTime = entry._time
+ if (!lastTime || compareTime(entry._time, lastTime) > 0) lastTime = entry._time
pending.push(entry)
timer ??= setTimeout(flush, FLUSH_MS)
}, signal)
@@ -88,7 +93,7 @@ export function useLogTail(
controller.abort()
clearTimeout(timer)
}
- }, [client, root, query, enabled, key, cap, retryDelayMs])
+ }, [client, root, query, session, key, cap, retryDelayMs])
const current = snap.key === key ? snap : { key, entries: [], state: "connecting" as const, error: null }
return { entries: current.entries, state: current.state, error: current.error }
From 290490769eacf8906256290d3078bba02e4e9d04 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:47:13 +0700
Subject: [PATCH 018/114] fix(dashboard): let the reader scroll up while logs
stream
The viewer jumped to the newest line on every 100 ms flush, so a line
could not be read on a busy application. It now follows the bottom
only while the reader is already there, and says when a refused tail
has stopped instead of showing it as connecting.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/LogLines.test.tsx | 34 +++++++++++++++++++
.../src/components/observability/LogLines.tsx | 34 +++++++++++++++++++
.../observability/LogsView.test.tsx | 12 +++++++
.../src/components/observability/LogsView.tsx | 33 +++++-------------
4 files changed, 88 insertions(+), 25 deletions(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
new file mode 100644
index 000000000..775c5fcc6
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
@@ -0,0 +1,34 @@
+import { describe, it, expect } from "vitest"
+import { fireEvent, render, screen } from "@testing-library/react"
+import { LogLines } from "./LogLines.tsx"
+
+const entry = (n: number) => ({ _time: `2026-11-04T10:00:0${n}Z`, _msg: `line ${n}`, _stream_id: "s" })
+
+function sized(list: HTMLElement, scrollHeight: number) {
+ Object.defineProperty(list, "scrollHeight", { configurable: true, value: scrollHeight })
+ Object.defineProperty(list, "clientHeight", { configurable: true, value: 100 })
+}
+
+describe("LogLines", () => {
+ it("keeps following the newest line while the reader is at the bottom", () => {
+ const { rerender } = render( )
+ const list = screen.getByRole("list")
+ sized(list, 500)
+
+ rerender( )
+
+ expect(list.scrollTop).toBe(500)
+ })
+
+ it("leaves the position alone once the reader scrolls up", () => {
+ const { rerender } = render( )
+ const list = screen.getByRole("list")
+ sized(list, 500)
+ list.scrollTop = 120
+ fireEvent.scroll(list)
+
+ rerender( )
+
+ expect(list.scrollTop).toBe(120)
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
new file mode 100644
index 000000000..a6f4b5861
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
@@ -0,0 +1,34 @@
+import { useEffect, useRef } from "react"
+import { CONTAINER_FIELD, POD_FIELD } from "../../lib/observability/logs.ts"
+import type { LogEntry } from "../../lib/observability/proxy.ts"
+
+const BOTTOM_SLACK_PX = 24
+
+export function LogLines({ entries }: { entries: LogEntry[] }) {
+ const ref = useRef(null)
+ const pinned = useRef(true)
+
+ useEffect(() => {
+ if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight
+ }, [entries])
+
+ if (entries.length === 0) return No log lines for this range and filter.
+ return (
+ {
+ const el = e.currentTarget
+ pinned.current = el.scrollHeight - el.scrollTop - el.clientHeight <= BOTTOM_SLACK_PX
+ }}
+ className="max-h-[60vh] overflow-auto rounded-md bg-slate-950 p-3 font-mono text-xs leading-5 text-slate-100"
+ >
+ {entries.map((e, i) => (
+
+ {new Date(e._time).toLocaleTimeString()} {" "}
+ {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`} {" "}
+ {e._msg}
+
+ ))}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index fae3f6a4b..526efeb23 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -81,6 +81,18 @@ describe("LogsView", () => {
expect(screen.getByRole("button", { name: "Follow" })).toHaveAttribute("aria-pressed", "true")
})
+ it("says the tail stopped when the proxy refuses the query", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("tail") ? new Response("pipe not allowed in tail", { status: 400 }) : defaultHandler(endpoint, params),
+ )
+ renderWithK8sProvider( , { client })
+
+ await userEvent.click(screen.getByRole("button", { name: "Follow" }))
+
+ expect(await screen.findByText(/Stopped/)).toBeInTheDocument()
+ expect(screen.getByText("pipe not allowed in tail")).toBeInTheDocument()
+ })
+
it("renders a cluster without the proxy as not available", async () => {
const { client } = clientWith(() => new Response("404 page not found", { status: 404 }))
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index 707b037b9..f21f2eb86 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -1,4 +1,4 @@
-import { useEffect, useRef, useState } from "react"
+import { useState } from "react"
import { useQuery } from "@tanstack/react-query"
import { useK8sClient } from "@cozystack/k8s-client"
import { Button, Spinner } from "@cozystack/ui"
@@ -14,15 +14,16 @@ import {
isScopeWide,
logsRoot,
queryLogs,
- type LogEntry,
type ObservabilityScope,
} from "../../lib/observability/proxy.ts"
import { useLogTail } from "../../lib/observability/use-log-tail.ts"
import { ProxyNotice } from "./ProxyNotice.tsx"
import { RangePicker } from "./RangePicker.tsx"
+import { LogLines } from "./LogLines.tsx"
const HISTORY_LIMIT = 500
const FOLLOW_CAP = 1000
+const TAIL_STATE_LABEL = { connecting: "Connecting…", live: "Live", reconnecting: "Reconnecting…", stopped: "Stopped" }
function windowFor(range: LogRangeKey) {
const end = Date.now()
@@ -50,25 +51,6 @@ function Picker({ label, value, options, onChange }: { label: string; value: str
)
}
-function LogLines({ entries }: { entries: LogEntry[] }) {
- const ref = useRef(null)
- useEffect(() => {
- if (ref.current) ref.current.scrollTop = ref.current.scrollHeight
- }, [entries])
- if (entries.length === 0) return No log lines for this range and filter.
- return (
-
- {entries.map((e, i) => (
-
- {new Date(e._time).toLocaleTimeString()} {" "}
- {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`} {" "}
- {e._msg}
-
- ))}
-
- )
-}
-
export function LogsView({ scope }: { scope: ObservabilityScope }) {
const client = useK8sClient()
const root = logsRoot(scope)
@@ -77,7 +59,8 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
const [filter, setFilter] = useState("")
const [pod, setPod] = useState("")
const [container, setContainer] = useState("")
- const [follow, setFollow] = useState(false)
+ const [followRun, setFollowRun] = useState(null)
+ const follow = followRun !== null
const query = buildLogsQuery({ filter, pod, container })
const pickerQuery = (field: string, narrowTo?: string) => ({
@@ -100,7 +83,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
enabled: !follow,
retry: false,
})
- const tail = useLogTail(root, query, follow, { cap: FOLLOW_CAP })
+ const tail = useLogTail(root, query, followRun, { cap: FOLLOW_CAP })
const wide = [pods.error, history.error, tail.error].find(isScopeWide)
if (wide) {
@@ -135,13 +118,13 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
{ setPod(v); setContainer("") }} />
{!follow && }
- setFollow(!follow)}>
+ setFollowRun(follow ? null : Date.now())}>
Follow
{follow && (
- {tail.state === "live" ? "Live" : tail.state === "reconnecting" ? "Reconnecting…" : "Connecting…"}
+ {TAIL_STATE_LABEL[tail.state]}
{` · newest ${FOLLOW_CAP} lines kept`}
)}
From b05323f6bee913256c3e4ae6eedc764689ec508c Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:47:22 +0700
Subject: [PATCH 019/114] docs(dashboard): note the observability proxy client
in the console guide
The next session needs to know why one module bypasses the typed
Kubernetes client and where the charting dependency is loaded.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
packages/system/dashboard/images/console/CLAUDE.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/packages/system/dashboard/images/console/CLAUDE.md b/packages/system/dashboard/images/console/CLAUDE.md
index 2acf80504..96419d118 100644
--- a/packages/system/dashboard/images/console/CLAUDE.md
+++ b/packages/system/dashboard/images/console/CLAUDE.md
@@ -32,6 +32,8 @@ Implications:
- **RJSF** (`@rjsf/core` + `validator-ajv8`) for application config forms.
- **Vitest + jsdom + @testing-library/react** for tests.
- **Monaco** for YAML view, **noVNC** for the VM console tab.
+- **uPlot** for the Metrics tab charts, lazy-loaded with that tab.
+- Metrics and logs come from the observability proxy, not kube-apiserver: `lib/observability/proxy.ts` reads its native Prometheus JSON and VictoriaLogs NDJSON through `K8sClient.raw`, under the paths nginx routes to it ahead of `/apis`.
## Dev loop
From 84ed3aaefd34159549534dd8ff6f7e0e3111c3c9 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:50:33 +0700
Subject: [PATCH 020/114] fix(dashboard): follow the newest line again after
the log list clears
A reader who scrolled up stayed unpinned across a new follow run or a
new filter, so the fresh list sat at its top while lines streamed in
below. An empty list now means the reader is back at the bottom.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../components/observability/LogLines.test.tsx | 16 ++++++++++++++++
.../src/components/observability/LogLines.tsx | 3 ++-
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
index 775c5fcc6..7110fe464 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
@@ -31,4 +31,20 @@ describe("LogLines", () => {
expect(list.scrollTop).toBe(120)
})
+
+ it("follows again once the list is cleared for a new run", () => {
+ const { rerender } = render( )
+ const first = screen.getByRole("list")
+ sized(first, 500)
+ first.scrollTop = 120
+ fireEvent.scroll(first)
+
+ rerender( )
+ rerender( )
+ const second = screen.getByRole("list")
+ sized(second, 300)
+ rerender( )
+
+ expect(second.scrollTop).toBe(300)
+ })
})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
index a6f4b5861..452faac72 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
@@ -9,7 +9,8 @@ export function LogLines({ entries }: { entries: LogEntry[] }) {
const pinned = useRef(true)
useEffect(() => {
- if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight
+ if (entries.length === 0) pinned.current = true
+ else if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight
}, [entries])
if (entries.length === 0) return No log lines for this range and filter.
From 1b4ce7433e1acbb7d5b1d5ec6a4634fe2cd1b730 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:50:33 +0700
Subject: [PATCH 021/114] fix(dashboard): keep retrying a log tail refused for
load
The proxy answers a tail over its rate or concurrent-tail cap with a
4xx, and a reconnect can arrive before the previous tail's slot is
freed. Treating that as final left Follow stopped for good, while the
existing backoff already waits long enough for the cap to clear.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../console/src/lib/observability/use-log-tail.test.tsx | 8 ++++++++
.../apps/console/src/lib/observability/use-log-tail.ts | 3 ++-
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
index 46df47970..794f30216 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
@@ -67,6 +67,14 @@ describe("useLogTail", () => {
expect(new URL(raw.mock.calls[1][0], "http://x").searchParams.get("since")).toBe(T3)
})
+ it("retries a tail refused for load instead of stopping", async () => {
+ const { wrapper } = setup(new Response("too many concurrent requests", { status: 422 }), ndjson(line(T1, "a")))
+
+ const { result } = renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper })
+
+ await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a"]))
+ })
+
it("does not connect while disabled", () => {
const { raw, wrapper } = setup()
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
index ddc835821..292f93e9e 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
@@ -14,7 +14,8 @@ interface TailSnapshot {
const FLUSH_MS = 100
const MAX_RETRY_MS = 30_000
-const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forbidden", "not-available", "limit"])
+// A 422 or 429 on tail is load (rate or concurrent-tail caps), which backoff outlasts.
+const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forbidden", "not-available"])
const sleep = (ms: number, signal: AbortSignal) =>
new Promise((resolve) => {
From ebdd74ce4e010cc347d1ddae312b2c20dfeb39fa Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:53:23 +0700
Subject: [PATCH 022/114] fix(dashboard): name the proxy's real load status for
a refused tail
The proxy caps rate and concurrent tails with 429, so the retry
reasoning and its test now say so. Leaving Follow also remounts the
log list, so the history it falls back to opens at the newest line
even when the reader had scrolled up while following.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../apps/console/src/components/observability/LogsView.tsx | 2 +-
.../apps/console/src/lib/observability/use-log-tail.test.tsx | 2 +-
.../console/apps/console/src/lib/observability/use-log-tail.ts | 3 ++-
3 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index f21f2eb86..a27ca41c5 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -129,7 +129,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
)}
{error ? : null}
- {!follow && history.isLoading ? : }
+ {!follow && history.isLoading ? : }
)
}
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
index 794f30216..0c2306613 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
@@ -68,7 +68,7 @@ describe("useLogTail", () => {
})
it("retries a tail refused for load instead of stopping", async () => {
- const { wrapper } = setup(new Response("too many concurrent requests", { status: 422 }), ndjson(line(T1, "a")))
+ const { wrapper } = setup(new Response("too many concurrent requests", { status: 429 }), ndjson(line(T1, "a")))
const { result } = renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper })
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
index 292f93e9e..53cbaf856 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
@@ -14,7 +14,8 @@ interface TailSnapshot {
const FLUSH_MS = 100
const MAX_RETRY_MS = 30_000
-// A 422 or 429 on tail is load (rate or concurrent-tail caps), which backoff outlasts.
+// Load refusals retry: the proxy caps rate and concurrent tails with 429 (422 in
+// its first builds), and a reconnect can race the previous tail's freed slot.
const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forbidden", "not-available"])
const sleep = (ms: number, signal: AbortSignal) =>
From 2b823ab79fa178f22b338ad654962bc9c334a30e Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:53:50 +0700
Subject: [PATCH 023/114] fix(dashboard): rate over at least 5m and four steps
Long metric ranges are served by the longterm store, which keeps one
sample per 5m, so a rate window narrower than that can hold no samples
and the chart comes back empty. Four steps also keep every sample
between two points inside some window.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/lib/observability/metrics.test.ts | 17 +++++++++++++----
.../console/src/lib/observability/metrics.ts | 6 +++---
2 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts
index 0803940bf..0598c0c7f 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts
@@ -32,10 +32,19 @@ describe("METRIC_PANELS", () => {
)
})
- it("widens the rate window with the step on long ranges", () => {
- const [query] = METRIC_PANELS.find((p) => p.id === "cpu")!.queries({ start: 0, end: 1, step: 2520 })
-
- expect(query.expr).toContain("[5040s]")
+ // Long ranges are served by the longterm store, deduplicated at 5m, where a
+ // narrower window can hold no samples and the rate comes back empty.
+ it.each([
+ [15, "300s"],
+ [75, "300s"],
+ [90, "360s"],
+ [2520, "10080s"],
+ ])("rates over max(5m, 4 x step) for step %is", (step, window) => {
+ for (const panel of METRIC_PANELS) {
+ for (const q of panel.queries({ start: 0, end: 1, step })) {
+ if (q.expr.includes("rate(")) expect(q.expr).toContain(`[${window}]`)
+ }
+ }
})
it("reads memory as working set per pod", () => {
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
index b23a51e27..d7182b8d6 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
@@ -23,10 +23,10 @@ export function rangeWindow(range: RangeKey, nowSeconds: number): RangeWindow {
return { start: nowSeconds - span, end: nowSeconds, step: Math.max(MIN_STEP, Math.round(span / POINTS)) }
}
-// A rate needs several scrapes inside its window, and on long ranges the window
-// must also cover the step or samples between points are skipped.
+// The longterm store keeps one sample per 5m, so a narrower window can be empty;
+// 4 x step also keeps samples between points from being skipped.
function rateWindow(step: number): string {
- return `${Math.max(300, step * 2)}s`
+ return `${Math.max(300, step * 4)}s`
}
const CONTAINERS = 'container!="",container!="POD"'
From 3b1302ba8e8d44a5294b28302a775f805bd40e2a Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:45:36 +0700
Subject: [PATCH 024/114] feat(dashboard): add an accessible Select to the
console
The console has no select primitive: form widgets fall back to the
native element, and the shared Dropdown has no listbox roles, no
keyboard navigation and no label association. Base UI, already a
dependency, ships one; this wraps it the same way dialog.tsx wraps the
Base UI dialog so pickers get a labelled, keyboard-driven listbox that
matches the console's buttons.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../console/src/components/ui/select.test.tsx | 64 +++++++++++
.../apps/console/src/components/ui/select.tsx | 107 ++++++++++++++++++
2 files changed, 171 insertions(+)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx
new file mode 100644
index 000000000..fb5b84e6f
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx
@@ -0,0 +1,64 @@
+import { describe, it, expect, vi } from "vitest"
+import { render, screen } from "@testing-library/react"
+import userEvent from "@testing-library/user-event"
+import { SelectItem, SelectLabel, SelectPopup, SelectRoot, SelectTrigger, SelectValue } from "./select.tsx"
+
+const items = [
+ { value: "", label: "All" },
+ { value: "db-0", label: "db-0" },
+ { value: "db-1", label: "db-1" },
+]
+
+function PodSelect({ value = "", onValueChange = () => {} }: { value?: string; onValueChange?: (v: string) => void }) {
+ return (
+ onValueChange(v ?? "")}>
+ Pod
+
+
+
+
+ {items.map((i) => (
+
+ {i.label}
+
+ ))}
+
+
+ )
+}
+
+describe("Select", () => {
+ it("names the trigger by its label and shows the chosen item", () => {
+ render( )
+
+ expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("db-1")
+ })
+
+ it("shows the label of an empty-string value", () => {
+ render( )
+
+ expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("All")
+ })
+
+ it("lists the items and reports the one clicked", async () => {
+ const onValueChange = vi.fn()
+ render( )
+
+ await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
+ await userEvent.click(await screen.findByRole("option", { name: "db-1" }))
+
+ expect(onValueChange).toHaveBeenCalledWith("db-1")
+ })
+
+ it("can be driven from the keyboard", async () => {
+ const onValueChange = vi.fn()
+ render( )
+
+ screen.getByRole("combobox", { name: "Pod" }).focus()
+ await userEvent.keyboard("{ArrowDown}")
+ expect(await screen.findByRole("listbox")).toBeInTheDocument()
+ await userEvent.keyboard("{ArrowDown}{Enter}")
+
+ expect(onValueChange).toHaveBeenCalledWith("db-0")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx
new file mode 100644
index 000000000..f256dba45
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx
@@ -0,0 +1,107 @@
+import { Select as SelectPrimitive } from "@base-ui/react/select"
+import { Check, ChevronDown } from "lucide-react"
+import { cn } from "@cozystack/ui"
+
+const SelectRoot = SelectPrimitive.Root
+
+function SelectLabel({
+ className,
+ ...props
+}: SelectPrimitive.Label.Props) {
+ return (
+
+ )
+}
+
+function SelectTrigger({
+ className,
+ children,
+ ...props
+}: SelectPrimitive.Trigger.Props) {
+ return (
+
+ {children}
+
+
+
+
+ )
+}
+
+function SelectValue({
+ className,
+ ...props
+}: SelectPrimitive.Value.Props) {
+ return (
+
+ )
+}
+
+function SelectPopup({
+ className,
+ children,
+ ...props
+}: SelectPrimitive.Popup.Props) {
+ return (
+
+
+
+ {children}
+
+
+
+ )
+}
+
+function SelectItem({
+ className,
+ children,
+ ...props
+}: SelectPrimitive.Item.Props) {
+ return (
+
+
+
+
+ {children}
+
+ )
+}
+
+export {
+ SelectRoot,
+ SelectLabel,
+ SelectTrigger,
+ SelectValue,
+ SelectPopup,
+ SelectItem,
+}
From f500aa955ac65384b527bf61167e77bf5dd02a3e Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:46:15 +0700
Subject: [PATCH 025/114] feat(dashboard): add a text Input matching the
console select
Every text field in the console restyles a bare input by hand, each a
little differently. A toolbar that mixes a text filter with the new
Select needs the two at the same height, border and focus ring, so the
styling lives in one place next to the Select instead of being copied.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../console/src/components/ui/input.test.tsx | 21 +++++++++++++++++++
.../apps/console/src/components/ui/input.tsx | 17 +++++++++++++++
2 files changed, 38 insertions(+)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx
new file mode 100644
index 000000000..e65a70a3a
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx
@@ -0,0 +1,21 @@
+import { describe, it, expect, vi } from "vitest"
+import { render, screen } from "@testing-library/react"
+import userEvent from "@testing-library/user-event"
+import { Input } from "./input.tsx"
+
+describe("Input", () => {
+ it("is a labelled text box that reports what is typed", async () => {
+ const onValueChange = vi.fn()
+ render( )
+
+ await userEvent.type(screen.getByRole("textbox", { name: "Filter" }), "x")
+
+ expect(onValueChange).toHaveBeenCalledWith("x", expect.anything())
+ })
+
+ it("keeps the caller's classes next to its own", () => {
+ render( )
+
+ expect(screen.getByRole("textbox", { name: "Filter" })).toHaveClass("font-mono", "rounded-lg")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx
new file mode 100644
index 000000000..b079e6b35
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx
@@ -0,0 +1,17 @@
+import { Input as InputPrimitive } from "@base-ui/react/input"
+import { cn } from "@cozystack/ui"
+
+function Input({ className, ...props }: InputPrimitive.Props) {
+ return (
+
+ )
+}
+
+export { Input }
From b88ad09678a234ff351173a1cc0674b9721d795f Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:47:25 +0700
Subject: [PATCH 026/114] fix(dashboard): keep the log console on screen when
it has no lines
With no matching lines, while loading, and while Follow waits for the
first line, the dark log console vanished and left a line of small
grey text or a bare spinner, so the page jumped and an empty result
looked like a broken view. The console frame now keeps a fixed height
in every state and says inside it what is going on.
The pod and container pickers move to the console Select and the
filter to the console Input, so the toolbar is keyboard and screen
reader friendly and a long pod name truncates instead of widening it.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/EmptyState.test.tsx | 17 +++++
.../components/observability/EmptyState.tsx | 25 ++++++++
.../observability/LogLines.test.tsx | 35 ++++++++---
.../src/components/observability/LogLines.tsx | 44 +++++++------
.../observability/LogsView.test.tsx | 50 ++++++++++++++-
.../src/components/observability/LogsView.tsx | 62 +++++++++++++------
6 files changed, 184 insertions(+), 49 deletions(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx
new file mode 100644
index 000000000..8872f4d43
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx
@@ -0,0 +1,17 @@
+import { describe, it, expect } from "vitest"
+import { render, screen } from "@testing-library/react"
+import { EmptyState } from "./EmptyState.tsx"
+
+describe("EmptyState", () => {
+ it("announces its message as a status", () => {
+ render( )
+
+ expect(screen.getByRole("status")).toHaveTextContent("No data for this range.Pick a longer range.")
+ })
+
+ it("marks itself busy while something is loading", () => {
+ render( )
+
+ expect(screen.getByRole("status")).toHaveAttribute("aria-busy", "true")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx
new file mode 100644
index 000000000..0cf58224b
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx
@@ -0,0 +1,25 @@
+import type { ReactNode } from "react"
+import { cn, Spinner } from "@cozystack/ui"
+
+interface EmptyStateProps {
+ title: ReactNode
+ hint?: ReactNode
+ busy?: boolean
+ tone?: "light" | "dark"
+ className?: string
+}
+
+export function EmptyState({ title, hint, busy, tone = "light", className }: EmptyStateProps) {
+ const dark = tone === "dark"
+ return (
+
+ {busy &&
}
+
{title}
+ {hint &&
{hint}
}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
index 7110fe464..dd8c503eb 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
@@ -10,40 +10,59 @@ function sized(list: HTMLElement, scrollHeight: number) {
}
describe("LogLines", () => {
+ it("keeps the console frame and shows the placeholder inside it when there are no lines", () => {
+ render( )
+
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(frame).toHaveTextContent("Waiting for new log lines…")
+ expect(screen.queryByRole("list")).toBeNull()
+ })
+
+ it("puts the lines in the same frame once they arrive", () => {
+ const { rerender } = render( )
+ const frame = screen.getByRole("region", { name: "Log lines" })
+
+ rerender( )
+
+ expect(screen.getByRole("region", { name: "Log lines" })).toBe(frame)
+ expect(frame).toContainElement(screen.getByRole("list"))
+ expect(frame).not.toHaveTextContent("Waiting")
+ })
+
it("keeps following the newest line while the reader is at the bottom", () => {
- const { rerender } = render( )
+ const { rerender } = render( )
const list = screen.getByRole("list")
sized(list, 500)
- rerender( )
+ rerender( )
expect(list.scrollTop).toBe(500)
})
it("leaves the position alone once the reader scrolls up", () => {
- const { rerender } = render( )
+ const { rerender } = render( )
const list = screen.getByRole("list")
sized(list, 500)
list.scrollTop = 120
fireEvent.scroll(list)
- rerender( )
+ rerender( )
expect(list.scrollTop).toBe(120)
})
it("follows again once the list is cleared for a new run", () => {
- const { rerender } = render( )
+ const { rerender } = render( )
const first = screen.getByRole("list")
sized(first, 500)
first.scrollTop = 120
fireEvent.scroll(first)
- rerender( )
- rerender( )
+ rerender( )
+ rerender( )
const second = screen.getByRole("list")
sized(second, 300)
- rerender( )
+ rerender( )
expect(second.scrollTop).toBe(300)
})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
index 452faac72..14b941ab2 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
@@ -1,10 +1,10 @@
-import { useEffect, useRef } from "react"
+import { useEffect, useRef, type ReactNode } from "react"
import { CONTAINER_FIELD, POD_FIELD } from "../../lib/observability/logs.ts"
import type { LogEntry } from "../../lib/observability/proxy.ts"
const BOTTOM_SLACK_PX = 24
-export function LogLines({ entries }: { entries: LogEntry[] }) {
+export function LogLines({ entries, placeholder }: { entries: LogEntry[]; placeholder: ReactNode }) {
const ref = useRef(null)
const pinned = useRef(true)
@@ -13,23 +13,31 @@ export function LogLines({ entries }: { entries: LogEntry[] }) {
else if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight
}, [entries])
- if (entries.length === 0) return No log lines for this range and filter.
return (
- {
- const el = e.currentTarget
- pinned.current = el.scrollHeight - el.scrollTop - el.clientHeight <= BOTTOM_SLACK_PX
- }}
- className="max-h-[60vh] overflow-auto rounded-md bg-slate-950 p-3 font-mono text-xs leading-5 text-slate-100"
+
- {entries.map((e, i) => (
-
- {new Date(e._time).toLocaleTimeString()} {" "}
- {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`} {" "}
- {e._msg}
-
- ))}
-
+ {entries.length === 0 ? (
+ placeholder
+ ) : (
+ {
+ const el = e.currentTarget
+ pinned.current = el.scrollHeight - el.scrollTop - el.clientHeight <= BOTTOM_SLACK_PX
+ }}
+ className="h-full overflow-auto p-3"
+ >
+ {entries.map((e, i) => (
+
+ {new Date(e._time).toLocaleTimeString()} {" "}
+ {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`} {" "}
+ {e._msg}
+
+ ))}
+
+ )}
+
)
}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 526efeb23..31c6fac0e 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -1,5 +1,5 @@
import { describe, it, expect, vi } from "vitest"
-import { screen, waitFor } from "@testing-library/react"
+import { screen, waitFor, within } from "@testing-library/react"
import userEvent from "@testing-library/user-event"
import { K8sClient } from "@cozystack/k8s-client"
import { renderWithK8sProvider } from "../../test-utils/render.tsx"
@@ -47,9 +47,9 @@ describe("LogsView", () => {
it("narrows to the picked pod with a stream filter", async () => {
const { client, raw } = clientWith(defaultHandler)
renderWithK8sProvider( , { client })
- await screen.findByRole("option", { name: "db-1" })
- await userEvent.selectOptions(screen.getByLabelText("Pod"), "db-1")
+ await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
+ await userEvent.click(await screen.findByRole("option", { name: "db-1" }))
await waitFor(() =>
expect(
@@ -58,6 +58,50 @@ describe("LogsView", () => {
)
})
+ it("offers All next to every pod and starts on it", async () => {
+ const { client } = clientWith(defaultHandler)
+ renderWithK8sProvider( , { client })
+ const pod = screen.getByRole("combobox", { name: "Pod" })
+ expect(pod).toHaveTextContent("All")
+
+ await userEvent.click(pod)
+
+ expect(await screen.findByRole("option", { name: "db-1" })).toBeInTheDocument()
+ expect(screen.getAllByRole("option").map((o) => o.textContent)).toEqual(["All", "db-0", "db-1"])
+ })
+
+ it("keeps the log console and says so inside it when nothing matches", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query") ? new Response("") : defaultHandler(endpoint, params),
+ )
+
+ renderWithK8sProvider( , { client })
+
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(await within(frame).findByText("No log lines for this range and filter.")).toBeInTheDocument()
+ })
+
+ it("shows loading inside the log console", () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query") ? new Promise(() => {}) : defaultHandler(endpoint, params),
+ )
+
+ renderWithK8sProvider( , { client })
+
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(within(frame).getByRole("status")).toHaveAttribute("aria-busy", "true")
+ })
+
+ it("says it is waiting while Follow has no line yet", async () => {
+ const { client } = clientWith(defaultHandler)
+ renderWithK8sProvider( , { client })
+
+ await userEvent.click(screen.getByRole("button", { name: "Follow" }))
+
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(within(frame).getByText("Waiting for new log lines…")).toBeInTheDocument()
+ })
+
it("applies the LogsQL filter on submit", async () => {
const { client, raw } = clientWith(defaultHandler)
renderWithK8sProvider( , { client })
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index a27ca41c5..d2a000010 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -1,7 +1,9 @@
import { useState } from "react"
import { useQuery } from "@tanstack/react-query"
import { useK8sClient } from "@cozystack/k8s-client"
-import { Button, Spinner } from "@cozystack/ui"
+import { Button } from "@cozystack/ui"
+import { Input } from "../ui/input.tsx"
+import { SelectItem, SelectLabel, SelectPopup, SelectRoot, SelectTrigger, SelectValue } from "../ui/select.tsx"
import {
CONTAINER_FIELD,
LOG_RANGES,
@@ -20,6 +22,7 @@ import { useLogTail } from "../../lib/observability/use-log-tail.ts"
import { ProxyNotice } from "./ProxyNotice.tsx"
import { RangePicker } from "./RangePicker.tsx"
import { LogLines } from "./LogLines.tsx"
+import { EmptyState } from "./EmptyState.tsx"
const HISTORY_LIMIT = 500
const FOLLOW_CAP = 1000
@@ -30,27 +33,42 @@ function windowFor(range: LogRangeKey) {
return { start: new Date(end - LOG_RANGES[range] * 1000).toISOString(), end: new Date(end).toISOString() }
}
+const ALL = ""
+
function Picker({ label, value, options, onChange }: { label: string; value: string; options: string[]; onChange: (v: string) => void }) {
+ const items = [{ value: ALL, label: "All" }, ...options.map((o) => ({ value: o, label: o }))]
return (
-
- {label}
- onChange(e.target.value)}
- className="rounded-md border border-slate-200 bg-white px-2 py-1 text-xs text-slate-900"
- >
- All
- {options.map((o) => (
-
- {o}
-
+ onChange(v ?? ALL)}>
+
+ {label}
+
+
+
+
+
+ {items.map((item) => (
+
+ {item.label}
+
))}
-
-
+
+
)
}
+function placeholderFor({ follow, stopped, loading, failed }: { follow: boolean; stopped: boolean; loading: boolean; failed: boolean }) {
+ if (follow) {
+ return stopped ? (
+
+ ) : (
+
+ )
+ }
+ if (loading) return
+ if (failed) return
+ return
+}
+
export function LogsView({ scope }: { scope: ObservabilityScope }) {
const client = useK8sClient()
const root = logsRoot(scope)
@@ -107,12 +125,12 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
setFilter(draft)
}}
>
- setDraft(e.target.value)}
+ onValueChange={setDraft}
placeholder='LogsQL filter, e.g. error or "connection refused"'
- className="w-full rounded-md border border-slate-200 px-2.5 py-1.5 font-mono text-xs"
+ className="font-mono"
/>
{ setPod(v); setContainer("") }} />
@@ -129,7 +147,11 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
)}
{error ? : null}
- {!follow && history.isLoading ? : }
+
)
}
From f5ebb43c4bdce908887029568065eb27785ff833 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:48:43 +0700
Subject: [PATCH 027/114] fix(dashboard): match the range picker to the log
toolbar controls
Next to the new Select and Input the range picker was shorter, lighter
bordered and had no focus ring, so the toolbar looked uneven and a
keyboard user could not see which range had focus. It now shares their
height, border and focus ring, and its buttons are grouped under one
accessible name.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/RangePicker.test.tsx | 23 +++++++++++++++++++
.../components/observability/RangePicker.tsx | 4 ++--
2 files changed, 25 insertions(+), 2 deletions(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx
new file mode 100644
index 000000000..9787963b2
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx
@@ -0,0 +1,23 @@
+import { describe, it, expect, vi } from "vitest"
+import { render, screen, within } from "@testing-library/react"
+import userEvent from "@testing-library/user-event"
+import { RangePicker } from "./RangePicker.tsx"
+
+describe("RangePicker", () => {
+ it("groups the ranges under one name and marks the chosen one pressed", () => {
+ render( {}} />)
+
+ const group = screen.getByRole("group", { name: "Time range" })
+ expect(within(group).getByRole("button", { name: "6h" })).toHaveAttribute("aria-pressed", "true")
+ expect(within(group).getByRole("button", { name: "1h" })).toHaveAttribute("aria-pressed", "false")
+ })
+
+ it("reports the range clicked", async () => {
+ const onChange = vi.fn()
+ render( )
+
+ await userEvent.click(screen.getByRole("button", { name: "1h" }))
+
+ expect(onChange).toHaveBeenCalledWith("1h")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
index 22030c706..95d245176 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
@@ -10,7 +10,7 @@ export function RangePicker({
onChange: (value: K) => void
}) {
return (
-
+
{options.map((key) => (
({
aria-pressed={value === key}
onClick={() => onChange(key)}
className={cn(
- "rounded px-2.5 py-1 text-xs",
+ "h-full rounded-md px-2.5 text-xs outline-none focus-visible:ring-2 focus-visible:ring-slate-400",
value === key ? "bg-slate-900 text-white" : "text-slate-600 hover:bg-slate-100",
)}
>
From 05b0ea0c4558c5363cc38d79b7549548de7fe504 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:49:39 +0700
Subject: [PATCH 028/114] fix(dashboard): hold each metrics chart's place while
empty or loading
A metrics panel with no series, or still loading, shrank to one line of
small text or a bare spinner, so the tab jumped by the chart's height
when data arrived and an empty range looked like a broken panel. Both
states now fill the chart's slot and say what is going on, the same way
the log console does.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/MetricsView.test.tsx | 16 +++++++++++++++-
.../src/components/observability/MetricsView.tsx | 9 ++++++---
2 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
index f09e15201..b4c7fc3a4 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
@@ -22,7 +22,7 @@ function matrix(pod: string) {
})
}
-function clientWith(handler: (url: URL) => Response) {
+function clientWith(handler: (url: URL) => Response | Promise) {
const client = new K8sClient()
const raw = vi.spyOn(client, "raw").mockImplementation(async (path) => handler(new URL(path, "http://x")))
return { client, raw }
@@ -84,5 +84,19 @@ describe("MetricsView", () => {
renderWithK8sProvider( , { client })
await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(3))
+ for (const status of screen.getAllByRole("status")) expect(status).toHaveClass("h-[230px]")
+ })
+
+ it("holds each chart's place while it loads", () => {
+ const { client } = clientWith(() => new Promise(() => {}))
+
+ renderWithK8sProvider( , { client })
+
+ const statuses = screen.getAllByRole("status")
+ expect(statuses).toHaveLength(3)
+ for (const status of statuses) {
+ expect(status).toHaveAttribute("aria-busy", "true")
+ expect(status).toHaveClass("h-[230px]")
+ }
})
})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
index 35c8e8417..34d3b33a0 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
@@ -1,7 +1,7 @@
import { useState } from "react"
import { useQueries } from "@tanstack/react-query"
import { useK8sClient } from "@cozystack/k8s-client"
-import { Section, Spinner } from "@cozystack/ui"
+import { Section } from "@cozystack/ui"
import {
METRIC_PANELS,
RANGES,
@@ -13,8 +13,11 @@ import { isScopeWide, metricsRoot, queryRange, type ObservabilityScope } from ".
import { ProxyNotice } from "./ProxyNotice.tsx"
import { TimeSeriesChart } from "./TimeSeriesChart.tsx"
import { RangePicker } from "./RangePicker.tsx"
+import { EmptyState } from "./EmptyState.tsx"
const REFRESH_MS = 60_000
+// The uPlot canvas plus its one-row legend, so a placeholder takes the chart's place without a jump.
+const CHART_SLOT = "h-[230px] rounded-md border border-dashed border-slate-200 bg-slate-50"
export function MetricsView({ scope }: { scope: ObservabilityScope }) {
const client = useK8sClient()
@@ -57,9 +60,9 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
{error ? (
) : isLoading || !data ? (
-
+
) : data.series.length === 0 ? (
- No data for this range.
+
) : (
)}
From 6740ed445899fdd689ef7676840bc2d7a74936bc Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:53:47 +0700
Subject: [PATCH 029/114] fix(dashboard): offer All once when a log stream has
no pod name
VictoriaLogs can return an empty value for the pod or container field
when some streams lack the label. That empty value is the picker's own
"All" sentinel, so the list showed two selected "All" entries with the
same key. Empty values are dropped from the field values.
The tests now also pin what the log console says when Follow stops
before any line and when the history query fails, so those messages
cannot silently swap with the empty-result one.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../components/observability/LogsView.test.tsx | 18 ++++++++++++++++++
.../src/components/observability/LogsView.tsx | 2 +-
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 31c6fac0e..47b6b4ef9 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -135,6 +135,8 @@ describe("LogsView", () => {
expect(await screen.findByText(/Stopped/)).toBeInTheDocument()
expect(screen.getByText("pipe not allowed in tail")).toBeInTheDocument()
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(within(frame).getByText("Follow stopped before any line arrived.")).toBeInTheDocument()
})
it("renders a cluster without the proxy as not available", async () => {
@@ -165,6 +167,22 @@ describe("LogsView", () => {
expect(await screen.findByText("cannot parse query")).toBeInTheDocument()
expect(screen.getByLabelText("LogsQL filter")).toBeInTheDocument()
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(within(frame).getByText("No log lines to show.")).toBeInTheDocument()
+ })
+
+ it("offers All once when some lines carry no pod name", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("stream_field_values") && params.get("field") === "kubernetes_pod_name"
+ ? values("", "db-0")
+ : defaultHandler(endpoint, params),
+ )
+ renderWithK8sProvider( , { client })
+
+ await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
+
+ await screen.findByRole("option", { name: "db-0" })
+ expect(screen.getAllByRole("option").map((o) => o.textContent)).toEqual(["All", "db-0"])
})
it("names the source pod and container of each line", async () => {
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index d2a000010..1976c7dfc 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -36,7 +36,7 @@ function windowFor(range: LogRangeKey) {
const ALL = ""
function Picker({ label, value, options, onChange }: { label: string; value: string; options: string[]; onChange: (v: string) => void }) {
- const items = [{ value: ALL, label: "All" }, ...options.map((o) => ({ value: o, label: o }))]
+ const items = [{ value: ALL, label: "All" }, ...options.filter((o) => o !== ALL).map((o) => ({ value: o, label: o }))]
return (
onChange(v ?? ALL)}>
From 554dad87bb18ece7adf5afd02c050a3520cf759a Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:53:58 +0700
Subject: [PATCH 030/114] fix(dashboard): give the range picker the toolbar's
focus ring offset
The range buttons drew their focus ring flush against the button while
the Select and Input next to them offset it, so keyboard focus looked
different from one control to the next. The empty metrics hint also
repeated its title; it now suggests the one thing that can help.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../apps/console/src/components/observability/MetricsView.tsx | 2 +-
.../apps/console/src/components/observability/RangePicker.tsx | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
index 34d3b33a0..ce7bcbe38 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
@@ -62,7 +62,7 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
) : isLoading || !data ? (
) : data.series.length === 0 ? (
-
+
) : (
)}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
index 95d245176..49abd8b1d 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
@@ -18,7 +18,7 @@ export function RangePicker({
aria-pressed={value === key}
onClick={() => onChange(key)}
className={cn(
- "h-full rounded-md px-2.5 text-xs outline-none focus-visible:ring-2 focus-visible:ring-slate-400",
+ "h-full rounded-md px-2.5 text-xs outline-none focus-visible:ring-2 focus-visible:ring-slate-400 focus-visible:ring-offset-2",
value === key ? "bg-slate-900 text-white" : "text-slate-600 hover:bg-slate-100",
)}
>
From c9a15cb8aed81e3b72d4e703c381474c6dc4db2b Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:56:01 +0700
Subject: [PATCH 031/114] fix(dashboard): suggest a longer metrics range only
when there is one
With the longest range already picked, the empty panel still told the
reader to try a longer range, which the picker cannot offer.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/components/observability/MetricsView.test.tsx | 11 +++++++++++
.../src/components/observability/MetricsView.tsx | 6 ++++--
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
index b4c7fc3a4..d3f1fc1dd 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
@@ -87,6 +87,17 @@ describe("MetricsView", () => {
for (const status of screen.getAllByRole("status")) expect(status).toHaveClass("h-[230px]")
})
+ it("suggests a longer range only while one exists", async () => {
+ const { client } = clientWith(() => Response.json({ status: "success", data: { resultType: "matrix", result: [] } }))
+ renderWithK8sProvider( , { client })
+ await waitFor(() => expect(screen.getAllByText("Try a longer range.")).toHaveLength(3))
+
+ await userEvent.click(screen.getByRole("button", { name: "7d" }))
+
+ await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(3))
+ expect(screen.queryByText("Try a longer range.")).toBeNull()
+ })
+
it("holds each chart's place while it loads", () => {
const { client } = clientWith(() => new Promise(() => {}))
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
index ce7bcbe38..fac2a6b91 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
@@ -23,6 +23,8 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
const client = useK8sClient()
const root = metricsRoot(scope)
const [range, setRange] = useState("1h")
+ const rangeKeys = Object.keys(RANGES) as RangeKey[]
+ const longestRange = range === rangeKeys[rangeKeys.length - 1]
const results = useQueries({
queries: METRIC_PANELS.map((panel) => ({
@@ -48,7 +50,7 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
Resource consumption
-
+
{wide ? (
@@ -62,7 +64,7 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
) : isLoading || !data ? (
) : data.series.length === 0 ? (
-
+
) : (
)}
From fca5ca78de81666be249769a8496d4f0567cc022 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 19:04:29 +0700
Subject: [PATCH 032/114] fix(dashboard): treat an open log tail as live before
its first line
The tail counted as live, cleared its error and reset its backoff only
when a line arrived. A quiet application therefore showed "Connecting"
forever, a 429 warning stuck after the retry succeeded, and every idle
close by the proxy (its tail idle timeout) raised the backoff towards
30s and reconnected without since, losing lines written in the gap.
The tail now reports when the response opens. That marks it live,
clears the error and resets the backoff; a clean close reconnects at
the base delay; and with no line seen yet the reconnect resumes from
the time the stream opened.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/lib/observability/proxy.test.ts | 18 +++++++++
.../console/src/lib/observability/proxy.ts | 2 +
.../lib/observability/use-log-tail.test.tsx | 37 +++++++++++++++++++
.../src/lib/observability/use-log-tail.ts | 19 +++++++---
4 files changed, 71 insertions(+), 5 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
index 69a8c7c8c..2809905d8 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
@@ -194,6 +194,24 @@ describe("tailLogs", () => {
expect(err).toMatchObject({ kind: "forbidden" })
})
+
+ it("says the stream opened before any line arrives", async () => {
+ const { client } = clientReturning(new Response(""))
+ const onOpen = vi.fn()
+
+ await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}, undefined, onOpen)
+
+ expect(onOpen).toHaveBeenCalledTimes(1)
+ })
+
+ it("does not say the stream opened when the tail is refused", async () => {
+ const { client } = clientReturning(new Response("", { status: 429 }))
+ const onOpen = vi.fn()
+
+ await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}, undefined, onOpen).catch(() => {})
+
+ expect(onOpen).not.toHaveBeenCalled()
+ })
})
describe("isScopeWide", () => {
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
index c9fce3d4f..3b5567f6b 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -202,8 +202,10 @@ export async function tailLogs(
params: { query: string; since?: string },
onEntry: (entry: LogEntry) => void,
signal?: AbortSignal,
+ onOpen?: () => void,
): Promise {
const res = await proxyGet(client, `${root}select/logsql/tail`, params, signal)
+ onOpen?.()
if (!res.body) return
const reader = res.body.getReader()
const decoder = new TextDecoder()
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
index 0c2306613..b5232d18e 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
@@ -10,6 +10,10 @@ function ndjson(...lines: string[]): Response {
return new Response(lines.map((l) => `${l}\n`).join(""))
}
+function openStream(): Response {
+ return new Response(new ReadableStream({ start() {} }))
+}
+
function setup(...responses: Response[]) {
const client = new K8sClient()
const raw = vi.spyOn(client, "raw")
@@ -75,6 +79,39 @@ describe("useLogTail", () => {
await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a"]))
})
+ it("goes live and clears a load warning once the stream opens, before any line", async () => {
+ const { wrapper } = setup(new Response("too many concurrent requests", { status: 429 }), openStream())
+
+ const { result } = renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper })
+
+ await waitFor(() => expect(result.current.state).toBe("live"))
+ expect(result.current.error).toBeNull()
+ })
+
+ it("reconnects after an idle close at the base delay, from when the quiet stream opened", async () => {
+ const delays: number[] = []
+ const realSetTimeout = globalThis.setTimeout
+ const spy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((fn: () => void, ms?: number) => {
+ if (ms !== undefined && ms % 7 === 0 && ms > 0) delays.push(ms)
+ return realSetTimeout(fn, ms)
+ }) as typeof setTimeout)
+ const refused = () => new Response("", { status: 429 })
+ const { raw, wrapper } = setup(refused(), refused(), ndjson())
+
+ try {
+ renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 7 }), { wrapper })
+ await waitFor(() => expect(raw).toHaveBeenCalledTimes(4))
+ } finally {
+ spy.mockRestore()
+ }
+
+ expect(delays).toEqual([7, 14, 7])
+ expect(new URL(raw.mock.calls[2][0], "http://x").searchParams.has("since")).toBe(false)
+ const since = new URL(raw.mock.calls[3][0], "http://x").searchParams.get("since")
+ expect(since).not.toBeNull()
+ expect(new Date(since!).toISOString()).toBe(since)
+ })
+
it("does not connect while disabled", () => {
const { raw, wrapper } = setup()
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
index 53cbaf856..2e3b4d706 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
@@ -66,15 +66,22 @@ export function useLogTail(
}
const run = async () => {
- for (let attempt = 0; !signal.aborted; attempt++) {
+ let failures = 0
+ let openedAt: string | undefined
+ while (!signal.aborted) {
resumedFrom = lastTime
+ let delay = retryDelayMs
try {
- await tailLogs(client, root, { query, since: lastTime }, (entry) => {
- attempt = 0
+ await tailLogs(client, root, { query, since: lastTime ?? openedAt }, (entry) => {
if (!lastTime || compareTime(entry._time, lastTime) > 0) lastTime = entry._time
pending.push(entry)
timer ??= setTimeout(flush, FLUSH_MS)
- }, signal)
+ }, signal, () => {
+ failures = 0
+ // With no line yet, the next reconnect resumes from here so the gap is not lost.
+ openedAt = new Date().toISOString()
+ update({ state: "live", error: null })
+ })
flush()
} catch (err) {
flush()
@@ -84,10 +91,12 @@ export function useLogTail(
return
}
update({ error: err })
+ delay = Math.min(retryDelayMs * 2 ** failures, MAX_RETRY_MS)
+ failures++
}
if (signal.aborted) return
update({ state: "reconnecting" })
- await sleep(Math.min(retryDelayMs * 2 ** attempt, MAX_RETRY_MS), signal)
+ await sleep(delay, signal)
}
}
void run()
From 881eea524693248cdcfb4421bad190cbf7b586c8 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 19:04:58 +0700
Subject: [PATCH 033/114] fix(dashboard): suggest a longer log range only when
there is one
On 24h, the longest range the log view offers, the empty console still
told the reader to try a longer range. It now suggests only a broader
filter there, as the metrics panels already do.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/components/observability/LogsView.test.tsx | 14 ++++++++++++++
.../src/components/observability/LogsView.tsx | 11 +++++++----
2 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 47b6b4ef9..1b18dd208 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -81,6 +81,20 @@ describe("LogsView", () => {
expect(await within(frame).findByText("No log lines for this range and filter.")).toBeInTheDocument()
})
+ it("suggests a longer range only while one exists", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query") ? new Response("") : defaultHandler(endpoint, params),
+ )
+ renderWithK8sProvider( , { client })
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(await within(frame).findByText("Try a longer range or a broader filter.")).toBeInTheDocument()
+
+ await userEvent.click(screen.getByRole("button", { name: "24h" }))
+
+ expect(await within(frame).findByText("Try a broader filter.")).toBeInTheDocument()
+ expect(within(frame).queryByText(/longer range/)).toBeNull()
+ })
+
it("shows loading inside the log console", () => {
const { client } = clientWith((endpoint, params) =>
endpoint.endsWith("query") ? new Promise(() => {}) : defaultHandler(endpoint, params),
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index 1976c7dfc..a0a73cc43 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -56,7 +56,10 @@ function Picker({ label, value, options, onChange }: { label: string; value: str
)
}
-function placeholderFor({ follow, stopped, loading, failed }: { follow: boolean; stopped: boolean; loading: boolean; failed: boolean }) {
+const RANGE_KEYS = Object.keys(LOG_RANGES) as LogRangeKey[]
+const LONGEST_RANGE = RANGE_KEYS[RANGE_KEYS.length - 1]
+
+function placeholderFor({ follow, stopped, loading, failed, longestRange }: { follow: boolean; stopped: boolean; loading: boolean; failed: boolean; longestRange: boolean }) {
if (follow) {
return stopped ? (
@@ -66,7 +69,7 @@ function placeholderFor({ follow, stopped, loading, failed }: { follow: boolean;
}
if (loading) return
if (failed) return
- return
+ return
}
export function LogsView({ scope }: { scope: ObservabilityScope }) {
@@ -135,7 +138,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
{ setPod(v); setContainer("") }} />
- {!follow && }
+ {!follow && }
setFollowRun(follow ? null : Date.now())}>
Follow
@@ -150,7 +153,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
)
From 9498c9bff520d73fe200f5043bc7efce9eb901ef Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 19:05:42 +0700
Subject: [PATCH 034/114] fix(dashboard): keep the proxy's own advice on a
limit refusal
The notice appended "Choose a shorter range." to every 422, so a
message that already ended in "use a shorter range" said it twice and
one that ended in "raise step" got advice that contradicted it. The
console's own advice is now added only when the proxy gave none.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/components/observability/ProxyNotice.test.tsx | 9 +++++++++
.../console/src/components/observability/ProxyNotice.tsx | 3 ++-
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
index fd001bcae..6e85ced33 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
@@ -29,6 +29,15 @@ describe("ProxyNotice", () => {
expect(screen.getByRole("alert")).toHaveTextContent(text)
})
+ it.each([
+ "the application has more than 50 pods in this window; use a shorter range",
+ "1200 points per series exceed the limit of 1000; raise step",
+ ])("does not add advice to a limit message that already carries its own: %s", (message) => {
+ render( )
+
+ expect(screen.getByRole("alert").textContent).toBe(`${message}.`)
+ })
+
it("shows the server message for a rejected query", () => {
render( )
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
index a60b831b0..ee68e4633 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
@@ -20,7 +20,8 @@ function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: bool
case "forbidden":
return { quiet: false, text: `You do not have permission to read ${lower} here.` }
case "limit":
- return { quiet: false, text: `${error.message}. Choose a shorter range.` }
+ // The proxy puts its own remedy after "; " (e.g. "use a shorter range", "raise step").
+ return { quiet: false, text: error.message.includes("; ") ? `${error.message}.` : `${error.message}. Choose a shorter range.` }
case "rate-limited":
return { quiet: false, text: "Too many requests. Try again in a moment." }
case "timeout":
From 76441871b8c3ae73f40ecf29fd454a468f1e0573 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 19:06:59 +0700
Subject: [PATCH 035/114] fix(dashboard): keep caller headers of any shape in
the k8s client
The client spread init.headers as a plain object, so a Headers
instance or a list of pairs passed by a caller was silently dropped
and the request went out without them. Headers are now built with the
Headers constructor, which accepts every shape fetch does.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../k8s-client/client.headers.test.ts | 31 +++++++++++++++++++
.../__tests__/k8s-client/client.raw.test.ts | 20 +++++++++++-
.../console/packages/k8s-client/src/client.ts | 14 +++------
3 files changed, 55 insertions(+), 10 deletions(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts
diff --git a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts
new file mode 100644
index 000000000..37985fca5
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts
@@ -0,0 +1,31 @@
+import { describe, it, expect, vi, afterEach } from "vitest"
+import { K8sClient } from "@cozystack/k8s-client"
+
+afterEach(() => {
+ vi.unstubAllGlobals()
+})
+
+describe("K8sClient request headers", () => {
+ it("lets a method's own content type replace the JSON default", async () => {
+ const fetchMock = vi.fn(async () => new Response("{}"))
+ vi.stubGlobal("fetch", fetchMock)
+ const client = new K8sClient({ getToken: async () => "t0k" })
+
+ await client.patch("apps", "v1", "deployments", "web", { spec: {} }, "default", "merge")
+
+ const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
+ const headers = new Headers(init.headers)
+ expect(headers.get("Content-Type")).toBe("application/merge-patch+json")
+ expect(headers.get("Authorization")).toBe("Bearer t0k")
+ })
+
+ it("sends JSON by default", async () => {
+ const fetchMock = vi.fn(async () => new Response("{}"))
+ vi.stubGlobal("fetch", fetchMock)
+
+ await new K8sClient().getApiGroups()
+
+ const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
+ expect(new Headers(init.headers).get("Content-Type")).toBe("application/json")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
index ab6c056f5..0ac0d75ae 100644
--- a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
@@ -24,7 +24,25 @@ describe("K8sClient.raw", () => {
const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
expect(url).toBe("/base/apis/x")
- expect(init.headers).toEqual({ Accept: "application/json", Authorization: "Bearer t0k" })
+ const headers = new Headers(init.headers)
+ expect(headers.get("Accept")).toBe("application/json")
+ expect(headers.get("Authorization")).toBe("Bearer t0k")
+ })
+
+ it.each([
+ ["a Headers instance", new Headers({ Accept: "application/x-ndjson" })],
+ ["a tuple list", [["Accept", "application/x-ndjson"]] as [string, string][]],
+ ])("keeps caller headers passed as %s", async (_, given) => {
+ const fetchMock = vi.fn(async () => new Response("{}"))
+ vi.stubGlobal("fetch", fetchMock)
+ const client = new K8sClient({ getToken: async () => "t0k" })
+
+ await client.raw("/x", { headers: given })
+
+ const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
+ const headers = new Headers(init.headers)
+ expect(headers.get("Accept")).toBe("application/x-ndjson")
+ expect(headers.get("Authorization")).toBe("Bearer t0k")
})
it("routes a 401 through the client's unauthorized handler", async () => {
diff --git a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
index 04c528760..64af1131d 100644
--- a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
+++ b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
@@ -50,14 +50,12 @@ export class K8sClient {
}
private async request(path: string, init?: RequestInit): Promise {
- const headers: Record = {
- "Content-Type": "application/json",
- ...(init?.headers as Record),
- }
+ const headers = new Headers(init?.headers)
+ if (!headers.has("Content-Type")) headers.set("Content-Type", "application/json")
if (this.getToken) {
const token = await this.getToken()
- headers["Authorization"] = `Bearer ${token}`
+ headers.set("Authorization", `Bearer ${token}`)
}
let res: Response
@@ -247,11 +245,9 @@ export class K8sClient {
* Kubernetes JSON, such as the observability proxy's streamed NDJSON.
*/
async raw(path: string, init?: RequestInit): Promise {
- const headers: Record = {
- ...(init?.headers as Record),
- }
+ const headers = new Headers(init?.headers)
if (this.getToken) {
- headers["Authorization"] = `Bearer ${await this.getToken()}`
+ headers.set("Authorization", `Bearer ${await this.getToken()}`)
}
const res = await fetch(`${this.baseUrl}${path}`, { ...init, headers })
if (res.status === 401) this.handleUnauthorized()
From 2a473086cf6cd1d245e014ed56ed44f9594038cd Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 19:11:05 +0700
Subject: [PATCH 036/114] fix(dashboard): advise a shorter range unless the
proxy already did
Any "; " in a 422 message was taken as the proxy's own remedy, but the
proxy passes vmselect's 422 bodies through, and their advice (raise
the -search.max* flags) is for operators. Those lost the one thing a
tenant can do. Only the remedies the proxy itself appends now replace
the console's advice.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/components/observability/ProxyNotice.test.tsx | 7 +++++++
.../src/components/observability/ProxyNotice.tsx | 10 ++++++++--
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
index 6e85ced33..c4b4e89c1 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
@@ -38,6 +38,13 @@ describe("ProxyNotice", () => {
expect(screen.getByRole("alert").textContent).toBe(`${message}.`)
})
+ it("still advises a shorter range for a store limit whose own advice is for operators", () => {
+ const message = "the number of matching timeseries exceeds 300000; either narrow down the search or increase -search.max* command-line flag values"
+ render( )
+
+ expect(screen.getByRole("alert").textContent).toBe(`${message}. Choose a shorter range.`)
+ })
+
it("shows the server message for a rejected query", () => {
render( )
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
index ee68e4633..1fb30c8ff 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
@@ -5,6 +5,10 @@ interface ProxyNoticeProps {
error: unknown
}
+// Remedies the observability proxy itself appends; vmselect's 422 text carries
+// advice for operators (raise -search.max* flags), so it still gets ours.
+const PROXY_REMEDIES = ["; use a shorter range", "; raise step"]
+
function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: boolean; text: string } {
const lower = what.toLowerCase()
if (!(error instanceof ProxyError)) {
@@ -20,8 +24,10 @@ function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: bool
case "forbidden":
return { quiet: false, text: `You do not have permission to read ${lower} here.` }
case "limit":
- // The proxy puts its own remedy after "; " (e.g. "use a shorter range", "raise step").
- return { quiet: false, text: error.message.includes("; ") ? `${error.message}.` : `${error.message}. Choose a shorter range.` }
+ return {
+ quiet: false,
+ text: PROXY_REMEDIES.some((r) => error.message.endsWith(r)) ? `${error.message}.` : `${error.message}. Choose a shorter range.`,
+ }
case "rate-limited":
return { quiet: false, text: "Too many requests. Try again in a moment." }
case "timeout":
From 1e1fdb740c229f7233e3be4993d063c2fd5cc5fb Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 00:14:24 +0700
Subject: [PATCH 037/114] fix(dashboard): show log rows a pipe stripped of
their timestamp
A LogsQL pipe such as `fields _msg` or `stats` returns rows without
_time, and the console dropped every one of them, so a successful
query read as "no lines matched". Such rows are now kept in the order
the store sent them, shown without a timestamp, and never
deduplicated on a tail resume. A row without _msg shows its remaining
fields instead of an empty line.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/LogLines.test.tsx | 12 +++++++
.../src/components/observability/LogLines.tsx | 33 +++++++++++++++----
.../observability/LogsView.test.tsx | 13 ++++++++
.../src/lib/observability/logs.test.ts | 6 ++++
.../console/src/lib/observability/logs.ts | 5 +--
.../src/lib/observability/proxy.test.ts | 20 ++++++++++-
.../console/src/lib/observability/proxy.ts | 10 +++---
.../src/lib/observability/use-log-tail.ts | 2 +-
8 files changed, 86 insertions(+), 15 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
index dd8c503eb..a0f519852 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
@@ -29,6 +29,18 @@ describe("LogLines", () => {
expect(frame).not.toHaveTextContent("Waiting")
})
+ it("shows a line without _time with no timestamp and no empty source", () => {
+ render( )
+
+ expect(screen.getByRole("listitem")).toHaveTextContent(/^visible message$/)
+ })
+
+ it("shows the remaining fields of a row without _msg", () => {
+ render( )
+
+ expect(screen.getByRole("listitem")).toHaveTextContent(/^db-0\/ rows=42$/)
+ })
+
it("keeps following the newest line while the reader is at the bottom", () => {
const { rerender } = render( )
const list = screen.getByRole("list")
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
index 14b941ab2..f29af017c 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
@@ -3,6 +3,22 @@ import { CONTAINER_FIELD, POD_FIELD } from "../../lib/observability/logs.ts"
import type { LogEntry } from "../../lib/observability/proxy.ts"
const BOTTOM_SLACK_PX = 24
+const SHOWN_APART = new Set(["_time", "_msg", "_stream_id", POD_FIELD, CONTAINER_FIELD])
+
+function source(e: LogEntry): string | null {
+ const pod = e[POD_FIELD]
+ const container = e[CONTAINER_FIELD]
+ return pod || container ? `${pod ?? ""}/${container ?? ""}` : null
+}
+
+// A `fields` or `stats` pipe can leave a row without _msg; its other fields are then the content.
+function message(e: LogEntry): string {
+ if (e._msg !== undefined) return e._msg
+ return Object.entries(e)
+ .filter(([k, v]) => v !== undefined && !SHOWN_APART.has(k))
+ .map(([k, v]) => `${k}=${v}`)
+ .join(" ")
+}
export function LogLines({ entries, placeholder }: { entries: LogEntry[]; placeholder: ReactNode }) {
const ref = useRef(null)
@@ -29,13 +45,16 @@ export function LogLines({ entries, placeholder }: { entries: LogEntry[]; placeh
}}
className="h-full overflow-auto p-3"
>
- {entries.map((e, i) => (
-
- {new Date(e._time).toLocaleTimeString()} {" "}
- {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`} {" "}
- {e._msg}
-
- ))}
+ {entries.map((e, i) => {
+ const from = source(e)
+ return (
+
+ {e._time !== undefined && <>{new Date(e._time).toLocaleTimeString()} {" "}>}
+ {from !== null && <>{from} {" "}>}
+ {message(e)}
+
+ )
+ })}
)}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 1b18dd208..770c8a263 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -199,6 +199,19 @@ describe("LogsView", () => {
expect(screen.getAllByRole("option").map((o) => o.textContent)).toEqual(["All", "db-0"])
})
+ it("shows rows of a query whose pipe removed _time", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query") ? new Response('{"_msg":"visible message"}\n') : defaultHandler(endpoint, params),
+ )
+ renderWithK8sProvider( , { client })
+
+ await userEvent.type(screen.getByLabelText("LogsQL filter"), "* | fields _msg{Enter}")
+
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(await within(frame).findByText("visible message")).toBeInTheDocument()
+ expect(within(frame).queryByText(/No log lines/)).toBeNull()
+ })
+
it("names the source pod and container of each line", async () => {
const { client } = clientWith(defaultHandler)
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts
index 25d26a23a..62d1ac249 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts
@@ -57,6 +57,12 @@ describe("mergeTail", () => {
expect(out).toHaveLength(2)
})
+ it("never deduplicates lines without _time, even on a resume", () => {
+ const out = mergeTail([{ _msg: "a" }], [{ _msg: "a" }], 10, "2026-01-01T00:00:02Z")
+
+ expect(out).toHaveLength(2)
+ })
+
it("does not deduplicate outside a resume", () => {
const out = mergeTail([e("t1", "a")], [e("t1", "a")], 10)
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts
index a6255e541..eb24a5f96 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts
@@ -61,12 +61,13 @@ export function compareTime(a: string, b: string): number {
const entryKey = (e: LogEntry) => `${e._stream_id ?? ""}\u0000${e._time}`
// (_stream_id, _time) is not unique in VictoriaLogs, so dedup applies only to
-// lines a resumed tail replays at or before the resume point.
+// lines a resumed tail replays at or before the resume point, and never to
+// lines a pipe stripped of _time.
export function mergeTail(buffer: LogEntry[], incoming: LogEntry[], cap: number, resumedFrom?: string): LogEntry[] {
let fresh = incoming
if (resumedFrom) {
const seen = new Set(buffer.map(entryKey))
- fresh = incoming.filter((e) => compareTime(e._time, resumedFrom) > 0 || !seen.has(entryKey(e)))
+ fresh = incoming.filter((e) => e._time === undefined || compareTime(e._time, resumedFrom) > 0 || !seen.has(entryKey(e)))
}
const merged = buffer.concat(fresh)
return merged.length > cap ? merged.slice(merged.length - cap) : merged
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
index 2809905d8..f2ba8d7c3 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
@@ -122,6 +122,15 @@ describe("queryLogs", () => {
expect(entries.map((e) => e._msg)).toEqual(["a", "b"])
})
+ it("keeps rows a pipe stripped of _time, in the order the store sent them", async () => {
+ const ndjson = ['{"_msg":"visible message"}', '{"_msg":"second"}', "[1]", "null", "{torn"].join("\n")
+ const { client } = clientReturning(new Response(ndjson))
+
+ const entries = await queryLogs(client, logsRoot(ns), { query: "* | fields _msg", start: "s", end: "e", limit: 100 })
+
+ expect(entries).toEqual([{ _msg: "visible message" }, { _msg: "second" }])
+ })
+
it("keeps a plain-text error body as the message", async () => {
const { client } = clientReturning(new Response("cannot parse query", { status: 400 }))
@@ -170,7 +179,7 @@ describe("tailLogs", () => {
},
})
const { client, raw } = clientReturning(new Response(body))
- const seen: string[] = []
+ const seen: (string | undefined)[] = []
await tailLogs(client, logsRoot(ns), { query: "*", since: "t0" }, (e) => seen.push(e._msg))
@@ -179,6 +188,15 @@ describe("tailLogs", () => {
expect(seen).toEqual(["a", "b"])
})
+ it("delivers a line without _time", async () => {
+ const { client } = clientReturning(new Response('{"_msg":"untimed"}\n'))
+ const seen: (string | undefined)[] = []
+
+ await tailLogs(client, logsRoot(ns), { query: "* | fields _msg" }, (e) => seen.push(e._msg))
+
+ expect(seen).toEqual(["untimed"])
+ })
+
it("omits since on the first connect", async () => {
const { client, raw } = clientReturning(new Response(""))
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
index 3b5567f6b..c28bd12d4 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -140,9 +140,10 @@ export async function queryRange(
return body.data?.result ?? []
}
+// A pipe such as `fields _msg` or `stats` can drop _time and _msg.
export interface LogEntry {
- _time: string
- _msg: string
+ _time?: string
+ _msg?: string
_stream_id?: string
[field: string]: string | undefined
}
@@ -151,7 +152,7 @@ function parseLine(line: string): LogEntry | null {
if (!line.trim()) return null
try {
const value: unknown = JSON.parse(line)
- if (value && typeof value === "object" && "_time" in value) return value as LogEntry
+ if (value && typeof value === "object" && !Array.isArray(value)) return value as LogEntry
} catch {
// a torn or non-JSON line carries nothing to show
}
@@ -173,7 +174,8 @@ export async function queryLogs(
): Promise {
const res = await proxyGet(client, `${root}select/logsql/query`, { ...params }, signal)
const entries = (await res.text()).split("\n").map(parseLine).filter((e) => e !== null)
- return entries.sort((a, b) => compareTime(a._time, b._time))
+ if (entries.some((e) => e._time === undefined)) return entries
+ return entries.sort((a, b) => compareTime(a._time ?? "", b._time ?? ""))
}
export interface FieldValuesParams {
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
index 2e3b4d706..d55bd5708 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
@@ -73,7 +73,7 @@ export function useLogTail(
let delay = retryDelayMs
try {
await tailLogs(client, root, { query, since: lastTime ?? openedAt }, (entry) => {
- if (!lastTime || compareTime(entry._time, lastTime) > 0) lastTime = entry._time
+ if (entry._time && (!lastTime || compareTime(entry._time, lastTime) > 0)) lastTime = entry._time
pending.push(entry)
timer ??= setTimeout(flush, FLUSH_MS)
}, signal, () => {
From 8e6048eb4a2556cc9af587b4df3259cface49b37 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 00:16:04 +0700
Subject: [PATCH 038/114] fix(dashboard): make metric samples readable without
a pointer
The charts are a canvas and their legend showed dashes until a point
was hovered, so keyboard and screen-reader users could not read any
sample. The legend now shows each series' latest value when nothing is
hovered, and every panel has a Show data toggle that opens a focusable
table of all samples, newest first, in the chart's units.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/MetricsView.test.tsx | 27 ++++++++++-
.../components/observability/MetricsView.tsx | 36 ++++++++++++--
.../observability/SeriesTable.test.tsx | 37 +++++++++++++++
.../components/observability/SeriesTable.tsx | 47 +++++++++++++++++++
.../observability/TimeSeriesChart.test.tsx | 17 ++++++-
.../observability/TimeSeriesChart.tsx | 4 +-
6 files changed, 160 insertions(+), 8 deletions(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
index d3f1fc1dd..8e7829ed5 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
@@ -1,5 +1,5 @@
import { describe, it, expect, vi } from "vitest"
-import { screen, waitFor } from "@testing-library/react"
+import { screen, waitFor, within } from "@testing-library/react"
import userEvent from "@testing-library/user-event"
import { K8sClient } from "@cozystack/k8s-client"
import { renderWithK8sProvider } from "../../test-utils/render.tsx"
@@ -98,6 +98,31 @@ describe("MetricsView", () => {
expect(screen.queryByText("Try a longer range.")).toBeNull()
})
+ it("shows a panel's samples as a table on request", async () => {
+ const { client } = clientWith(() => matrix("db-0"))
+ renderWithK8sProvider( , { client })
+ await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3))
+ const toggle = screen.getByRole("button", { name: "Show data for CPU (cores)" })
+ expect(toggle).toHaveAttribute("aria-expanded", "false")
+
+ await userEvent.click(toggle)
+
+ const table = screen.getByRole("region", { name: "CPU (cores) data" })
+ expect(toggle).toHaveAttribute("aria-expanded", "true")
+ expect(toggle).toHaveAttribute("aria-controls", table.id)
+ expect(toggle).toHaveTextContent("Hide data")
+ expect(within(table).getByRole("columnheader", { name: "db-0" })).toBeInTheDocument()
+ expect(screen.queryByRole("region", { name: "Memory (working set) data" })).toBeNull()
+ })
+
+ it("offers no table while a panel has no samples", () => {
+ const { client } = clientWith(() => new Promise(() => {}))
+
+ renderWithK8sProvider( , { client })
+
+ expect(screen.getByRole("button", { name: "Show data for Network" })).toBeDisabled()
+ })
+
it("holds each chart's place while it loads", () => {
const { client } = clientWith(() => new Promise(() => {}))
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
index fac2a6b91..4663b2ae3 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
@@ -1,7 +1,7 @@
-import { useState } from "react"
+import { useId, useState } from "react"
import { useQueries } from "@tanstack/react-query"
import { useK8sClient } from "@cozystack/k8s-client"
-import { Section } from "@cozystack/ui"
+import { Button, Section } from "@cozystack/ui"
import {
METRIC_PANELS,
RANGES,
@@ -14,6 +14,7 @@ import { ProxyNotice } from "./ProxyNotice.tsx"
import { TimeSeriesChart } from "./TimeSeriesChart.tsx"
import { RangePicker } from "./RangePicker.tsx"
import { EmptyState } from "./EmptyState.tsx"
+import { SeriesTable } from "./SeriesTable.tsx"
const REFRESH_MS = 60_000
// The uPlot canvas plus its one-row legend, so a placeholder takes the chart's place without a jump.
@@ -23,6 +24,8 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
const client = useK8sClient()
const root = metricsRoot(scope)
const [range, setRange] = useState("1h")
+ const [tables, setTables] = useState>(new Set())
+ const idBase = useId()
const rangeKeys = Object.keys(RANGES) as RangeKey[]
const longestRange = range === rangeKeys[rangeKeys.length - 1]
@@ -57,8 +60,30 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
) : (
METRIC_PANELS.map((panel, i) => {
const { data, error, isLoading } = results[i]
+ const hasSeries = !error && !isLoading && !!data && data.series.length > 0
+ const tableOpen = hasSeries && tables.has(panel.id)
+ const tableId = `${idBase}-${panel.id}-data`
+ const toggle = (
+
+ setTables((prev) => {
+ const next = new Set(prev)
+ if (!next.delete(panel.id)) next.add(panel.id)
+ return next
+ })
+ }
+ >
+ {tableOpen ? "Hide data" : "Show data"}{" "}
+ for {panel.title}
+
+ )
return (
-
+
{error ? (
) : isLoading || !data ? (
@@ -66,7 +91,10 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
) : data.series.length === 0 ? (
) : (
-
+ <>
+
+ {tableOpen && }
+ >
)}
)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx
new file mode 100644
index 000000000..0b3364268
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx
@@ -0,0 +1,37 @@
+import { describe, it, expect } from "vitest"
+import { render, screen, within } from "@testing-library/react"
+import { SeriesTable } from "./SeriesTable.tsx"
+
+const data = {
+ timestamps: [1_700_000_000, 1_700_000_060],
+ series: [
+ { label: "db-0", values: [1, 2] },
+ { label: "db-1", values: [null, 3] },
+ ],
+}
+
+describe("SeriesTable", () => {
+ it("is a named region the keyboard can reach", () => {
+ render( )
+
+ const region = screen.getByRole("region", { name: "Memory data" })
+ expect(region).toHaveAttribute("id", "t")
+ expect(region).toHaveAttribute("tabindex", "0")
+ })
+
+ it("has a time column and one column per series", () => {
+ render( )
+
+ expect(screen.getAllByRole("columnheader").map((h) => h.textContent)).toEqual(["Time", "db-0", "db-1"])
+ })
+
+ it("lists every sample newest first in the chart's units", () => {
+ render( `${v} B`} />)
+
+ const rows = screen.getAllByRole("row").slice(1)
+ expect(rows).toHaveLength(2)
+ expect(within(rows[0]).getAllByRole("cell").map((c) => c.textContent)).toEqual(["2 B", "3 B"])
+ expect(within(rows[1]).getAllByRole("cell").map((c) => c.textContent)).toEqual(["1 B", "–"])
+ expect(within(rows[0]).getByRole("rowheader")).toHaveTextContent(new Date(1_700_000_060_000).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }))
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx
new file mode 100644
index 000000000..57cf755a0
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx
@@ -0,0 +1,47 @@
+import type { ChartData } from "../../lib/observability/metrics.ts"
+
+interface SeriesTableProps {
+ id: string
+ title: string
+ data: ChartData
+ format: (value: number) => string
+}
+
+const formatTime = (seconds: number) =>
+ new Date(seconds * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit" })
+
+export function SeriesTable({ id, title, data, format }: SeriesTableProps) {
+ const newestFirst = data.timestamps.map((t, i) => ({ t, i })).reverse()
+ return (
+
+
+ {`${title}, newest first`}
+
+
+ Time
+ {data.series.map((s, j) => (
+ {s.label}
+ ))}
+
+
+
+ {newestFirst.map(({ t, i }) => (
+
+ {formatTime(t)}
+ {data.series.map((s, j) => {
+ const v = s.values[i]
+ return {v === null ? "–" : format(v)}
+ })}
+
+ ))}
+
+
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
index 57cf6fb99..3df8125e3 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
@@ -1,12 +1,15 @@
import { describe, it, expect, vi, beforeEach } from "vitest"
import { render } from "@testing-library/react"
-const created: { opts: { series: { label?: string }[] }; data: unknown[]; destroy: ReturnType }[] = []
+type LegendValue = (u: unknown, v: number | null, sidx: number, idx: number | null) => string
+type Opts = { series: { label?: string; value?: LegendValue }[] }
+
+const created: { opts: Opts; data: unknown[]; destroy: ReturnType }[] = []
vi.mock("uplot", () => ({
default: class {
destroy = vi.fn()
- constructor(opts: { series: { label?: string }[] }, data: unknown[]) {
+ constructor(opts: Opts, data: unknown[]) {
created.push({ opts, data, destroy: this.destroy })
}
setSize() {}
@@ -36,6 +39,16 @@ describe("TimeSeriesChart", () => {
expect(created[0].opts.series.slice(1).map((s) => s.label)).toEqual(["db-0", "db-1"])
})
+ it("shows each series' latest value in the legend while nothing is hovered", () => {
+ render( `${v} B`} />)
+
+ const legend = created[0].opts.series[1].value!
+
+ expect(legend(null, null, 1, null)).toBe("2 B")
+ expect(legend(null, 1, 1, 0)).toBe("1 B")
+ expect(legend(null, null, 1, 2)).toBe("–")
+ })
+
it("destroys the chart on unmount", () => {
const { unmount } = render( )
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
index 683df2dff..a6251d40f 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
@@ -18,6 +18,7 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) {
const el = ref.current
if (!el) return
const fmt = (v: number | null) => (v === null ? "–" : format(v))
+ const latest = data.series.map((s) => s.values.findLast((v) => v !== null) ?? null)
const chart = new uPlot(
{
width: el.clientWidth || 600,
@@ -30,7 +31,8 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) {
label: s.label,
stroke: COLORS[i % COLORS.length],
width: 1.5,
- value: (_: uPlot, v: number | null) => fmt(v),
+ // Without a hovered point the legend shows the latest sample instead of a dash.
+ value: (_: uPlot, v: number | null, sidx: number, idx: number | null) => fmt(idx === null ? latest[sidx - 1] : v),
})),
],
axes: [{}, { size: 84, values: (_: uPlot, splits: number[]) => splits.map(format) }],
From e65d827cf9eca51f742200e76172b62224003871 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 00:16:49 +0700
Subject: [PATCH 039/114] fix(dashboard): move the tenant logs page under
observability/
The fixed /console/logs route came before the generic :plural route,
so an application kind with the plural "logs" would have its list page
replaced by the logs page. A two-segment path cannot shadow an
application list, the same way the backups and migration pages avoid
it.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/routes/ConsolePage.routing.test.tsx | 24 +++++++++++++++++++
.../apps/console/src/routes/ConsolePage.tsx | 2 +-
.../src/routes/sidebar-sections.test.tsx | 2 +-
.../console/src/routes/sidebar-sections.tsx | 2 +-
4 files changed, 27 insertions(+), 3 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx
index 7c6d1209c..cf6982823 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx
@@ -70,4 +70,28 @@ describe("ConsolePage routing", () => {
await waitFor(() => expect(client.list).toHaveBeenCalled())
expect(screen.queryByText(/Cluster-scoped capacity/i)).toBeNull()
})
+
+ it("serves the tenant logs page under observability/", async () => {
+ renderWithK8sProvider(
+
+
+ ,
+ { client: makeClient(), initialRoute: "/observability/logs" },
+ )
+
+ expect(await screen.findByText(/^Every application in/)).toBeInTheDocument()
+ })
+
+ it("leaves /logs to an application kind whose plural is logs", async () => {
+ const client = makeClient()
+ renderWithK8sProvider(
+
+
+ ,
+ { client, initialRoute: "/logs" },
+ )
+
+ await waitFor(() => expect(client.list).toHaveBeenCalled())
+ expect(screen.queryByText(/^Every application in/)).toBeNull()
+ })
})
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx
index 5e204e84f..2b2aed3e4 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx
@@ -97,7 +97,7 @@ export function ConsolePage() {
path="migration/vmimporttasks/:name"
element={ }
/>
- } />
+ } />
} />
} />
} />
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx
index da40081bc..1fcc3cde4 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx
@@ -117,7 +117,7 @@ describe("useConsoleSidebarSections — admin areas moved out", () => {
wrapper: makeWrapper(client),
})
await waitFor(() => expect(result.current.length).toBeGreaterThan(0))
- expect(findItem(result.current, "Logs")?.to).toBe("/console/logs")
+ expect(findItem(result.current, "Logs")?.to).toBe("/console/observability/logs")
expect(result.current.find((s) => s.title === "Observability")?.items).toHaveLength(1)
})
})
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx
index 4cec3a94c..0cf1ce089 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx
@@ -121,7 +121,7 @@ export function useConsoleSidebarSections(): SidebarSection[] {
const observabilitySection: SidebarSection = {
title: "Observability",
- items: [{ label: "Logs", to: "/console/logs", icon: ScrollText }],
+ items: [{ label: "Logs", to: "/console/observability/logs", icon: ScrollText }],
}
return [...categorySections, backupsSection, migrationSection, observabilitySection]
From 52087d92dcaae1316463973643ccbeaf37869a43 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 00:18:25 +0700
Subject: [PATCH 040/114] fix(dashboard): keep the logs view when only the pod
list is refused
The pod picker's values query counted as a scope-wide refusal, so a
proxy that refused only that path replaced the whole view, filter and
lines included, with a notice even though the log query itself worked.
A real scope-wide refusal still reaches the history or tail query and
shows the notice from there.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/components/observability/LogsView.test.tsx | 12 ++++++++++++
.../src/components/observability/LogsView.tsx | 3 ++-
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 770c8a263..86428733d 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -212,6 +212,18 @@ describe("LogsView", () => {
expect(within(frame).queryByText(/No log lines/)).toBeNull()
})
+ it("keeps the view when only the pod list is refused", async () => {
+ const status = JSON.stringify({ kind: "Status", apiVersion: "v1", status: "Failure", code: 404 })
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("stream_field_values") ? new Response(status, { status: 404 }) : defaultHandler(endpoint, params),
+ )
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
+ expect(screen.queryByText("Logs are not available on this cluster.")).toBeNull()
+ })
+
it("names the source pod and container of each line", async () => {
const { client } = clientWith(defaultHandler)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index a0a73cc43..c7a1f5d58 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -106,7 +106,8 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
})
const tail = useLogTail(root, query, followRun, { cap: FOLLOW_CAP })
- const wide = [pods.error, history.error, tail.error].find(isScopeWide)
+ // The pod list is optional: refusing it alone must not hide lines the query can still read.
+ const wide = [history.error, tail.error].find(isScopeWide)
if (wide) {
return (
From ef7312579d9872272baf407f622dea7e5fd05944 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 00:18:33 +0700
Subject: [PATCH 041/114] fix(dashboard): offer a retry when the log query
fails
The console does not refetch on window focus and must not poll, so a
failed history query stayed failed until the filter or range changed,
even after the store recovered. The failed state now has a Retry
button that runs the same query again.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/components/observability/EmptyState.tsx | 4 +++-
.../src/components/observability/LogsView.test.tsx | 13 +++++++++++++
.../src/components/observability/LogsView.tsx | 14 +++++++++++---
3 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx
index 0cf58224b..5b41ef13d 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx
@@ -4,12 +4,13 @@ import { cn, Spinner } from "@cozystack/ui"
interface EmptyStateProps {
title: ReactNode
hint?: ReactNode
+ action?: ReactNode
busy?: boolean
tone?: "light" | "dark"
className?: string
}
-export function EmptyState({ title, hint, busy, tone = "light", className }: EmptyStateProps) {
+export function EmptyState({ title, hint, action, busy, tone = "light", className }: EmptyStateProps) {
const dark = tone === "dark"
return (
}
{title}
{hint && {hint}
}
+ {action && {action}
}
)
}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 86428733d..8b60b7d35 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -224,6 +224,19 @@ describe("LogsView", () => {
expect(screen.queryByText("Logs are not available on this cluster.")).toBeNull()
})
+ it("retries a failed history query on request", async () => {
+ let calls = 0
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query") && calls++ === 0 ? new Response("upstream down", { status: 502 }) : defaultHandler(endpoint, params),
+ )
+ renderWithK8sProvider( , { client })
+
+ await userEvent.click(await screen.findByRole("button", { name: "Retry" }))
+
+ expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
+ expect(screen.queryByRole("button", { name: "Retry" })).toBeNull()
+ })
+
it("names the source pod and container of each line", async () => {
const { client } = clientWith(defaultHandler)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index c7a1f5d58..b8d77003a 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -59,7 +59,7 @@ function Picker({ label, value, options, onChange }: { label: string; value: str
const RANGE_KEYS = Object.keys(LOG_RANGES) as LogRangeKey[]
const LONGEST_RANGE = RANGE_KEYS[RANGE_KEYS.length - 1]
-function placeholderFor({ follow, stopped, loading, failed, longestRange }: { follow: boolean; stopped: boolean; loading: boolean; failed: boolean; longestRange: boolean }) {
+function placeholderFor({ follow, stopped, loading, failed, longestRange, retry }: { follow: boolean; stopped: boolean; loading: boolean; failed: boolean; longestRange: boolean; retry: () => void }) {
if (follow) {
return stopped ? (
@@ -68,7 +68,15 @@ function placeholderFor({ follow, stopped, loading, failed, longestRange }: { fo
)
}
if (loading) return
- if (failed) return
+ if (failed) {
+ return (
+ Retry}
+ />
+ )
+ }
return
}
@@ -154,7 +162,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
void history.refetch() })}
/>
)
From 7d43c3bcd6470c1282a539e05609e9a96de25067 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 00:18:38 +0700
Subject: [PATCH 042/114] fix(dashboard): open each new log result at its
newest line
The console keeps a query's previous data as placeholder while the next
one loads, so the log list was never remounted between queries. After
a reader scrolled up, a new filter or range opened at the old scroll
offset instead of the newest lines. The list is now keyed on the query
and range as well.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/LogsView.test.tsx | 25 ++++++++++++++++---
.../src/components/observability/LogsView.tsx | 3 ++-
2 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 8b60b7d35..1a703cceb 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -86,13 +86,13 @@ describe("LogsView", () => {
endpoint.endsWith("query") ? new Response("") : defaultHandler(endpoint, params),
)
renderWithK8sProvider( , { client })
- const frame = screen.getByRole("region", { name: "Log lines" })
- expect(await within(frame).findByText("Try a longer range or a broader filter.")).toBeInTheDocument()
+ const frame = () => screen.getByRole("region", { name: "Log lines" })
+ expect(await within(frame()).findByText("Try a longer range or a broader filter.")).toBeInTheDocument()
await userEvent.click(screen.getByRole("button", { name: "24h" }))
- expect(await within(frame).findByText("Try a broader filter.")).toBeInTheDocument()
- expect(within(frame).queryByText(/longer range/)).toBeNull()
+ await waitFor(() => expect(within(frame()).getByText("Try a broader filter.")).toBeInTheDocument())
+ expect(within(frame()).queryByText(/longer range/)).toBeNull()
})
it("shows loading inside the log console", () => {
@@ -237,6 +237,23 @@ describe("LogsView", () => {
expect(screen.queryByRole("button", { name: "Retry" })).toBeNull()
})
+ it("opens a new history result at its newest line, not the old scroll position", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query")
+ ? new Response(`${line("2026-11-04T10:00:01Z", `result for ${params.get("query")}`)}\n`)
+ : defaultHandler(endpoint, params),
+ )
+ const { queryClient } = renderWithK8sProvider( , { client })
+ queryClient.setDefaultOptions({ queries: { ...queryClient.getDefaultOptions().queries, placeholderData: (prev: unknown) => prev } })
+ await screen.findByText("result for *")
+ const before = screen.getByRole("list")
+
+ await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}")
+
+ await screen.findByText("result for error")
+ expect(screen.getByRole("list")).not.toBe(before)
+ })
+
it("names the source pod and container of each line", async () => {
const { client } = clientWith(defaultHandler)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index b8d77003a..1a25d4229 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -160,7 +160,8 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
)}
{error ? : null}
void history.refetch() })}
/>
From 601c9ece3f0ce24792d1dc1f9dce862a6826e876 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 00:18:47 +0700
Subject: [PATCH 043/114] docs(dashboard): note why the metrics panels refresh
on an interval
The console rule against refetchInterval assumes a watch is available,
which holds for Kubernetes lists but not for the proxy's Prometheus
query API. Record the exception so the metrics panels' refresh does
not read as a rule violation.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
packages/system/dashboard/images/console/CLAUDE.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/packages/system/dashboard/images/console/CLAUDE.md b/packages/system/dashboard/images/console/CLAUDE.md
index 96419d118..036e527d6 100644
--- a/packages/system/dashboard/images/console/CLAUDE.md
+++ b/packages/system/dashboard/images/console/CLAUDE.md
@@ -81,6 +81,7 @@ the apiserver checks that user's RBAC.
to a chunked-encoding WATCH against the same `resourceVersion`. Don't
poll. Don't add `refetchInterval`. If you need a one-shot, pass
`{ watch: false }`.
+ The one exception is the metrics panels in `components/observability`: the observability proxy serves the Prometheus query API, which has no watch, so they refresh on a fixed interval instead. Log lines stream through the tail endpoint and are not polled.
4. **Tenant scoping.** Most resources live in `tenant-` namespaces.
Pull the active tenant from `useTenantContext()` — never read the
namespace from a URL param or guess it.
From 046df7fe969f2b0b6d62ced127ed601463d9dfc8 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 00:53:06 +0700
Subject: [PATCH 044/114] fix(dashboard): say the legend shows the latest
sample
In the browser the legend's time still read "--" next to the latest
values, and the data table repeated the same minute on every row at
the one-hour range, whose step is 15 seconds. The legend's time now
reads "latest" until a point is hovered, and the table shows seconds.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/components/observability/SeriesTable.test.tsx | 2 +-
.../src/components/observability/SeriesTable.tsx | 2 +-
.../components/observability/TimeSeriesChart.test.tsx | 10 ++++++++++
.../src/components/observability/TimeSeriesChart.tsx | 3 ++-
4 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx
index 0b3364268..e85041c0a 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx
@@ -32,6 +32,6 @@ describe("SeriesTable", () => {
expect(rows).toHaveLength(2)
expect(within(rows[0]).getAllByRole("cell").map((c) => c.textContent)).toEqual(["2 B", "3 B"])
expect(within(rows[1]).getAllByRole("cell").map((c) => c.textContent)).toEqual(["1 B", "–"])
- expect(within(rows[0]).getByRole("rowheader")).toHaveTextContent(new Date(1_700_000_060_000).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" }))
+ expect(within(rows[0]).getByRole("rowheader")).toHaveTextContent(new Date(1_700_000_060_000).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" }))
})
})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx
index 57cf755a0..4cc382781 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx
@@ -8,7 +8,7 @@ interface SeriesTableProps {
}
const formatTime = (seconds: number) =>
- new Date(seconds * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit" })
+ new Date(seconds * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit", second: "2-digit" })
export function SeriesTable({ id, title, data, format }: SeriesTableProps) {
const newestFirst = data.timestamps.map((t, i) => ({ t, i })).reverse()
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
index 3df8125e3..13cbe933d 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
@@ -13,6 +13,7 @@ vi.mock("uplot", () => ({
created.push({ opts, data, destroy: this.destroy })
}
setSize() {}
+ static fmtDate = () => (d: Date) => d.toISOString()
},
}))
@@ -49,6 +50,15 @@ describe("TimeSeriesChart", () => {
expect(legend(null, null, 1, 2)).toBe("–")
})
+ it("labels the legend's time as latest while nothing is hovered", () => {
+ render( )
+
+ const time = created[0].opts.series[0].value!
+
+ expect(time(null, null, 0, null)).toBe("latest")
+ expect(time(null, 2, 0, 1)).toBe(new Date(2000).toISOString())
+ })
+
it("destroys the chart on unmount", () => {
const { unmount } = render( )
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
index a6251d40f..063b42b12 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
@@ -19,6 +19,7 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) {
if (!el) return
const fmt = (v: number | null) => (v === null ? "–" : format(v))
const latest = data.series.map((s) => s.values.findLast((v) => v !== null) ?? null)
+ const stamp = uPlot.fmtDate("{M}/{D}/{YY} {h}:{mm}:{ss} {AA}")
const chart = new uPlot(
{
width: el.clientWidth || 600,
@@ -26,7 +27,7 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) {
scales: { x: { time: true } },
legend: { live: true },
series: [
- {},
+ { value: (_: uPlot, v: number | null, _sidx: number, idx: number | null) => (idx === null || v === null ? "latest" : stamp(new Date(v * 1000))) },
...data.series.map((s, i) => ({
label: s.label,
stroke: COLORS[i % COLORS.length],
From bfa85e57bfe9a619d6c6bcba31c842171a4d848d Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 00:56:26 +0700
Subject: [PATCH 045/114] fix(dashboard): show sample times the same way in
legend and table
The legend used a fixed US date format while the data table used the
reader's locale, so one panel showed the same instant two ways. Both
now use one locale-aware format. A test also pins that the data
toggle stays disabled for a panel with no series or an error.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../components/observability/MetricsView.test.tsx | 15 +++++++++++++++
.../src/components/observability/SeriesTable.tsx | 7 ++-----
.../observability/TimeSeriesChart.test.tsx | 4 ++--
.../components/observability/TimeSeriesChart.tsx | 5 ++---
.../apps/console/src/lib/observability/metrics.ts | 4 ++++
5 files changed, 25 insertions(+), 10 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
index 8e7829ed5..ece4d70ba 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
@@ -123,6 +123,21 @@ describe("MetricsView", () => {
expect(screen.getByRole("button", { name: "Show data for Network" })).toBeDisabled()
})
+ it("offers no table for a panel with no series or an error", async () => {
+ const { client } = clientWith((url) =>
+ url.searchParams.get("query")?.includes("memory")
+ ? Response.json({ status: "error", error: "too many series" }, { status: 422 })
+ : Response.json({ status: "success", data: { resultType: "matrix", result: [] } }),
+ )
+ renderWithK8sProvider( , { client })
+
+ await screen.findByText("too many series. Choose a shorter range.")
+ await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(2))
+ for (const title of ["CPU (cores)", "Memory (working set)", "Network"]) {
+ expect(screen.getByRole("button", { name: `Show data for ${title}` })).toBeDisabled()
+ }
+ })
+
it("holds each chart's place while it loads", () => {
const { client } = clientWith(() => new Promise(() => {}))
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx
index 4cc382781..5e9d88ba1 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx
@@ -1,4 +1,4 @@
-import type { ChartData } from "../../lib/observability/metrics.ts"
+import { formatSampleTime, type ChartData } from "../../lib/observability/metrics.ts"
interface SeriesTableProps {
id: string
@@ -7,9 +7,6 @@ interface SeriesTableProps {
format: (value: number) => string
}
-const formatTime = (seconds: number) =>
- new Date(seconds * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit", second: "2-digit" })
-
export function SeriesTable({ id, title, data, format }: SeriesTableProps) {
const newestFirst = data.timestamps.map((t, i) => ({ t, i })).reverse()
return (
@@ -33,7 +30,7 @@ export function SeriesTable({ id, title, data, format }: SeriesTableProps) {
{newestFirst.map(({ t, i }) => (
- {formatTime(t)}
+ {formatSampleTime(t)}
{data.series.map((s, j) => {
const v = s.values[i]
return {v === null ? "–" : format(v)}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
index 13cbe933d..bb76edb5a 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
@@ -13,11 +13,11 @@ vi.mock("uplot", () => ({
created.push({ opts, data, destroy: this.destroy })
}
setSize() {}
- static fmtDate = () => (d: Date) => d.toISOString()
},
}))
const { TimeSeriesChart } = await import("./TimeSeriesChart.tsx")
+const { formatSampleTime } = await import("../../lib/observability/metrics.ts")
beforeEach(() => {
created.length = 0
@@ -56,7 +56,7 @@ describe("TimeSeriesChart", () => {
const time = created[0].opts.series[0].value!
expect(time(null, null, 0, null)).toBe("latest")
- expect(time(null, 2, 0, 1)).toBe(new Date(2000).toISOString())
+ expect(time(null, 2, 0, 1)).toBe(formatSampleTime(2))
})
it("destroys the chart on unmount", () => {
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
index 063b42b12..607d44593 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
@@ -1,7 +1,7 @@
import { useEffect, useRef } from "react"
import uPlot from "uplot"
import "uplot/dist/uPlot.min.css"
-import type { ChartData } from "../../lib/observability/metrics.ts"
+import { formatSampleTime, type ChartData } from "../../lib/observability/metrics.ts"
const COLORS = ["#2563eb", "#16a34a", "#d97706", "#9333ea", "#dc2626", "#0891b2", "#4b5563", "#db2777"]
const HEIGHT = 200
@@ -19,7 +19,6 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) {
if (!el) return
const fmt = (v: number | null) => (v === null ? "–" : format(v))
const latest = data.series.map((s) => s.values.findLast((v) => v !== null) ?? null)
- const stamp = uPlot.fmtDate("{M}/{D}/{YY} {h}:{mm}:{ss} {AA}")
const chart = new uPlot(
{
width: el.clientWidth || 600,
@@ -27,7 +26,7 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) {
scales: { x: { time: true } },
legend: { live: true },
series: [
- { value: (_: uPlot, v: number | null, _sidx: number, idx: number | null) => (idx === null || v === null ? "latest" : stamp(new Date(v * 1000))) },
+ { value: (_: uPlot, v: number | null, _sidx: number, idx: number | null) => (idx === null || v === null ? "latest" : formatSampleTime(v)) },
...data.series.map((s, i) => ({
label: s.label,
stroke: COLORS[i % COLORS.length],
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
index d7182b8d6..2f77e251f 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
@@ -93,6 +93,10 @@ export const METRIC_PANELS: MetricPanel[] = [
},
]
+export function formatSampleTime(seconds: number): string {
+ return new Date(seconds * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit", second: "2-digit" })
+}
+
export interface ChartSeries {
label: string
values: (number | null)[]
From 311cb1c90b751bcc24cf962e68676a6997680c4d Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 06:10:02 +0700
Subject: [PATCH 046/114] fix(dashboard): skip tail dedup for rows without a
stream ID
A pipe such as `delete _stream_id` leaves rows from different streams
that share a timestamp with the same dedup key, so a resumed tail
dropped a record it had never shown. Only rows that still carry both
_stream_id and _time are deduplicated now, as the accepted logs design
specifies.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../console/src/lib/observability/logs.test.ts | 18 ++++++++++++++++++
.../apps/console/src/lib/observability/logs.ts | 15 ++++++++++-----
2 files changed, 28 insertions(+), 5 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts
index 62d1ac249..278da6890 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts
@@ -63,6 +63,24 @@ describe("mergeTail", () => {
expect(out).toHaveLength(2)
})
+ it("never deduplicates lines a pipe stripped of _stream_id, even on a resume", () => {
+ const t = "2026-01-01T00:00:02Z"
+ const buffer = [{ _time: t, _msg: "from stream 1" }]
+
+ const out = mergeTail(buffer, [{ _time: t, _msg: "from stream 2" }], 10, t)
+
+ expect(out.map((x) => x._msg)).toEqual(["from stream 1", "from stream 2"])
+ })
+
+ it("deduplicates only the complete rows of a mixed resume", () => {
+ const t = "2026-01-01T00:00:02Z"
+ const buffer = [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }]
+
+ const out = mergeTail(buffer, [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }], 10, t)
+
+ expect(out.map((x) => x._msg)).toEqual(["seen", "no stream", "no time", "no stream", "no time"])
+ })
+
it("does not deduplicate outside a resume", () => {
const out = mergeTail([e("t1", "a")], [e("t1", "a")], 10)
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts
index eb24a5f96..b1005a1cd 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts
@@ -58,16 +58,21 @@ export function compareTime(a: string, b: string): number {
return x < y ? -1 : x > y ? 1 : 0
}
-const entryKey = (e: LogEntry) => `${e._stream_id ?? ""}\u0000${e._time}`
+// Null when a pipe removed either identity field: such a row cannot be told from another stream's.
+const entryKey = (e: LogEntry) =>
+ e._stream_id !== undefined && e._time !== undefined ? `${e._stream_id}\u0000${e._time}` : null
// (_stream_id, _time) is not unique in VictoriaLogs, so dedup applies only to
-// lines a resumed tail replays at or before the resume point, and never to
-// lines a pipe stripped of _time.
+// lines a resumed tail replays at or before the resume point, and only to
+// lines that still carry both fields.
export function mergeTail(buffer: LogEntry[], incoming: LogEntry[], cap: number, resumedFrom?: string): LogEntry[] {
let fresh = incoming
if (resumedFrom) {
- const seen = new Set(buffer.map(entryKey))
- fresh = incoming.filter((e) => e._time === undefined || compareTime(e._time, resumedFrom) > 0 || !seen.has(entryKey(e)))
+ const seen = new Set(buffer.map(entryKey).filter((k) => k !== null))
+ fresh = incoming.filter((e) => {
+ const key = entryKey(e)
+ return key === null || compareTime(e._time ?? "", resumedFrom) > 0 || !seen.has(key)
+ })
}
const merged = buffer.concat(fresh)
return merged.length > cap ? merged.slice(merged.length - cap) : merged
From 9c111b3a95f77731b8296310cfd0d32a431b50d6 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 06:11:34 +0700
Subject: [PATCH 047/114] fix(dashboard): keep placeholder results within one
observability scope
The provider keeps a query's previous result while its key changes, so
switching tenants showed the first tenant's log lines and pod names
under the second tenant's heading until its request returned. Every
observability query now keeps the previous result only while the
proxy root is the same, so a new filter or range still shows the old
lines, but a new tenant or application starts empty. Tests render with
the provider's own query defaults.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/LogsView.test.tsx | 49 ++++++++++++++++++-
.../src/components/observability/LogsView.tsx | 3 ++
.../observability/MetricsView.test.tsx | 20 +++++++-
.../components/observability/MetricsView.tsx | 3 +-
.../console/src/lib/observability/proxy.ts | 8 +++
5 files changed, 78 insertions(+), 5 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 1a703cceb..fe73e4164 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -1,7 +1,7 @@
import { describe, it, expect, vi } from "vitest"
-import { screen, waitFor, within } from "@testing-library/react"
+import { render, screen, waitFor, within } from "@testing-library/react"
import userEvent from "@testing-library/user-event"
-import { K8sClient } from "@cozystack/k8s-client"
+import { K8sClient, K8sProvider } from "@cozystack/k8s-client"
import { renderWithK8sProvider } from "../../test-utils/render.tsx"
import { LogsView } from "./LogsView.tsx"
@@ -30,6 +30,51 @@ function defaultHandler(endpoint: string, params?: URLSearchParams): Response |
return new Promise(() => {})
}
+// The provider's own QueryClient keeps the previous result while a key changes; it is
+// shared across tests, so each test below uses namespaces no other test reads.
+function renderWithProductionDefaults(client: K8sClient, namespace: string) {
+ const view = (ns: string) => (
+
+
+
+ )
+ const result = render(view(namespace))
+ return { switchTo: (ns: string) => result.rerender(view(ns)) }
+}
+
+describe("LogsView with the production query defaults", () => {
+ it("drops the previous tenant's lines and pods while the next tenant loads", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.includes("/tenant-switch-b/") ? new Promise(() => {}) : defaultHandler(endpoint, params),
+ )
+ const { switchTo } = renderWithProductionDefaults(client, "tenant-switch-a")
+ expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
+ await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
+ await screen.findByRole("option", { name: "db-1" })
+ await userEvent.keyboard("{Escape}")
+
+ switchTo("tenant-switch-b")
+
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(within(frame).getByText("Loading log lines…")).toBeInTheDocument()
+ expect(screen.queryByText("ready to accept connections")).toBeNull()
+ await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
+ expect((await screen.findAllByRole("option")).map((o) => o.textContent)).toEqual(["All"])
+ })
+
+ it("keeps the current lines while a new filter in the same tenant loads", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query") && params.get("query") === "slow" ? new Promise(() => {}) : defaultHandler(endpoint, params),
+ )
+ renderWithProductionDefaults(client, "tenant-filter-a")
+ expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
+
+ await userEvent.type(screen.getByLabelText("LogsQL filter"), "slow{Enter}")
+
+ expect(screen.getByText("ready to accept connections")).toBeInTheDocument()
+ })
+})
+
describe("LogsView", () => {
it("shows recent lines from the application root", async () => {
const { client, raw } = clientWith(defaultHandler)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index 1a25d4229..c2dc8ca5c 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -14,6 +14,7 @@ import {
import {
fieldValues,
isScopeWide,
+ keepWithinScope,
logsRoot,
queryLogs,
type ObservabilityScope,
@@ -101,6 +102,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
query: buildLogsQuery({ filter: "", pod: narrowTo }),
...windowFor(range),
}, signal),
+ placeholderData: keepWithinScope(root),
retry: false,
})
const pods = useQuery(pickerQuery(POD_FIELD))
@@ -110,6 +112,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
queryKey: ["observability", "logs", root, query, range],
queryFn: ({ signal }) => queryLogs(client, root, { query, limit: HISTORY_LIMIT, ...windowFor(range) }, signal),
enabled: !follow,
+ placeholderData: keepWithinScope(root),
retry: false,
})
const tail = useLogTail(root, query, followRun, { cap: FOLLOW_CAP })
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
index ece4d70ba..fccb6f43d 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
@@ -1,7 +1,7 @@
import { describe, it, expect, vi } from "vitest"
-import { screen, waitFor, within } from "@testing-library/react"
+import { render, screen, waitFor, within } from "@testing-library/react"
import userEvent from "@testing-library/user-event"
-import { K8sClient } from "@cozystack/k8s-client"
+import { K8sClient, K8sProvider } from "@cozystack/k8s-client"
import { renderWithK8sProvider } from "../../test-utils/render.tsx"
import type { ChartData } from "../../lib/observability/metrics.ts"
@@ -138,6 +138,22 @@ describe("MetricsView", () => {
}
})
+ it("drops the previous application's charts while the next one loads, with the production query defaults", async () => {
+ const { client } = clientWith((url) => (url.pathname.includes("/switch-b/") ? new Promise(() => {}) : matrix("db-0")))
+ const view = (name: string) => (
+
+
+
+ )
+ const { rerender } = render(view("switch-a"))
+ await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3))
+
+ rerender(view("switch-b"))
+
+ expect(screen.queryAllByTestId("chart")).toHaveLength(0)
+ expect(screen.getAllByText("Loading…")).toHaveLength(3)
+ })
+
it("holds each chart's place while it loads", () => {
const { client } = clientWith(() => new Promise(() => {}))
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
index 4663b2ae3..07980aacf 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
@@ -9,7 +9,7 @@ import {
toChartData,
type RangeKey,
} from "../../lib/observability/metrics.ts"
-import { isScopeWide, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts"
+import { isScopeWide, keepWithinScope, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts"
import { ProxyNotice } from "./ProxyNotice.tsx"
import { TimeSeriesChart } from "./TimeSeriesChart.tsx"
import { RangePicker } from "./RangePicker.tsx"
@@ -42,6 +42,7 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
)
return toChartData(series)
},
+ placeholderData: keepWithinScope(root),
retry: false,
refetchInterval: REFRESH_MS,
})),
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
index c28bd12d4..3986c6a3d 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -20,6 +20,14 @@ export function logsRoot(scope: ObservabilityScope): string {
return app ? `${app}/log/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/logs/`
}
+// The provider keeps a query's previous result while its key changes; across
+// scopes that would show one tenant's or application's data under another.
+// Every observability query key carries its root at index 2.
+export function keepWithinScope(root: string) {
+ return (previous: T | undefined, previousQuery?: { queryKey: readonly unknown[] }): T | undefined =>
+ previousQuery?.queryKey[2] === root ? previous : undefined
+}
+
export type ProxyErrorKind =
| "bad-request"
| "unauthorized"
From 711a82eaaf3b073c4e69b6bca94c235adf9a891f Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 06:15:06 +0700
Subject: [PATCH 048/114] refactor(dashboard): drop the scoped placeholder from
metrics panels
useQueries creates a new observer for every new key, so the metrics
panels never carried a previous result across scopes and the scoped
placeholder there did nothing. The switch test for the panels stays
and passes without it; the helper's comment now says which queries it
is for.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../console/src/components/observability/MetricsView.tsx | 3 +--
.../console/apps/console/src/lib/observability/proxy.ts | 5 +++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
index 07980aacf..4663b2ae3 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
@@ -9,7 +9,7 @@ import {
toChartData,
type RangeKey,
} from "../../lib/observability/metrics.ts"
-import { isScopeWide, keepWithinScope, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts"
+import { isScopeWide, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts"
import { ProxyNotice } from "./ProxyNotice.tsx"
import { TimeSeriesChart } from "./TimeSeriesChart.tsx"
import { RangePicker } from "./RangePicker.tsx"
@@ -42,7 +42,6 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
)
return toChartData(series)
},
- placeholderData: keepWithinScope(root),
retry: false,
refetchInterval: REFRESH_MS,
})),
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
index 3986c6a3d..d33052f70 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -20,8 +20,9 @@ export function logsRoot(scope: ObservabilityScope): string {
return app ? `${app}/log/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/logs/`
}
-// The provider keeps a query's previous result while its key changes; across
-// scopes that would show one tenant's or application's data under another.
+// The provider's default keeps a useQuery's previous result while its key
+// changes; across scopes that would show one tenant's or application's data
+// under another. (useQueries starts a new observer per key, so it never does.)
// Every observability query key carries its root at index 2.
export function keepWithinScope(root: string) {
return (previous: T | undefined, previousQuery?: { queryKey: readonly unknown[] }): T | undefined =>
From 89194b4f88e4a7399709a48946419745be7326ec Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 06:17:48 +0700
Subject: [PATCH 049/114] fix(dashboard): reset the log filters when the tenant
or app changes
LogsView kept its filter, pod and container across a scope switch, so
the first request for the new tenant still asked for the previous
tenant's pod before the picker fell back to All. The view's state is
now keyed on the proxy root, so a new tenant or application starts
with no filter and All selected.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/LogsView.test.tsx | 20 +++++++++++++++++++
.../src/components/observability/LogsView.tsx | 7 ++++++-
2 files changed, 26 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index fe73e4164..a0df67c68 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -62,6 +62,26 @@ describe("LogsView with the production query defaults", () => {
expect((await screen.findAllByRole("option")).map((o) => o.textContent)).toEqual(["All"])
})
+ it("starts the next tenant with no filter, pod or container of the previous one", async () => {
+ const { client, raw } = clientWith(defaultHandler)
+ const { switchTo } = renderWithProductionDefaults(client, "tenant-reset-a")
+ await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}")
+ await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
+ await userEvent.click(await screen.findByRole("option", { name: "db-1" }))
+ await waitFor(() =>
+ expect(raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("query") === '{kubernetes_pod_name="db-1"} (error)')).toBe(true),
+ )
+
+ switchTo("tenant-reset-b")
+
+ await waitFor(() => expect(raw.mock.calls.some(([p]) => p.includes("/tenant-reset-b/") && p.includes("select/logsql/query?"))).toBe(true))
+ const first = raw.mock.calls.find(([p]) => p.includes("/tenant-reset-b/") && p.includes("select/logsql/query?"))!
+ expect(new URL(first[0], "http://x").searchParams.get("query")).toBe("*")
+ expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("All")
+ expect(screen.getByRole("combobox", { name: "Container" })).toHaveTextContent("All")
+ expect(screen.getByLabelText("LogsQL filter")).toHaveValue("")
+ })
+
it("keeps the current lines while a new filter in the same tenant loads", async () => {
const { client } = clientWith((endpoint, params) =>
endpoint.endsWith("query") && params.get("query") === "slow" ? new Promise(() => {}) : defaultHandler(endpoint, params),
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index c2dc8ca5c..8c05a8743 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -81,9 +81,14 @@ function placeholderFor({ follow, stopped, loading, failed, longestRange, retry
return
}
+// Filter, pod and container belong to one scope; a new tenant or application starts clean.
export function LogsView({ scope }: { scope: ObservabilityScope }) {
- const client = useK8sClient()
const root = logsRoot(scope)
+ return
+}
+
+function ScopedLogsView({ root }: { root: string }) {
+ const client = useK8sClient()
const [range, setRange] = useState("1h")
const [draft, setDraft] = useState("")
const [filter, setFilter] = useState("")
From 32a466fb141b27833737fa6bc634f428376ad4ce Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 18:10:35 +0700
Subject: [PATCH 050/114] fix(dashboard): scope the logs and metrics tabs to
the selected app
The provider keeps a query's previous result while its key changes, so
while the next application's GET is pending the detail page still holds
the previous instance. The Logs and Metrics tabs took the application
name from that instance, so their proxy root did not change and the
previous application's lines and series stayed under the new heading
until the GET completed.
Take the name from the route instead, which is the identity the user
selected, and cover the pending GET at the route level with the real
page, router and production query defaults.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
...plicationDetailPage.observability.test.tsx | 113 ++++++++++++++++++
.../routes/detail/ApplicationDetailPage.tsx | 4 +-
.../console/src/routes/detail/LogsTab.tsx | 9 +-
.../console/src/routes/detail/MetricsTab.tsx | 9 +-
4 files changed, 125 insertions(+), 10 deletions(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.observability.test.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.observability.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.observability.test.tsx
new file mode 100644
index 000000000..e16bf1665
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.observability.test.tsx
@@ -0,0 +1,113 @@
+import { describe, it, expect, vi, beforeEach } from "vitest"
+import { act, render, screen, waitFor } from "@testing-library/react"
+import { createMemoryRouter, RouterProvider } from "react-router"
+import { K8sProvider } from "@cozystack/k8s-client"
+import { TenantContext, type TenantContextValue } from "../../lib/tenant-context.tsx"
+import type { ChartData } from "../../lib/observability/metrics.ts"
+
+// Keep the consoles' and the chart's browser-only libraries out of jsdom; the
+// Logs and Metrics tabs, the page and the providers stay real.
+vi.mock("./VncTab.tsx", () => ({ VncTab: () => null }))
+vi.mock("./SerialTab.tsx", () => ({ SerialTab: () => null }))
+vi.mock("../../components/observability/TimeSeriesChart.tsx", () => ({
+ TimeSeriesChart: ({ data }: { data: ChartData }) => (
+ {data.series.map((s) => s.label).join(",")}
+ ),
+}))
+
+const { ApplicationDetailPage } = await import("./ApplicationDetailPage.tsx")
+
+const NS = "tenant-test"
+const APPS = `/apis/apps.cozystack.io/v1alpha1/namespaces/${NS}/postgreses`
+
+const tenant: TenantContextValue = {
+ tenants: [],
+ selectedTenant: "test",
+ selectTenant: () => {},
+ tenantNamespace: NS,
+ isLoading: false,
+ error: null,
+}
+
+const definitions = {
+ items: [{ metadata: { name: "postgres" }, spec: { application: { plural: "postgreses", kind: "Postgres" } } }],
+}
+
+const instance = (name: string) => ({
+ apiVersion: "apps.cozystack.io/v1alpha1",
+ kind: "Postgres",
+ metadata: { name, namespace: NS },
+})
+
+const logLine = (name: string) =>
+ `${JSON.stringify({ _time: "2026-11-04T10:00:01Z", _msg: `${name} is ready`, _stream_id: "s", kubernetes_pod_name: `${name}-0`, kubernetes_container_name: "postgres" })}\n`
+
+const matrix = (name: string) =>
+ Response.json({ status: "success", data: { resultType: "matrix", result: [{ metric: { pod: `${name}-0` }, values: [[1, "1"]] }] } })
+
+// app-b's own GET never answers, so the page keeps the placeholder of app-a
+// that the production query defaults hand it; its proxy reads do answer.
+function handle(url: URL): Response | Promise {
+ const path = url.pathname
+ if (path.endsWith("/applicationdefinitions")) return Response.json(definitions)
+ if (path === `${APPS}/app-a`) return Response.json(instance("app-a"))
+ if (path === `${APPS}/app-b`) return new Promise(() => {})
+ const app = path.match(/\/postgreses\/(app-[ab])\//)?.[1]
+ if (app && path.endsWith("/select/logsql/query")) return new Response(logLine(app))
+ if (app && path.endsWith("/stream_field_values")) return Response.json({ values: [] })
+ if (app && path.includes("/metrics/api/v1/query_range")) return matrix(app)
+ return Response.json({ items: [] })
+}
+
+const fetchMock = vi.fn(async (input: RequestInfo | URL) => handle(new URL(String(input), "http://x")))
+
+function renderAt(path: string) {
+ const router = createMemoryRouter(
+ [{ path: "/console/:plural/:name/*", element: }],
+ { initialEntries: [path] },
+ )
+ // No queryClient: the provider's production defaults keep a query's previous
+ // result while its key changes, which is what exposes the stale instance.
+ render(
+
+
+
+
+ ,
+ )
+ return { go: (to: string) => act(() => router.navigate(to)) }
+}
+
+const requested = (prefix: string) =>
+ fetchMock.mock.calls.some(([input]) => new URL(String(input), "http://x").pathname.startsWith(prefix))
+
+beforeEach(() => {
+ vi.stubGlobal("fetch", fetchMock)
+ fetchMock.mockClear()
+})
+
+describe("ApplicationDetailPage observability while the next application loads", () => {
+ it("shows the selected application's logs, not the previous one's", async () => {
+ const { go } = renderAt("/console/postgreses/app-a/logs")
+ expect(await screen.findByText("app-a is ready")).toBeInTheDocument()
+
+ await go("/console/postgreses/app-b/logs")
+
+ expect(screen.getByRole("heading", { name: "app-b" })).toBeInTheDocument()
+ expect(screen.queryByText("app-a is ready")).toBeNull()
+ await waitFor(() => expect(requested(`${APPS}/app-b/log/`)).toBe(true))
+ expect(await screen.findByText("app-b is ready")).toBeInTheDocument()
+ })
+
+ it("shows the selected application's metrics, not the previous one's", async () => {
+ const { go } = renderAt("/console/postgreses/app-a/metrics")
+ await waitFor(() => expect(screen.getAllByTestId("chart")[0]).toHaveTextContent("app-a-0"))
+
+ await go("/console/postgreses/app-b/metrics")
+
+ expect(screen.getByRole("heading", { name: "app-b" })).toBeInTheDocument()
+ expect(screen.queryAllByText("app-a-0", { exact: false })).toEqual([])
+ await waitFor(() => expect(requested(`${APPS}/app-b/metrics/`)).toBe(true))
+ await waitFor(() => expect(screen.getAllByTestId("chart")[0]).toHaveTextContent("app-b-0"))
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
index ca499aac1..ab68f7c49 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
@@ -273,11 +273,11 @@ export function ApplicationDetailPage() {
path="metrics"
element={
}>
-
+
}
/>
- } />
+ } />
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx
index 4ac5f878d..64cbcc89e 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx
@@ -1,15 +1,16 @@
-import type { ApplicationDefinition, ApplicationInstance } from "@cozystack/types"
+import type { ApplicationDefinition } from "@cozystack/types"
import { useTenantContext } from "../../lib/tenant-context.tsx"
import { LogsView } from "../../components/observability/LogsView.tsx"
interface LogsTabProps {
ad: ApplicationDefinition
- instance: ApplicationInstance
+ // The route's name: while the next application loads, the page still holds the previous instance.
+ name: string
}
-export function LogsTab({ ad, instance }: LogsTabProps) {
+export function LogsTab({ ad, name }: LogsTabProps) {
const { tenantNamespace } = useTenantContext()
const plural = ad.spec?.application.plural
if (!tenantNamespace || !plural) return null
- return
+ return
}
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx
index a928e633d..992cfc89b 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx
@@ -1,15 +1,16 @@
-import type { ApplicationDefinition, ApplicationInstance } from "@cozystack/types"
+import type { ApplicationDefinition } from "@cozystack/types"
import { useTenantContext } from "../../lib/tenant-context.tsx"
import { MetricsView } from "../../components/observability/MetricsView.tsx"
interface MetricsTabProps {
ad: ApplicationDefinition
- instance: ApplicationInstance
+ // The route's name: while the next application loads, the page still holds the previous instance.
+ name: string
}
-export function MetricsTab({ ad, instance }: MetricsTabProps) {
+export function MetricsTab({ ad, name }: MetricsTabProps) {
const { tenantNamespace } = useTenantContext()
const plural = ad.spec?.application.plural
if (!tenantNamespace || !plural) return null
- return
+ return
}
From 2e5a3bd6a7bcbd85f77e1043f782951ce37411cd Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Mon, 9 Nov 2026 15:47:34 +0700
Subject: [PATCH 051/114] fix(dashboard): keep the tenant logs page off
application URLs
Every console path of one or two segments belongs to an application
list or detail page, so /console/observability/logs took the Overview
of an application named logs whose kind has the plural observability.
Kubernetes names must start with an alphanumeric character, so the
reserved "-" segment cannot collide with any application.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/routes/ConsolePage.routing.test.tsx | 18 ++++++++++++++++--
.../apps/console/src/routes/ConsolePage.tsx | 2 +-
.../src/routes/sidebar-sections.test.tsx | 2 +-
.../console/src/routes/sidebar-sections.tsx | 2 +-
4 files changed, 19 insertions(+), 5 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx
index cf6982823..8e71ff98a 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx
@@ -71,12 +71,12 @@ describe("ConsolePage routing", () => {
expect(screen.queryByText(/Cluster-scoped capacity/i)).toBeNull()
})
- it("serves the tenant logs page under observability/", async () => {
+ it("serves the tenant logs page under the reserved -/ segment", async () => {
renderWithK8sProvider(
,
- { client: makeClient(), initialRoute: "/observability/logs" },
+ { client: makeClient(), initialRoute: "/-/observability/logs" },
)
expect(await screen.findByText(/^Every application in/)).toBeInTheDocument()
@@ -94,4 +94,18 @@ describe("ConsolePage routing", () => {
await waitFor(() => expect(client.list).toHaveBeenCalled())
expect(screen.queryByText(/^Every application in/)).toBeNull()
})
+
+ // No ApplicationDefinition plural or application name can be "-", so only
+ // routes under it are safe from shadowing an application's own pages.
+ it("leaves /observability/logs to an application named logs of kind observability", async () => {
+ renderWithK8sProvider(
+
+
+ ,
+ { client: makeClient(), initialRoute: "/observability/logs" },
+ )
+
+ expect(await screen.findByText("Loading…")).toBeInTheDocument()
+ expect(screen.queryByText(/^Every application in/)).toBeNull()
+ })
})
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx
index 2b2aed3e4..c78ae619e 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx
@@ -97,7 +97,7 @@ export function ConsolePage() {
path="migration/vmimporttasks/:name"
element={ }
/>
- } />
+ } />
} />
} />
} />
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx
index 1fcc3cde4..e678c7dd4 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx
@@ -117,7 +117,7 @@ describe("useConsoleSidebarSections — admin areas moved out", () => {
wrapper: makeWrapper(client),
})
await waitFor(() => expect(result.current.length).toBeGreaterThan(0))
- expect(findItem(result.current, "Logs")?.to).toBe("/console/observability/logs")
+ expect(findItem(result.current, "Logs")?.to).toBe("/console/-/observability/logs")
expect(result.current.find((s) => s.title === "Observability")?.items).toHaveLength(1)
})
})
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx
index 0cf1ce089..51c9ea00a 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx
@@ -121,7 +121,7 @@ export function useConsoleSidebarSections(): SidebarSection[] {
const observabilitySection: SidebarSection = {
title: "Observability",
- items: [{ label: "Logs", to: "/console/observability/logs", icon: ScrollText }],
+ items: [{ label: "Logs", to: "/console/-/observability/logs", icon: ScrollText }],
}
return [...categorySections, backupsSection, migrationSection, observabilitySection]
From e623530c479ad92afd9c8a707aa12cd645f0ae93 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Mon, 9 Nov 2026 10:29:20 +0100
Subject: [PATCH 052/114] fix(kubernetes): detach volumes when the infra PVC is
missing
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Assisted-by: LLM
---
.../images/kubevirt-csi-driver/controller.go | 3 +-
.../kubevirt-csi-driver/controller_test.go | 77 +++++++++++++++++--
2 files changed, 71 insertions(+), 9 deletions(-)
diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go
index 9029c9ff3..b61121758 100644
--- a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go
+++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go
@@ -424,7 +424,8 @@ func (w *WrappedControllerService) ControllerUnpublishVolume(ctx context.Context
pvc, err := w.infraClient.CoreV1().PersistentVolumeClaims(w.infraNamespace).Get(ctx, dvName, metav1.GetOptions{})
if err != nil {
if errors.IsNotFound(err) {
- return &csi.ControllerUnpublishVolumeResponse{}, nil
+ // A failed PVC creation can still leave a hotplug entry in the VM.
+ return w.ControllerService.ControllerUnpublishVolume(ctx, req)
}
return nil, err
}
diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go
index 46e2e1dce..73b8ffc7d 100644
--- a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go
+++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go
@@ -310,13 +310,14 @@ const (
testVolumeName = "pvc-0"
)
-// teardownClient models the CAPI/CAPK teardown race: the VM is gone while its
-// VMI still reports the hot-plugged volume. The VMI drops the volume once
-// RemoveVolumeFromVMI is called, or when the garbage collector deletes a
-// VMI whose controlling VM is gone.
+// teardownClient keeps hotplug state until removal or garbage collection,
+// including when the VMI outlives its VM during CAPI/CAPK teardown.
type teardownClient struct {
kubevirtclient.Client
+ vm *kubevirtv1.VirtualMachine
vmiOwners []metav1.OwnerReference
+ detached bool
+ vmRemovals int
vmiRemovals int
lookups int
}
@@ -328,6 +329,9 @@ func (c *teardownClient) vmiGarbageCollected() bool {
func (c *teardownClient) GetWorkloadManagingVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachine, error) {
c.lookups++
+ if c.vm != nil {
+ return c.vm, nil
+ }
return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachines"}, testVMName)
}
@@ -336,14 +340,27 @@ func (c *teardownClient) GetVirtualMachine(context.Context, string, string) (*ku
vmi := &kubevirtv1.VirtualMachineInstance{
ObjectMeta: metav1.ObjectMeta{Name: testVMName, Namespace: testInfraNamespace, OwnerReferences: c.vmiOwners},
}
- if c.vmiRemovals == 0 {
- vmi.Status.VolumeStatus = []kubevirtv1.VolumeStatus{{Name: testVolumeName, HotplugVolume: &kubevirtv1.HotplugVolumeStatus{}}}
+ if !c.detached && c.vmRemovals == 0 && c.vmiRemovals == 0 {
+ vmi.Status.VolumeStatus = []kubevirtv1.VolumeStatus{{Name: testVolumeName, Phase: kubevirtv1.VolumePending, Reason: "FailedPvcNotFound", HotplugVolume: &kubevirtv1.HotplugVolumeStatus{}}}
}
return vmi, nil
}
+func (c *teardownClient) EnsureVolumeRemovedVM(context.Context, string, string, string) (bool, error) {
+ return c.vm == nil || len(c.vm.Spec.Template.Spec.Volumes) == 0, nil
+}
+
func (c *teardownClient) EnsureVolumeRemovedVMI(context.Context, string, string, string) (bool, error) {
- return c.vmiRemovals > 0, nil
+ return c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0, nil
+}
+
+func (c *teardownClient) RemoveVolumeFromVM(_ context.Context, namespace, name string, opts *kubevirtv1.RemoveVolumeOptions) error {
+ if namespace != testInfraNamespace || name != testVMName || opts.Name != testVolumeName {
+ return errors.New("unexpected hotplug removal target")
+ }
+ c.vmRemovals++
+ c.vm.Spec.Template.Spec.Volumes = nil
+ return nil
}
func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *kubevirtv1.RemoveVolumeOptions) error {
@@ -352,7 +369,7 @@ func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *k
}
func (c *teardownClient) EnsureVolumeRemoved(context.Context, string, string, string, time.Duration) error {
- if c.vmiRemovals > 0 || c.vmiGarbageCollected() {
+ if c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0 || c.vmiGarbageCollected() {
return nil
}
return errors.New("timed out waiting for the VMI to drop the volume")
@@ -410,6 +427,50 @@ func TestControllerUnpublishVolumeHonoursVMIHotplugFallback(t *testing.T) {
}
}
+func TestControllerUnpublishVolumeMissingPVC(t *testing.T) {
+ tests := []struct {
+ name string
+ vmGone bool
+ detached bool
+ wantVMCalls int
+ wantVMICalls int
+ }{
+ {name: "pending hotplug volume", wantVMCalls: 1},
+ {name: "already detached", detached: true},
+ {name: "VM gone, VMI still has volume", vmGone: true, wantVMICalls: 1},
+ {name: "VM gone, no attached volume", vmGone: true, detached: true},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ client := &teardownClient{detached: tt.detached}
+ if !tt.vmGone {
+ client.vm = &kubevirtv1.VirtualMachine{Spec: kubevirtv1.VirtualMachineSpec{Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{}}}
+ if !tt.detached {
+ client.vm.Spec.Template.Spec.Volumes = []kubevirtv1.Volume{{
+ Name: testVolumeName,
+ VolumeSource: kubevirtv1.VolumeSource{DataVolume: &kubevirtv1.DataVolumeSource{Name: testVolumeName, Hotpluggable: true}},
+ }}
+ }
+ }
+ w := &WrappedControllerService{
+ ControllerService: service.NewControllerService(client, testInfraNamespace, nil, util.StorageClassEnforcement{AllowAll: true}, true),
+ infraClient: fake.NewClientset(),
+ infraNamespace: testInfraNamespace,
+ }
+ req := &csi.ControllerUnpublishVolumeRequest{VolumeId: testVolumeName, NodeId: testInfraNamespace + "/" + testVMName}
+ if _, err := w.ControllerUnpublishVolume(context.Background(), req); err != nil {
+ t.Fatalf("ControllerUnpublishVolume() error = %v", err)
+ }
+ if client.vmRemovals != tt.wantVMCalls || client.vmiRemovals != tt.wantVMICalls {
+ t.Errorf("VM/VMI removevolume calls = %d/%d, want %d/%d", client.vmRemovals, client.vmiRemovals, tt.wantVMCalls, tt.wantVMICalls)
+ }
+ if client.vm != nil && len(client.vm.Spec.Template.Spec.Volumes) != 0 {
+ t.Error("hotplug volume remains in VM spec")
+ }
+ })
+ }
+}
+
func nfsCNP(owners ...string) *unstructured.Unstructured {
refs := make([]interface{}, 0, len(owners))
for _, owner := range owners {
From 5f6679e4cff70b6e37535920427004259af8adac Mon Sep 17 00:00:00 2001
From: Mattia Eleuteri
Date: Mon, 9 Nov 2026 10:52:28 +0100
Subject: [PATCH 053/114] fix(ingress-nginx): raise client keepalive_time above
the 1h default
ingress-nginx has no ConfigMap key for the client-side keepalive_time,
so nginx applies its 1h default and closes every HTTP/2 connection
that reaches that age with a GOAWAY. Clients that multiplex a long
operation over a single connection see it fail: a kopia repository
maintenance run by Velero against SeaweedFS lasted just over an hour
and died while listing blobs, with "http2: server sent GOAWAY and
closed the connection; ErrCode=NO_ERROR", because the listing is not
retried.
Set keepalive_time to 12h through http-snippet. Idle connections are
still closed by keep-alive after 75s, so this only lengthens the life
of connections that are actually in use.
Assisted-by: LLM
Signed-off-by: Mattia Eleuteri
---
.../tests/client_keepalive_time_test.yaml | 26 +++++++++++++++++++
packages/system/ingress-nginx/values.yaml | 7 +++++
2 files changed, 33 insertions(+)
create mode 100644 packages/system/ingress-nginx/tests/client_keepalive_time_test.yaml
diff --git a/packages/system/ingress-nginx/tests/client_keepalive_time_test.yaml b/packages/system/ingress-nginx/tests/client_keepalive_time_test.yaml
new file mode 100644
index 000000000..bb492e57c
--- /dev/null
+++ b/packages/system/ingress-nginx/tests/client_keepalive_time_test.yaml
@@ -0,0 +1,26 @@
+suite: ingress-nginx client keepalive_time
+release:
+ name: ingress-nginx
+ namespace: cozy-ingress-nginx
+templates:
+ - charts/ingress-nginx/templates/controller-configmap.yaml
+tests:
+ - it: raises the client keepalive_time above the nginx 1h default
+ asserts:
+ - equal:
+ path: data.http-snippet
+ value: keepalive_time 12h;
+ - it: keeps the snippet when a tenant ingress sets its own config keys
+ # The tenant ingress HelmRelease passes controller.config for whitelist,
+ # Cloudflare and PROXY protocol; Helm merges maps, so those keys must not
+ # drop the default snippet.
+ set:
+ ingress-nginx.controller.config.use-proxy-protocol: "true"
+ ingress-nginx.controller.config.whitelist-source-range: 10.0.0.0/8
+ asserts:
+ - equal:
+ path: data.http-snippet
+ value: keepalive_time 12h;
+ - equal:
+ path: data.use-proxy-protocol
+ value: "true"
diff --git a/packages/system/ingress-nginx/values.yaml b/packages/system/ingress-nginx/values.yaml
index fca145843..1cb7b58da 100644
--- a/packages/system/ingress-nginx/values.yaml
+++ b/packages/system/ingress-nginx/values.yaml
@@ -47,6 +47,13 @@ ingress-nginx:
upstream-keepalive-timeout: "60"
upstream-keepalive-connections: "320"
ssl-session-tickets: "true"
+ # Client-side keepalive_time has no ConfigMap key, so nginx keeps its
+ # 1h default and sends GOAWAY to any HTTP/2 connection older than that.
+ # A long S3 operation (kopia maintenance behind Velero) multiplexes its
+ # requests over one connection, and the in-flight object listing fails
+ # instead of being retried. Idle connections still close after
+ # keep-alive (75s).
+ http-snippet: "keepalive_time 12h;"
resources:
limits:
cpu: "1"
From a1e28cccddce7a76a57da0b95b0fe80cf5b276ff Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Mon, 9 Nov 2026 12:51:14 +0100
Subject: [PATCH 054/114] test(kubernetes): cover missing VMI during volume
unpublish
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Assisted-by: LLM
---
.../kubevirt-csi-driver/controller_test.go | 49 +++++++++++++------
1 file changed, 34 insertions(+), 15 deletions(-)
diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go
index 73b8ffc7d..c6029f17b 100644
--- a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go
+++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go
@@ -314,12 +314,15 @@ const (
// including when the VMI outlives its VM during CAPI/CAPK teardown.
type teardownClient struct {
kubevirtclient.Client
- vm *kubevirtv1.VirtualMachine
- vmiOwners []metav1.OwnerReference
- detached bool
- vmRemovals int
- vmiRemovals int
- lookups int
+ vm *kubevirtv1.VirtualMachine
+ vmiOwners []metav1.OwnerReference
+ detached bool
+ vmiGone bool
+ vmiGoneAfterCheck bool
+ vmiLookups int
+ vmRemovals int
+ vmiRemovals int
+ lookups int
}
func (c *teardownClient) vmiGarbageCollected() bool {
@@ -337,6 +340,10 @@ func (c *teardownClient) GetWorkloadManagingVirtualMachine(context.Context, stri
func (c *teardownClient) GetVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachineInstance, error) {
c.lookups++
+ c.vmiLookups++
+ if c.vmiGone {
+ return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachineinstances"}, testVMName)
+ }
vmi := &kubevirtv1.VirtualMachineInstance{
ObjectMeta: metav1.ObjectMeta{Name: testVMName, Namespace: testInfraNamespace, OwnerReferences: c.vmiOwners},
}
@@ -351,7 +358,11 @@ func (c *teardownClient) EnsureVolumeRemovedVM(context.Context, string, string,
}
func (c *teardownClient) EnsureVolumeRemovedVMI(context.Context, string, string, string) (bool, error) {
- return c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0, nil
+ removed := c.vmiGone || c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0
+ if c.vmiGoneAfterCheck {
+ c.vmiGone = true
+ }
+ return removed, nil
}
func (c *teardownClient) RemoveVolumeFromVM(_ context.Context, namespace, name string, opts *kubevirtv1.RemoveVolumeOptions) error {
@@ -369,7 +380,7 @@ func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *k
}
func (c *teardownClient) EnsureVolumeRemoved(context.Context, string, string, string, time.Duration) error {
- if c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0 || c.vmiGarbageCollected() {
+ if c.vmiGone || c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0 || c.vmiGarbageCollected() {
return nil
}
return errors.New("timed out waiting for the VMI to drop the volume")
@@ -429,20 +440,25 @@ func TestControllerUnpublishVolumeHonoursVMIHotplugFallback(t *testing.T) {
func TestControllerUnpublishVolumeMissingPVC(t *testing.T) {
tests := []struct {
- name string
- vmGone bool
- detached bool
- wantVMCalls int
- wantVMICalls int
+ name string
+ vmGone bool
+ detached bool
+ vmiGone bool
+ vmiGoneAfterCheck bool
+ wantVMILookups int
+ wantVMCalls int
+ wantVMICalls int
}{
{name: "pending hotplug volume", wantVMCalls: 1},
{name: "already detached", detached: true},
- {name: "VM gone, VMI still has volume", vmGone: true, wantVMICalls: 1},
+ {name: "VM gone, VMI still has volume", vmGone: true, wantVMICalls: 1, wantVMILookups: 1},
{name: "VM gone, no attached volume", vmGone: true, detached: true},
+ {name: "VM and VMI gone", vmGone: true, vmiGone: true},
+ {name: "VMI disappears after attachment check", vmGone: true, vmiGoneAfterCheck: true, wantVMILookups: 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- client := &teardownClient{detached: tt.detached}
+ client := &teardownClient{detached: tt.detached, vmiGone: tt.vmiGone, vmiGoneAfterCheck: tt.vmiGoneAfterCheck}
if !tt.vmGone {
client.vm = &kubevirtv1.VirtualMachine{Spec: kubevirtv1.VirtualMachineSpec{Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{}}}
if !tt.detached {
@@ -464,6 +480,9 @@ func TestControllerUnpublishVolumeMissingPVC(t *testing.T) {
if client.vmRemovals != tt.wantVMCalls || client.vmiRemovals != tt.wantVMICalls {
t.Errorf("VM/VMI removevolume calls = %d/%d, want %d/%d", client.vmRemovals, client.vmiRemovals, tt.wantVMCalls, tt.wantVMICalls)
}
+ if client.vmiLookups != tt.wantVMILookups {
+ t.Errorf("GetVirtualMachine calls = %d, want %d", client.vmiLookups, tt.wantVMILookups)
+ }
if client.vm != nil && len(client.vm.Spec.Template.Spec.Volumes) != 0 {
t.Error("hotplug volume remains in VM spec")
}
From 4be80657beaf164067a0e08675f951452bd8b3dd Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Tue, 10 Nov 2026 00:41:10 +0700
Subject: [PATCH 055/114] test(dashboard): prove /observability/logs opens the
application page
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The list page shows the same "Loading…" text as the detail page, so the
previous check could not tell which route matched. A missing instance
makes only the detail page answer with its own name, and the GET it
issues names the plural and the application.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/routes/ConsolePage.routing.test.tsx | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx
index 8e71ff98a..c3b014f4d 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx
@@ -98,14 +98,26 @@ describe("ConsolePage routing", () => {
// No ApplicationDefinition plural or application name can be "-", so only
// routes under it are safe from shadowing an application's own pages.
it("leaves /observability/logs to an application named logs of kind observability", async () => {
+ const client = makeClient()
+ vi.mocked(client.list).mockImplementation(async (_g, _v, plural) => ({
+ apiVersion: "v1",
+ kind: `${plural}List`,
+ metadata: {},
+ items:
+ plural === "tenantnamespaces"
+ ? [{ apiVersion: "core.cozystack.io/v1alpha1", kind: "TenantNamespace", metadata: { name: "tenant-root" } }]
+ : [],
+ }) as K8sList)
+ const get = vi.spyOn(client, "get").mockRejectedValue(new Error("not found"))
renderWithK8sProvider(
,
- { client: makeClient(), initialRoute: "/observability/logs" },
+ { client, initialRoute: "/observability/logs" },
)
- expect(await screen.findByText("Loading…")).toBeInTheDocument()
+ expect(await screen.findByText(/^Application/)).toHaveTextContent("Application logs not found.")
+ expect(get).toHaveBeenCalledWith(expect.anything(), expect.anything(), "observability", "logs", expect.anything())
expect(screen.queryByText(/^Every application in/)).toBeNull()
})
})
From 7eff63b028cf769fcd1237e9f7fdd3dc568588a9 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Tue, 10 Nov 2026 00:51:01 +0700
Subject: [PATCH 056/114] refactor(dashboard): drop the scope-keyed placeholder
from the logs view
The logs view remounts under a key per scope, so its queries start with
no previous result to carry over and the scope check never had anything
to refuse. The key is the one guard, and the tenant-switch tests fail
without it.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../console/src/components/observability/LogsView.tsx | 5 +----
.../console/apps/console/src/lib/observability/proxy.ts | 9 ---------
2 files changed, 1 insertion(+), 13 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index 8c05a8743..05535e12e 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -14,7 +14,6 @@ import {
import {
fieldValues,
isScopeWide,
- keepWithinScope,
logsRoot,
queryLogs,
type ObservabilityScope,
@@ -81,7 +80,7 @@ function placeholderFor({ follow, stopped, loading, failed, longestRange, retry
return
}
-// Filter, pod and container belong to one scope; a new tenant or application starts clean.
+// Filter, pod, container and the lines already read belong to one scope; a new tenant or application starts clean.
export function LogsView({ scope }: { scope: ObservabilityScope }) {
const root = logsRoot(scope)
return
@@ -107,7 +106,6 @@ function ScopedLogsView({ root }: { root: string }) {
query: buildLogsQuery({ filter: "", pod: narrowTo }),
...windowFor(range),
}, signal),
- placeholderData: keepWithinScope(root),
retry: false,
})
const pods = useQuery(pickerQuery(POD_FIELD))
@@ -117,7 +115,6 @@ function ScopedLogsView({ root }: { root: string }) {
queryKey: ["observability", "logs", root, query, range],
queryFn: ({ signal }) => queryLogs(client, root, { query, limit: HISTORY_LIMIT, ...windowFor(range) }, signal),
enabled: !follow,
- placeholderData: keepWithinScope(root),
retry: false,
})
const tail = useLogTail(root, query, followRun, { cap: FOLLOW_CAP })
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
index d33052f70..c28bd12d4 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -20,15 +20,6 @@ export function logsRoot(scope: ObservabilityScope): string {
return app ? `${app}/log/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/logs/`
}
-// The provider's default keeps a useQuery's previous result while its key
-// changes; across scopes that would show one tenant's or application's data
-// under another. (useQueries starts a new observer per key, so it never does.)
-// Every observability query key carries its root at index 2.
-export function keepWithinScope(root: string) {
- return (previous: T | undefined, previousQuery?: { queryKey: readonly unknown[] }): T | undefined =>
- previousQuery?.queryKey[2] === root ? previous : undefined
-}
-
export type ProxyErrorKind =
| "bad-request"
| "unauthorized"
From 48f459e103f4b3a3bc6a79e3932160cee659c788 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Tue, 10 Nov 2026 00:54:51 +0700
Subject: [PATCH 057/114] fix(dashboard): say the logs are loading while an
empty result stands in
The previous result stays on screen while a new range or filter loads.
When that result was empty, the query reports success, so the view said
no lines matched the range it had not read yet.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../components/observability/LogsView.test.tsx | 17 +++++++++++++++++
.../src/components/observability/LogsView.tsx | 2 +-
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index a0df67c68..47cbf1c22 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -93,6 +93,23 @@ describe("LogsView with the production query defaults", () => {
expect(screen.getByText("ready to accept connections")).toBeInTheDocument()
})
+
+ it("says it is loading, not that nothing matched, while a longer range replaces an empty one", async () => {
+ const { client } = clientWith((endpoint, params) => {
+ if (!endpoint.endsWith("query")) return defaultHandler(endpoint, params)
+ return Date.parse(params.get("start") ?? "") < Date.now() - 2 * 3600_000
+ ? new Promise(() => {})
+ : new Response("")
+ })
+ renderWithProductionDefaults(client, "tenant-range-a")
+ const frame = () => screen.getByRole("region", { name: "Log lines" })
+ expect(await within(frame()).findByText("No log lines for this range and filter.")).toBeInTheDocument()
+
+ await userEvent.click(screen.getByRole("button", { name: "24h" }))
+
+ await waitFor(() => expect(within(frame()).getByRole("status")).toHaveAttribute("aria-busy", "true"))
+ expect(within(frame()).queryByText("No log lines for this range and filter.")).toBeNull()
+ })
})
describe("LogsView", () => {
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index 05535e12e..1563d90b5 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -168,7 +168,7 @@ function ScopedLogsView({ root }: { root: string }) {
// Previous results stay as placeholder data while a new query loads, so only a new key resets the scroll pin.
key={followRun ?? `history\n${query}\n${range}`}
entries={entries}
- placeholder={placeholderFor({ follow, stopped: tail.state === "stopped", loading: history.isLoading, failed: Boolean(error), longestRange: range === LONGEST_RANGE, retry: () => void history.refetch() })}
+ placeholder={placeholderFor({ follow, stopped: tail.state === "stopped", loading: history.isLoading || (history.isPlaceholderData && history.isFetching), failed: Boolean(error), longestRange: range === LONGEST_RANGE, retry: () => void history.refetch() })}
/>
)
From 051442b3187abbe0eb73e21924ee1eebd1b1e701 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Tue, 10 Nov 2026 00:58:29 +0700
Subject: [PATCH 058/114] fix(dashboard): mark previous log lines while a new
query replaces them
The previous result stays on screen while a new range or filter loads,
with nothing saying those lines answer the old query. A status line now
says so until the new result arrives. The empty case reads a placeholder
as loading whether or not the request is in flight, so a paused query
does not claim that nothing matched either.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/LogsView.test.tsx | 31 +++++++++++++++++++
.../src/components/observability/LogsView.tsx | 7 ++++-
2 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 47cbf1c22..be8bff1a8 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -110,6 +110,22 @@ describe("LogsView with the production query defaults", () => {
await waitFor(() => expect(within(frame()).getByRole("status")).toHaveAttribute("aria-busy", "true"))
expect(within(frame()).queryByText("No log lines for this range and filter.")).toBeNull()
})
+
+ it("marks the lines of the previous range as standing in while a longer range loads", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query") && Date.parse(params.get("start") ?? "") < Date.now() - 2 * 3600_000
+ ? new Promise(() => {})
+ : defaultHandler(endpoint, params),
+ )
+ renderWithProductionDefaults(client, "tenant-range-b")
+ expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
+ expect(screen.queryByText(/previous lines/)).toBeNull()
+
+ await userEvent.click(screen.getByRole("button", { name: "24h" }))
+
+ expect(await screen.findByText(/previous lines/)).toHaveAttribute("aria-busy", "true")
+ expect(screen.getByText("ready to accept connections")).toBeInTheDocument()
+ })
})
describe("LogsView", () => {
@@ -319,6 +335,21 @@ describe("LogsView", () => {
expect(screen.queryByRole("button", { name: "Retry" })).toBeNull()
})
+ it("says it is loading while a retry runs", async () => {
+ let calls = 0
+ const { client } = clientWith((endpoint, params) => {
+ if (!endpoint.endsWith("query")) return defaultHandler(endpoint, params)
+ return calls++ === 0 ? new Response("upstream down", { status: 502 }) : new Promise(() => {})
+ })
+ renderWithK8sProvider( , { client })
+
+ await userEvent.click(await screen.findByRole("button", { name: "Retry" }))
+
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ await waitFor(() => expect(within(frame).getByRole("status")).toHaveAttribute("aria-busy", "true"))
+ expect(within(frame).queryByRole("button", { name: "Retry" })).toBeNull()
+ })
+
it("opens a new history result at its newest line, not the old scroll position", async () => {
const { client } = clientWith((endpoint, params) =>
endpoint.endsWith("query")
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index 1563d90b5..a4376df47 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -163,12 +163,17 @@ function ScopedLogsView({ root }: { root: string }) {
{` · newest ${FOLLOW_CAP} lines kept`}
)}
+ {!follow && history.isPlaceholderData && entries.length > 0 && (
+
+ Showing the previous lines until the new range or filter loads…
+
+ )}
{error ? : null}
void history.refetch() })}
+ placeholder={placeholderFor({ follow, stopped: tail.state === "stopped", loading: history.isLoading || history.isPlaceholderData, failed: Boolean(error), longestRange: range === LONGEST_RANGE, retry: () => void history.refetch() })}
/>
)
From 7105364161b9b0dea4afaae49681647c32edf1ca Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Tue, 10 Nov 2026 01:02:17 +0700
Subject: [PATCH 059/114] test(dashboard): pin when the previous-lines note
goes away
The note must leave as soon as the new result lands and must not appear
over live lines while following; neither was exercised, since the only
test kept the longer range pending forever.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/LogsView.test.tsx | 39 +++++++++++++++++++
1 file changed, 39 insertions(+)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index be8bff1a8..faba85a4e 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -126,6 +126,45 @@ describe("LogsView with the production query defaults", () => {
expect(await screen.findByText(/previous lines/)).toHaveAttribute("aria-busy", "true")
expect(screen.getByText("ready to accept connections")).toBeInTheDocument()
})
+
+ it("drops the standing-in note once the longer range answers", async () => {
+ let answer: (r: Response) => void = () => {}
+ const longer = new Promise((resolve) => { answer = resolve })
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query") && Date.parse(params.get("start") ?? "") < Date.now() - 2 * 3600_000
+ ? longer
+ : defaultHandler(endpoint, params),
+ )
+ renderWithProductionDefaults(client, "tenant-range-c")
+ expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
+ await userEvent.click(screen.getByRole("button", { name: "24h" }))
+ expect(await screen.findByText(/previous lines/)).toBeInTheDocument()
+
+ answer(new Response(`${line("2026-11-03T12:00:00Z", "checkpoint complete")}\n`))
+
+ expect(await screen.findByText("checkpoint complete")).toBeInTheDocument()
+ expect(screen.queryByText(/previous lines/)).toBeNull()
+ expect(screen.queryByText("ready to accept connections")).toBeNull()
+ })
+
+ it("shows no standing-in note while following", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("tail")
+ ? new Response(`${line("2026-11-04T10:00:02Z", "live line")}\n`)
+ : endpoint.endsWith("query") && params.get("query")?.includes("error")
+ ? new Promise(() => {})
+ : defaultHandler(endpoint, params),
+ )
+ renderWithProductionDefaults(client, "tenant-follow-a")
+ expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
+
+ await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}")
+ expect(await screen.findByText(/previous lines/)).toBeInTheDocument()
+ await userEvent.click(screen.getByRole("button", { name: "Follow" }))
+
+ expect(await screen.findByText("live line")).toBeInTheDocument()
+ expect(screen.queryByText(/previous lines/)).toBeNull()
+ })
})
describe("LogsView", () => {
From 4f3260e5d432410ca8a3397d16af189e255faab4 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 04:58:49 +0700
Subject: [PATCH 060/114] feat(observability-proxy): rebuild metrics queries
from an allowlist
vmselect ORs repeated extra_filters, so a client-supplied value would
widen the tenant scope. The upstream request is therefore built from
scratch: only query, start, end, step, time, match[] and a clamped
timeout are read from the client, and the namespace selector is sent
as the single extra_filters[] value. The query itself is not parsed;
vmselect ANDs that filter into every selector.
The label endpoints always carry a match[] (the namespace selector
when the client sent none), because a vmselect started with
-search.ignoreExtraFiltersAtLabelsAPI applies extra filters there only
when a match[] is present.
Range, points per series and step are bounded with 422 per the cost
limits of the metrics proposal. Times are accepted only in the two
forms the Prometheus API defines, since vmselect guesses the unit of a
numeric timestamp from its digit count. The window start picks the
short-term store when it lies within that store's retention.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/metricsparams.go | 284 ++++++++++++++++++
.../observabilityproxy/metricsparams_test.go | 246 +++++++++++++++
2 files changed, 530 insertions(+)
create mode 100644 internal/observabilityproxy/metricsparams.go
create mode 100644 internal/observabilityproxy/metricsparams_test.go
diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go
new file mode 100644
index 000000000..0e24b6597
--- /dev/null
+++ b/internal/observabilityproxy/metricsparams.go
@@ -0,0 +1,284 @@
+/*
+Copyright 2026 The Cozystack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package observabilityproxy
+
+import (
+ "fmt"
+ "net/http"
+ "net/url"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// MetricsLimits bound what one metrics request may make vmselect do.
+type MetricsLimits struct {
+ MaxRange time.Duration
+ MaxPoints int
+ MaxTimeout time.Duration
+}
+
+// MetricsConfig names the vmselect stores of the root tenant and the
+// retention that decides between them.
+type MetricsConfig struct {
+ RootTenant string
+ ShortTermStore string
+ LongTermStore string
+ ShortTermRetention time.Duration
+ Limits MetricsLimits
+ // UpstreamURL maps a store name to its Prometheus API base URL, without
+ // a trailing slash; nil means the in-cluster vmselect Service.
+ UpstreamURL func(store string) string
+}
+
+// store picks the short-term store when everything from start on is still
+// within its retention, so a live graph never waits on the downsampled one.
+func (c MetricsConfig) store(start, now time.Time) string {
+ if now.Sub(start) <= c.ShortTermRetention {
+ return c.ShortTermStore
+ }
+ return c.LongTermStore
+}
+
+// maxMetricsMatchers bounds match[]: vmselect runs one index search per
+// matcher and has no cap of its own.
+const maxMetricsMatchers = 20
+
+// labelsDefaultWindow is vmselect's own default for the label and series
+// APIs (defaultStep in app/vmselect/prometheus); the proxy sends it
+// explicitly so the store choice and the range check see the same window.
+const labelsDefaultWindow = 5 * time.Minute
+
+var labelName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)
+
+type metricsRequest struct {
+ path string
+ params url.Values
+ // start is the earliest instant the request reads, for the store choice.
+ start time.Time
+}
+
+// namespaceSelector quotes ns the way MetricsQL unquotes a label value
+// (strconv.Unquote in metricsql's extractStringValue).
+func namespaceSelector(ns string) string {
+ return "{namespace=" + strconv.Quote(ns) + "}"
+}
+
+// BuildMetricsParams rebuilds the upstream parameters from an allowlist and
+// adds the namespace scope as the only extra_filters[] value. vmselect ORs
+// repeated extra_filters, so a client value would widen the scope; it ANDs
+// the single one into every selector of the query and every match[], which
+// is what lets the query pass through unparsed.
+func BuildMetricsParams(endpoint string, in url.Values, namespace string, now time.Time, lim MetricsLimits) (metricsRequest, *HTTPError) {
+ req := metricsRequest{path: endpoint, params: url.Values{}}
+ out := req.params
+ out.Set("extra_filters[]", namespaceSelector(namespace))
+ timeout := lim.MaxTimeout
+ if v := in.Get("timeout"); v != "" {
+ d, herr := parseMetricsDuration("timeout", v)
+ if herr != nil {
+ return req, herr
+ }
+ timeout = min(d, lim.MaxTimeout)
+ }
+ out.Set("timeout", formatSeconds(timeout))
+
+ switch endpoint {
+ case "api/v1/query":
+ if herr := copyQuery(out, in); herr != nil {
+ return req, herr
+ }
+ req.start = now
+ if v := in.Get("time"); v != "" {
+ t, herr := parseMetricsTime("time", v)
+ if herr != nil {
+ return req, herr
+ }
+ req.start = t
+ out.Set("time", formatTime(t))
+ }
+ if v := in.Get("step"); v != "" {
+ step, herr := parseStep(v, lim)
+ if herr != nil {
+ return req, herr
+ }
+ out.Set("step", formatSeconds(step))
+ }
+ return req, nil
+
+ case "api/v1/query_range":
+ if herr := copyQuery(out, in); herr != nil {
+ return req, herr
+ }
+ for _, k := range []string{"start", "end", "step"} {
+ if in.Get(k) == "" {
+ return req, errorf(http.StatusBadRequest, "%s is required", k)
+ }
+ }
+ start, end, herr := metricsWindow(in, now, lim)
+ if herr != nil {
+ return req, herr
+ }
+ step, herr := parseStep(in.Get("step"), lim)
+ if herr != nil {
+ return req, herr
+ }
+ if points := end.Sub(start)/step + 1; points > time.Duration(lim.MaxPoints) {
+ return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints)
+ }
+ req.start = start
+ out.Set("start", formatTime(start))
+ out.Set("end", formatTime(end))
+ out.Set("step", formatSeconds(step))
+ return req, nil
+ }
+
+ name, isLabelValues := strings.CutPrefix(endpoint, "api/v1/label/")
+ if isLabelValues {
+ var ok bool
+ if name, ok = strings.CutSuffix(name, "/values"); !ok || strings.Contains(name, "/") {
+ return req, errorf(http.StatusNotFound, "not found")
+ }
+ if !labelName.MatchString(name) {
+ return req, errorf(http.StatusBadRequest, "invalid label name %q", name)
+ }
+ } else if endpoint != "api/v1/series" && endpoint != "api/v1/labels" {
+ return req, errorf(http.StatusNotFound, "not found")
+ }
+ start, end, herr := metricsWindow(in, now, lim)
+ if herr != nil {
+ return req, herr
+ }
+ req.start = start
+ out.Set("start", formatTime(start))
+ out.Set("end", formatTime(end))
+
+ matchers := in["match[]"]
+ if len(matchers) > maxMetricsMatchers {
+ return req, errorf(http.StatusUnprocessableEntity, "at most %d match[] are allowed", maxMetricsMatchers)
+ }
+ for _, m := range matchers {
+ if m == "" {
+ return req, errorf(http.StatusBadRequest, "empty match[]")
+ }
+ out.Add("match[]", m)
+ }
+ if len(matchers) == 0 {
+ if endpoint == "api/v1/series" {
+ return req, errorf(http.StatusBadRequest, "match[] is required")
+ }
+ // vmselect started with -search.ignoreExtraFiltersAtLabelsAPI drops
+ // extra_filters[] on the label APIs unless a match[] is present
+ // (getCommonParamsInternal), so the scope is repeated here.
+ out.Set("match[]", namespaceSelector(namespace))
+ }
+ return req, nil
+}
+
+func copyQuery(out, in url.Values) *HTTPError {
+ q := in.Get("query")
+ if q == "" {
+ return errorf(http.StatusBadRequest, "query is required")
+ }
+ out.Set("query", q)
+ return nil
+}
+
+// metricsWindow reads start and end with vmselect's label API defaults:
+// end is now and start is five minutes before end.
+func metricsWindow(in url.Values, now time.Time, lim MetricsLimits) (time.Time, time.Time, *HTTPError) {
+ end := now
+ if v := in.Get("end"); v != "" {
+ t, herr := parseMetricsTime("end", v)
+ if herr != nil {
+ return time.Time{}, time.Time{}, herr
+ }
+ end = t
+ }
+ start := end.Add(-labelsDefaultWindow)
+ if v := in.Get("start"); v != "" {
+ t, herr := parseMetricsTime("start", v)
+ if herr != nil {
+ return time.Time{}, time.Time{}, herr
+ }
+ start = t
+ }
+ if start.After(end) {
+ return time.Time{}, time.Time{}, errorf(http.StatusBadRequest, "start is after end")
+ }
+ if end.Sub(start) > lim.MaxRange {
+ return time.Time{}, time.Time{}, errorf(http.StatusUnprocessableEntity, "range %s exceeds the limit of %s", end.Sub(start), lim.MaxRange)
+ }
+ return start, end, nil
+}
+
+// parseStep refuses a step above the range limit: vmselect aligns the
+// window outward to whole steps, and an implicit rollup window is one step,
+// so a huge step reads far more than the range it was sent with.
+func parseStep(v string, lim MetricsLimits) (time.Duration, *HTTPError) {
+ step, herr := parseMetricsDuration("step", v)
+ if herr != nil {
+ return 0, herr
+ }
+ if step > lim.MaxRange {
+ return 0, errorf(http.StatusUnprocessableEntity, "step %s exceeds the limit of %s", step, lim.MaxRange)
+ }
+ return step, nil
+}
+
+// parseMetricsDuration takes Prometheus float seconds or a duration such as
+// 1m30s. vmselect works in milliseconds and refuses anything shorter.
+func parseMetricsDuration(key, v string) (time.Duration, *HTTPError) {
+ d, err := parseDuration(v)
+ if err != nil || d < time.Millisecond {
+ return 0, errorf(http.StatusBadRequest, "invalid %s %q", key, v)
+ }
+ return d, nil
+}
+
+// parseMetricsTime takes the two forms the Prometheus API defines: float
+// unix seconds and RFC3339. vmselect would also guess milliseconds from the
+// digit count and accept relative times; a guessed unit silently moves the
+// window, so those are refused.
+func parseMetricsTime(key, v string) (time.Time, *HTTPError) {
+ if sec, frac, ok := strings.Cut(v, "."); isDigits(sec) && (!ok || isDigits(frac)) {
+ if len(sec) > 10 {
+ return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q: not unix seconds", key, v)
+ }
+ secs, _ := strconv.ParseInt(sec, 10, 64)
+ var nanos int64
+ if ok {
+ nanos, _ = strconv.ParseInt((frac + "000000000")[:9], 10, 64)
+ }
+ return time.Unix(secs, nanos).UTC(), nil
+ }
+ t, err := time.Parse(time.RFC3339Nano, v)
+ if err != nil || t.Before(time.Unix(0, 0)) {
+ return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q", key, v)
+ }
+ return t.UTC(), nil
+}
+
+func formatTime(t time.Time) string {
+ ms := t.UnixMilli()
+ return fmt.Sprintf("%d.%03d", ms/1000, ms%1000)
+}
+
+func formatSeconds(d time.Duration) string {
+ return strconv.FormatFloat(float64(d.Milliseconds())/1000, 'f', -1, 64)
+}
diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go
new file mode 100644
index 000000000..be1d69c38
--- /dev/null
+++ b/internal/observabilityproxy/metricsparams_test.go
@@ -0,0 +1,246 @@
+/*
+Copyright 2026 The Cozystack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package observabilityproxy
+
+import (
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+)
+
+func testMetricsLimits() MetricsLimits {
+ return MetricsLimits{MaxRange: 30 * 24 * time.Hour, MaxPoints: 11000, MaxTimeout: 30 * time.Second}
+}
+
+const wantMetricsScope = `{namespace="tenant-a"}`
+
+func unix(t time.Time) string { return strconv.FormatInt(t.Unix(), 10) }
+
+func buildMetrics(t *testing.T, endpoint string, in url.Values) metricsRequest {
+ t.Helper()
+ req, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits())
+ if err != nil {
+ t.Fatalf("BuildMetricsParams(%s, %v): %v", endpoint, in, err)
+ }
+ return req
+}
+
+func metricsStatus(endpoint string, in url.Values) int {
+ _, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits())
+ if err == nil {
+ return http.StatusOK
+ }
+ return err.Code
+}
+
+func TestBuildMetricsParamsAllowlist(t *testing.T) {
+ hostile := url.Values{
+ "extra_filters": {`{namespace=~".+"}`},
+ "extra_filters[]": {`{namespace="tenant-b"}`, `{namespace=~".*"}`},
+ "extra_label": {"namespace=tenant-b"},
+ "match": {`{namespace="tenant-b"}`},
+ "nocache": {"1"},
+ "limit": {"1"},
+ "AccountID": {"7"},
+ "latency_offset": {"1h"},
+ "round_digits": {"2"},
+ }
+ cases := map[string]url.Values{
+ "api/v1/query": {"query": {"up"}, "time": {unix(testNow)}, "step": {"30s"}},
+ "api/v1/query_range": {"query": {"up"}, "start": {unix(testNow.Add(-time.Hour))}, "end": {unix(testNow)}, "step": {"60"}},
+ "api/v1/series": {"match[]": {"up"}},
+ "api/v1/labels": {"match[]": {"up"}},
+ "api/v1/label/pod/values": {"match[]": {"up"}},
+ }
+ allowed := map[string]bool{"query": true, "start": true, "end": true, "step": true, "time": true, "match[]": true, "timeout": true, "extra_filters[]": true}
+ for endpoint, in := range cases {
+ t.Run(endpoint, func(t *testing.T) {
+ for k, v := range hostile {
+ in[k] = v
+ }
+ req := buildMetrics(t, endpoint, in)
+ if req.path != endpoint {
+ t.Errorf("path = %q", req.path)
+ }
+ for k := range req.params {
+ if !allowed[k] {
+ t.Errorf("parameter %q forwarded: %v", k, req.params)
+ }
+ }
+ if got := req.params["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope {
+ t.Errorf("extra_filters[] = %q, want exactly [%s]", got, wantMetricsScope)
+ }
+ for _, m := range req.params["match[]"] {
+ if strings.Contains(m, "tenant-b") {
+ t.Errorf("client match without brackets forwarded as match[]: %q", m)
+ }
+ }
+ })
+ }
+}
+
+func TestNamespaceSelectorEscapes(t *testing.T) {
+ for _, ns := range []string{"tenant-a", `a"} or {namespace=~".+`, `back\slash`, "new\nline"} {
+ sel := namespaceSelector(ns)
+ lit, ok := strings.CutPrefix(sel, "{namespace=")
+ lit, ok2 := strings.CutSuffix(lit, "}")
+ if !ok || !ok2 {
+ t.Fatalf("selector %q has the wrong shape", sel)
+ }
+ // MetricsQL reads a double-quoted label value with strconv.Unquote
+ // (metricsql extractStringValue).
+ got, err := strconv.Unquote(lit)
+ if err != nil || got != ns {
+ t.Errorf("selector %q decodes to %q, %v; want %q", sel, got, err, ns)
+ }
+ }
+}
+
+func TestBuildMetricsParamsLabelEndpointsAlwaysMatch(t *testing.T) {
+ for _, endpoint := range []string{"api/v1/labels", "api/v1/label/__name__/values"} {
+ req := buildMetrics(t, endpoint, url.Values{})
+ if got := req.params["match[]"]; len(got) != 1 || got[0] != wantMetricsScope {
+ t.Errorf("%s without match[] sends %q, want the namespace selector", endpoint, got)
+ }
+ req = buildMetrics(t, endpoint, url.Values{"match[]": {"up", `{job="x"}`}})
+ if got := req.params["match[]"]; len(got) != 2 || got[0] != "up" || got[1] != `{job="x"}` {
+ t.Errorf("%s client match[] = %q, want them kept", endpoint, got)
+ }
+ }
+ if got := metricsStatus("api/v1/series", url.Values{}); got != http.StatusBadRequest {
+ t.Errorf("series without match[] = %d, want 400", got)
+ }
+ if got := metricsStatus("api/v1/series", url.Values{"match[]": {""}}); got != http.StatusBadRequest {
+ t.Errorf("empty match[] = %d, want 400", got)
+ }
+ many := url.Values{}
+ for i := range maxMetricsMatchers + 1 {
+ many.Add("match[]", "up"+strconv.Itoa(i))
+ }
+ if got := metricsStatus("api/v1/labels", many); got != http.StatusUnprocessableEntity {
+ t.Errorf("%d match[] = %d, want 422", maxMetricsMatchers+1, got)
+ }
+}
+
+func TestBuildMetricsParamsLimits(t *testing.T) {
+ ago := func(d time.Duration) string { return unix(testNow.Add(-d)) }
+ day := 24 * time.Hour
+ rng := func(start, end, step string) url.Values {
+ return url.Values{"query": {"up"}, "start": {start}, "end": {end}, "step": {step}}
+ }
+ tests := []struct {
+ name string
+ endpoint string
+ in url.Values
+ want int
+ }{
+ {"range at the limit", "api/v1/query_range", rng(ago(30*day), ago(0), "1h"), http.StatusOK},
+ {"range above the limit", "api/v1/query_range", rng(ago(31*day), ago(0), "1h"), http.StatusUnprocessableEntity},
+ {"points at the limit", "api/v1/query_range", rng(ago(10999*time.Minute), ago(0), "1m"), http.StatusOK},
+ {"points above the limit", "api/v1/query_range", rng(ago(11000*time.Minute), ago(0), "1m"), http.StatusUnprocessableEntity},
+ {"step above the range limit", "api/v1/query_range", rng(ago(time.Hour), ago(0), "31d"), http.StatusUnprocessableEntity},
+ {"zero step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0"), http.StatusBadRequest},
+ {"sub-millisecond step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0.0001"), http.StatusBadRequest},
+ {"missing step", "api/v1/query_range", url.Values{"query": {"up"}, "start": {ago(time.Hour)}, "end": {ago(0)}}, http.StatusBadRequest},
+ {"missing start", "api/v1/query_range", url.Values{"query": {"up"}, "end": {ago(0)}, "step": {"1m"}}, http.StatusBadRequest},
+ {"missing query", "api/v1/query_range", url.Values{"start": {ago(time.Hour)}, "end": {ago(0)}, "step": {"1m"}}, http.StatusBadRequest},
+ {"inverted range", "api/v1/query_range", rng(ago(0), ago(time.Hour), "1m"), http.StatusBadRequest},
+ {"millisecond timestamp", "api/v1/query_range", rng(ago(time.Hour)+"000", ago(0)+"000", "1m"), http.StatusBadRequest},
+ {"relative time", "api/v1/query_range", rng("-1h", "now", "1m"), http.StatusBadRequest},
+ {"rfc3339", "api/v1/query_range", rng("2026-11-04T11:00:00Z", "2026-11-04T12:00:00.5Z", "15s"), http.StatusOK},
+ {"instant without time", "api/v1/query", url.Values{"query": {"up"}}, http.StatusOK},
+ {"instant step above the range limit", "api/v1/query", url.Values{"query": {"up"}, "step": {"31d"}}, http.StatusUnprocessableEntity},
+ {"instant bad time", "api/v1/query", url.Values{"query": {"up"}, "time": {"yesterday"}}, http.StatusBadRequest},
+ {"negative timeout", "api/v1/query", url.Values{"query": {"up"}, "timeout": {"-1s"}}, http.StatusBadRequest},
+ {"series range above the limit", "api/v1/series", url.Values{"match[]": {"up"}, "start": {ago(31 * day)}}, http.StatusUnprocessableEntity},
+ {"labels inverted range", "api/v1/labels", url.Values{"start": {ago(0)}, "end": {ago(time.Hour)}}, http.StatusBadRequest},
+ {"invalid label name", "api/v1/label/a-b/values", url.Values{}, http.StatusBadRequest},
+ {"unknown endpoint", "api/v1/status/tsdb", url.Values{}, http.StatusNotFound},
+ {"export", "api/v1/export", url.Values{"match[]": {"up"}}, http.StatusNotFound},
+ {"label without values", "api/v1/label/pod", url.Values{}, http.StatusNotFound},
+ {"root", "", url.Values{}, http.StatusNotFound},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := metricsStatus(tt.endpoint, tt.in); got != tt.want {
+ t.Fatalf("status = %d, want %d", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestBuildMetricsParamsCanonicalForms(t *testing.T) {
+ req := buildMetrics(t, "api/v1/query_range", url.Values{
+ "query": {"up"}, "start": {"2026-11-04T11:00:00.25Z"}, "end": {unix(testNow)}, "step": {"1m30s"},
+ })
+ want := url.Values{
+ "query": {"up"}, "start": {"1793790000.250"}, "end": {"1793793600.000"}, "step": {"90"},
+ "timeout": {"30"}, "extra_filters[]": {wantMetricsScope},
+ }
+ if req.params.Encode() != want.Encode() {
+ t.Errorf("params = %v\nwant %v", req.params, want)
+ }
+
+ for in, want := range map[string]string{"": "30", "10s": "10", "5m": "30", "1.5": "1.5"} {
+ v := url.Values{"query": {"up"}}
+ if in != "" {
+ v.Set("timeout", in)
+ }
+ if got := buildMetrics(t, "api/v1/query", v).params.Get("timeout"); got != want {
+ t.Errorf("timeout %q forwarded as %q, want %q", in, got, want)
+ }
+ }
+
+ req = buildMetrics(t, "api/v1/labels", url.Values{})
+ if req.params.Get("end") != "1793793600.000" || req.params.Get("start") != "1793793300.000" {
+ t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end"))
+ }
+ if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow) {
+ t.Errorf("instant query without time: params %v, start %v", q.params, q.start)
+ }
+}
+
+func TestMetricsStoreByRange(t *testing.T) {
+ cfg := MetricsConfig{ShortTermStore: "shortterm", LongTermStore: "longterm", ShortTermRetention: 72 * time.Hour}
+ tests := []struct {
+ name string
+ endpoint string
+ in url.Values
+ want string
+ }{
+ {"instant now", "api/v1/query", url.Values{"query": {"up"}}, "shortterm"},
+ {"instant within retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-71 * time.Hour))}}, "shortterm"},
+ {"instant past retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-73 * time.Hour))}}, "longterm"},
+ {"live graph", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-6 * time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}, "shortterm"},
+ {"range fits exactly", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-72 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "shortterm"},
+ {"range past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-7 * 24 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"},
+ {"old window ending long ago", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-5 * 24 * time.Hour))}, "end": {unix(testNow.Add(-4 * 24 * time.Hour))}, "step": {"600"}}, "longterm"},
+ {"labels default window", "api/v1/labels", url.Values{}, "shortterm"},
+ {"series past retention", "api/v1/series", url.Values{"match[]": {"up"}, "start": {unix(testNow.Add(-4 * 24 * time.Hour))}}, "longterm"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ req := buildMetrics(t, tt.endpoint, tt.in)
+ if got := cfg.store(req.start, testNow); got != tt.want {
+ t.Fatalf("store = %s, want %s", got, tt.want)
+ }
+ })
+ }
+}
From 47409db70c303fb36ad31ac3d34465c8bcbbf428 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:00:14 +0700
Subject: [PATCH 061/114] feat(observability-proxy): serve the namespace
metrics root
The namespace metrics root shares the logs root's authorization path:
the SubjectAccessReview comes from the path alone, so a POST query,
which Grafana sends by default, is authorized exactly like a GET. Both
roots also share the per-user rate and concurrency limits and the way
an unreachable store (502) is told apart from a slow one (504); the
upstream call moves into one helper so the two stay identical.
Every namespace resolves to the root tenant's vmselect for now,
because the series the graphs need are only scraped there.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/server.go | 103 ++++++++---
.../observabilityproxy/server_metrics_test.go | 163 ++++++++++++++++++
internal/observabilityproxy/server_test.go | 17 +-
3 files changed, 259 insertions(+), 24 deletions(-)
create mode 100644 internal/observabilityproxy/server_metrics_test.go
diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go
index e980818d9..3e24be109 100644
--- a/internal/observabilityproxy/server.go
+++ b/internal/observabilityproxy/server.go
@@ -25,6 +25,7 @@ import (
"log/slog"
"math"
"net/http"
+ "net/url"
"os"
"strconv"
"strings"
@@ -56,6 +57,7 @@ type Config struct {
// means the in-cluster Service of the "generic" logs storage.
UpstreamURL func(store string) string
Logs LogsLimits
+ Metrics MetricsConfig
Limiter LimiterConfig
RequestTimeout time.Duration
TailIdleTimeout time.Duration
@@ -84,6 +86,12 @@ func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus.
return fmt.Sprintf("http://vlselect-generic.%s.svc.%s:9471", store, domain)
}
}
+ if cfg.Metrics.UpstreamURL == nil {
+ domain, root := cfg.ClusterDomain, cfg.Metrics.RootTenant
+ cfg.Metrics.UpstreamURL = func(store string) string {
+ return fmt.Sprintf("http://vmselect-%s.%s.svc.%s:8481/select/0/prometheus", store, root, domain)
+ }
+ }
s := &Server{
cfg: cfg,
auth: auth,
@@ -167,7 +175,7 @@ func writeError(w http.ResponseWriter, e *HTTPError) {
// serve answers a root it has no handler for before authenticating, so an
// unknown path costs no review calls.
func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTTPError {
- if route.Kind != NamespaceLogs {
+ if route.Kind != NamespaceLogs && route.Kind != NamespaceMetrics {
return errorf(http.StatusNotFound, "not found")
}
user, herr := s.authorize(r, route)
@@ -182,6 +190,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
w.Header().Set("Retry-After", strconv.FormatInt(int64(math.Ceil(wait.Seconds())), 10))
return errorf(http.StatusTooManyRequests, "request rate limit exceeded")
}
+ if route.Kind == NamespaceMetrics {
+ return s.serveMetrics(w, r, route, user)
+ }
endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/")
if !ok {
return errorf(http.StatusNotFound, "not found")
@@ -226,29 +237,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
ctx, stop = context.WithTimeout(ctx, limit)
defer stop()
}
- req, err := http.NewRequestWithContext(ctx, http.MethodPost,
- s.cfg.UpstreamURL(store)+"/select/logsql/"+endpoint, strings.NewReader(params.Encode()))
- if err != nil {
- return errorf(http.StatusInternalServerError, "building upstream request: %v", err)
- }
- req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- req.Header.Set("AccountID", "0")
- req.Header.Set("ProjectID", "0")
+ tenant := http.Header{}
+ tenant.Set("AccountID", "0")
+ tenant.Set("ProjectID", "0")
if tail {
// vlselect holds even the headers of a gzip-encoded tail until the
// first line; an explicit value stops the transport from asking for gzip.
- req.Header.Set("Accept-Encoding", "identity")
+ tenant.Set("Accept-Encoding", "identity")
}
- resp, err := s.client.Do(req)
- switch {
- case err == nil:
- case r.Context().Err() != nil:
- return errorf(http.StatusBadGateway, "client went away")
- case errors.Is(context.Cause(ctx), context.DeadlineExceeded):
- return errorf(http.StatusGatewayTimeout, "log store did not answer within %s", limit)
- default:
- slog.Warn("log store unreachable", "store", store, "error", err)
- return errorf(http.StatusBadGateway, "log store unreachable")
+ resp, herr := s.post(ctx, r, s.cfg.UpstreamURL(store)+"/select/logsql/"+endpoint, params, "log store", tenant)
+ if herr != nil {
+ return herr
}
defer func() { _ = resp.Body.Close() }()
@@ -280,6 +279,70 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
return nil
}
+func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError {
+ r.Body = http.MaxBytesReader(w, r.Body, maxFormBytes)
+ if err := r.ParseForm(); err != nil {
+ return errorf(http.StatusBadRequest, "invalid form: %v", err)
+ }
+ now := s.now()
+ req, herr := BuildMetricsParams(route.Endpoint, r.Form, route.Namespace, now, s.cfg.Metrics.Limits)
+ if herr != nil {
+ return herr
+ }
+ release, ok := s.limits.acquire(user)
+ if !ok {
+ return errorf(http.StatusUnprocessableEntity, "too many concurrent requests")
+ }
+ defer release()
+
+ ctx := r.Context()
+ if s.cfg.RequestTimeout > 0 {
+ var cancel context.CancelFunc
+ ctx, cancel = context.WithTimeout(ctx, s.cfg.RequestTimeout)
+ defer cancel()
+ }
+ store := s.cfg.Metrics.store(req.start, now)
+ resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/"+req.path, req.params, "metrics store", nil)
+ if herr != nil {
+ return herr
+ }
+ defer func() { _ = resp.Body.Close() }()
+ if ct := resp.Header.Get("Content-Type"); ct != "" {
+ w.Header().Set("Content-Type", ct)
+ }
+ w.WriteHeader(resp.StatusCode)
+ if _, err := io.Copy(w, resp.Body); err != nil && !errors.Is(err, context.Canceled) {
+ slog.Debug("copying upstream response", "error", err)
+ }
+ return nil
+}
+
+// post sends params as a form with only the given headers, so nothing the
+// client sent reaches the store, and tells an unreachable store (502) from
+// one that missed its deadline or, for a tail, its idle timeout (504).
+func (s *Server) post(ctx context.Context, r *http.Request, target string, params url.Values, what string, header http.Header) (*http.Response, *HTTPError) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, strings.NewReader(params.Encode()))
+ if err != nil {
+ return nil, errorf(http.StatusInternalServerError, "building upstream request: %v", err)
+ }
+ for k, v := range header {
+ req.Header[k] = v
+ }
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ resp, err := s.client.Do(req)
+ switch {
+ case err == nil:
+ return resp, nil
+ case r.Context().Err() != nil:
+ return nil, errorf(http.StatusBadGateway, "client went away")
+ case errors.Is(context.Cause(ctx), context.DeadlineExceeded):
+ return nil, errorf(http.StatusGatewayTimeout, "%s did not answer in time", what)
+ default:
+ slog.Warn(what+" unreachable", "url", target, "error", err)
+ return nil, errorf(http.StatusBadGateway, "%s unreachable", what)
+ }
+}
+
// concurrencyRetryAfter is a hint only: a slot frees when another request
// of the user ends, which the proxy cannot predict, and one second keeps
// a polling client from retrying in a tight loop.
diff --git a/internal/observabilityproxy/server_metrics_test.go b/internal/observabilityproxy/server_metrics_test.go
new file mode 100644
index 000000000..887790d34
--- /dev/null
+++ b/internal/observabilityproxy/server_metrics_test.go
@@ -0,0 +1,163 @@
+/*
+Copyright 2026 The Cozystack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package observabilityproxy
+
+import (
+ "io"
+ "net/http"
+ "net/url"
+ "testing"
+ "time"
+
+ authorizationv1 "k8s.io/api/authorization/v1"
+)
+
+const metricsRoot = "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/"
+
+func promUpstream(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = io.WriteString(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
+}
+
+func TestServerMetricsRebuildsUpstreamRequest(t *testing.T) {
+ h := newHarness(t, Config{}, promUpstream)
+ hostile := url.Values{
+ "query": {"sum(rate(container_cpu_usage_seconds_total[5m]))"},
+ "start": {unix(testNow.Add(-time.Hour))},
+ "end": {unix(testNow)},
+ "step": {"60"},
+ "extra_filters": {`{namespace=~".+"}`},
+ "extra_filters[]": {`{namespace="tenant-b"}`, `{namespace=~".*"}`},
+ "extra_label": {"namespace=tenant-b"},
+ "nocache": {"1"},
+ }
+ headers := map[string]string{"AccountID": "9", "X-Forwarded-For": "1.2.3.4", "Cookie": "kc-access=secret"}
+
+ resp := h.do(http.MethodGet, metricsRoot+"api/v1/query_range?"+hostile.Encode(), "good", nil, headers)
+ _ = resp.Body.Close()
+ resp = h.do(http.MethodPost, metricsRoot+"api/v1/query_range?extra_filters[]="+url.QueryEscape(`{namespace="x"}`), "good", hostile, headers)
+ body, _ := io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+ if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Type") != "application/json" {
+ t.Fatalf("POST answered %d %s %q", resp.StatusCode, resp.Header.Get("Content-Type"), body)
+ }
+
+ reqs := h.requests()
+ if len(reqs) != 2 {
+ t.Fatalf("upstream saw %d requests", len(reqs))
+ }
+ if reqs[0].form.Encode() != reqs[1].form.Encode() {
+ t.Fatalf("GET and POST built different upstream requests:\n%v\n%v", reqs[0].form, reqs[1].form)
+ }
+ if n := h.kube.sars.Load(); n != 1 {
+ t.Fatalf("SubjectAccessReviews = %d, want 1: POST must reuse the GET decision", n)
+ }
+ want := authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "metrics", Namespace: "tenant-a"}
+ if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want {
+ t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want)
+ }
+ for _, r := range reqs {
+ if r.path != "/api/v1/query_range" {
+ t.Errorf("upstream path = %s", r.path)
+ }
+ if got := r.form["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope {
+ t.Errorf("extra_filters[] = %q", got)
+ }
+ if r.form.Has("extra_filters") || r.form.Has("extra_label") || r.form.Has("nocache") {
+ t.Errorf("client parameter forwarded: %v", r.form)
+ }
+ for _, k := range []string{"Authorization", "AccountID", "X-Forwarded-For", "Cookie"} {
+ if r.headers.Get(k) != "" {
+ t.Errorf("client header %s forwarded", k)
+ }
+ }
+ }
+}
+
+func TestServerMetricsStoreByRange(t *testing.T) {
+ h := newHarness(t, Config{}, promUpstream)
+ for _, path := range []string{
+ "api/v1/query?query=up",
+ "api/v1/query_range?query=up&step=600&start=" + unix(testNow.Add(-7*24*time.Hour)) + "&end=" + unix(testNow),
+ "api/v1/label/pod/values",
+ } {
+ if got := h.status(http.MethodGet, metricsRoot+path, "good"); got != http.StatusOK {
+ t.Fatalf("%s answered %d", path, got)
+ }
+ }
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ if want := []string{"shortterm", "longterm", "shortterm"}; len(h.stores) != 3 || h.stores[0] != want[0] || h.stores[1] != want[1] || h.stores[2] != want[2] {
+ t.Fatalf("stores = %v, want %v", h.stores, want)
+ }
+ if r := h.got[2]; r.path != "/api/v1/label/pod/values" || r.form.Get("match[]") != wantMetricsScope {
+ t.Fatalf("label values upstream = %s %v", r.path, r.form)
+ }
+}
+
+func TestServerMetricsStatusCodes(t *testing.T) {
+ h := newHarness(t, Config{}, promUpstream)
+ tooLong := "api/v1/query_range?query=up&step=3600&start=" + unix(testNow.Add(-31*24*time.Hour)) + "&end=" + unix(testNow)
+ if got := h.status(http.MethodGet, metricsRoot+tooLong, "good"); got != http.StatusUnprocessableEntity {
+ t.Errorf("range above the limit answered %d, want 422", got)
+ }
+ if got := h.status(http.MethodGet, metricsRoot+"api/v1/series", "good"); got != http.StatusBadRequest {
+ t.Errorf("series without match[] answered %d, want 400", got)
+ }
+ if got := h.status(http.MethodPut, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusMethodNotAllowed {
+ t.Errorf("PUT answered %d, want 405", got)
+ }
+ if n := len(h.requests()); n != 0 {
+ t.Errorf("upstream saw %d requests for refused ones", n)
+ }
+
+ h2 := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) {
+ http.Error(w, `{"status":"error","errorType":"422"}`, http.StatusUnprocessableEntity)
+ })
+ if got := h2.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusUnprocessableEntity {
+ t.Errorf("upstream 422 answered as %d", got)
+ }
+ h2.upstream.Close()
+ if got := h2.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusBadGateway {
+ t.Errorf("unreachable upstream answered %d, want 502", got)
+ }
+
+ release := make(chan struct{})
+ t.Cleanup(func() { close(release) })
+ h3 := newHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(_ http.ResponseWriter, r *http.Request) {
+ select {
+ case <-release:
+ case <-r.Context().Done():
+ }
+ })
+ if got := h3.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusGatewayTimeout {
+ t.Errorf("slow upstream answered %d, want 504", got)
+ }
+
+ h4 := newHarness(t, Config{Limiter: LimiterConfig{RatePerUser: 0.001, RateBurst: 1}}, promUpstream)
+ h4.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good")
+ if got := h4.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusUnprocessableEntity {
+ t.Errorf("rate-limited request answered %d, want 422", got)
+ }
+}
+
+func TestDefaultMetricsUpstreamURL(t *testing.T) {
+ s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil)
+ if got, want := s.cfg.Metrics.UpstreamURL("shortterm"), "http://vmselect-shortterm.tenant-root.svc.cozy.local:8481/select/0/prometheus"; got != want {
+ t.Fatalf("upstream = %s, want %s", got, want)
+ }
+}
diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go
index df3e23bc4..2670d0e3e 100644
--- a/internal/observabilityproxy/server_test.go
+++ b/internal/observabilityproxy/server_test.go
@@ -52,12 +52,12 @@ type mapStores map[string]string
func (m mapStores) Resolve(ns string) (string, error) { return m[ns], nil }
// newHarness wires the proxy to a fake apiserver that knows tokens "good"
-// (alice) and "bob-token" (bob), both allowed list logs in tenant-a only,
-// and to an upstream served by handler.
+// (alice) and "bob-token" (bob), both allowed list logs and metrics in
+// tenant-a only, and to an upstream served by handler.
func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness {
h := &harness{t: t}
h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, func(_ string, a *authorizationv1.ResourceAttributes) bool {
- return a.Namespace == "tenant-a" && a.Resource == "logs"
+ return a.Namespace == "tenant-a" && (a.Resource == "logs" || a.Resource == "metrics")
})
h.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseForm(); err != nil {
@@ -80,6 +80,13 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness {
h.mu.Unlock()
return h.upstream.URL
}
+ cfg.Metrics.UpstreamURL = cfg.UpstreamURL
+ if cfg.Metrics.ShortTermStore == "" {
+ cfg.Metrics.ShortTermStore, cfg.Metrics.LongTermStore, cfg.Metrics.ShortTermRetention = "shortterm", "longterm", 72*time.Hour
+ }
+ if cfg.Metrics.Limits == (MetricsLimits{}) {
+ cfg.Metrics.Limits = testMetricsLimits()
+ }
if cfg.Logs == (LogsLimits{}) {
cfg.Logs = testLimits()
}
@@ -144,7 +151,9 @@ func TestServerStatusCodes(t *testing.T) {
{"other namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/logs/select/logsql/query", "good", http.StatusForbidden},
{"invalid namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/Tenant/logs/select/logsql/query", "good", http.StatusBadRequest},
{"unknown root", http.MethodGet, "/api/v1/namespaces/tenant-a/pods", "good", http.StatusNotFound},
- {"namespace metrics not served yet", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query", "good", http.StatusNotFound},
+ {"unknown metrics endpoint", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/status/tsdb", "good", http.StatusNotFound},
+ {"metrics in another namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/metrics/api/v1/query?query=up", "good", http.StatusForbidden},
+ {"metrics without token", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query?query=up", "", http.StatusUnauthorized},
{"application log not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusNotFound},
{"application metrics not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusNotFound},
{"tenant_ids not proxied", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/tenant_ids", "good", http.StatusNotFound},
From cd3d889ee8b489e36034c64dcdd1a56798fedfac Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:00:45 +0700
Subject: [PATCH 062/114] feat(observability-proxy): add metrics store and cost
limit flags
The store names and the short-term retention mirror metricsStorages in
the monitoring package, which an operator can rename or resize; they
are flags so the chart can follow such a change without a rebuild. The
cost limit defaults are the ones the metrics proposal settles on: 30d
range, 11000 points per series, 30s timeout.
Store names become part of a Service host name, so they are checked
as DNS labels at startup.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
cmd/observability-proxy/main.go | 29 +++++++++++++++++++++++++---
cmd/observability-proxy/main_test.go | 18 +++++++++++++++++
2 files changed, 44 insertions(+), 3 deletions(-)
diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go
index 558908671..6acfb1e08 100644
--- a/cmd/observability-proxy/main.go
+++ b/cmd/observability-proxy/main.go
@@ -14,9 +14,10 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-// Command observability-proxy serves tenant logs from the VictoriaLogs
-// stores behind a Kubernetes TokenReview and SubjectAccessReview, with the
-// tenant's namespace scope forced into every upstream request.
+// Command observability-proxy serves tenant logs and metrics from the
+// VictoriaLogs and VictoriaMetrics stores behind a Kubernetes TokenReview
+// and SubjectAccessReview, with the tenant's namespace scope forced into
+// every upstream request.
package main
import (
@@ -37,6 +38,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
+ "k8s.io/apimachinery/pkg/util/validation"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
@@ -56,6 +58,7 @@ type options struct {
tailIdleTimeout time.Duration
shutdownTimeout time.Duration
logs observabilityproxy.LogsLimits
+ metrics observabilityproxy.MetricsConfig
limiter observabilityproxy.LimiterConfig
}
@@ -81,6 +84,14 @@ func parseFlags(args []string) (options, error) {
fs.IntVar(&o.logs.DefaultLimit, "logs-default-limit", 1000, "entries returned when a request sets no limit")
fs.IntVar(&o.logs.MaxLimit, "logs-max-limit", 10000, "largest limit a request may set")
fs.DurationVar(&o.logs.MaxTailResume, "logs-max-tail-resume", 15*time.Minute, "how far back a tail may resume with since or start_offset")
+ // Store names and retention follow metricsStorages in
+ // packages/system/monitoring/values.yaml.
+ fs.StringVar(&o.metrics.ShortTermStore, "metrics-shortterm-store", "shortterm", "VMCluster name of the short-term metrics store of the root tenant")
+ fs.StringVar(&o.metrics.LongTermStore, "metrics-longterm-store", "longterm", "VMCluster name of the long-term metrics store of the root tenant")
+ fs.DurationVar(&o.metrics.ShortTermRetention, "metrics-shortterm-retention", 72*time.Hour, "retention of the short-term store; older windows are read from the long-term one")
+ fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use")
+ fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return")
+ fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped")
fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables")
fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables")
fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user")
@@ -94,6 +105,13 @@ func parseFlags(args []string) (options, error) {
if fs.NArg() > 0 {
return o, fmt.Errorf("unexpected argument %q", fs.Arg(0))
}
+ o.metrics.RootTenant = o.rootTenant
+ for _, store := range []string{o.metrics.ShortTermStore, o.metrics.LongTermStore} {
+ // The name becomes part of a Service host name.
+ if errs := validation.IsDNS1123Label("vmselect-" + store); store == "" || len(errs) > 0 {
+ return o, fmt.Errorf("invalid metrics store name %q", store)
+ }
+ }
switch {
case o.rootTenant == "":
return o, errors.New("--root-tenant must not be empty")
@@ -110,6 +128,10 @@ func parseFlags(args []string) (options, error) {
return o, errors.New("--logs-max-range must be at least 3s")
case o.logs.DefaultLimit <= 0 || o.logs.MaxLimit < o.logs.DefaultLimit:
return o, errors.New("--logs-default-limit must be positive and not above --logs-max-limit")
+ case o.metrics.ShortTermRetention <= 0 || o.metrics.Limits.MaxRange <= 0 || o.metrics.Limits.MaxTimeout <= 0:
+ return o, errors.New("--metrics-shortterm-retention, --metrics-max-range and --metrics-max-timeout must be positive")
+ case o.metrics.Limits.MaxPoints < 2:
+ return o, errors.New("--metrics-max-points must be at least 2")
case o.logs.MaxTailResume < 0:
return o, errors.New("--logs-max-tail-resume must not be negative")
case o.requestTimeout < 0 || o.tailIdleTimeout < 0 || o.authCacheTTL < 0 || o.shutdownTimeout < 0:
@@ -167,6 +189,7 @@ func run(o options) error {
proxy := observabilityproxy.NewServer(observabilityproxy.Config{
ClusterDomain: o.clusterDomain,
Logs: o.logs,
+ Metrics: o.metrics,
Limiter: o.limiter,
RequestTimeout: o.requestTimeout,
TailIdleTimeout: o.tailIdleTimeout,
diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go
index ea678461c..fe7d3ea47 100644
--- a/cmd/observability-proxy/main_test.go
+++ b/cmd/observability-proxy/main_test.go
@@ -32,6 +32,18 @@ func TestParseFlagsDefaults(t *testing.T) {
if o.authCacheTTL != 10*time.Second || o.logs.MaxTailResume != 15*time.Minute || o.logs.MaxRange != 7*24*time.Hour {
t.Errorf("limit defaults changed: %+v", o)
}
+ // Retention and store names of packages/system/monitoring/values.yaml;
+ // cost limits of the metrics proposal (Design 7).
+ m := o.metrics
+ if m.ShortTermStore != "shortterm" || m.LongTermStore != "longterm" || m.ShortTermRetention != 72*time.Hour {
+ t.Errorf("metrics store defaults changed: %+v", m)
+ }
+ if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second {
+ t.Errorf("metrics limit defaults changed: %+v", m.Limits)
+ }
+ if m.RootTenant != o.rootTenant {
+ t.Errorf("metrics root tenant = %q, want --root-tenant", m.RootTenant)
+ }
}
func TestParseFlagsOverrides(t *testing.T) {
@@ -69,6 +81,12 @@ func TestParseFlagsInvalid(t *testing.T) {
{"--max-concurrent-per-user=-1"},
{"--max-tails-per-user=-1"},
{"--max-tails=-1"},
+ {"--metrics-shortterm-store="},
+ {"--metrics-longterm-store=Long.Term"},
+ {"--metrics-shortterm-retention=0s"},
+ {"--metrics-max-range=0s"},
+ {"--metrics-max-points=1"},
+ {"--metrics-max-timeout=0s"},
{"--unknown"},
{"unexpected", "--root-tenant=tenant-x"},
} {
From 5630de4ab13077ef9a1861d29de2add980f477e3 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:01:25 +0700
Subject: [PATCH 063/114] feat(observability-proxy): grant tenant viewers the
metrics root
The proxy authorizes the namespace metrics root with a
SubjectAccessReview for list on cozystack.io metrics, which no role
grants yet. Aggregating it into the tenant view role reaches every
tenant group and ancestor subject without a tenant chart change.
The Deployment passes the metrics store names and short-term
retention from values, so a platform that renames or resizes its
metricsStorages can keep the proxy's store choice in step.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../templates/deployment.yaml | 3 ++
.../templates/tenant-clusterroles.yaml | 8 ++---
.../tests/deployment_test.yaml | 29 +++++++++++++++++++
.../observability-proxy/tests/rbac_test.yaml | 6 ++--
.../system/observability-proxy/values.yaml | 7 +++++
5 files changed, 46 insertions(+), 7 deletions(-)
diff --git a/packages/system/observability-proxy/templates/deployment.yaml b/packages/system/observability-proxy/templates/deployment.yaml
index 581c15fde..d49de140b 100644
--- a/packages/system/observability-proxy/templates/deployment.yaml
+++ b/packages/system/observability-proxy/templates/deployment.yaml
@@ -38,6 +38,9 @@ spec:
- --cluster-domain={{ $clusterDomain }}
- --root-tenant={{ .Values.observabilityProxy.rootTenant }}
- --auth-cache-ttl={{ .Values.observabilityProxy.authCacheTTL }}
+ - --metrics-shortterm-store={{ .Values.observabilityProxy.metrics.shorttermStore }}
+ - --metrics-longterm-store={{ .Values.observabilityProxy.metrics.longtermStore }}
+ - --metrics-shortterm-retention={{ .Values.observabilityProxy.metrics.shorttermRetention }}
ports:
- name: http
containerPort: 8080
diff --git a/packages/system/observability-proxy/templates/tenant-clusterroles.yaml b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml
index 23460352f..a2bf94f3c 100644
--- a/packages/system/observability-proxy/templates/tenant-clusterroles.yaml
+++ b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml
@@ -1,7 +1,7 @@
# Aggregated into cozy-tenant-view (and consequently use, admin, super-admin).
-# No API server serves cozystack.io logs: the rule exists only for the
-# SubjectAccessReview the observability proxy asks before it answers a read
-# under /apis/cozystack.io/v1alpha1/namespaces//logs/.
+# No API server serves cozystack.io logs or metrics: the rule exists only for
+# the SubjectAccessReview the observability proxy asks before it answers a
+# read under /apis/cozystack.io/v1alpha1/namespaces//{logs,metrics}/.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
@@ -10,5 +10,5 @@ metadata:
rbac.cozystack.io/aggregate-to-tenant-view: "true"
rules:
- apiGroups: ["cozystack.io"]
- resources: ["logs"]
+ resources: ["logs", "metrics"]
verbs: ["list"]
diff --git a/packages/system/observability-proxy/tests/deployment_test.yaml b/packages/system/observability-proxy/tests/deployment_test.yaml
index ddf9d34b8..a5e948920 100644
--- a/packages/system/observability-proxy/tests/deployment_test.yaml
+++ b/packages/system/observability-proxy/tests/deployment_test.yaml
@@ -40,6 +40,35 @@ tests:
path: spec.template.spec.containers[0].args
content: --auth-cache-ttl=10s
+ - it: passes the metrics store names and short-term retention
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: --metrics-shortterm-store=shortterm
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: --metrics-longterm-store=longterm
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: --metrics-shortterm-retention=72h
+
+ - it: follows renamed metrics stores
+ set:
+ observabilityProxy.metrics:
+ shorttermStore: fast
+ longtermStore: slow
+ shorttermRetention: 24h
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: --metrics-shortterm-store=fast
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: --metrics-longterm-store=slow
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: --metrics-shortterm-retention=24h
+
- it: falls back to cozy.local when the platform passes no cluster domain
asserts:
- contains:
diff --git a/packages/system/observability-proxy/tests/rbac_test.yaml b/packages/system/observability-proxy/tests/rbac_test.yaml
index af339b5d0..ef96c65d7 100644
--- a/packages/system/observability-proxy/tests/rbac_test.yaml
+++ b/packages/system/observability-proxy/tests/rbac_test.yaml
@@ -2,7 +2,7 @@ suite: observability-proxy RBAC
# The proxy authenticates and authorizes every request through the API server,
# so a dropped verb here turns every request into an error at runtime rather
# than a render failure. The tenant role is what lets a namespace viewer pass
-# the SubjectAccessReview the proxy asks for the namespace logs root.
+# the SubjectAccessReview the proxy asks for the namespace logs and metrics roots.
templates:
- templates/rbac.yaml
- templates/tenant-clusterroles.yaml
@@ -50,7 +50,7 @@ tests:
name: observability-proxy
namespace: cozy-observability-proxy
- - it: aggregates list on cozystack.io logs into the tenant view role
+ - it: aggregates list on cozystack.io logs and metrics into the tenant view role
template: templates/tenant-clusterroles.yaml
asserts:
- isKind:
@@ -63,5 +63,5 @@ tests:
path: rules
value:
- apiGroups: ["cozystack.io"]
- resources: ["logs"]
+ resources: ["logs", "metrics"]
verbs: ["list"]
diff --git a/packages/system/observability-proxy/values.yaml b/packages/system/observability-proxy/values.yaml
index 919e450a9..67334a237 100644
--- a/packages/system/observability-proxy/values.yaml
+++ b/packages/system/observability-proxy/values.yaml
@@ -7,6 +7,13 @@ observabilityProxy:
replicas: 2
rootTenant: tenant-root
authCacheTTL: 10s
+ # Names and retention of the root tenant's metricsStorages
+ # (packages/system/monitoring/values.yaml). The retention is a Go
+ # duration: 3d is written 72h.
+ metrics:
+ shorttermStore: shortterm
+ longtermStore: longterm
+ shorttermRetention: 72h
resources:
requests:
cpu: 10m
From 1ac7fd29bf30a15d9dedcd365649718dd8157484 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:06:54 +0700
Subject: [PATCH 064/114] fix(observability-proxy): round the metrics step up
to whole milliseconds
vmselect reads step and timeout in whole milliseconds and drops the
remainder, so a step of 1.9ms passed the points check as 1.9ms and
then ran as 1ms, close to twice the points the limit allows. The
duration is now rounded up before the limits are checked, so they see
the value vmselect will use.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/metricsparams.go | 6 ++++--
internal/observabilityproxy/metricsparams_test.go | 10 ++++++++++
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go
index 0e24b6597..e74d2cbb5 100644
--- a/internal/observabilityproxy/metricsparams.go
+++ b/internal/observabilityproxy/metricsparams.go
@@ -242,13 +242,15 @@ func parseStep(v string, lim MetricsLimits) (time.Duration, *HTTPError) {
}
// parseMetricsDuration takes Prometheus float seconds or a duration such as
-// 1m30s. vmselect works in milliseconds and refuses anything shorter.
+// 1m30s. vmselect works in whole milliseconds and refuses anything shorter;
+// the value is rounded up to the millisecond so the limits are checked
+// against what vmselect will use, not against a fraction it drops.
func parseMetricsDuration(key, v string) (time.Duration, *HTTPError) {
d, err := parseDuration(v)
if err != nil || d < time.Millisecond {
return 0, errorf(http.StatusBadRequest, "invalid %s %q", key, v)
}
- return d, nil
+ return (d + time.Millisecond - 1).Truncate(time.Millisecond), nil
}
// parseMetricsTime takes the two forms the Prometheus API defines: float
diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go
index be1d69c38..ac9ff0ada 100644
--- a/internal/observabilityproxy/metricsparams_test.go
+++ b/internal/observabilityproxy/metricsparams_test.go
@@ -155,6 +155,7 @@ func TestBuildMetricsParamsLimits(t *testing.T) {
{"range above the limit", "api/v1/query_range", rng(ago(31*day), ago(0), "1h"), http.StatusUnprocessableEntity},
{"points at the limit", "api/v1/query_range", rng(ago(10999*time.Minute), ago(0), "1m"), http.StatusOK},
{"points above the limit", "api/v1/query_range", rng(ago(11000*time.Minute), ago(0), "1m"), http.StatusUnprocessableEntity},
+ {"points above the limit after rounding the step up", "api/v1/query_range", rng(ago(22*time.Second), ago(0), "0.0019"), http.StatusUnprocessableEntity},
{"step above the range limit", "api/v1/query_range", rng(ago(time.Hour), ago(0), "31d"), http.StatusUnprocessableEntity},
{"zero step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0"), http.StatusBadRequest},
{"sub-millisecond step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0.0001"), http.StatusBadRequest},
@@ -198,6 +199,15 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) {
t.Errorf("params = %v\nwant %v", req.params, want)
}
+ // vmselect drops a sub-millisecond remainder; the points check has to
+ // see the step it will use.
+ for in, want := range map[string]string{"60.0001": "60.001", "0.0019": "0.002"} {
+ req = buildMetrics(t, "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-20 * time.Second))}, "end": {unix(testNow)}, "step": {in}})
+ if got := req.params.Get("step"); got != want {
+ t.Errorf("step %s forwarded as %q, want %s", in, got, want)
+ }
+ }
+
for in, want := range map[string]string{"": "30", "10s": "10", "5m": "30", "1.5": "1.5"} {
v := url.Values{"query": {"up"}}
if in != "" {
From 81cd278f63bc69cd6ecd7ee164deffb5bae779a1 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:13:30 +0700
Subject: [PATCH 065/114] test(observability-proxy): cover metrics concurrency
and flag overrides
The per-user concurrency cap on the metrics root had no test, so
dropping it from the handler went unnoticed. The new test holds one
request in the upstream, expects the same user's second one to be
refused and another user's to go through. Overrides of the metrics
flags were only checked for their defaults and refusals.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
cmd/observability-proxy/main_test.go | 15 +++++++
.../observabilityproxy/server_metrics_test.go | 40 +++++++++++++++++++
2 files changed, 55 insertions(+)
diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go
index fe7d3ea47..be89e20d5 100644
--- a/cmd/observability-proxy/main_test.go
+++ b/cmd/observability-proxy/main_test.go
@@ -60,6 +60,21 @@ func TestParseFlagsOverrides(t *testing.T) {
}
}
+func TestParseFlagsMetricsOverrides(t *testing.T) {
+ o, err := parseFlags([]string{
+ "--root-tenant=tenant-x", "--metrics-shortterm-store=fast", "--metrics-longterm-store=slow",
+ "--metrics-shortterm-retention=24h", "--metrics-max-range=168h", "--metrics-max-points=500", "--metrics-max-timeout=10s",
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ m := o.metrics
+ if m.RootTenant != "tenant-x" || m.ShortTermStore != "fast" || m.LongTermStore != "slow" || m.ShortTermRetention != 24*time.Hour ||
+ m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second {
+ t.Errorf("overrides not applied: %+v", m)
+ }
+}
+
func TestParseFlagsInvalid(t *testing.T) {
for _, args := range [][]string{
{"--root-tenant="},
diff --git a/internal/observabilityproxy/server_metrics_test.go b/internal/observabilityproxy/server_metrics_test.go
index 887790d34..981644668 100644
--- a/internal/observabilityproxy/server_metrics_test.go
+++ b/internal/observabilityproxy/server_metrics_test.go
@@ -20,6 +20,7 @@ import (
"io"
"net/http"
"net/url"
+ "sync"
"testing"
"time"
@@ -155,6 +156,45 @@ func TestServerMetricsStatusCodes(t *testing.T) {
}
}
+func TestServerMetricsConcurrency(t *testing.T) {
+ entered, release := make(chan struct{}, 3), make(chan struct{})
+ h := newHarness(t, Config{Limiter: LimiterConfig{MaxConcurrentPerUser: 1}}, func(w http.ResponseWriter, _ *http.Request) {
+ entered <- struct{}{}
+ <-release
+ promUpstream(w, nil)
+ })
+ // Registered after the harness so it runs before the servers close.
+ var once sync.Once
+ t.Cleanup(func() { once.Do(func() { close(release) }) })
+ held := make(chan int, 2)
+ hold := func(token string) {
+ go func() { held <- h.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", token) }()
+ select {
+ case <-entered:
+ case <-time.After(2 * time.Second):
+ t.Fatalf("request of %s did not reach the upstream", token)
+ }
+ }
+ hold("good")
+ refused := make(chan int, 1)
+ go func() { refused <- h.status(http.MethodPost, metricsRoot+"api/v1/query?query=up", "good") }()
+ select {
+ case got := <-refused:
+ if got != http.StatusUnprocessableEntity {
+ t.Fatalf("concurrent request answered %d, want 422", got)
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("concurrent request of the same user reached the upstream")
+ }
+ hold("bob-token")
+ once.Do(func() { close(release) })
+ for range 2 {
+ if got := <-held; got != http.StatusOK {
+ t.Fatalf("held request answered %d", got)
+ }
+ }
+}
+
func TestDefaultMetricsUpstreamURL(t *testing.T) {
s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil)
if got, want := s.cfg.Metrics.UpstreamURL("shortterm"), "http://vmselect-shortterm.tenant-root.svc.cozy.local:8481/select/0/prometheus"; got != want {
From 6d6f05a1c7980cf369d607732daadb29df4351d8 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:17:15 +0700
Subject: [PATCH 066/114] fix(observability-proxy): refuse metrics durations
vmselect would refuse
Rounding a duration up to the millisecond overflows for values within
a millisecond of the largest one, which turned such a step or timeout
negative and let it past the range, points and timeout limits. vmselect
refuses durations above 100 years itself, so the proxy now answers 400
for them before rounding.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/metricsparams.go | 7 ++++++-
internal/observabilityproxy/metricsparams_test.go | 2 ++
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go
index e74d2cbb5..fe9c8aafd 100644
--- a/internal/observabilityproxy/metricsparams.go
+++ b/internal/observabilityproxy/metricsparams.go
@@ -241,13 +241,18 @@ func parseStep(v string, lim MetricsLimits) (time.Duration, *HTTPError) {
return step, nil
}
+// maxMetricsDuration is vmselect's own bound on step and timeout
+// (maxDurationMsecs in lib/httputil); it also keeps the rounding below
+// from overflowing.
+const maxMetricsDuration = 100 * 365 * 24 * time.Hour
+
// parseMetricsDuration takes Prometheus float seconds or a duration such as
// 1m30s. vmselect works in whole milliseconds and refuses anything shorter;
// the value is rounded up to the millisecond so the limits are checked
// against what vmselect will use, not against a fraction it drops.
func parseMetricsDuration(key, v string) (time.Duration, *HTTPError) {
d, err := parseDuration(v)
- if err != nil || d < time.Millisecond {
+ if err != nil || d < time.Millisecond || d > maxMetricsDuration {
return 0, errorf(http.StatusBadRequest, "invalid %s %q", key, v)
}
return (d + time.Millisecond - 1).Truncate(time.Millisecond), nil
diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go
index ac9ff0ada..fc403191b 100644
--- a/internal/observabilityproxy/metricsparams_test.go
+++ b/internal/observabilityproxy/metricsparams_test.go
@@ -156,6 +156,8 @@ func TestBuildMetricsParamsLimits(t *testing.T) {
{"points at the limit", "api/v1/query_range", rng(ago(10999*time.Minute), ago(0), "1m"), http.StatusOK},
{"points above the limit", "api/v1/query_range", rng(ago(11000*time.Minute), ago(0), "1m"), http.StatusUnprocessableEntity},
{"points above the limit after rounding the step up", "api/v1/query_range", rng(ago(22*time.Second), ago(0), "0.0019"), http.StatusUnprocessableEntity},
+ {"overflowing step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "2562047h47m16.854775807s"), http.StatusBadRequest},
+ {"overflowing timeout", "api/v1/query", url.Values{"query": {"up"}, "timeout": {"2562047h47m16.854775807s"}}, http.StatusBadRequest},
{"step above the range limit", "api/v1/query_range", rng(ago(time.Hour), ago(0), "31d"), http.StatusUnprocessableEntity},
{"zero step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0"), http.StatusBadRequest},
{"sub-millisecond step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0.0001"), http.StatusBadRequest},
From 19030c7a1a286f724f92b91628973bfc26aed3aa Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:25:58 +0700
Subject: [PATCH 067/114] fix(observability-proxy): answer 429 to the metrics
concurrency limit
The logs root answers 429 when a user has too many requests in flight,
so clients back off instead of treating the refusal as a bad query.
The metrics root kept its own 422 for the same limit; it now matches.
The rate limit is shared and already answers 429 with Retry-After,
which the metrics test now expects too.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/server.go | 2 +-
internal/observabilityproxy/server_metrics_test.go | 11 +++++++----
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go
index 3e24be109..4760f7421 100644
--- a/internal/observabilityproxy/server.go
+++ b/internal/observabilityproxy/server.go
@@ -291,7 +291,7 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout
}
release, ok := s.limits.acquire(user)
if !ok {
- return errorf(http.StatusUnprocessableEntity, "too many concurrent requests")
+ return errorf(http.StatusTooManyRequests, "too many concurrent requests")
}
defer release()
diff --git a/internal/observabilityproxy/server_metrics_test.go b/internal/observabilityproxy/server_metrics_test.go
index 981644668..a1c07d944 100644
--- a/internal/observabilityproxy/server_metrics_test.go
+++ b/internal/observabilityproxy/server_metrics_test.go
@@ -151,8 +151,11 @@ func TestServerMetricsStatusCodes(t *testing.T) {
h4 := newHarness(t, Config{Limiter: LimiterConfig{RatePerUser: 0.001, RateBurst: 1}}, promUpstream)
h4.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good")
- if got := h4.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusUnprocessableEntity {
- t.Errorf("rate-limited request answered %d, want 422", got)
+ limited := h4.do(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good", nil, nil)
+ _ = limited.Body.Close()
+ if limited.StatusCode != http.StatusTooManyRequests || limited.Header.Get("Retry-After") != "1000" {
+ t.Errorf("rate-limited request answered %d with Retry-After %q, want 429 with 1000",
+ limited.StatusCode, limited.Header.Get("Retry-After"))
}
}
@@ -180,8 +183,8 @@ func TestServerMetricsConcurrency(t *testing.T) {
go func() { refused <- h.status(http.MethodPost, metricsRoot+"api/v1/query?query=up", "good") }()
select {
case got := <-refused:
- if got != http.StatusUnprocessableEntity {
- t.Fatalf("concurrent request answered %d, want 422", got)
+ if got != http.StatusTooManyRequests {
+ t.Fatalf("concurrent request answered %d, want 429", got)
}
case <-time.After(2 * time.Second):
t.Fatal("concurrent request of the same user reached the upstream")
From 560e4d7ba5c2bb7123fd1e1e618b1995fdaf3578 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:40:06 +0700
Subject: [PATCH 068/114] fix(observability-proxy): count the query lookback in
the store choice
The short-term store was picked from the request window alone, so an
instant increase(x[7d]) or x offset 10d went to the store that keeps
72h and silently returned a wrong number.
A lexical scan of the query now bounds how far back it reads: the sum
of every [...] window and subquery step, positive offsets, and the
step and silence interval vmselect reads before each rollup. An @
modifier, step-multiple units or any duration the scan cannot read
make the bound unknown and pick the long-term store, which receives
the same writes. The scan sees quoted label values too, so a false
hit only moves a query to the long-term store. The query is still
forwarded unchanged.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observabilityproxy/metricslookback.go | 172 ++++++++++++++++++
internal/observabilityproxy/metricsparams.go | 15 +-
.../observabilityproxy/metricsparams_test.go | 109 ++++++++++-
3 files changed, 288 insertions(+), 8 deletions(-)
create mode 100644 internal/observabilityproxy/metricslookback.go
diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go
new file mode 100644
index 000000000..17a586bb9
--- /dev/null
+++ b/internal/observabilityproxy/metricslookback.go
@@ -0,0 +1,172 @@
+/*
+Copyright 2026 The Cozystack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package observabilityproxy
+
+import (
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// instantDefaultStep is the step vmselect uses for an instant query without
+// one (defaultStep in app/vmselect/prometheus); it is also the implicit
+// rollup window of a bare selector.
+const instantDefaultStep = 5 * time.Minute
+
+// silenceInterval is how far before a rollup or subquery window vmselect
+// starts reading (maxSilenceInterval in app/vmselect/promql: the value of
+// -search.minStalenessInterval, 5m when unset; cozystack does not set it).
+const silenceInterval = 5 * time.Minute
+
+// queryDurationPart is the subset of the MetricsQL duration grammar
+// (scanSingleDuration in metricsql) the scan understands. Step multiples
+// (5i, $__interval), negative parts and upper-case units are left out on
+// purpose: anything else makes the bound unknown.
+var (
+ queryDurationPart = regexp.MustCompile(`(\d+(?:\.\d+)?)(ms|s|m|h|d|w|y)`)
+ queryDuration = regexp.MustCompile(`^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d|w|y))+$`)
+ querySeconds = regexp.MustCompile(`^\d+(?:\.\d+)?$`)
+)
+
+var queryDurationUnits = map[string]time.Duration{
+ "ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour,
+ "d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour,
+}
+
+// queryStart is the earliest instant a query evaluated from start may read;
+// zero when the bound is unknown.
+func queryStart(query string, start time.Time, step time.Duration) time.Time {
+ lookback, ok := queryLookback(query, step)
+ if !ok {
+ return time.Time{}
+ }
+ return start.Add(-lookback)
+}
+
+// queryLookback is an upper bound on how far before its evaluation window
+// a query reads, from a lexical scan that never parses the query. Every
+// [...] and offset counts, including ones inside quoted label values: an
+// extra hit only makes the bound larger or unknown, never smaller. Windows
+// and offsets are summed rather than maxed, which also covers nesting.
+// vmselect starts a rollup max(window, step) plus the silence interval
+// before the first point, a subquery window+step+silence before it and
+// aligns both down to a step; each step term uses the largest step seen.
+func queryLookback(query string, step time.Duration) (time.Duration, bool) {
+ if strings.Contains(query, "@") {
+ return 0, false
+ }
+ var sum float64
+ subqueries := 0
+ maxStep := float64(step)
+ for rest := query; ; {
+ open := strings.IndexByte(rest, '[')
+ if open < 0 {
+ break
+ }
+ closing := strings.IndexByte(rest[open:], ']')
+ if closing < 0 {
+ return 0, false
+ }
+ window, subStep, isSubquery := strings.Cut(rest[open+1:open+closing], ":")
+ rest = rest[open+closing+1:]
+ if isSubquery {
+ subqueries++
+ if strings.TrimSpace(window) == "" {
+ window = "0s"
+ }
+ if strings.TrimSpace(subStep) != "" {
+ d, ok := parseQueryDuration(subStep)
+ if !ok {
+ return 0, false
+ }
+ maxStep = max(maxStep, d)
+ }
+ }
+ d, ok := parseQueryDuration(window)
+ if !ok {
+ return 0, false
+ }
+ sum += d
+ }
+ lower := strings.ToLower(query)
+ for i := 0; ; {
+ k := strings.Index(lower[i:], "offset")
+ if k < 0 {
+ break
+ }
+ k += i
+ i = k + len("offset")
+ if k > 0 && continuesIdent(lower[k-1], false) || i < len(lower) && continuesIdent(lower[i], true) {
+ continue
+ }
+ arg := strings.TrimLeft(lower[i:], " \t\n\r")
+ negative := strings.HasPrefix(arg, "-")
+ if negative {
+ arg = strings.TrimLeft(arg[1:], " \t\n\r")
+ }
+ end := strings.IndexFunc(arg, func(r rune) bool {
+ return !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.')
+ })
+ if end < 0 {
+ end = len(arg)
+ }
+ d, ok := parseQueryDuration(arg[:end])
+ if !ok {
+ return 0, false
+ }
+ if !negative {
+ sum += d
+ }
+ }
+ n := float64(subqueries)
+ sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval)
+ if sum > float64(maxMetricsDuration) {
+ return 0, false
+ }
+ return time.Duration(sum), true
+}
+
+// continuesIdent reports whether c, next to "offset", makes it part of an
+// identifier rather than the keyword. Digits and dots only continue an
+// identifier after it, so before it they still count as a keyword hit.
+func continuesIdent(c byte, after bool) bool {
+ return c >= 'a' && c <= 'z' || c == '_' || c == ':' || c >= 0x80 ||
+ after && (c >= '0' && c <= '9' || c == '.' || c == '\\')
+}
+
+// parseQueryDuration returns nanoseconds as a float so that a sum of huge
+// durations cannot wrap; the caller compares it against the bound.
+func parseQueryDuration(s string) (float64, bool) {
+ s = strings.TrimSpace(s)
+ if querySeconds.MatchString(s) {
+ f, err := strconv.ParseFloat(s, 64)
+ return f * float64(time.Second), err == nil
+ }
+ if !queryDuration.MatchString(s) {
+ return 0, false
+ }
+ var d float64
+ for _, m := range queryDurationPart.FindAllStringSubmatch(s, -1) {
+ f, err := strconv.ParseFloat(m[1], 64)
+ if err != nil {
+ return 0, false
+ }
+ d += f * float64(queryDurationUnits[m[2]])
+ }
+ return d, true
+}
diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go
index fe9c8aafd..117f1b573 100644
--- a/internal/observabilityproxy/metricsparams.go
+++ b/internal/observabilityproxy/metricsparams.go
@@ -69,7 +69,8 @@ var labelName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)
type metricsRequest struct {
path string
params url.Values
- // start is the earliest instant the request reads, for the store choice.
+ // start is the earliest instant the request reads, for the store choice;
+ // zero when that is unknown, which picks the long-term store.
start time.Time
}
@@ -103,22 +104,24 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
if herr := copyQuery(out, in); herr != nil {
return req, herr
}
- req.start = now
+ at := now
if v := in.Get("time"); v != "" {
t, herr := parseMetricsTime("time", v)
if herr != nil {
return req, herr
}
- req.start = t
+ at = t
out.Set("time", formatTime(t))
}
+ step := instantDefaultStep
if v := in.Get("step"); v != "" {
- step, herr := parseStep(v, lim)
- if herr != nil {
+ var herr *HTTPError
+ if step, herr = parseStep(v, lim); herr != nil {
return req, herr
}
out.Set("step", formatSeconds(step))
}
+ req.start = queryStart(out.Get("query"), at, step)
return req, nil
case "api/v1/query_range":
@@ -141,7 +144,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
if points := end.Sub(start)/step + 1; points > time.Duration(lim.MaxPoints) {
return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints)
}
- req.start = start
+ req.start = queryStart(out.Get("query"), start, step)
out.Set("start", formatTime(start))
out.Set("end", formatTime(end))
out.Set("step", formatSeconds(step))
diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go
index fc403191b..54e5acd04 100644
--- a/internal/observabilityproxy/metricsparams_test.go
+++ b/internal/observabilityproxy/metricsparams_test.go
@@ -224,7 +224,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) {
if req.params.Get("end") != "1793793600.000" || req.params.Get("start") != "1793793300.000" {
t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end"))
}
- if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow) {
+ if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute)) {
t.Errorf("instant query without time: params %v, start %v", q.params, q.start)
}
}
@@ -241,7 +241,8 @@ func TestMetricsStoreByRange(t *testing.T) {
{"instant within retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-71 * time.Hour))}}, "shortterm"},
{"instant past retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-73 * time.Hour))}}, "longterm"},
{"live graph", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-6 * time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}, "shortterm"},
- {"range fits exactly", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-72 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "shortterm"},
+ {"range fits with lookback", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-71 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "shortterm"},
+ {"range lookback past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-72 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"},
{"range past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-7 * 24 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"},
{"old window ending long ago", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-5 * 24 * time.Hour))}, "end": {unix(testNow.Add(-4 * 24 * time.Hour))}, "step": {"600"}}, "longterm"},
{"labels default window", "api/v1/labels", url.Values{}, "shortterm"},
@@ -256,3 +257,107 @@ func TestMetricsStoreByRange(t *testing.T) {
})
}
}
+
+func TestQueryLookback(t *testing.T) {
+ tests := []struct {
+ query string
+ step time.Duration
+ want time.Duration
+ }{
+ {`up`, 5 * time.Minute, 15 * time.Minute},
+ {`increase(x[7d])`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute},
+ {`rate(x[300])`, time.Minute, 12 * time.Minute},
+ {`rate(x[ 1h30m ])`, time.Minute, 97 * time.Minute},
+ {`max_over_time(x[1h:5m])`, time.Minute, 95 * time.Minute},
+ {`max_over_time(x[1h:])`, time.Minute, 75 * time.Minute},
+ {`max_over_time(max_over_time(rate(x[5m])[1h:1m])[1d:1h])`, time.Minute, 33*time.Hour + 20*time.Minute},
+ {`x offset 10d`, 5 * time.Minute, 10*24*time.Hour + 15*time.Minute},
+ {`sum(x OFFSET 2h)`, 5 * time.Minute, 135 * time.Minute},
+ {`rate(x[5m] offset 1h)`, time.Minute, 72 * time.Minute},
+ {`x offset -1h`, 5 * time.Minute, 15 * time.Minute},
+ {`x offset - 1h`, 5 * time.Minute, 15 * time.Minute},
+ {`node_offset_seconds + offset_total`, 5 * time.Minute, 15 * time.Minute},
+ {`x{a="[1h]"}`, 5 * time.Minute, 75 * time.Minute},
+ {`x{a="]"}[7d]`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute},
+ }
+ for _, tt := range tests {
+ t.Run(tt.query, func(t *testing.T) {
+ got, ok := queryLookback(tt.query, tt.step)
+ if !ok || got != tt.want {
+ t.Fatalf("queryLookback = %s, %t; want %s, true", got, ok, tt.want)
+ }
+ })
+ }
+}
+
+func TestQueryLookbackUnknown(t *testing.T) {
+ for _, q := range []string{
+ `x @ 123`,
+ `x @ start()`,
+ `x{a="a@b"}`,
+ `rate(x[5i])`,
+ `rate(x[$__interval])`,
+ `x[1h-5m]`,
+ `x[7D]`,
+ `x[1.d]`,
+ `x[]`,
+ `rate(x[7d)`,
+ `x offset`,
+ `x offset w`,
+ `WITH (w = 7d) rate(x[w])`,
+ `x{a="["}`,
+ `x{a="[5m"}[7d]`,
+ `x{path="/offset/1"}`,
+ `x[99999999999y]`,
+ `x offset 99999999999y`,
+ } {
+ t.Run(q, func(t *testing.T) {
+ if got, ok := queryLookback(q, 5*time.Minute); ok {
+ t.Fatalf("queryLookback = %s, true; want unknown", got)
+ }
+ })
+ }
+}
+
+func TestMetricsStoreByLookback(t *testing.T) {
+ cfg := MetricsConfig{ShortTermStore: "shortterm", LongTermStore: "longterm", ShortTermRetention: 72 * time.Hour}
+ rangeFrom := func(q string, ago time.Duration, step string) url.Values {
+ return url.Values{"query": {q}, "start": {unix(testNow.Add(-ago))}, "end": {unix(testNow)}, "step": {step}}
+ }
+ tests := []struct {
+ name string
+ endpoint string
+ in url.Values
+ want string
+ }{
+ {"instant short window", "api/v1/query", url.Values{"query": {"rate(x[5m])"}}, "shortterm"},
+ {"instant 7d window", "api/v1/query", url.Values{"query": {"increase(x[7d])"}}, "longterm"},
+ {"instant offset 10d", "api/v1/query", url.Values{"query": {"x offset 10d"}}, "longterm"},
+ {"instant negative offset", "api/v1/query", url.Values{"query": {"x offset -1h"}}, "shortterm"},
+ {"instant at timestamp", "api/v1/query", url.Values{"query": {"x @ 123"}}, "longterm"},
+ {"instant at start", "api/v1/query", url.Values{"query": {"x @ start()"}}, "longterm"},
+ {"instant step window", "api/v1/query", url.Values{"query": {"rate(x[5i])"}}, "longterm"},
+ {"instant step past retention", "api/v1/query", url.Values{"query": {"up"}, "step": {"259200"}}, "longterm"},
+ {"instant subquery past retention", "api/v1/query", url.Values{"query": {"max_over_time(rate(x[5m])[4d:1h])"}}, "longterm"},
+ {"quoted at sign", "api/v1/query", url.Values{"query": {`x{a="a@b"}`}}, "longterm"},
+ {"quoted open bracket hides window", "api/v1/query", url.Values{"query": {`rate(x{a="[5m"}[7d])`}}, "longterm"},
+ {"quoted close bracket before window", "api/v1/query", url.Values{"query": {`rate(x{a="]"}[7d])`}}, "longterm"},
+ {"quoted offset before offset", "api/v1/query", url.Values{"query": {`x{a="offset 1m"} offset 10d`}}, "longterm"},
+ {"range short window", "api/v1/query_range", rangeFrom("rate(x[1h])", 6*time.Hour, "60"), "shortterm"},
+ {"range 7d window", "api/v1/query_range", rangeFrom("increase(x[7d])", 6*time.Hour, "60"), "longterm"},
+ {"range offset past retention", "api/v1/query_range", rangeFrom("x offset 3d", time.Hour, "60"), "longterm"},
+ {"range lookback fits", "api/v1/query_range", rangeFrom("rate(x[5m])", 70*time.Hour, "600"), "shortterm"},
+ {"range lookback crosses retention", "api/v1/query_range", rangeFrom("rate(x[5m])", 71*time.Hour+50*time.Minute, "600"), "longterm"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ req := buildMetrics(t, tt.endpoint, tt.in)
+ if got := cfg.store(req.start, testNow); got != tt.want {
+ t.Fatalf("store = %s, want %s", got, tt.want)
+ }
+ if q := req.params.Get("query"); q != tt.in.Get("query") {
+ t.Fatalf("query forwarded as %q, want it unchanged", q)
+ }
+ })
+ }
+}
From 2fad05b01f1dc35ed7c0e9d3a2c8467f00f81d88 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:46:37 +0700
Subject: [PATCH 069/114] fix(observability-proxy): count the latency offset in
instant lookback
vmselect moves an instant query evaluated near the current time back by
-search.latencyOffset (30s). With a tiny explicit step the lookback
bound did not cover that shift, so a window ending at the retention
edge could still be sent to the short-term store.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/metricslookback.go | 6 ++++++
internal/observabilityproxy/metricsparams.go | 2 +-
internal/observabilityproxy/metricsparams_test.go | 3 ++-
3 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go
index 17a586bb9..a1de3ca32 100644
--- a/internal/observabilityproxy/metricslookback.go
+++ b/internal/observabilityproxy/metricslookback.go
@@ -33,6 +33,12 @@ const instantDefaultStep = 5 * time.Minute
// -search.minStalenessInterval, 5m when unset; cozystack does not set it).
const silenceInterval = 5 * time.Minute
+// latencyOffset is how far vmselect moves an instant query evaluated within
+// it of the current time back (-search.latencyOffset, 30s by default and
+// not set by cozystack); the proxy forwards neither nocache nor
+// latency_offset.
+const latencyOffset = 30 * time.Second
+
// queryDurationPart is the subset of the MetricsQL duration grammar
// (scanSingleDuration in metricsql) the scan understands. Step multiples
// (5i, $__interval), negative parts and upper-case units are left out on
diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go
index 117f1b573..5aae58479 100644
--- a/internal/observabilityproxy/metricsparams.go
+++ b/internal/observabilityproxy/metricsparams.go
@@ -121,7 +121,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti
}
out.Set("step", formatSeconds(step))
}
- req.start = queryStart(out.Get("query"), at, step)
+ req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step)
return req, nil
case "api/v1/query_range":
diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go
index 54e5acd04..4b034c661 100644
--- a/internal/observabilityproxy/metricsparams_test.go
+++ b/internal/observabilityproxy/metricsparams_test.go
@@ -224,7 +224,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) {
if req.params.Get("end") != "1793793600.000" || req.params.Get("start") != "1793793300.000" {
t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end"))
}
- if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute)) {
+ if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-30*time.Second)) {
t.Errorf("instant query without time: params %v, start %v", q.params, q.start)
}
}
@@ -338,6 +338,7 @@ func TestMetricsStoreByLookback(t *testing.T) {
{"instant at start", "api/v1/query", url.Values{"query": {"x @ start()"}}, "longterm"},
{"instant step window", "api/v1/query", url.Values{"query": {"rate(x[5i])"}}, "longterm"},
{"instant step past retention", "api/v1/query", url.Values{"query": {"up"}, "step": {"259200"}}, "longterm"},
+ {"instant latency offset at the edge", "api/v1/query", url.Values{"query": {"default_rollup(x[71h54m59s])"}, "step": {"0.001"}}, "longterm"},
{"instant subquery past retention", "api/v1/query", url.Values{"query": {"max_over_time(rate(x[5m])[4d:1h])"}}, "longterm"},
{"quoted at sign", "api/v1/query", url.Values{"query": {`x{a="a@b"}`}}, "longterm"},
{"quoted open bracket hides window", "api/v1/query", url.Values{"query": {`rate(x{a="[5m"}[7d])`}}, "longterm"},
From 1ead35a37ae90294ae3134c786ba41c66d0d5a11 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:52:21 +0700
Subject: [PATCH 070/114] fix(observability-proxy): bound implicit subqueries
and WITH templates
The lookback scan assumed every window and offset applies once and
explicitly. A rollup function or offset over anything but a selector is
an implicit subquery in vmselect, so every "(" now counts as one level;
a WITH template can apply one window several times, so WITH makes the
bound unknown; and an offset written with digit separators was read as
its first digits only, so "_" now ends in an unknown bound instead.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observabilityproxy/metricslookback.go | 40 +++++++++++++++----
.../observabilityproxy/metricsparams_test.go | 27 ++++++++-----
2 files changed, 50 insertions(+), 17 deletions(-)
diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go
index a1de3ca32..ed7e9d166 100644
--- a/internal/observabilityproxy/metricslookback.go
+++ b/internal/observabilityproxy/metricslookback.go
@@ -66,12 +66,15 @@ func queryStart(query string, start time.Time, step time.Duration) time.Time {
// queryLookback is an upper bound on how far before its evaluation window
// a query reads, from a lexical scan that never parses the query. Every
-// [...] and offset counts, including ones inside quoted label values: an
-// extra hit only makes the bound larger or unknown, never smaller. Windows
-// and offsets are summed rather than maxed, which also covers nesting.
+// [...], offset and "(" counts, including ones inside quoted label values:
+// an extra hit only makes the bound larger or unknown, never smaller.
+// Windows and offsets are summed rather than maxed, which covers nesting;
+// WITH is refused because a template applies one window several times.
// vmselect starts a rollup max(window, step) plus the silence interval
// before the first point, a subquery window+step+silence before it and
-// aligns both down to a step; each step term uses the largest step seen.
+// aligns both down to a step. A rollup function or offset over anything
+// but a selector is an implicit subquery (getRollupExprArg), so every "("
+// is counted as one. Each step term uses the largest step seen.
func queryLookback(query string, step time.Duration) (time.Duration, bool) {
if strings.Contains(query, "@") {
return 0, false
@@ -117,7 +120,7 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
}
k += i
i = k + len("offset")
- if k > 0 && continuesIdent(lower[k-1], false) || i < len(lower) && continuesIdent(lower[i], true) {
+ if !isKeyword(lower, k, i) {
continue
}
arg := strings.TrimLeft(lower[i:], " \t\n\r")
@@ -126,7 +129,7 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
arg = strings.TrimLeft(arg[1:], " \t\n\r")
}
end := strings.IndexFunc(arg, func(r rune) bool {
- return !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.')
+ return !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.' || r == '_')
})
if end < 0 {
end = len(arg)
@@ -139,7 +142,10 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
sum += d
}
}
- n := float64(subqueries)
+ if hasKeyword(lower, "with") {
+ return 0, false
+ }
+ n := float64(subqueries + strings.Count(query, "("))
sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval)
if sum > float64(maxMetricsDuration) {
return 0, false
@@ -147,6 +153,26 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
return time.Duration(sum), true
}
+// isKeyword reports whether lower[start:end] stands on its own rather than
+// inside an identifier.
+func isKeyword(lower string, start, end int) bool {
+ return !(start > 0 && continuesIdent(lower[start-1], false) || end < len(lower) && continuesIdent(lower[end], true))
+}
+
+func hasKeyword(lower, word string) bool {
+ for i := 0; ; {
+ k := strings.Index(lower[i:], word)
+ if k < 0 {
+ return false
+ }
+ k += i
+ i = k + len(word)
+ if isKeyword(lower, k, i) {
+ return true
+ }
+ }
+}
+
// continuesIdent reports whether c, next to "offset", makes it part of an
// identifier rather than the keyword. Digits and dots only continue an
// identifier after it, so before it they still count as a keyword hit.
diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go
index 4b034c661..4a8565564 100644
--- a/internal/observabilityproxy/metricsparams_test.go
+++ b/internal/observabilityproxy/metricsparams_test.go
@@ -265,18 +265,19 @@ func TestQueryLookback(t *testing.T) {
want time.Duration
}{
{`up`, 5 * time.Minute, 15 * time.Minute},
- {`increase(x[7d])`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute},
- {`rate(x[300])`, time.Minute, 12 * time.Minute},
- {`rate(x[ 1h30m ])`, time.Minute, 97 * time.Minute},
- {`max_over_time(x[1h:5m])`, time.Minute, 95 * time.Minute},
- {`max_over_time(x[1h:])`, time.Minute, 75 * time.Minute},
- {`max_over_time(max_over_time(rate(x[5m])[1h:1m])[1d:1h])`, time.Minute, 33*time.Hour + 20*time.Minute},
+ {`increase(x[7d])`, 5 * time.Minute, 7*24*time.Hour + 35*time.Minute},
+ {`rate(x[300])`, time.Minute, 20 * time.Minute},
+ {`rate(x[ 1h30m ])`, time.Minute, 105 * time.Minute},
+ {`max_over_time(x[1h:5m])`, time.Minute, 115 * time.Minute},
+ {`max_over_time(x[1h:])`, time.Minute, 83 * time.Minute},
+ {`max_over_time(max_over_time(rate(x[5m])[1h:1m])[1d:1h])`, time.Minute, 42*time.Hour + 35*time.Minute},
+ {`rate(rate(x))`, time.Hour, 2*(3*time.Hour+5*time.Minute) + 2*time.Hour + 5*time.Minute},
{`x offset 10d`, 5 * time.Minute, 10*24*time.Hour + 15*time.Minute},
- {`sum(x OFFSET 2h)`, 5 * time.Minute, 135 * time.Minute},
- {`rate(x[5m] offset 1h)`, time.Minute, 72 * time.Minute},
+ {`sum(x OFFSET 2h)`, 5 * time.Minute, 155 * time.Minute},
+ {`rate(x[5m] offset 1h)`, time.Minute, 80 * time.Minute},
{`x offset -1h`, 5 * time.Minute, 15 * time.Minute},
{`x offset - 1h`, 5 * time.Minute, 15 * time.Minute},
- {`node_offset_seconds + offset_total`, 5 * time.Minute, 15 * time.Minute},
+ {`node_offset_seconds + offset_total + with_x`, 5 * time.Minute, 15 * time.Minute},
{`x{a="[1h]"}`, 5 * time.Minute, 75 * time.Minute},
{`x{a="]"}[7d]`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute},
}
@@ -310,6 +311,9 @@ func TestQueryLookbackUnknown(t *testing.T) {
`x{path="/offset/1"}`,
`x[99999999999y]`,
`x offset 99999999999y`,
+ `x offset 3_000_000`,
+ `WITH (f(q) = max_over_time(q[2d:1h])) f(f(x))`,
+ `with (s(m) = m offset 2d) s(s(x))`,
} {
t.Run(q, func(t *testing.T) {
if got, ok := queryLookback(q, 5*time.Minute); ok {
@@ -338,8 +342,11 @@ func TestMetricsStoreByLookback(t *testing.T) {
{"instant at start", "api/v1/query", url.Values{"query": {"x @ start()"}}, "longterm"},
{"instant step window", "api/v1/query", url.Values{"query": {"rate(x[5i])"}}, "longterm"},
{"instant step past retention", "api/v1/query", url.Values{"query": {"up"}, "step": {"259200"}}, "longterm"},
- {"instant latency offset at the edge", "api/v1/query", url.Values{"query": {"default_rollup(x[71h54m59s])"}, "step": {"0.001"}}, "longterm"},
+ {"instant latency offset at the edge", "api/v1/query", url.Values{"query": {"x[71h54m59s]"}, "step": {"0.001"}}, "longterm"},
{"instant subquery past retention", "api/v1/query", url.Values{"query": {"max_over_time(rate(x[5m])[4d:1h])"}}, "longterm"},
+ {"instant offset with underscores", "api/v1/query", url.Values{"query": {"x offset 3_000_000"}}, "longterm"},
+ {"instant WITH template", "api/v1/query", url.Values{"query": {"WITH (f(q) = max_over_time(q[2d:1h])) f(f(x))"}}, "longterm"},
+ {"instant implicit subqueries", "api/v1/query", url.Values{"query": {"rate(rate(rate(rate(rate(rate(x))))))"}, "step": {"43200"}}, "longterm"},
{"quoted at sign", "api/v1/query", url.Values{"query": {`x{a="a@b"}`}}, "longterm"},
{"quoted open bracket hides window", "api/v1/query", url.Values{"query": {`rate(x{a="[5m"}[7d])`}}, "longterm"},
{"quoted close bracket before window", "api/v1/query", url.Values{"query": {`rate(x{a="]"}[7d])`}}, "longterm"},
From 0d3a0487e470152817e15796e83d4ca84d80d4a6 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 00:31:11 +0700
Subject: [PATCH 071/114] fix(observability-proxy): bound the metrics request
lifetime
The metrics root parsed its form before taking a concurrency slot,
under no read deadline, and wrote its answer under no write deadline,
so a client that sent its body slowly or stopped reading held the
handler outside every limit. A store answer that broke off after the
status line still ended as a clean 200.
The metrics handler now follows the logs root: the slot is taken
before the body is read under the request timeout, the answer must be
taken by the request deadline, and a failed copy cuts the connection.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/server.go | 23 +++++---
.../server_deadline_test.go | 56 +++++++++++++++++++
2 files changed, 70 insertions(+), 9 deletions(-)
diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go
index 4760f7421..50b6ac89e 100644
--- a/internal/observabilityproxy/server.go
+++ b/internal/observabilityproxy/server.go
@@ -280,20 +280,19 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
}
func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError {
- r.Body = http.MaxBytesReader(w, r.Body, maxFormBytes)
- if err := r.ParseForm(); err != nil {
- return errorf(http.StatusBadRequest, "invalid form: %v", err)
+ release, ok := s.limits.acquire(user)
+ if !ok {
+ return errorf(http.StatusTooManyRequests, "too many concurrent requests")
+ }
+ defer release()
+ if herr := s.readForm(w, r); herr != nil {
+ return herr
}
now := s.now()
req, herr := BuildMetricsParams(route.Endpoint, r.Form, route.Namespace, now, s.cfg.Metrics.Limits)
if herr != nil {
return herr
}
- release, ok := s.limits.acquire(user)
- if !ok {
- return errorf(http.StatusTooManyRequests, "too many concurrent requests")
- }
- defer release()
ctx := r.Context()
if s.cfg.RequestTimeout > 0 {
@@ -310,9 +309,15 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout
if ct := resp.Header.Get("Content-Type"); ct != "" {
w.Header().Set("Content-Type", ct)
}
+ // As for a bounded logs answer: the client must take it by the request
+ // deadline, and a break-off cuts the connection.
+ if deadline, ok := ctx.Deadline(); ok {
+ _ = http.NewResponseController(w).SetWriteDeadline(deadline)
+ }
w.WriteHeader(resp.StatusCode)
- if _, err := io.Copy(w, resp.Body); err != nil && !errors.Is(err, context.Canceled) {
+ if _, err := io.Copy(w, resp.Body); err != nil {
slog.Debug("copying upstream response", "error", err)
+ return errAborted
}
return nil
}
diff --git a/internal/observabilityproxy/server_deadline_test.go b/internal/observabilityproxy/server_deadline_test.go
index 7415b46fa..03d03f218 100644
--- a/internal/observabilityproxy/server_deadline_test.go
+++ b/internal/observabilityproxy/server_deadline_test.go
@@ -444,3 +444,59 @@ func TestServerStalledReviewAnswersInTime(t *testing.T) {
})
}
}
+
+// TestServerMetricsRequestLifetime repeats the body, write and break-off
+// checks on the metrics root, which has its own handler.
+func TestServerMetricsRequestLifetime(t *testing.T) {
+ query := metricsRoot + "api/v1/query?query=up"
+ t.Run("slow body", func(t *testing.T) {
+ h := newHarness(t, Config{
+ RequestTimeout: 200 * time.Millisecond,
+ Limiter: LimiterConfig{MaxConcurrentPerUser: 1},
+ }, promUpstream)
+ pipe := servePipe(t, h.server)
+ conn := pipe.send(t, "POST "+metricsRoot+"api/v1/query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n"+
+ "Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 100\r\n\r\nquery=")
+ eventually(t, time.Second, func() bool {
+ return h.status(http.MethodGet, query, "good") == http.StatusTooManyRequests
+ }, "a metrics request with an unfinished body does not hold a concurrency slot")
+ if resp := readResponse(t, conn); resp.StatusCode != http.StatusRequestTimeout {
+ t.Fatalf("unfinished body answered %d, want 408", resp.StatusCode)
+ }
+ })
+ t.Run("stalled client", func(t *testing.T) {
+ // Larger than the server's write buffer, so the handler itself
+ // blocks on the client.
+ big := `{"status":"success","data":"` + strings.Repeat("x", 64<<10) + `"}`
+ h := newHarness(t, Config{
+ RequestTimeout: 200 * time.Millisecond,
+ Limiter: LimiterConfig{MaxConcurrentPerUser: 1},
+ }, func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, big) })
+ pipe := servePipe(t, h.server)
+ pipe.send(t, "GET "+query+" HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n")
+ eventually(t, time.Second, func() bool {
+ return h.status(http.MethodGet, query, "good") == http.StatusTooManyRequests
+ }, "the stalled metrics request never held its slot")
+ eventually(t, 2*time.Second, func() bool {
+ return h.status(http.MethodGet, query, "good") == http.StatusOK
+ }, "a metrics client that stopped reading keeps its concurrency slot")
+ })
+ t.Run("truncated answer", func(t *testing.T) {
+ h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Length", "100")
+ _, _ = io.WriteString(w, `{"status":"success",`)
+ })
+ // Nothing has been flushed yet when the copy fails, so the
+ // connection may close before the status line.
+ req, _ := http.NewRequest(http.MethodGet, h.proxy.URL+query, nil)
+ req.Header.Set("Authorization", "Bearer good")
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return
+ }
+ defer func() { _ = resp.Body.Close() }()
+ if _, err := io.ReadAll(resp.Body); err == nil {
+ t.Fatal("a truncated store answer reached the client as a complete response")
+ }
+ })
+}
From 6f5b0b666e068345d9d1814c3bffd7fa09b040ec Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 00:31:54 +0700
Subject: [PATCH 072/114] fix(observability-proxy): check metrics points on
millisecond bounds
The points limit was checked on the parsed bounds, but vmselect gets
both cut to the millisecond. A start cut back by more than the end
widened the forwarded range, so a 1ms step admitted at 11000 points
reached vmselect as 11001.
Times are now cut to the millisecond when they are parsed, so the
range and points checks and the forwarded request see one window.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/metricsparams.go | 7 ++++---
internal/observabilityproxy/metricsparams_test.go | 3 +++
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go
index 5aae58479..44761a2ea 100644
--- a/internal/observabilityproxy/metricsparams.go
+++ b/internal/observabilityproxy/metricsparams.go
@@ -264,7 +264,8 @@ func parseMetricsDuration(key, v string) (time.Duration, *HTTPError) {
// parseMetricsTime takes the two forms the Prometheus API defines: float
// unix seconds and RFC3339. vmselect would also guess milliseconds from the
// digit count and accept relative times; a guessed unit silently moves the
-// window, so those are refused.
+// window, so those are refused. The result is cut to the millisecond, as
+// formatTime sends it, so the limits see the window vmselect will use.
func parseMetricsTime(key, v string) (time.Time, *HTTPError) {
if sec, frac, ok := strings.Cut(v, "."); isDigits(sec) && (!ok || isDigits(frac)) {
if len(sec) > 10 {
@@ -275,13 +276,13 @@ func parseMetricsTime(key, v string) (time.Time, *HTTPError) {
if ok {
nanos, _ = strconv.ParseInt((frac + "000000000")[:9], 10, 64)
}
- return time.Unix(secs, nanos).UTC(), nil
+ return time.UnixMilli(time.Unix(secs, nanos).UnixMilli()).UTC(), nil
}
t, err := time.Parse(time.RFC3339Nano, v)
if err != nil || t.Before(time.Unix(0, 0)) {
return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q", key, v)
}
- return t.UTC(), nil
+ return time.UnixMilli(t.UnixMilli()).UTC(), nil
}
func formatTime(t time.Time) string {
diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go
index 4a8565564..729af997a 100644
--- a/internal/observabilityproxy/metricsparams_test.go
+++ b/internal/observabilityproxy/metricsparams_test.go
@@ -156,6 +156,9 @@ func TestBuildMetricsParamsLimits(t *testing.T) {
{"points at the limit", "api/v1/query_range", rng(ago(10999*time.Minute), ago(0), "1m"), http.StatusOK},
{"points above the limit", "api/v1/query_range", rng(ago(11000*time.Minute), ago(0), "1m"), http.StatusUnprocessableEntity},
{"points above the limit after rounding the step up", "api/v1/query_range", rng(ago(22*time.Second), ago(0), "0.0019"), http.StatusUnprocessableEntity},
+ // 10.9999s admits 11000 points, but vmselect gets both bounds cut to
+ // the millisecond: 11.000s, 11001 points.
+ {"points above the limit once the bounds are cut to milliseconds", "api/v1/query_range", rng("2026-11-04T11:59:49.0009Z", "2026-11-04T12:00:00.0008Z", "0.001"), http.StatusUnprocessableEntity},
{"overflowing step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "2562047h47m16.854775807s"), http.StatusBadRequest},
{"overflowing timeout", "api/v1/query", url.Values{"query": {"up"}, "timeout": {"2562047h47m16.854775807s"}}, http.StatusBadRequest},
{"step above the range limit", "api/v1/query_range", rng(ago(time.Hour), ago(0), "31d"), http.StatusUnprocessableEntity},
From eaadba4adddd8988c84b959de0f202bfb581d935 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 00:44:11 +0700
Subject: [PATCH 073/114] docs(observability-proxy): state what the metrics
limits bound
The comment claimed the limits bound everything a request makes
vmselect do, but windows and offsets inside the expression pass
through unchecked: they are bounded by the store's retention and by
vmselect's own series and sample limits, not by the proxy. A reader
taking the comment at its word would rely on a cost cap that does not
exist.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/metricsparams.go | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go
index 44761a2ea..414821302 100644
--- a/internal/observabilityproxy/metricsparams.go
+++ b/internal/observabilityproxy/metricsparams.go
@@ -26,7 +26,10 @@ import (
"time"
)
-// MetricsLimits bound what one metrics request may make vmselect do.
+// MetricsLimits bound the window, resolution and timeout a metrics request
+// asks vmselect for. Windows and offsets inside the expression are not
+// limited here: they cannot read past the store's retention, and vmselect
+// caps the series and samples one query may touch.
type MetricsLimits struct {
MaxRange time.Duration
MaxPoints int
From b4ddb93b6f5e376ee298a6a4371ad33d1f03b433 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 00:55:03 +0700
Subject: [PATCH 074/114] docs(observability-proxy): name the timeout as the
bound on rollup work
vmselect's sample limit counts raw samples read, not the work of a
rollup that revisits each sample across overlapping windows, so for
that work the forwarded timeout is the limit that applies.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/metricsparams.go | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go
index 414821302..5b4169050 100644
--- a/internal/observabilityproxy/metricsparams.go
+++ b/internal/observabilityproxy/metricsparams.go
@@ -28,8 +28,9 @@ import (
// MetricsLimits bound the window, resolution and timeout a metrics request
// asks vmselect for. Windows and offsets inside the expression are not
-// limited here: they cannot read past the store's retention, and vmselect
-// caps the series and samples one query may touch.
+// limited here: they cannot read past the store's retention, vmselect caps
+// the series and samples one query may touch, and MaxTimeout bounds the
+// work of rollups over long overlapping windows.
type MetricsLimits struct {
MaxRange time.Duration
MaxPoints int
From 843d360ee08597da49d89b0d98e08b9db8848f58 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 01:24:53 +0700
Subject: [PATCH 075/114] fix(observability-proxy): send Retry-After with the
metrics concurrency 429
The metrics root has its own admission, so the hint added to the logs
root did not reach it, and a console graph polling into the limit had
no back-off hint.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/server.go | 2 +-
internal/observabilityproxy/server_metrics_test.go | 12 ++++++++----
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go
index 50b6ac89e..eb540a271 100644
--- a/internal/observabilityproxy/server.go
+++ b/internal/observabilityproxy/server.go
@@ -282,7 +282,7 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT
func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError {
release, ok := s.limits.acquire(user)
if !ok {
- return errorf(http.StatusTooManyRequests, "too many concurrent requests")
+ return tooManyConcurrent(w)
}
defer release()
if herr := s.readForm(w, r); herr != nil {
diff --git a/internal/observabilityproxy/server_metrics_test.go b/internal/observabilityproxy/server_metrics_test.go
index a1c07d944..ea47cf782 100644
--- a/internal/observabilityproxy/server_metrics_test.go
+++ b/internal/observabilityproxy/server_metrics_test.go
@@ -179,12 +179,16 @@ func TestServerMetricsConcurrency(t *testing.T) {
}
}
hold("good")
- refused := make(chan int, 1)
- go func() { refused <- h.status(http.MethodPost, metricsRoot+"api/v1/query?query=up", "good") }()
+ refused := make(chan *http.Response, 1)
+ go func() {
+ resp := h.do(http.MethodPost, metricsRoot+"api/v1/query?query=up", "good", nil, nil)
+ _ = resp.Body.Close()
+ refused <- resp
+ }()
select {
case got := <-refused:
- if got != http.StatusTooManyRequests {
- t.Fatalf("concurrent request answered %d, want 429", got)
+ if got.StatusCode != http.StatusTooManyRequests || got.Header.Get("Retry-After") == "" {
+ t.Fatalf("concurrent request answered %d with Retry-After %q, want 429 with a value", got.StatusCode, got.Header.Get("Retry-After"))
}
case <-time.After(2 * time.Second):
t.Fatal("concurrent request of the same user reached the upstream")
From 3a5bcf2567fd6fc00da6085ac505b447c6a4c3e2 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 01:26:41 +0700
Subject: [PATCH 076/114] fix(observability-proxy): send durations vmselect
reads back exactly
vmselect reads step and timeout as float seconds multiplied by 1000
and truncated, so a step the proxy checked as 1001ms and sent as
"1.001" arrived as 1000ms. The points limit was then checked against
a coarser step than vmselect used: a limit of 11000 let 11010 points
through, and many steps between 1ms and 100000s were affected.
Durations are now written so that the truncation lands on the
milliseconds the limits were checked with: the shortest form when it
reads back exactly, otherwise with half a millisecond added. Step and
timeout share the formatter, and a sweep test mirrors vmselect's
parsing.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/metricsparams.go | 12 +++-
.../observabilityproxy/metricsparams_test.go | 66 +++++++++++++++++++
2 files changed, 77 insertions(+), 1 deletion(-)
diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go
index 5b4169050..54c46f527 100644
--- a/internal/observabilityproxy/metricsparams.go
+++ b/internal/observabilityproxy/metricsparams.go
@@ -294,6 +294,16 @@ func formatTime(t time.Time) string {
return fmt.Sprintf("%d.%03d", ms/1000, ms%1000)
}
+// formatSeconds writes a whole-millisecond duration so that vmselect reads
+// back exactly those milliseconds. vmselect multiplies the float seconds
+// by 1e3 and truncates (httputil.GetDuration), so "1.001" arrives as
+// 1000ms; where the shortest form truncates low, half a millisecond more
+// lands the truncation on the intended value without reaching the next.
func formatSeconds(d time.Duration) string {
- return strconv.FormatFloat(float64(d.Milliseconds())/1000, 'f', -1, 64)
+ ms := d.Milliseconds()
+ s := strconv.FormatFloat(float64(ms)/1000, 'f', -1, 64)
+ if secs, _ := strconv.ParseFloat(s, 64); int64(secs*1e3) != ms {
+ s = fmt.Sprintf("%d.%03d5", ms/1000, ms%1000)
+ }
+ return s
}
diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go
index 729af997a..a4957a19a 100644
--- a/internal/observabilityproxy/metricsparams_test.go
+++ b/internal/observabilityproxy/metricsparams_test.go
@@ -17,6 +17,7 @@ limitations under the License.
package observabilityproxy
import (
+ "math"
"net/http"
"net/url"
"strconv"
@@ -372,3 +373,68 @@ func TestMetricsStoreByLookback(t *testing.T) {
})
}
}
+
+// vmselectMillis repeats how vmselect reads a duration argument
+// (httputil.GetDuration in lib/httputil/duration.go, v1.136.22 and
+// v1.139.0): float seconds, multiplied by 1e3 and truncated.
+func vmselectMillis(t *testing.T, v string) int64 {
+ t.Helper()
+ secs, err := strconv.ParseFloat(v, 64)
+ if err != nil {
+ t.Fatalf("vmselect cannot parse %q: %v", v, err)
+ }
+ return int64(secs * 1e3)
+}
+
+// TestFormatSecondsVmselectExact sweeps every step up to 2000s and the
+// large end of the range: vmselect must read back the milliseconds the
+// limits were checked with.
+func TestFormatSecondsVmselectExact(t *testing.T) {
+ check := func(ms int64) {
+ if got := vmselectMillis(t, formatSeconds(time.Duration(ms)*time.Millisecond)); got != ms {
+ t.Fatalf("%dms formatted as %q reaches vmselect as %dms", ms, formatSeconds(time.Duration(ms)*time.Millisecond), got)
+ }
+ }
+ for ms := int64(1); ms <= 2_000_000; ms++ {
+ check(ms)
+ }
+ for ms := maxMetricsDuration.Milliseconds() - 100_000; ms <= maxMetricsDuration.Milliseconds(); ms++ {
+ check(ms)
+ }
+ // A spread of the range in between, on a stride no power of ten divides.
+ for ms := int64(2_000_001); ms < maxMetricsDuration.Milliseconds(); ms += 3_153_599_999 {
+ check(ms)
+ }
+ // Whole seconds keep their plain form.
+ if got := formatSeconds(90 * time.Second); got != "90" {
+ t.Errorf("90s formatted as %q", got)
+ }
+}
+
+// TestBuildMetricsParamsPointsAsVmselectCounts: a 1.001s step is 1001ms
+// to the proxy; vmselect must count the same 11000 points at the limit.
+func TestBuildMetricsParamsPointsAsVmselectCounts(t *testing.T) {
+ end := testNow
+ step := 1001 * time.Millisecond
+ rng := func(points int64) url.Values {
+ start := end.Add(-time.Duration(points-1) * step)
+ return url.Values{"query": {"up"}, "start": {start.Format(time.RFC3339Nano)}, "end": {end.Format(time.RFC3339Nano)}, "step": {"1.001"}}
+ }
+ req := buildMetrics(t, "api/v1/query_range", rng(11000))
+ stepMs := vmselectMillis(t, req.params.Get("step"))
+ // Times, unlike durations, are rounded (timeutil.ParseTimeMsec).
+ timeMs := func(v string) int64 {
+ secs, err := strconv.ParseFloat(v, 64)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return int64(math.Round(secs * 1e3))
+ }
+ startMs, endMs := timeMs(req.params.Get("start")), timeMs(req.params.Get("end"))
+ if points := (endMs-startMs)/stepMs + 1; stepMs != 1001 || points != 11000 {
+ t.Fatalf("vmselect reads step %dms and %d points, want 1001ms and 11000", stepMs, points)
+ }
+ if got := metricsStatus("api/v1/query_range", rng(11001)); got != http.StatusUnprocessableEntity {
+ t.Fatalf("11001 points answered %d, want 422", got)
+ }
+}
From d2feb9b7c2f15a62fa82f84329d6a7843be8d6f2 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 01:39:09 +0700
Subject: [PATCH 077/114] test(observability-proxy): pin the metrics
concurrency Retry-After value
The metrics concurrency test only checked that a Retry-After was
present, while the rate limit test pins its value, so a change to the
promised hint passed. It now expects the literal value.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/server_metrics_test.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/internal/observabilityproxy/server_metrics_test.go b/internal/observabilityproxy/server_metrics_test.go
index ea47cf782..1ef5e761e 100644
--- a/internal/observabilityproxy/server_metrics_test.go
+++ b/internal/observabilityproxy/server_metrics_test.go
@@ -187,8 +187,8 @@ func TestServerMetricsConcurrency(t *testing.T) {
}()
select {
case got := <-refused:
- if got.StatusCode != http.StatusTooManyRequests || got.Header.Get("Retry-After") == "" {
- t.Fatalf("concurrent request answered %d with Retry-After %q, want 429 with a value", got.StatusCode, got.Header.Get("Retry-After"))
+ if got.StatusCode != http.StatusTooManyRequests || got.Header.Get("Retry-After") != "1" {
+ t.Fatalf("concurrent request answered %d with Retry-After %q, want 429 with 1", got.StatusCode, got.Header.Get("Retry-After"))
}
case <-time.After(2 * time.Second):
t.Fatal("concurrent request of the same user reached the upstream")
From ec251cbd3b1db23a26b1961c968eace8d2807db0 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 02:21:21 +0700
Subject: [PATCH 078/114] test(observability-proxy): watch the metrics slot
instead of probing
The metrics slow-body and stalled-client tests probed the slot with
requests of their own, which competed for the one slot under test and
made them fail at random, under one shuffle seed reliably. They now
read the limiter's slot count, as the logs tests do; 300 runs of the
test and 20 shuffled runs of the package pass.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observabilityproxy/server_deadline_test.go | 15 ++++++---------
1 file changed, 6 insertions(+), 9 deletions(-)
diff --git a/internal/observabilityproxy/server_deadline_test.go b/internal/observabilityproxy/server_deadline_test.go
index 03d03f218..abe1b3703 100644
--- a/internal/observabilityproxy/server_deadline_test.go
+++ b/internal/observabilityproxy/server_deadline_test.go
@@ -457,12 +457,12 @@ func TestServerMetricsRequestLifetime(t *testing.T) {
pipe := servePipe(t, h.server)
conn := pipe.send(t, "POST "+metricsRoot+"api/v1/query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n"+
"Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 100\r\n\r\nquery=")
- eventually(t, time.Second, func() bool {
- return h.status(http.MethodGet, query, "good") == http.StatusTooManyRequests
- }, "a metrics request with an unfinished body does not hold a concurrency slot")
+ eventually(t, time.Second, func() bool { return slotsHeld(h) == 1 },
+ "a metrics request with an unfinished body does not hold a concurrency slot")
if resp := readResponse(t, conn); resp.StatusCode != http.StatusRequestTimeout {
t.Fatalf("unfinished body answered %d, want 408", resp.StatusCode)
}
+ eventually(t, time.Second, func() bool { return slotsHeld(h) == 0 }, "the slot of an unfinished body was not released")
})
t.Run("stalled client", func(t *testing.T) {
// Larger than the server's write buffer, so the handler itself
@@ -474,12 +474,9 @@ func TestServerMetricsRequestLifetime(t *testing.T) {
}, func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, big) })
pipe := servePipe(t, h.server)
pipe.send(t, "GET "+query+" HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n")
- eventually(t, time.Second, func() bool {
- return h.status(http.MethodGet, query, "good") == http.StatusTooManyRequests
- }, "the stalled metrics request never held its slot")
- eventually(t, 2*time.Second, func() bool {
- return h.status(http.MethodGet, query, "good") == http.StatusOK
- }, "a metrics client that stopped reading keeps its concurrency slot")
+ eventually(t, time.Second, func() bool { return slotsHeld(h) == 1 }, "the stalled metrics request never held its slot")
+ eventually(t, 2*time.Second, func() bool { return slotsHeld(h) == 0 },
+ "a metrics client that stopped reading keeps its concurrency slot")
})
t.Run("truncated answer", func(t *testing.T) {
h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) {
From 65944fcb79171fdf542192cb243fcd685231214d Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 02:23:09 +0700
Subject: [PATCH 079/114] fix(observability-proxy): send metrics times vmselect
cannot misread
Times went to vmselect as "%d.%03d" unix seconds. vmselect reads a
number without "-" as a unix time whose unit it guesses from its size,
so a time in the first 24.8 days of 1970 arrived as a thousand times
as many seconds: a window the proxy checked as 8.8 days and 30 points
reached vmselect as 2001..2025 and 30000 points, and the label and
series APIs, which have no points check, scanned the whole retention.
Times past 2038 were read as microseconds.
Times now go out as RFC3339 with milliseconds, which vmselect parses
as a date and never as a guessed unit, and times outside 1970..2262,
which vmselect cannot hold as nanoseconds, are refused. A test repeats
vmselect's parsing of the exact strings the proxy sends.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/metricsparams.go | 24 +++-
.../observabilityproxy/metricsparams_test.go | 15 +--
.../observabilityproxy/vmselect_parse_test.go | 117 ++++++++++++++++++
3 files changed, 140 insertions(+), 16 deletions(-)
create mode 100644 internal/observabilityproxy/vmselect_parse_test.go
diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go
index 54c46f527..8beb347d0 100644
--- a/internal/observabilityproxy/metricsparams.go
+++ b/internal/observabilityproxy/metricsparams.go
@@ -18,6 +18,7 @@ package observabilityproxy
import (
"fmt"
+ "math"
"net/http"
"net/url"
"regexp"
@@ -280,18 +281,33 @@ func parseMetricsTime(key, v string) (time.Time, *HTTPError) {
if ok {
nanos, _ = strconv.ParseInt((frac + "000000000")[:9], 10, 64)
}
- return time.UnixMilli(time.Unix(secs, nanos).UnixMilli()).UTC(), nil
+ return metricsTimeInRange(key, v, time.Unix(secs, nanos))
}
t, err := time.Parse(time.RFC3339Nano, v)
- if err != nil || t.Before(time.Unix(0, 0)) {
+ if err != nil {
return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q", key, v)
}
+ return metricsTimeInRange(key, v, t)
+}
+
+// maxMetricsTime is the last instant vmselect can read: it takes an
+// RFC3339 time through time.UnixNano (timeutil.ParseTimeAt).
+var maxMetricsTime = time.Unix(0, math.MaxInt64)
+
+func metricsTimeInRange(key, v string, t time.Time) (time.Time, *HTTPError) {
+ if t.Before(time.Unix(0, 0)) || !t.Before(maxMetricsTime) {
+ return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q: outside 1970..2262", key, v)
+ }
return time.UnixMilli(t.UnixMilli()).UTC(), nil
}
+// formatTime writes RFC3339 with milliseconds. vmselect reads a number
+// without "-" as a unix time whose unit it guesses from its size
+// (timeutil.getUnixTimestampNanoseconds): "1000000.000" is taken as
+// 1000000000 seconds, and anything past 2038 in milliseconds as
+// microseconds. A dated form never reaches that guess.
func formatTime(t time.Time) string {
- ms := t.UnixMilli()
- return fmt.Sprintf("%d.%03d", ms/1000, ms%1000)
+ return t.UTC().Format("2006-01-02T15:04:05.000Z07:00")
}
// formatSeconds writes a whole-millisecond duration so that vmselect reads
diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go
index a4957a19a..a4a435567 100644
--- a/internal/observabilityproxy/metricsparams_test.go
+++ b/internal/observabilityproxy/metricsparams_test.go
@@ -17,7 +17,6 @@ limitations under the License.
package observabilityproxy
import (
- "math"
"net/http"
"net/url"
"strconv"
@@ -198,7 +197,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) {
"query": {"up"}, "start": {"2026-11-04T11:00:00.25Z"}, "end": {unix(testNow)}, "step": {"1m30s"},
})
want := url.Values{
- "query": {"up"}, "start": {"1793790000.250"}, "end": {"1793793600.000"}, "step": {"90"},
+ "query": {"up"}, "start": {"2026-11-04T11:00:00.250Z"}, "end": {"2026-11-04T12:00:00.000Z"}, "step": {"90"},
"timeout": {"30"}, "extra_filters[]": {wantMetricsScope},
}
if req.params.Encode() != want.Encode() {
@@ -225,7 +224,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) {
}
req = buildMetrics(t, "api/v1/labels", url.Values{})
- if req.params.Get("end") != "1793793600.000" || req.params.Get("start") != "1793793300.000" {
+ if req.params.Get("end") != "2026-11-04T12:00:00.000Z" || req.params.Get("start") != "2026-11-04T11:55:00.000Z" {
t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end"))
}
if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-30*time.Second)) {
@@ -422,15 +421,7 @@ func TestBuildMetricsParamsPointsAsVmselectCounts(t *testing.T) {
}
req := buildMetrics(t, "api/v1/query_range", rng(11000))
stepMs := vmselectMillis(t, req.params.Get("step"))
- // Times, unlike durations, are rounded (timeutil.ParseTimeMsec).
- timeMs := func(v string) int64 {
- secs, err := strconv.ParseFloat(v, 64)
- if err != nil {
- t.Fatal(err)
- }
- return int64(math.Round(secs * 1e3))
- }
- startMs, endMs := timeMs(req.params.Get("start")), timeMs(req.params.Get("end"))
+ startMs, endMs := vmselectTimeMillis(t, req.params.Get("start")), vmselectTimeMillis(t, req.params.Get("end"))
if points := (endMs-startMs)/stepMs + 1; stepMs != 1001 || points != 11000 {
t.Fatalf("vmselect reads step %dms and %d points, want 1001ms and 11000", stepMs, points)
}
diff --git a/internal/observabilityproxy/vmselect_parse_test.go b/internal/observabilityproxy/vmselect_parse_test.go
new file mode 100644
index 000000000..6d75bbf7f
--- /dev/null
+++ b/internal/observabilityproxy/vmselect_parse_test.go
@@ -0,0 +1,117 @@
+/*
+Copyright 2026 The Cozystack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package observabilityproxy
+
+import (
+ "math"
+ "net/url"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+)
+
+// vmselectTimeMillis repeats how vmselect v1.136.0-cluster reads a time
+// argument (httputil.GetTime -> timeutil.ParseTimeMsec -> ParseTimeAt in
+// lib/timeutil/time.go) for the forms a client or the proxy may send: a
+// value without "-" is a number whose unit vmselect guesses from its size
+// (getUnixTimestampNanoseconds), anything else with a zone is RFC3339 read
+// through time.UnixNano. Relative and partial dates are left out; the
+// proxy refuses them.
+func vmselectTimeMillis(t *testing.T, s string) int64 {
+ t.Helper()
+ if strings.Contains(s, "-") {
+ ts, err := time.Parse(time.RFC3339, s)
+ if err != nil {
+ t.Fatalf("vmselect cannot parse %q: %v", s, err)
+ }
+ return int64(math.Round(float64(ts.UnixNano()) / 1e6))
+ }
+ intPart, frac, _ := strings.Cut(s, ".")
+ n, err := strconv.ParseInt(intPart+frac, 10, 64)
+ if err != nil {
+ t.Fatalf("vmselect cannot parse %q: %v", s, err)
+ }
+ for e := len(frac); e%3 != 0; e++ {
+ n *= 10
+ }
+ var nanos int64
+ switch {
+ case n < 1<<31:
+ nanos = n * 1e9
+ case n < 1e3*(1<<31):
+ nanos = n * 1e6
+ case n < 1e6*(1<<31):
+ nanos = n * 1e3
+ default:
+ nanos = n
+ }
+ return int64(math.Round(float64(nanos) / 1e6))
+}
+
+// TestVmselectReadsProxyTimes sends times across the range the proxy
+// accepts, in both client forms, and checks that vmselect reads the
+// forwarded value as the same instant the proxy validated. A number such
+// as "1000000.000" is seconds to the proxy but milliseconds-as-seconds
+// guesswork to vmselect, so the proxy must not forward one.
+func TestVmselectReadsProxyTimes(t *testing.T) {
+ for _, client := range []string{
+ "0", "1000000", "1762400", "2147483", "2147484", "1000000.5",
+ "1793793600", "1793793600.123", "2147483647", "2147483648", "9223372035",
+ "1970-01-01T00:00:00Z", "1970-01-12T13:46:40.001Z", "2026-11-04T12:00:00.123Z", "2262-01-01T00:00:00Z",
+ } {
+ want, herr := parseMetricsTime("time", client)
+ if herr != nil {
+ t.Fatalf("%q refused: %v", client, herr)
+ }
+ req := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}, "time": {client}})
+ sent := req.params.Get("time")
+ if got := vmselectTimeMillis(t, sent); got != want.UnixMilli() {
+ t.Errorf("client time %q forwarded as %q reaches vmselect as %dms, want %dms", client, sent, got, want.UnixMilli())
+ }
+ }
+}
+
+// TestVmselectReadsSecondsWithFraction is the window that passed the proxy as
+// 8.8 days and 30 points while vmselect, reading "1000000.000" as a
+// thousand times that many seconds, saw 2001..2025 and 30000 points.
+func TestVmselectReadsSecondsWithFraction(t *testing.T) {
+ in := url.Values{"query": {"up"}, "start": {"1000000"}, "end": {"1762400"}, "step": {"25414"}}
+ req := buildMetrics(t, "api/v1/query_range", in)
+ start, end := vmselectTimeMillis(t, req.params.Get("start")), vmselectTimeMillis(t, req.params.Get("end"))
+ step := vmselectMillis(t, req.params.Get("step"))
+ if start != 1000000*1000 || end != 1762400*1000 {
+ t.Fatalf("vmselect reads %dms..%dms, want 1000000s..1762400s", start, end)
+ }
+ if points := (end-start)/step + 1; points != 30 {
+ t.Fatalf("vmselect counts %d points, want the 30 the proxy checked", points)
+ }
+ labels := buildMetrics(t, "api/v1/labels", url.Values{"start": {"1000000"}, "end": {"1000300"}})
+ if got := vmselectTimeMillis(t, labels.params.Get("start")); got != 1000000*1000 {
+ t.Fatalf("label API start reaches vmselect as %dms, want 1000000s", got)
+ }
+}
+
+func TestParseMetricsTimeRange(t *testing.T) {
+ // vmselect reads an RFC3339 time through time.UnixNano, which is
+ // undefined from 2262-04-11 on.
+ for _, v := range []string{"9223372037", "9999999999", "2262-04-12T00:00:00Z", "9999-01-01T00:00:00Z"} {
+ if _, herr := parseMetricsTime("time", v); herr == nil {
+ t.Errorf("%q accepted, want refused", v)
+ }
+ }
+}
From 89fbdb136f8fc1e7b4af07d9f81f534e4dd26f51 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 02:32:21 +0700
Subject: [PATCH 080/114] test(observability-proxy): cover the 1970 bound of
metrics times
Only the upper half of the refused 1970..2262 range had a test, so
the lower check could be dropped unnoticed. vmselect clamps a time
before 1970 to 0, which would answer for a window other than the one
asked for.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/vmselect_parse_test.go | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/internal/observabilityproxy/vmselect_parse_test.go b/internal/observabilityproxy/vmselect_parse_test.go
index 6d75bbf7f..1fb23d18f 100644
--- a/internal/observabilityproxy/vmselect_parse_test.go
+++ b/internal/observabilityproxy/vmselect_parse_test.go
@@ -109,7 +109,9 @@ func TestVmselectReadsSecondsWithFraction(t *testing.T) {
func TestParseMetricsTimeRange(t *testing.T) {
// vmselect reads an RFC3339 time through time.UnixNano, which is
// undefined from 2262-04-11 on.
- for _, v := range []string{"9223372037", "9999999999", "2262-04-12T00:00:00Z", "9999-01-01T00:00:00Z"} {
+ // Before 1970 vmselect clamps to 0, which would quietly answer for a
+ // window other than the one asked for.
+ for _, v := range []string{"9223372037", "9999999999", "2262-04-12T00:00:00Z", "9999-01-01T00:00:00Z", "1969-12-31T23:59:59Z", "1969-12-31T23:59:59.999Z"} {
if _, herr := parseMetricsTime("time", v); herr == nil {
t.Errorf("%q accepted, want refused", v)
}
From 0c5d1cd93f6220b5497c0c6c479c112ffa3961d9 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 02:32:42 +0700
Subject: [PATCH 081/114] fix(observability-proxy): keep the vmselect timeout
within the request's
The timeout forwarded to vmselect could be set longer than the proxy's
own request timeout. The proxy then cut a query vmselect was still
allowed to run and answered 504 for a store that was within its time.
Such a combination now stops the proxy at startup.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
cmd/observability-proxy/main.go | 4 ++++
cmd/observability-proxy/main_test.go | 3 +++
2 files changed, 7 insertions(+)
diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go
index 6acfb1e08..a8ac36efd 100644
--- a/cmd/observability-proxy/main.go
+++ b/cmd/observability-proxy/main.go
@@ -134,6 +134,10 @@ func parseFlags(args []string) (options, error) {
return o, errors.New("--metrics-max-points must be at least 2")
case o.logs.MaxTailResume < 0:
return o, errors.New("--logs-max-tail-resume must not be negative")
+ // Past the request timeout the proxy cuts a query vmselect is still
+ // allowed to run, and answers 504 for a store that is within its time.
+ case o.requestTimeout > 0 && o.metrics.Limits.MaxTimeout > o.requestTimeout:
+ return o, errors.New("--metrics-max-timeout must not exceed --request-timeout")
case o.requestTimeout < 0 || o.tailIdleTimeout < 0 || o.authCacheTTL < 0 || o.shutdownTimeout < 0:
return o, errors.New("timeouts and the auth cache TTL must not be negative")
// NaN passes every comparison and +Inf never empties a bucket; either
diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go
index be89e20d5..f27317515 100644
--- a/cmd/observability-proxy/main_test.go
+++ b/cmd/observability-proxy/main_test.go
@@ -103,6 +103,9 @@ func TestParseFlagsInvalid(t *testing.T) {
{"--metrics-max-points=1"},
{"--metrics-max-timeout=0s"},
{"--unknown"},
+ // The proxy would cut the request while vmselect is still within
+ // the timeout it was given.
+ {"--request-timeout=10s", "--metrics-max-timeout=30s"},
{"unexpected", "--root-tenant=tenant-x"},
} {
if _, err := parseFlags(args); err == nil {
From 3e2472280e776beb8060bd65d11a251aa9206474 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Thu, 5 Nov 2026 02:38:50 +0700
Subject: [PATCH 082/114] fix(observability-proxy): keep the vmselect timeout
below the request's
An equal vmselect timeout still lost the race: the proxy's deadline
starts before the request reaches vmselect, so the proxy cut first
and answered 504 for a store within its time. The vmselect timeout
must now be strictly below a set request timeout, the flag help says
so, and tests pin the accepted side too, including no request timeout.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
cmd/observability-proxy/main.go | 11 ++++++-----
cmd/observability-proxy/main_test.go | 14 ++++++++++++++
2 files changed, 20 insertions(+), 5 deletions(-)
diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go
index a8ac36efd..a3687fe08 100644
--- a/cmd/observability-proxy/main.go
+++ b/cmd/observability-proxy/main.go
@@ -91,7 +91,7 @@ func parseFlags(args []string) (options, error) {
fs.DurationVar(&o.metrics.ShortTermRetention, "metrics-shortterm-retention", 72*time.Hour, "retention of the short-term store; older windows are read from the long-term one")
fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use")
fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return")
- fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped")
+ fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set")
fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables")
fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables")
fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user")
@@ -134,10 +134,11 @@ func parseFlags(args []string) (options, error) {
return o, errors.New("--metrics-max-points must be at least 2")
case o.logs.MaxTailResume < 0:
return o, errors.New("--logs-max-tail-resume must not be negative")
- // Past the request timeout the proxy cuts a query vmselect is still
- // allowed to run, and answers 504 for a store that is within its time.
- case o.requestTimeout > 0 && o.metrics.Limits.MaxTimeout > o.requestTimeout:
- return o, errors.New("--metrics-max-timeout must not exceed --request-timeout")
+ // The proxy's deadline starts before the request reaches vmselect, so
+ // at an equal or longer timeout it cuts a query vmselect is still
+ // allowed to run and answers 504 for a store that is within its time.
+ case o.requestTimeout > 0 && o.metrics.Limits.MaxTimeout >= o.requestTimeout:
+ return o, errors.New("--metrics-max-timeout must be below --request-timeout")
case o.requestTimeout < 0 || o.tailIdleTimeout < 0 || o.authCacheTTL < 0 || o.shutdownTimeout < 0:
return o, errors.New("timeouts and the auth cache TTL must not be negative")
// NaN passes every comparison and +Inf never empties a bucket; either
diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go
index f27317515..6531550e9 100644
--- a/cmd/observability-proxy/main_test.go
+++ b/cmd/observability-proxy/main_test.go
@@ -106,6 +106,7 @@ func TestParseFlagsInvalid(t *testing.T) {
// The proxy would cut the request while vmselect is still within
// the timeout it was given.
{"--request-timeout=10s", "--metrics-max-timeout=30s"},
+ {"--request-timeout=30s", "--metrics-max-timeout=30s"},
{"unexpected", "--root-tenant=tenant-x"},
} {
if _, err := parseFlags(args); err == nil {
@@ -113,3 +114,16 @@ func TestParseFlagsInvalid(t *testing.T) {
}
}
}
+
+// The vmselect timeout must stay strictly below a request timeout, and is
+// free when the request has none.
+func TestParseFlagsTimeoutPairs(t *testing.T) {
+ for _, args := range [][]string{
+ {"--request-timeout=31s", "--metrics-max-timeout=30s"},
+ {"--request-timeout=0", "--metrics-max-timeout=10m"},
+ } {
+ if _, err := parseFlags(args); err != nil {
+ t.Errorf("parseFlags(%v) refused: %v", args, err)
+ }
+ }
+}
From 41b3237c07718e79f8bb1faf96b04d62d4e1c592 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Tue, 10 Nov 2026 00:46:05 +0700
Subject: [PATCH 083/114] refactor(observability-proxy): state the keyword and
duration tests positively
staticcheck flags the negated disjunctions; naming the duration
characters and spelling out both word boundaries reads the same way the
rule is stated.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/metricslookback.go | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go
index ed7e9d166..e55d189a5 100644
--- a/internal/observabilityproxy/metricslookback.go
+++ b/internal/observabilityproxy/metricslookback.go
@@ -128,9 +128,7 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
if negative {
arg = strings.TrimLeft(arg[1:], " \t\n\r")
}
- end := strings.IndexFunc(arg, func(r rune) bool {
- return !(r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.' || r == '_')
- })
+ end := strings.IndexFunc(arg, func(r rune) bool { return !inDuration(r) })
if end < 0 {
end = len(arg)
}
@@ -156,7 +154,12 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) {
// isKeyword reports whether lower[start:end] stands on its own rather than
// inside an identifier.
func isKeyword(lower string, start, end int) bool {
- return !(start > 0 && continuesIdent(lower[start-1], false) || end < len(lower) && continuesIdent(lower[end], true))
+ return (start == 0 || !continuesIdent(lower[start-1], false)) &&
+ (end == len(lower) || !continuesIdent(lower[end], true))
+}
+
+func inDuration(r rune) bool {
+ return r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.' || r == '_'
}
func hasKeyword(lower, word string) bool {
From 16bcaa447dc17d1060b5f47212c26a93e7666553 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Tue, 10 Nov 2026 03:38:59 +0700
Subject: [PATCH 084/114] test(observability-proxy): cover stalled reviews on
the metrics root
The metrics handler shares the bounded review calls with the logs
handler; a POST to it with an unread body must answer 503 as well
rather than wait on a stalled kube-apiserver.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
internal/observabilityproxy/server_deadline_test.go | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/internal/observabilityproxy/server_deadline_test.go b/internal/observabilityproxy/server_deadline_test.go
index abe1b3703..6b367229c 100644
--- a/internal/observabilityproxy/server_deadline_test.go
+++ b/internal/observabilityproxy/server_deadline_test.go
@@ -421,15 +421,20 @@ func stalledAPIServer(t *testing.T, stallTokens bool) kubernetes.Interface {
// a bounded review call ends the request.
func TestServerStalledReviewAnswersInTime(t *testing.T) {
for _, tc := range []struct {
- name string
+ name, path string
stallTokens bool
- }{{"token review", true}, {"subject access review", false}} {
+ }{
+ {"logs token review", logsRoot + "query", true},
+ {"logs subject access review", logsRoot + "query", false},
+ {"metrics token review", metricsRoot + "api/v1/query", true},
+ {"metrics subject access review", metricsRoot + "api/v1/query", false},
+ } {
t.Run(tc.name, func(t *testing.T) {
h := newHarness(t, Config{RequestTimeout: 200 * time.Millisecond}, nil)
h.server.auth = NewAuthorizer(stalledAPIServer(t, tc.stallTokens), 10*time.Second)
got := make(chan int, 1)
go func() {
- resp := h.do(http.MethodPost, logsRoot+"query", "good", url.Values{"query": {"*"}}, nil)
+ resp := h.do(http.MethodPost, tc.path, "good", url.Values{"query": {"up"}}, nil)
_ = resp.Body.Close()
got <- resp.StatusCode
}()
From b3288be9038656d74efa06a0e519deb426f21c9b Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Tue, 10 Nov 2026 04:03:06 +0700
Subject: [PATCH 085/114] test(observability-proxy): cite only sources the
repository has
The flag defaults test named a design document that is not in the
repository as the origin of the cost limits; the limits stand on their
own as flag defaults.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
cmd/observability-proxy/main_test.go | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go
index 6531550e9..bf7ac396b 100644
--- a/cmd/observability-proxy/main_test.go
+++ b/cmd/observability-proxy/main_test.go
@@ -32,8 +32,7 @@ func TestParseFlagsDefaults(t *testing.T) {
if o.authCacheTTL != 10*time.Second || o.logs.MaxTailResume != 15*time.Minute || o.logs.MaxRange != 7*24*time.Hour {
t.Errorf("limit defaults changed: %+v", o)
}
- // Retention and store names of packages/system/monitoring/values.yaml;
- // cost limits of the metrics proposal (Design 7).
+ // Retention and store names of packages/system/monitoring/values.yaml.
m := o.metrics
if m.ShortTermStore != "shortterm" || m.LongTermStore != "longterm" || m.ShortTermRetention != 72*time.Hour {
t.Errorf("metrics store defaults changed: %+v", m)
From f1dd2d17fc864ac5b8a7417fdb3d272b0fbc5c57 Mon Sep 17 00:00:00 2001
From: Mattia Eleuteri
Date: Wed, 11 Nov 2026 09:20:19 +0100
Subject: [PATCH 086/114] fix(backups): give the barman-cloud sidecar an
explicit CPU limit
The sidecar was left without a CPU limit so that a throttled sidecar
would slow WAL archiving down rather than fail it. In a tenant with
resourceQuotas that does not hold: the tenant LimitRange gives every
container a 250m CPU limit by default, so the sidecar gets it and
compresses full backups and WAL segments on a quarter of a core. A
1.6 GB database took 30 minutes to back up.
Set an explicit one-core limit in both places that build the sidecar
configuration, the backup driver and cozy-lib, the same way the memory
request and limit already override the LimitRange defaults.
Assisted-by: LLM
Signed-off-by: Mattia Eleuteri
---
internal/backupcontroller/cnpgstrategy_controller.go | 10 +++++++---
.../backupcontroller/cnpgstrategy_controller_test.go | 7 +++++--
.../apps/postgres/tests/barman_resources_test.yaml | 8 ++------
packages/library/cozy-lib/templates/_barman.tpl | 11 +++++++----
4 files changed, 21 insertions(+), 15 deletions(-)
diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go
index 19f255c6c..3e2f05c0d 100644
--- a/internal/backupcontroller/cnpgstrategy_controller.go
+++ b/internal/backupcontroller/cnpgstrategy_controller.go
@@ -2224,9 +2224,12 @@ func buildBarmanPlugin(objectStoreName, serverName string) cnpgtypes.PluginConfi
// high-water mark on a 38 MB database is 254 MiB, so 128Mi cannot hold it. A
// tenant that leaves resourceQuotas empty has no LimitRange, and there the
// sidecar had no requests and no limit at all. 256Mi holds the measured working
-// set and 1Gi is four times the measurement. No CPU limit is set, matching the
-// entityOperator precedent in packages/apps/kafka: a throttled sidecar stalls
-// WAL archiving instead of failing it.
+// set and 1Gi is four times the measurement. The CPU limit is explicit for the
+// same reason: left unset, that LimitRange gives the sidecar its 250m default,
+// so gzip compression of full backups and WAL segments runs on a quarter of a
+// core (a 1.6 GB database took 30 minutes to back up). One core keeps the
+// sidecar from starving the instance while removing that ceiling; it counts
+// against the tenant's limits.cpu budget, one core per instance pod.
//
// Checksum: since botocore ~1.36 the default (when_supported) attaches a
// flexible checksum to every PutObject, which non-AWS S3-compatible backends
@@ -2246,6 +2249,7 @@ func barmanSidecarConfiguration() *cnpgtypes.InstanceSidecarConfiguration {
corev1.ResourceMemory: resource.MustParse("256Mi"),
},
Limits: corev1.ResourceList{
+ corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go
index 551209fac..bc2827371 100644
--- a/internal/backupcontroller/cnpgstrategy_controller_test.go
+++ b/internal/backupcontroller/cnpgstrategy_controller_test.go
@@ -2152,8 +2152,11 @@ func TestApplyClusterPluginBackup_PatchesExistingCluster(t *testing.T) {
sc.Resources.Requests.Cpu().MilliValue() != 100 {
t.Errorf("ObjectStore leaves Barman subject to undersized tenant resource defaults: %+v", sc.Resources)
}
- if _, ok := sc.Resources.Limits[corev1.ResourceCPU]; ok {
- t.Errorf("ObjectStore caps sidecar CPU, which throttles WAL archiving instead of failing it: %+v", sc.Resources.Limits)
+ // Without an explicit CPU limit the tenant LimitRange caps the sidecar at
+ // its 250m default, and every backup and WAL upload compresses on a
+ // quarter of a core.
+ if sc.Resources.Limits.Cpu().MilliValue() != 1000 {
+ t.Errorf("ObjectStore leaves the sidecar CPU limit to the tenant LimitRange default: %+v", sc.Resources.Limits)
}
// The ObjectStore must be owner-referenced to the Cluster so Kubernetes GC
// removes it when the Cluster is deleted (no orphan in the platform flow,
diff --git a/packages/apps/postgres/tests/barman_resources_test.yaml b/packages/apps/postgres/tests/barman_resources_test.yaml
index 905fde2bb..1903fe8b1 100644
--- a/packages/apps/postgres/tests/barman_resources_test.yaml
+++ b/packages/apps/postgres/tests/barman_resources_test.yaml
@@ -13,7 +13,7 @@ set:
s3CredentialsSecret:
name: postgres-backup
tests:
- - it: sets resources for native backup without inheriting the tenant 128Mi limit
+ - it: sets resources for native backup without inheriting the tenant 128Mi and 250m limits
set:
backup.enabled: true
asserts:
@@ -24,15 +24,11 @@ tests:
cpu: 100m
memory: 256Mi
limits:
+ cpu: "1"
memory: 1Gi
documentSelector:
path: kind
value: ObjectStore
- - notExists:
- path: spec.instanceSidecarConfiguration.resources.limits.cpu
- documentSelector:
- path: kind
- value: ObjectStore
- equal:
path: spec.configuration.s3Credentials.accessKeyId.name
value: postgres-backup
diff --git a/packages/library/cozy-lib/templates/_barman.tpl b/packages/library/cozy-lib/templates/_barman.tpl
index c37b9673f..3b187dda6 100644
--- a/packages/library/cozy-lib/templates/_barman.tpl
+++ b/packages/library/cozy-lib/templates/_barman.tpl
@@ -13,10 +13,12 @@ LimitRange, which is cozy-keycloak and any tenant that leaves resourceQuotas
empty, gave the sidecar no requests and no limit at all, so nothing reserved
memory for it and nothing bounded it. The same request and limit settle both:
256Mi holds the measured working set and 1Gi is four times the measurement.
-No CPU limit is set, matching the entityOperator precedent in
-packages/apps/kafka: a throttled sidecar stalls WAL archiving instead of
-failing it. Inside a tenant the limit counts against the ResourceQuota
-limits.memory budget, 1Gi per instance pod.
+The CPU limit is explicit for the same reason: left unset, that LimitRange
+gives the sidecar its 250m default, so gzip compression of full backups and
+WAL segments runs on a quarter of a core (a 1.6 GB database took 30 minutes
+to back up). One core keeps the sidecar from starving the instance while
+removing that ceiling. Inside a tenant both limits count against the
+ResourceQuota budget, one core and 1Gi per instance pod.
Checksum: since botocore ~1.36 (early 2025) the default
RequestChecksumCalculation is when_supported, which attaches a flexible checksum
@@ -39,6 +41,7 @@ instanceSidecarConfiguration:
cpu: 100m
memory: 256Mi
limits:
+ cpu: "1"
memory: 1Gi
env:
- name: AWS_REQUEST_CHECKSUM_CALCULATION
From 6297ebd70865dfd8a5f084521715a7720c7b8971 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Fri, 13 Nov 2026 02:55:31 +0300
Subject: [PATCH 087/114] fix(linstor): run LINSTOR init containers on the
platform server image
piraeus-operator fills in its own default LINSTOR server image for every
container this package leaves unset, and two of those are init
containers: run-migration in the controller pod and
setup-lvm-configuration in the satellite pods. Only the main containers
were pinned to the platform build.
run-migration upgrades the LINSTOR database in place. When the operator
default is newer than the platform build, it migrates the schema ahead
of the controller, the controller then fails to read the migrated
database and crash-loops, and the migration cannot be rolled back. This
stayed hidden while the operator default was older than the version
built here; an operator update that moves the default past it exposes
it during the window between the operator and linstor releases
reconciling.
Pin both init containers to the same image as the main containers, so
the database moves only together with the controller.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
.../system/linstor/templates/cluster.yaml | 6 +++
.../linstor/templates/satellites-cozy.yaml | 3 ++
.../linstor/tests/server-image_test.yaml | 40 +++++++++++++++++++
3 files changed, 49 insertions(+)
create mode 100644 packages/system/linstor/tests/server-image_test.yaml
diff --git a/packages/system/linstor/templates/cluster.yaml b/packages/system/linstor/templates/cluster.yaml
index ba5154375..c648c0a8c 100644
--- a/packages/system/linstor/templates/cluster.yaml
+++ b/packages/system/linstor/templates/cluster.yaml
@@ -36,6 +36,12 @@ spec:
enabled: true
podTemplate:
spec:
+ {{- /* run-migration upgrades the database in place: left on the
+ operator's default image it migrates ahead of the controller,
+ irreversibly. */}}
+ initContainers:
+ - name: run-migration
+ image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
containers:
- name: linstor-controller
image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
diff --git a/packages/system/linstor/templates/satellites-cozy.yaml b/packages/system/linstor/templates/satellites-cozy.yaml
index 7897189f4..bb6240128 100644
--- a/packages/system/linstor/templates/satellites-cozy.yaml
+++ b/packages/system/linstor/templates/satellites-cozy.yaml
@@ -11,6 +11,9 @@ spec:
spec:
# host-network is recommended by Piraeus while it is not default in the upstream
hostNetwork: true
+ initContainers:
+ - name: setup-lvm-configuration
+ image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
containers:
- name: linstor-satellite
image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }}
diff --git a/packages/system/linstor/tests/server-image_test.yaml b/packages/system/linstor/tests/server-image_test.yaml
new file mode 100644
index 000000000..d63d19ae4
--- /dev/null
+++ b/packages/system/linstor/tests/server-image_test.yaml
@@ -0,0 +1,40 @@
+suite: every LINSTOR server container runs the platform image
+
+# piraeus-operator ships its own default for the LINSTOR server image and
+# applies it to every container this package leaves unset. The init
+# containers matter most: run-migration upgrades the LINSTOR database in
+# place, so an operator default newer than the platform build migrates the
+# schema ahead of the controller, which then refuses to start, and the
+# migration cannot be undone. Pinning them to the same image as the main
+# containers makes the database move only together with the controller.
+
+templates:
+ - templates/cluster.yaml
+ - templates/satellites-cozy.yaml
+
+set:
+ piraeusServer:
+ image:
+ repository: registry.example/piraeus-server
+ tag: v0.0.0-test
+
+tests:
+ - it: the controller migration runs the platform image
+ template: templates/cluster.yaml
+ asserts:
+ - equal:
+ path: spec.controller.podTemplate.spec.initContainers[?(@.name=="run-migration")].image
+ value: registry.example/piraeus-server:v0.0.0-test
+ - equal:
+ path: spec.controller.podTemplate.spec.containers[?(@.name=="linstor-controller")].image
+ value: registry.example/piraeus-server:v0.0.0-test
+
+ - it: the satellite LVM setup runs the platform image
+ template: templates/satellites-cozy.yaml
+ asserts:
+ - equal:
+ path: spec.podTemplate.spec.initContainers[?(@.name=="setup-lvm-configuration")].image
+ value: registry.example/piraeus-server:v0.0.0-test
+ - equal:
+ path: spec.podTemplate.spec.containers[?(@.name=="linstor-satellite")].image
+ value: registry.example/piraeus-server:v0.0.0-test
From acfae30136fa9dfdd1afeab5dead9401f74d92fe Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Thu, 12 Nov 2026 23:20:34 +0300
Subject: [PATCH 088/114] fix(linstor): update piraeus-operator to v2.12.1
The nfs-server DaemonSet runs piraeus-csi-nfs-server v1.10.3, which
boots systemd as PID 1 in a privileged container with the host /dev
mounted. Its getty units hang up the host's getty@tty1 about a second
after the pod starts; systemd restarts the host getty until it hits
the start limit and gives up, so login on the local or out-of-band
console stops working on nodes that run the NFS server. Seen on
bare-metal Ubuntu 24.04 nodes with Cozystack 1.6.5 and 1.7.0.
Upstream fixed the image by disabling the services the NFS server does
not need (piraeusdatastore/linstor-csi#417, reported as
piraeusdatastore/piraeus-operator#961); v1.10.6 includes the fix.
v2.12.1 is the current stable release. Its image set is LINSTOR
1.35.3 and LINSTOR CSI v1.13.2, which this repository builds itself;
both builds follow in the next commits. The CSI release line is what
carries IO QoS limits (qos.linbit.com/* StorageClass parameters,
added in v1.11.3), which operators have asked for.
Behaviour that changes with this range:
- The CSIDriver moves to fsGroupPolicy: File, so fsGroup is applied
to ReadWriteMany filesystem volumes too.
- The operator manages Secrets only in its own namespace, and the CSI
controller loses its cluster-wide read on Secrets; nothing in this
repository references snapshotter Secrets.
- The csi-health-monitor sidecar is removed.
- Volume group snapshots are enabled only when the
groupsnapshot.storage.k8s.io/v1 API is served; the snapshot
controller here still ships v1alpha1, so they stay off.
The csi-provisioner arguments are unchanged in this range, so the
restated strictTopology list stays complete.
The chart is vendored the way `make update` does it, which also drops
the stale CRD copy from the vendored chart; piraeus-operator-crds
installs the CRDs. Declare the local chart dependency so Helm can
validate the umbrella chart without changing its rendered resources.
This commit is not deployable on its own. The operator now hands the
CSI pods their client certificate through LS_USER_CERTIFICATE_FILE and
LS_USER_KEY_FILE, which the linstor-wait-until in the current
linstor-csi image does not read, so the CSI init containers retry
forever until the linstor-csi update in the next commit.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
.../linstor/tests/csi-topology_test.yaml | 2 +-
.../piraeus-operator-crds/templates/crds.yaml | 218 +-
packages/system/piraeus-operator/Chart.yaml | 4 +
.../charts/piraeus/Chart.yaml | 4 +-
.../charts/piraeus/templates/config.yaml | 42 +-
.../charts/piraeus/templates/crds.yaml | 2173 -----------------
.../charts/piraeus/templates/deployment.yaml | 18 +-
.../piraeus/templates/extra-objects.yaml | 15 +
.../piraeus/templates/metrics-service.yaml | 1 +
.../charts/piraeus/templates/rbac.yaml | 47 +-
.../piraeus/templates/rolebindings.yaml | 16 +
.../piraeus/templates/serviceaccount.yaml | 1 +
.../validating-webhook-configuration.yaml | 2 +
.../piraeus/templates/webhook-service.yaml | 1 +
.../charts/piraeus/values.yaml | 43 +
15 files changed, 372 insertions(+), 2215 deletions(-)
delete mode 100644 packages/system/piraeus-operator/charts/piraeus/templates/crds.yaml
create mode 100644 packages/system/piraeus-operator/charts/piraeus/templates/extra-objects.yaml
diff --git a/packages/system/linstor/tests/csi-topology_test.yaml b/packages/system/linstor/tests/csi-topology_test.yaml
index fbde97a2d..6dd9bb3ea 100644
--- a/packages/system/linstor/tests/csi-topology_test.yaml
+++ b/packages/system/linstor/tests/csi-topology_test.yaml
@@ -34,7 +34,7 @@ tests:
# Until there is, the re-check on a bump is a human obligation, stated in
# templates/cluster.yaml beside the list.
#
- # The list was last verified complete against piraeus-operator v2.10.2:
+ # The list was last verified complete against piraeus-operator v2.12.1:
# eleven upstream flags plus --strict-topology.
- it: the restated argument list is complete
template: templates/cluster.yaml
diff --git a/packages/system/piraeus-operator-crds/templates/crds.yaml b/packages/system/piraeus-operator-crds/templates/crds.yaml
index 5bd9212c6..d190bc2b9 100644
--- a/packages/system/piraeus-operator-crds/templates/crds.yaml
+++ b/packages/system/piraeus-operator-crds/templates/crds.yaml
@@ -151,13 +151,17 @@ spec:
*SecretName using the issuer configured here.
properties:
group:
- description: Group of the resource being referred to.
+ description: |-
+ Group of the issuer being referred to.
+ Defaults to 'cert-manager.io'.
type: string
kind:
- description: Kind of the resource being referred to.
+ description: |-
+ Kind of the issuer being referred to.
+ Defaults to 'Issuer'.
type: string
name:
- description: Name of the resource being referred to.
+ description: Name of the issuer being referred to.
type: string
required:
- name
@@ -201,6 +205,11 @@ spec:
type: object
x-kubernetes-map-type: atomic
x-kubernetes-preserve-unknown-fields: true
+ replicas:
+ description: Number of desired pods. Defaults to 1.
+ format: int32
+ minimum: 0
+ type: integer
type: object
csiController:
description: CSIController controls the deployment of the CSI Controller
@@ -320,13 +329,17 @@ spec:
issuer configured here.
properties:
group:
- description: Group of the resource being referred to.
+ description: |-
+ Group of the issuer being referred to.
+ Defaults to 'cert-manager.io'.
type: string
kind:
- description: Kind of the resource being referred to.
+ description: |-
+ Kind of the issuer being referred to.
+ Defaults to 'Issuer'.
type: string
name:
- description: Name of the resource being referred to.
+ description: Name of the issuer being referred to.
type: string
required:
- name
@@ -345,6 +358,18 @@ spec:
* Store credentials for accessing remotes for backups.
See https://linbit.com/drbd-user-guide/linstor-guide-1_0-en/#s-encrypt_commands for more information.
type: string
+ maxConcurrentEvacuations:
+ description: |-
+ MaxConcurrentEvacuations limits how many Satellites are evacuated at the same time when using the
+ "Evacuate" deletion policy.
+
+ This applies both to Satellites removed because their node no longer matches the cluster, and to nodes
+ removed through ClusterAPI. While the limit is reached, additional Satellites wait for a free slot before
+ their volumes are moved and, for ClusterAPI managed nodes, before the node is allowed to drain.
+ A value of 0 (the default) places no limit on the number of concurrent evacuations.
+ format: int32
+ minimum: 0
+ type: integer
nfsServer:
description: NFSServer controls the deployment of the LINSTOR CSI
NFS Server DaemonSet.
@@ -568,9 +593,10 @@ spec:
operator:
description: |-
Operator represents a key's relationship to the value.
- Valid operators are Exists and Equal. Defaults to Equal.
+ Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal.
Exists is equivalent to wildcard for value, so that a pod can
tolerate all taints of a particular category.
+ Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators).
type: string
tolerationSeconds:
description: |-
@@ -1054,13 +1080,17 @@ spec:
issuer configured here.
properties:
group:
- description: Group of the resource being referred to.
+ description: |-
+ Group of the issuer being referred to.
+ Defaults to 'cert-manager.io'.
type: string
kind:
- description: Kind of the resource being referred to.
+ description: |-
+ Kind of the issuer being referred to.
+ Defaults to 'Issuer'.
type: string
name:
- description: Name of the resource being referred to.
+ description: Name of the issuer being referred to.
type: string
required:
- name
@@ -1321,6 +1351,15 @@ spec:
x-kubernetes-list-map-keys:
- name
x-kubernetes-list-type: map
+ resourceNameSuffixSeparator:
+ description: |-
+ ResourceNameSuffixSeparator is inserted between the base resource name and the Satellite name.
+
+ Defaults to ".".
+ enum:
+ - .
+ - '-'
+ type: string
storagePools:
description: StoragePools is a list of storage pools to configure
on the node.
@@ -1347,18 +1386,66 @@ spec:
lvmPool:
description: Configures a LVM Volume Group as storage pool.
properties:
- volumeGroup:
+ externalLocking:
+ description: |-
+ ExternalLocking skips LINSTOR's internal locking for the shared VG, relying on an external
+ lock manager such as lvmlockd on the hosts instead.
+ type: boolean
+ physicalVolumeCreateArguments:
+ description: |-
+ PhysicalVolumeCreateArguments are arguments to pass to "pvcreate".
+ This has no effect on an existing VG, it only applies when the VG initially gets created.
+ items:
+ type: string
+ type: array
+ sharedSpace:
+ description: |-
+ SharedSpace marks the VG as shared between multiple nodes, identified by the given name.
+ All storage pools backed by the same physical storage must use the same shared space name.
+ A shared VG must be set up manually on the hosts, it cannot be created from source devices.
type: string
+ volumeGroup:
+ description: VolumeGroup is the name of the Volume Group
+ (VG) to use.
+ type: string
+ volumeGroupCreateArguments:
+ description: |-
+ VolumeGroupCreateArguments are arguments to pass to "vgcreate".
+ This has no effect on an existing VG, it only applies when the VG initially gets created.
+ items:
+ type: string
+ type: array
type: object
lvmThinPool:
description: Configures a LVM Thin Pool as storage pool.
properties:
+ logicalVolumeCreateArguments:
+ description: |-
+ LogicalVolumeCreateArguments are arguments to pass to "lvcreate".
+ This has no effect on an existing thinpool, it only applies when the thinpool initially gets created.
+ items:
+ type: string
+ type: array
+ physicalVolumeCreateArguments:
+ description: |-
+ PhysicalVolumeCreateArguments are arguments to pass to "pvcreate".
+ This has no effect on an existing VG, it only applies when the VG initially gets created.
+ items:
+ type: string
+ type: array
thinPool:
description: ThinPool is the name of the thinpool LV (without
VG prefix).
type: string
volumeGroup:
type: string
+ volumeGroupCreateArguments:
+ description: |-
+ VolumeGroupCreateArguments are arguments to pass to "vgcreate".
+ This has no effect on an existing VG, it only applies when the VG initially gets created.
+ items:
+ type: string
+ type: array
type: object
name:
description: Name of the storage pool in linstor.
@@ -1447,6 +1534,16 @@ spec:
zPool:
description: ZPool is the name of the ZFS zpool.
type: string
+ zpoolCreateArguments:
+ description: |-
+ ZpoolCreateArguments are arguments to pass to "zpool create".
+ This has no effect on an existing zpool, it only applies when the zpool initially gets created.
+
+ If not set, "-o failmode=continue" is automatically added to ensure a failed zpool does not stop replication
+ to and from other nodes in the cluster.
+ items:
+ type: string
+ type: array
type: object
zfsThinPool:
description: Configures a ZFS system based storage pool, allocating
@@ -1455,6 +1552,16 @@ spec:
zPool:
description: ZPool is the name of the ZFS zpool.
type: string
+ zpoolCreateArguments:
+ description: |-
+ ZpoolCreateArguments are arguments to pass to "zpool create".
+ This has no effect on an existing zpool, it only applies when the zpool initially gets created.
+
+ If not set, "-o failmode=continue" is automatically added to ensure a failed zpool does not stop replication
+ to and from other nodes in the cluster.
+ items:
+ type: string
+ type: array
type: object
required:
- name
@@ -1762,13 +1869,17 @@ spec:
issuer configured here.
properties:
group:
- description: Group of the resource being referred to.
+ description: |-
+ Group of the issuer being referred to.
+ Defaults to 'cert-manager.io'.
type: string
kind:
- description: Kind of the resource being referred to.
+ description: |-
+ Kind of the issuer being referred to.
+ Defaults to 'Issuer'.
type: string
name:
- description: Name of the resource being referred to.
+ description: Name of the issuer being referred to.
type: string
required:
- name
@@ -1925,6 +2036,15 @@ spec:
repository:
description: Repository used to pull workload images.
type: string
+ resourceNameSuffixSeparator:
+ description: |-
+ ResourceNameSuffixSeparator is inserted between the base resource name and the Satellite name.
+
+ Defaults to ".".
+ enum:
+ - .
+ - '-'
+ type: string
storagePools:
description: StoragePools is a list of storage pools to configure
on the node.
@@ -1951,18 +2071,66 @@ spec:
lvmPool:
description: Configures a LVM Volume Group as storage pool.
properties:
- volumeGroup:
+ externalLocking:
+ description: |-
+ ExternalLocking skips LINSTOR's internal locking for the shared VG, relying on an external
+ lock manager such as lvmlockd on the hosts instead.
+ type: boolean
+ physicalVolumeCreateArguments:
+ description: |-
+ PhysicalVolumeCreateArguments are arguments to pass to "pvcreate".
+ This has no effect on an existing VG, it only applies when the VG initially gets created.
+ items:
+ type: string
+ type: array
+ sharedSpace:
+ description: |-
+ SharedSpace marks the VG as shared between multiple nodes, identified by the given name.
+ All storage pools backed by the same physical storage must use the same shared space name.
+ A shared VG must be set up manually on the hosts, it cannot be created from source devices.
type: string
+ volumeGroup:
+ description: VolumeGroup is the name of the Volume Group
+ (VG) to use.
+ type: string
+ volumeGroupCreateArguments:
+ description: |-
+ VolumeGroupCreateArguments are arguments to pass to "vgcreate".
+ This has no effect on an existing VG, it only applies when the VG initially gets created.
+ items:
+ type: string
+ type: array
type: object
lvmThinPool:
description: Configures a LVM Thin Pool as storage pool.
properties:
+ logicalVolumeCreateArguments:
+ description: |-
+ LogicalVolumeCreateArguments are arguments to pass to "lvcreate".
+ This has no effect on an existing thinpool, it only applies when the thinpool initially gets created.
+ items:
+ type: string
+ type: array
+ physicalVolumeCreateArguments:
+ description: |-
+ PhysicalVolumeCreateArguments are arguments to pass to "pvcreate".
+ This has no effect on an existing VG, it only applies when the VG initially gets created.
+ items:
+ type: string
+ type: array
thinPool:
description: ThinPool is the name of the thinpool LV (without
VG prefix).
type: string
volumeGroup:
type: string
+ volumeGroupCreateArguments:
+ description: |-
+ VolumeGroupCreateArguments are arguments to pass to "vgcreate".
+ This has no effect on an existing VG, it only applies when the VG initially gets created.
+ items:
+ type: string
+ type: array
type: object
name:
description: Name of the storage pool in linstor.
@@ -2051,6 +2219,16 @@ spec:
zPool:
description: ZPool is the name of the ZFS zpool.
type: string
+ zpoolCreateArguments:
+ description: |-
+ ZpoolCreateArguments are arguments to pass to "zpool create".
+ This has no effect on an existing zpool, it only applies when the zpool initially gets created.
+
+ If not set, "-o failmode=continue" is automatically added to ensure a failed zpool does not stop replication
+ to and from other nodes in the cluster.
+ items:
+ type: string
+ type: array
type: object
zfsThinPool:
description: Configures a ZFS system based storage pool, allocating
@@ -2059,6 +2237,16 @@ spec:
zPool:
description: ZPool is the name of the ZFS zpool.
type: string
+ zpoolCreateArguments:
+ description: |-
+ ZpoolCreateArguments are arguments to pass to "zpool create".
+ This has no effect on an existing zpool, it only applies when the zpool initially gets created.
+
+ If not set, "-o failmode=continue" is automatically added to ensure a failed zpool does not stop replication
+ to and from other nodes in the cluster.
+ items:
+ type: string
+ type: array
type: object
required:
- name
diff --git a/packages/system/piraeus-operator/Chart.yaml b/packages/system/piraeus-operator/Chart.yaml
index 40b3fce28..7a1925e25 100644
--- a/packages/system/piraeus-operator/Chart.yaml
+++ b/packages/system/piraeus-operator/Chart.yaml
@@ -1,3 +1,7 @@
apiVersion: v2
name: cozy-piraeus-operator
version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process
+dependencies:
+ - name: piraeus
+ version: "*"
+ repository: file://charts/piraeus
diff --git a/packages/system/piraeus-operator/charts/piraeus/Chart.yaml b/packages/system/piraeus-operator/charts/piraeus/Chart.yaml
index 777722f3f..1565e6a27 100644
--- a/packages/system/piraeus-operator/charts/piraeus/Chart.yaml
+++ b/packages/system/piraeus-operator/charts/piraeus/Chart.yaml
@@ -3,8 +3,8 @@ name: piraeus
description: |
The Piraeus Operator manages software defined storage clusters using LINSTOR in Kubernetes.
type: application
-version: 2.10.2
-appVersion: "v2.10.2"
+version: 2.12.1
+appVersion: "v2.12.1"
maintainers:
- name: Piraeus Datastore
url: https://piraeus.io
diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/config.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/config.yaml
index 2e6963f0a..4788ffba0 100644
--- a/packages/system/piraeus-operator/charts/piraeus/templates/config.yaml
+++ b/packages/system/piraeus-operator/charts/piraeus/templates/config.yaml
@@ -3,6 +3,7 @@ apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "piraeus-operator.fullname" . }}-image-config
+ namespace: {{ .Release.Namespace }}
labels:
{{- include "piraeus-operator.labels" . | nindent 4 }}
data:
@@ -17,38 +18,38 @@ data:
# quay.io/piraeusdatastore/piraeus-server:v1.24.2
components:
linstor-controller:
- tag: v1.32.3
+ tag: v1.35.3
image: piraeus-server
linstor-satellite:
- tag: v1.32.3
+ tag: v1.35.3
image: piraeus-server
linstor-csi:
- tag: v1.10.3
+ tag: v1.13.2
image: piraeus-csi
nfs-server:
- tag: v1.10.3
+ tag: v1.13.2
image: piraeus-csi-nfs-server
drbd-reactor:
- tag: v1.10.0
+ tag: v1.12.0
image: drbd-reactor
ha-controller:
- tag: v1.3.1
+ tag: v1.3.4
image: piraeus-ha-controller
drbd-shutdown-guard:
- tag: v1.1.1
+ tag: v1.1.2
image: drbd-shutdown-guard
ktls-utils:
- tag: v1.2.1
+ tag: v1.5.0
image: ktls-utils
linstor-affinity-controller:
- tag: v1.3.0
+ tag: v1.5.0
image: linstor-affinity-controller
drbd-module-loader:
- tag: v9.2.16
+ tag: v9.3.4
# The special "match" attribute is used to select an image based on the node's reported OS.
# The operator will first check the k8s node's ".status.nodeInfo.osImage" field, and compare it against the list
# here. If one matches, that specific image name will be used instead of the fallback image.
- image: drbd9-noble # Fallback image: chose a recent kernel, which can hopefully compile whatever config is actually in use
+ image: drbd9-resolute # Fallback image: chose a recent kernel, which can hopefully compile whatever config is actually in use
match:
- osImage: Red Hat Enterprise Linux 8\.
image: drbd9-almalinux8
@@ -84,6 +85,10 @@ data:
image: drbd9-jammy
- osImage: Ubuntu 24\.04
image: drbd9-noble
+ - osImage: Ubuntu 26\.04
+ image: drbd9-resolute
+ - osImage: Debian GNU/Linux 13
+ image: drbd9-trixie
- osImage: Debian GNU/Linux 12
image: drbd9-bookworm
- osImage: Debian GNU/Linux 11
@@ -93,25 +98,22 @@ data:
base: registry.k8s.io/sig-storage
components:
csi-attacher:
- tag: v4.10.0
+ tag: v4.13.0
image: csi-attacher
csi-livenessprobe:
- tag: v2.17.0
+ tag: v2.20.0
image: livenessprobe
csi-provisioner:
- tag: v6.1.0
+ tag: v6.4.0
image: csi-provisioner
csi-snapshotter:
- tag: v8.4.0
+ tag: v8.7.0
image: csi-snapshotter
csi-resizer:
- tag: v2.0.0
+ tag: v2.3.0
image: csi-resizer
- csi-external-health-monitor-controller:
- tag: v0.16.0
- image: csi-external-health-monitor-controller
csi-node-driver-registrar:
- tag: v2.15.0
+ tag: v2.18.0
image: csi-node-driver-registrar
{{- range $idx, $value := .Values.imageConfigOverride }}
{{ add $idx 1 }}_helm_override.yaml: |
diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/crds.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/crds.yaml
deleted file mode 100644
index 5bd9212c6..000000000
--- a/packages/system/piraeus-operator/charts/piraeus/templates/crds.yaml
+++ /dev/null
@@ -1,2173 +0,0 @@
- {{ if .Values.installCRDs }}
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
- annotations:
- controller-gen.kubebuilder.io/version: v0.18.0
- name: linstorclusters.piraeus.io
-spec:
- group: piraeus.io
- names:
- kind: LinstorCluster
- listKind: LinstorClusterList
- plural: linstorclusters
- singular: linstorcluster
- scope: Cluster
- versions:
- - additionalPrinterColumns:
- - description: If the LINSTOR Cluster is available
- jsonPath: .status.conditions[?(@.type=='Available')].status
- name: Available
- type: string
- - description: If the LINSTOR Cluster is fully configured
- jsonPath: .status.conditions[?(@.type=='Configured')].status
- name: Configured
- type: string
- - description: The version of the LINSTOR Cluster
- jsonPath: .status.version
- name: Version
- priority: 10
- type: string
- - description: The number of running/expected Satellites
- jsonPath: .status.satellites
- name: Satellites
- type: string
- - description: The used capacity in all storage pools
- jsonPath: .status.capacity
- name: Used Capacity
- type: string
- - description: The number of volumes in the cluster
- jsonPath: .status.numberOfVolumes
- name: Volumes
- type: integer
- - description: The number of snapshots in the cluster
- jsonPath: .status.numberOfSnapshots
- name: Snapshots
- priority: 10
- type: integer
- - jsonPath: .metadata.creationTimestamp
- name: Age
- type: date
- name: v1
- schema:
- openAPIV3Schema:
- description: LinstorCluster is the Schema for the linstorclusters API
- properties:
- apiVersion:
- description: |-
- APIVersion defines the versioned schema of this representation of an object.
- Servers should convert recognized schemas to the latest internal value, and
- may reject unrecognized values.
- More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
- type: string
- kind:
- description: |-
- Kind is a string value representing the REST resource this object represents.
- Servers may infer this from the endpoint the client submits requests to.
- Cannot be updated.
- In CamelCase.
- More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
- type: string
- metadata:
- type: object
- spec:
- description: LinstorClusterSpec defines the desired state of LinstorCluster
- properties:
- affinityController:
- description: AffinityController controls the deployment of the Affinity
- Controller Deployment.
- properties:
- enabled:
- default: true
- description: Enable the component.
- type: boolean
- podTemplate:
- description: |-
- Template to apply to Pods of the component.
-
- The template is applied as a patch to the default deployment, so it can be "sparse", not listing any
- containers or volumes that should remain unchanged.
- See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates
- type: object
- x-kubernetes-map-type: atomic
- x-kubernetes-preserve-unknown-fields: true
- replicas:
- description: Number of desired pods. Defaults to 1.
- format: int32
- minimum: 0
- type: integer
- type: object
- apiTLS:
- description: |-
- ApiTLS secures the LINSTOR API.
-
- This configures the TLS key and certificate used to secure the LINSTOR API.
- nullable: true
- properties:
- affinityControllerSecretName:
- description: |-
- AffinityControllerSecretName references a secret holding the TLS key and certificate used by the Affinity
- Controller to monitor volume state. Defaults to "linstor-affinity-controller-tls".
- type: string
- apiSecretName:
- description: |-
- ApiSecretName references a secret holding the TLS key and certificate used to protect the API.
- Defaults to "linstor-api-tls".
- type: string
- caReference:
- description: |-
- CAReference configures the CA certificate to use when validating TLS certificates.
- If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate.
- properties:
- key:
- default: ca.crt
- description: |-
- Key to select in the resource.
- Defaults to ca.crt if not specified.
- type: string
- kind:
- default: Secret
- description: Kind of the resource containing the CA Certificate,
- either a ConfigMap or Secret.
- enum:
- - ConfigMap
- - Secret
- type: string
- name:
- description: Name of the resource containing the CA Certificate.
- type: string
- optional:
- description: Optional specifies whether the resource and its
- key must exist.
- type: boolean
- required:
- - name
- type: object
- certManager:
- description: |-
- CertManager references a cert-manager Issuer or ClusterIssuer.
- If set, cert-manager.io/Certificate resources will be created, provisioning the secrets referenced in
- *SecretName using the issuer configured here.
- properties:
- group:
- description: Group of the resource being referred to.
- type: string
- kind:
- description: Kind of the resource being referred to.
- type: string
- name:
- description: Name of the resource being referred to.
- type: string
- required:
- - name
- type: object
- clientSecretName:
- description: |-
- ClientSecretName references a secret holding the TLS key and certificate used by the operator to configure
- the cluster. Defaults to "linstor-client-tls".
- type: string
- csiControllerSecretName:
- description: |-
- CsiControllerSecretName references a secret holding the TLS key and certificate used by the CSI Controller
- to provision volumes. Defaults to "linstor-csi-controller-tls".
- type: string
- csiNodeSecretName:
- description: |-
- CsiNodeSecretName references a secret holding the TLS key and certificate used by the CSI Nodes to query
- the volume state. Defaults to "linstor-csi-node-tls".
- type: string
- nfsServerSecretName:
- description: |-
- NFSServerSecretName references a secret holding the TLS key and certificate used by the NFS Server to query
- the cluster state. Defaults to "linstor-csi-nfs-server-tls".
- type: string
- type: object
- controller:
- description: Controller controls the deployment of the LINSTOR Controller
- Deployment.
- properties:
- enabled:
- default: true
- description: Enable the component.
- type: boolean
- podTemplate:
- description: |-
- Template to apply to Pods of the component.
-
- The template is applied as a patch to the default deployment, so it can be "sparse", not listing any
- containers or volumes that should remain unchanged.
- See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates
- type: object
- x-kubernetes-map-type: atomic
- x-kubernetes-preserve-unknown-fields: true
- type: object
- csiController:
- description: CSIController controls the deployment of the CSI Controller
- Deployment.
- properties:
- enabled:
- default: true
- description: Enable the component.
- type: boolean
- podTemplate:
- description: |-
- Template to apply to Pods of the component.
-
- The template is applied as a patch to the default deployment, so it can be "sparse", not listing any
- containers or volumes that should remain unchanged.
- See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates
- type: object
- x-kubernetes-map-type: atomic
- x-kubernetes-preserve-unknown-fields: true
- replicas:
- description: Number of desired pods. Defaults to 1.
- format: int32
- minimum: 0
- type: integer
- type: object
- csiNode:
- description: CSINode controls the deployment of the CSI Node DaemonSet.
- properties:
- enabled:
- default: true
- description: Enable the component.
- type: boolean
- podTemplate:
- description: |-
- Template to apply to Pods of the component.
-
- The template is applied as a patch to the default deployment, so it can be "sparse", not listing any
- containers or volumes that should remain unchanged.
- See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates
- type: object
- x-kubernetes-map-type: atomic
- x-kubernetes-preserve-unknown-fields: true
- type: object
- externalController:
- description: |-
- ExternalController references an external controller.
- When set, the Operator will skip deploying a LINSTOR Controller and instead use the external cluster
- to register satellites.
- properties:
- url:
- description: URL of the external controller.
- minLength: 3
- type: string
- required:
- - url
- type: object
- highAvailabilityController:
- description: HighAvailabilityController controls the deployment of
- the High Availability Controller DaemonSet.
- properties:
- enabled:
- default: true
- description: Enable the component.
- type: boolean
- podTemplate:
- description: |-
- Template to apply to Pods of the component.
-
- The template is applied as a patch to the default deployment, so it can be "sparse", not listing any
- containers or volumes that should remain unchanged.
- See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates
- type: object
- x-kubernetes-map-type: atomic
- x-kubernetes-preserve-unknown-fields: true
- type: object
- internalTLS:
- description: |-
- InternalTLS secures the connection between LINSTOR Controller and Satellite.
-
- This configures the client certificate used when the Controller connects to a Satellite. This only has an effect
- when the Satellite is configured to for secure connections using `LinstorSatellite.spec.internalTLS`.
- nullable: true
- properties:
- caReference:
- description: |-
- CAReference configures the CA certificate to use when validating TLS certificates.
- If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate.
- properties:
- key:
- default: ca.crt
- description: |-
- Key to select in the resource.
- Defaults to ca.crt if not specified.
- type: string
- kind:
- default: Secret
- description: Kind of the resource containing the CA Certificate,
- either a ConfigMap or Secret.
- enum:
- - ConfigMap
- - Secret
- type: string
- name:
- description: Name of the resource containing the CA Certificate.
- type: string
- optional:
- description: Optional specifies whether the resource and its
- key must exist.
- type: boolean
- required:
- - name
- type: object
- certManager:
- description: |-
- CertManager references a cert-manager Issuer or ClusterIssuer.
- If set, a Certificate resource will be created, provisioning the secret references in SecretName using the
- issuer configured here.
- properties:
- group:
- description: Group of the resource being referred to.
- type: string
- kind:
- description: Kind of the resource being referred to.
- type: string
- name:
- description: Name of the resource being referred to.
- type: string
- required:
- - name
- type: object
- secretName:
- description: SecretName references a secret holding the TLS key
- and certificates.
- type: string
- type: object
- linstorPassphraseSecret:
- description: |-
- LinstorPassphraseSecret used to configure the LINSTOR master passphrase.
-
- The referenced secret must contain a single key "MASTER_PASSPHRASE". The master passphrase is used to
- * Derive encryption keys for volumes using the LUKS layer.
- * Store credentials for accessing remotes for backups.
- See https://linbit.com/drbd-user-guide/linstor-guide-1_0-en/#s-encrypt_commands for more information.
- type: string
- nfsServer:
- description: NFSServer controls the deployment of the LINSTOR CSI
- NFS Server DaemonSet.
- properties:
- enabled:
- default: true
- description: Enable the component.
- type: boolean
- podTemplate:
- description: |-
- Template to apply to Pods of the component.
-
- The template is applied as a patch to the default deployment, so it can be "sparse", not listing any
- containers or volumes that should remain unchanged.
- See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates
- type: object
- x-kubernetes-map-type: atomic
- x-kubernetes-preserve-unknown-fields: true
- type: object
- nodeAffinity:
- description: |-
- NodeAffinity selects the nodes on which LINSTOR Satellites will be deployed.
- See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
- properties:
- nodeSelectorTerms:
- description: Required. A list of node selector terms. The terms
- are ORed.
- items:
- description: |-
- A null or empty node selector term matches no objects. The requirements of
- them are ANDed.
- The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.
- properties:
- matchExpressions:
- description: A list of node selector requirements by node's
- labels.
- items:
- description: |-
- A node selector requirement is a selector that contains values, a key, and an operator
- that relates the key and values.
- properties:
- key:
- description: The label key that the selector applies
- to.
- type: string
- operator:
- description: |-
- Represents a key's relationship to a set of values.
- Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
- type: string
- values:
- description: |-
- An array of string values. If the operator is In or NotIn,
- the values array must be non-empty. If the operator is Exists or DoesNotExist,
- the values array must be empty. If the operator is Gt or Lt, the values
- array must have a single element, which will be interpreted as an integer.
- This array is replaced during a strategic merge patch.
- items:
- type: string
- type: array
- x-kubernetes-list-type: atomic
- required:
- - key
- - operator
- type: object
- type: array
- x-kubernetes-list-type: atomic
- matchFields:
- description: A list of node selector requirements by node's
- fields.
- items:
- description: |-
- A node selector requirement is a selector that contains values, a key, and an operator
- that relates the key and values.
- properties:
- key:
- description: The label key that the selector applies
- to.
- type: string
- operator:
- description: |-
- Represents a key's relationship to a set of values.
- Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
- type: string
- values:
- description: |-
- An array of string values. If the operator is In or NotIn,
- the values array must be non-empty. If the operator is Exists or DoesNotExist,
- the values array must be empty. If the operator is Gt or Lt, the values
- array must have a single element, which will be interpreted as an integer.
- This array is replaced during a strategic merge patch.
- items:
- type: string
- type: array
- x-kubernetes-list-type: atomic
- required:
- - key
- - operator
- type: object
- type: array
- x-kubernetes-list-type: atomic
- type: object
- x-kubernetes-map-type: atomic
- type: array
- x-kubernetes-list-type: atomic
- required:
- - nodeSelectorTerms
- type: object
- x-kubernetes-map-type: atomic
- nodeSelector:
- additionalProperties:
- type: string
- description: |-
- NodeSelector selects the nodes on which LINSTOR Satellites will be deployed.
- See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
- type: object
- patches:
- description: |-
- Patches is a list of kustomize patches to apply.
-
- See https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/ for how to create patches.
- items:
- description: Patch represent either a Strategic Merge Patch or a
- JSON patch and its targets.
- properties:
- options:
- description: Options is a list of options for the patch
- properties:
- allowKindChange:
- description: AllowKindChange allows kind changes to the
- resource.
- type: boolean
- allowNameChange:
- description: AllowNameChange allows name changes to the
- resource.
- type: boolean
- type: object
- patch:
- description: Patch is the content of a patch.
- minLength: 1
- type: string
- target:
- description: Target points to the resources that the patch is
- applied to
- properties:
- annotationSelector:
- description: |-
- AnnotationSelector is a string that follows the label selection expression
- https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api
- It matches against the resource annotations.
- type: string
- group:
- type: string
- kind:
- type: string
- labelSelector:
- description: |-
- LabelSelector is a string that follows the label selection expression
- https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api
- It matches against the resource labels.
- type: string
- name:
- description: Name of the resource.
- type: string
- namespace:
- description: Namespace the resource belongs to, if it can
- belong to a namespace.
- type: string
- version:
- type: string
- type: object
- required:
- - patch
- type: object
- type: array
- properties:
- description: |-
- Properties to apply on the cluster level.
-
- Use to create default settings for DRBD that should apply to all resources or to configure some other cluster
- wide default.
- items:
- properties:
- name:
- description: Name of the property to set.
- minLength: 1
- type: string
- value:
- description: Value to set the property to.
- type: string
- required:
- - name
- type: object
- type: array
- x-kubernetes-list-map-keys:
- - name
- x-kubernetes-list-type: map
- repository:
- description: Repository used to pull workload images.
- type: string
- tolerations:
- description: |-
- Tolerations selects the nodes on which LINSTOR Satellites will be deployed.
-
- The default tolerations for DaemonSets are automatically added.
- items:
- description: |-
- The pod this Toleration is attached to tolerates any taint that matches
- the triple using the matching operator .
- properties:
- effect:
- description: |-
- Effect indicates the taint effect to match. Empty means match all taint effects.
- When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
- type: string
- key:
- description: |-
- Key is the taint key that the toleration applies to. Empty means match all taint keys.
- If the key is empty, operator must be Exists; this combination means to match all values and all keys.
- type: string
- operator:
- description: |-
- Operator represents a key's relationship to the value.
- Valid operators are Exists and Equal. Defaults to Equal.
- Exists is equivalent to wildcard for value, so that a pod can
- tolerate all taints of a particular category.
- type: string
- tolerationSeconds:
- description: |-
- TolerationSeconds represents the period of time the toleration (which must be
- of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
- it is not set, which means tolerate the taint forever (do not evict). Zero and
- negative values will be treated as 0 (evict immediately) by the system.
- format: int64
- type: integer
- value:
- description: |-
- Value is the taint value the toleration matches to.
- If the operator is Exists, the value should be empty, otherwise just a regular string.
- type: string
- type: object
- type: array
- x-kubernetes-list-type: atomic
- type: object
- status:
- description: LinstorClusterStatus defines the observed state of LinstorCluster
- properties:
- availableCapacityBytes:
- description: The number of bytes in total in all storage pools in
- the LINSTOR Cluster.
- format: int64
- type: integer
- capacity:
- description: Capacity mirrors the information from TotalCapacityBytes
- and FreeCapacityBytes in a human-readable string
- type: string
- conditions:
- description: Current LINSTOR Cluster state
- items:
- description: Condition contains details for one aspect of the current
- state of this API Resource.
- properties:
- lastTransitionTime:
- description: |-
- lastTransitionTime is the last time the condition transitioned from one status to another.
- This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
- format: date-time
- type: string
- message:
- description: |-
- message is a human readable message indicating details about the transition.
- This may be an empty string.
- maxLength: 32768
- type: string
- observedGeneration:
- description: |-
- observedGeneration represents the .metadata.generation that the condition was set based upon.
- For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
- with respect to the current state of the instance.
- format: int64
- minimum: 0
- type: integer
- reason:
- description: |-
- reason contains a programmatic identifier indicating the reason for the condition's last transition.
- Producers of specific condition types may define expected values and meanings for this field,
- and whether the values are considered a guaranteed API.
- The value should be a CamelCase string.
- This field may not be empty.
- maxLength: 1024
- minLength: 1
- pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
- type: string
- status:
- description: status of the condition, one of True, False, Unknown.
- enum:
- - "True"
- - "False"
- - Unknown
- type: string
- type:
- description: type of condition in CamelCase or in foo.example.com/CamelCase.
- maxLength: 316
- pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
- type: string
- required:
- - lastTransitionTime
- - message
- - reason
- - status
- - type
- type: object
- type: array
- x-kubernetes-list-map-keys:
- - type
- x-kubernetes-list-type: map
- freeCapacityBytes:
- description: The number of bytes free in all storage pools in the
- LINSTOR Cluster.
- format: int64
- type: integer
- numberOfSnapshots:
- description: The number of snapshots in the LINSTOR Cluster.
- format: int32
- type: integer
- numberOfVolumes:
- description: The number of volumes in the LINSTOR Cluster.
- format: int32
- type: integer
- runningSatellites:
- description: The number of LINSTOR Satellites currently running.
- format: int32
- type: integer
- satellites:
- description: Satellites mirrors the information from ScheduledSatellites
- and RunningSatellites in a human-readable string
- type: string
- scheduledSatellites:
- description: The number of LINSTOR Satellites that are expected to
- run.
- format: int32
- type: integer
- version:
- description: The Version of the LINSTOR Cluster.
- type: string
- type: object
- type: object
- served: true
- storage: true
- subresources:
- status: {}
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
- annotations:
- controller-gen.kubebuilder.io/version: v0.18.0
- name: linstornodeconnections.piraeus.io
-spec:
- group: piraeus.io
- names:
- kind: LinstorNodeConnection
- listKind: LinstorNodeConnectionList
- plural: linstornodeconnections
- singular: linstornodeconnection
- scope: Cluster
- versions:
- - additionalPrinterColumns:
- - description: If the LINSTOR Node Connection is fully configured
- jsonPath: .status.conditions[?(@.type=='Configured')].status
- name: Configured
- type: string
- - jsonPath: .metadata.creationTimestamp
- name: Age
- type: date
- name: v1
- schema:
- openAPIV3Schema:
- description: LinstorNodeConnection is the Schema for the linstornodeconnections
- API
- properties:
- apiVersion:
- description: |-
- APIVersion defines the versioned schema of this representation of an object.
- Servers should convert recognized schemas to the latest internal value, and
- may reject unrecognized values.
- More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
- type: string
- kind:
- description: |-
- Kind is a string value representing the REST resource this object represents.
- Servers may infer this from the endpoint the client submits requests to.
- Cannot be updated.
- In CamelCase.
- More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
- type: string
- metadata:
- type: object
- spec:
- description: LinstorNodeConnectionSpec defines the desired state of LinstorNodeConnection
- properties:
- paths:
- description: Paths configure the network path used when connecting
- two nodes.
- items:
- properties:
- interface:
- description: Interface to use on both nodes.
- type: string
- name:
- description: Name of the path.
- type: string
- required:
- - interface
- - name
- type: object
- type: array
- x-kubernetes-list-map-keys:
- - name
- x-kubernetes-list-type: map
- properties:
- description: |-
- Properties to apply for the node connection.
-
- Use to create default settings for DRBD that should apply to all resources connections between a set of
- cluster nodes.
- items:
- properties:
- name:
- description: Name of the property to set.
- minLength: 1
- type: string
- value:
- description: Value to set the property to.
- type: string
- required:
- - name
- type: object
- type: array
- x-kubernetes-list-map-keys:
- - name
- x-kubernetes-list-type: map
- selector:
- description: |-
- Selector selects which pair of Satellites the connection should apply to.
- If not given, the connection will be applied to all connections.
- items:
- description: SelectorTerm matches pairs of nodes by checking that
- the nodes match all specified requirements.
- properties:
- matchLabels:
- description: MatchLabels is a list of match expressions that
- the node pairs must meet.
- items:
- properties:
- key:
- description: Key is the name of a node label.
- minLength: 1
- type: string
- op:
- default: Exists
- description: |-
- Op to apply to the label.
- Exists (default) checks for the presence of the label on both nodes in the pair.
- DoesNotExist checks that the label is not present on either node in the pair.
- In checks for the presence of the label value given by Values on both nodes in the pair.
- NotIn checks that both nodes in the pair do not have any of the label values given by Values.
- Same checks that the label value is equal in the node pair.
- NotSame checks that the label value is not equal in the node pair.
- enum:
- - Exists
- - DoesNotExist
- - In
- - NotIn
- - Same
- - NotSame
- type: string
- values:
- description: Values to match on, using the provided Op.
- items:
- type: string
- type: array
- required:
- - key
- type: object
- type: array
- required:
- - matchLabels
- type: object
- type: array
- type: object
- status:
- description: LinstorNodeConnectionStatus defines the observed state of
- LinstorNodeConnection
- properties:
- conditions:
- description: Current LINSTOR Node Connection state
- items:
- description: Condition contains details for one aspect of the current
- state of this API Resource.
- properties:
- lastTransitionTime:
- description: |-
- lastTransitionTime is the last time the condition transitioned from one status to another.
- This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
- format: date-time
- type: string
- message:
- description: |-
- message is a human readable message indicating details about the transition.
- This may be an empty string.
- maxLength: 32768
- type: string
- observedGeneration:
- description: |-
- observedGeneration represents the .metadata.generation that the condition was set based upon.
- For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
- with respect to the current state of the instance.
- format: int64
- minimum: 0
- type: integer
- reason:
- description: |-
- reason contains a programmatic identifier indicating the reason for the condition's last transition.
- Producers of specific condition types may define expected values and meanings for this field,
- and whether the values are considered a guaranteed API.
- The value should be a CamelCase string.
- This field may not be empty.
- maxLength: 1024
- minLength: 1
- pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
- type: string
- status:
- description: status of the condition, one of True, False, Unknown.
- enum:
- - "True"
- - "False"
- - Unknown
- type: string
- type:
- description: type of condition in CamelCase or in foo.example.com/CamelCase.
- maxLength: 316
- pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
- type: string
- required:
- - lastTransitionTime
- - message
- - reason
- - status
- - type
- type: object
- type: array
- x-kubernetes-list-map-keys:
- - type
- x-kubernetes-list-type: map
- type: object
- type: object
- served: true
- storage: true
- subresources:
- status: {}
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
- annotations:
- controller-gen.kubebuilder.io/version: v0.18.0
- name: linstorsatelliteconfigurations.piraeus.io
-spec:
- group: piraeus.io
- names:
- kind: LinstorSatelliteConfiguration
- listKind: LinstorSatelliteConfigurationList
- plural: linstorsatelliteconfigurations
- singular: linstorsatelliteconfiguration
- scope: Cluster
- versions:
- - additionalPrinterColumns:
- - description: The node selector used
- jsonPath: .spec.nodeSelector
- name: Selector
- type: string
- - description: If the Configuration was applied
- jsonPath: .status.conditions[?(@.type=='Applied')].status
- name: Applied
- type: string
- - description: Number of Satellites this Configuration has been applied to
- jsonPath: .status.matched
- name: Matched
- type: integer
- - description: Satellites this Configuration has been applied to
- jsonPath: .status.appliedTo
- name: Satellites
- priority: 10
- type: string
- - jsonPath: .metadata.creationTimestamp
- name: Age
- type: date
- name: v1
- schema:
- openAPIV3Schema:
- description: LinstorSatelliteConfiguration is the Schema for the linstorsatelliteconfigurations
- API
- properties:
- apiVersion:
- description: |-
- APIVersion defines the versioned schema of this representation of an object.
- Servers should convert recognized schemas to the latest internal value, and
- may reject unrecognized values.
- More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
- type: string
- kind:
- description: |-
- Kind is a string value representing the REST resource this object represents.
- Servers may infer this from the endpoint the client submits requests to.
- Cannot be updated.
- In CamelCase.
- More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
- type: string
- metadata:
- type: object
- spec:
- description: |-
- LinstorSatelliteConfigurationSpec defines a partial, desired state of a LinstorSatelliteSpec.
-
- All the LinstorSatelliteConfiguration resources with matching NodeSelector will
- be merged into a single LinstorSatelliteSpec.
- properties:
- deletionPolicy:
- description: |-
- DeletionPolicy configures the way LinstorSatellite resources are deleted.
-
- A LinstorSatellite may be deleted because:
- * It no longer matches the affinity and node selector of the LinstorCluster resource.
- * The node it references has been removed from Kubernetes.
- * It was manually deleted outside the Operator.
-
- A LinstorSatellite may store the last copy of a volume, in which case it is not desirable to unconditionally remove
- the satellite from the cluster. For this reason, the following deletion policies exist:
-
- * DeletionPolicyEvacuate will start evacuation of the LINSTOR Satellite and wait until it completes before removing the LinstorSatellite object, comparable to the "linstor node evacuate" command.
- * DeletionPolicyRetain will retain the LINSTOR Satellite, keeping it registered in LINSTOR, but removing associated Kubernetes resources.
- * DeletionPolicyDelete will remove the LINSTOR Satellite from the LINSTOR Cluster without prior eviction, comparable to the "linstor node lost" command.
- enum:
- - Evacuate
- - Retain
- - Delete
- type: string
- evacuationStrategy:
- description: EvacuationStrategy configures the evacuation of volumes
- from a Satellite when DeletionPolicy "Evacuate" is used.
- nullable: true
- properties:
- attachedVolumeReattachTimeout:
- default: 5m
- description: |-
- AttachedVolumeReattachTimeout configures how long evacuation waits for attached volumes to reattach on
- different nodes. Setting this to 0 disable this evacuation step.
- type: string
- unattachedVolumeAttachTimeout:
- default: 5m
- description: |-
- UnattachedVolumeAttachTimeout configures how long evacuation waits for unattached volumes to attach on
- different nodes. Setting this to 0 disable this evacuation step.
- type: string
- type: object
- internalTLS:
- description: |-
- InternalTLS configures secure communication for the LINSTOR Satellite.
-
- If set, the control traffic between LINSTOR Controller and Satellite will be encrypted using mTLS.
- nullable: true
- properties:
- caReference:
- description: |-
- CAReference configures the CA certificate to use when validating TLS certificates.
- If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate.
- properties:
- key:
- default: ca.crt
- description: |-
- Key to select in the resource.
- Defaults to ca.crt if not specified.
- type: string
- kind:
- default: Secret
- description: Kind of the resource containing the CA Certificate,
- either a ConfigMap or Secret.
- enum:
- - ConfigMap
- - Secret
- type: string
- name:
- description: Name of the resource containing the CA Certificate.
- type: string
- optional:
- description: Optional specifies whether the resource and its
- key must exist.
- type: boolean
- required:
- - name
- type: object
- certManager:
- description: |-
- CertManager references a cert-manager Issuer or ClusterIssuer.
- If set, a Certificate resource will be created, provisioning the secret references in SecretName using the
- issuer configured here.
- properties:
- group:
- description: Group of the resource being referred to.
- type: string
- kind:
- description: Kind of the resource being referred to.
- type: string
- name:
- description: Name of the resource being referred to.
- type: string
- required:
- - name
- type: object
- secretName:
- description: SecretName references a secret holding the TLS key
- and certificates.
- type: string
- tlsHandshakeDaemon:
- description: |-
- TLSHandshakeDaemon enables tlshd for establishing TLS sessions for use by DRBD.
-
- If enabled, adds a new sidecar to the LINSTOR Satellite that runs the tlshd handshake daemon.
- The daemon uses the TLS certificate and key to establish secure connections on behalf of DRBD.
- type: boolean
- type: object
- ipFamilies:
- description: |-
- IPFamilies configures the IP Family (IPv4 or IPv6) to use to connect to the LINSTOR Satellite.
-
- If set, the control traffic between LINSTOR Controller and Satellite will use only the given IP Family.
- If not set, the Operator will configure all families found in the Satellites Pods' Status.
- items:
- description: IPFamily represents the IP Family (IPv4 or IPv6).
- enum:
- - IPv4
- - IPv6
- type: string
- type: array
- nodeAffinity:
- description: |-
- NodeAffinity selects which LinstorSatellite resources this spec should be applied to.
- See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
- properties:
- nodeSelectorTerms:
- description: Required. A list of node selector terms. The terms
- are ORed.
- items:
- description: |-
- A null or empty node selector term matches no objects. The requirements of
- them are ANDed.
- The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.
- properties:
- matchExpressions:
- description: A list of node selector requirements by node's
- labels.
- items:
- description: |-
- A node selector requirement is a selector that contains values, a key, and an operator
- that relates the key and values.
- properties:
- key:
- description: The label key that the selector applies
- to.
- type: string
- operator:
- description: |-
- Represents a key's relationship to a set of values.
- Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
- type: string
- values:
- description: |-
- An array of string values. If the operator is In or NotIn,
- the values array must be non-empty. If the operator is Exists or DoesNotExist,
- the values array must be empty. If the operator is Gt or Lt, the values
- array must have a single element, which will be interpreted as an integer.
- This array is replaced during a strategic merge patch.
- items:
- type: string
- type: array
- x-kubernetes-list-type: atomic
- required:
- - key
- - operator
- type: object
- type: array
- x-kubernetes-list-type: atomic
- matchFields:
- description: A list of node selector requirements by node's
- fields.
- items:
- description: |-
- A node selector requirement is a selector that contains values, a key, and an operator
- that relates the key and values.
- properties:
- key:
- description: The label key that the selector applies
- to.
- type: string
- operator:
- description: |-
- Represents a key's relationship to a set of values.
- Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
- type: string
- values:
- description: |-
- An array of string values. If the operator is In or NotIn,
- the values array must be non-empty. If the operator is Exists or DoesNotExist,
- the values array must be empty. If the operator is Gt or Lt, the values
- array must have a single element, which will be interpreted as an integer.
- This array is replaced during a strategic merge patch.
- items:
- type: string
- type: array
- x-kubernetes-list-type: atomic
- required:
- - key
- - operator
- type: object
- type: array
- x-kubernetes-list-type: atomic
- type: object
- x-kubernetes-map-type: atomic
- type: array
- x-kubernetes-list-type: atomic
- required:
- - nodeSelectorTerms
- type: object
- x-kubernetes-map-type: atomic
- nodeSelector:
- additionalProperties:
- type: string
- description: |-
- NodeSelector selects which LinstorSatellite resources this spec should be applied to.
- See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
- type: object
- patches:
- description: |-
- Patches is a list of kustomize patches to apply.
-
- See https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/ for how to create patches.
- items:
- description: Patch represent either a Strategic Merge Patch or a
- JSON patch and its targets.
- properties:
- options:
- description: Options is a list of options for the patch
- properties:
- allowKindChange:
- description: AllowKindChange allows kind changes to the
- resource.
- type: boolean
- allowNameChange:
- description: AllowNameChange allows name changes to the
- resource.
- type: boolean
- type: object
- patch:
- description: Patch is the content of a patch.
- minLength: 1
- type: string
- target:
- description: Target points to the resources that the patch is
- applied to
- properties:
- annotationSelector:
- description: |-
- AnnotationSelector is a string that follows the label selection expression
- https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api
- It matches against the resource annotations.
- type: string
- group:
- type: string
- kind:
- type: string
- labelSelector:
- description: |-
- LabelSelector is a string that follows the label selection expression
- https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api
- It matches against the resource labels.
- type: string
- name:
- description: Name of the resource.
- type: string
- namespace:
- description: Namespace the resource belongs to, if it can
- belong to a namespace.
- type: string
- version:
- type: string
- type: object
- required:
- - patch
- type: object
- type: array
- podTemplate:
- description: |-
- Template to apply to Satellite Pods.
-
- The template is applied as a patch to the default resource, so it can be "sparse", not listing any
- containers or volumes that should remain unchanged.
- See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates
- type: object
- x-kubernetes-map-type: atomic
- x-kubernetes-preserve-unknown-fields: true
- properties:
- description: Properties is a list of properties to set on the node.
- items:
- properties:
- expandFrom:
- description: |-
- ExpandFrom can reference multiple resource fields at once.
- It either sets the property to an aggregate value based on matched resource fields, or expands to multiple
- properties.
- properties:
- delimiter:
- description: Delimiter used to join multiple key and value
- pairs together.
- type: string
- nameTemplate:
- description: |-
- NameTemplate defines how the property key is expanded.
- If set, the template is appended to the defined property name, creating multiple properties instead of one
- aggregate.
- * $1 is replaced with the matched key.
- * $2 is replaced with the matched value.
- type: string
- nodeFieldRef:
- description: Select a field of the node. Supports `metadata.name`,
- `metadata.labels['']`, `metadata.annotations['']`.
- minLength: 1
- type: string
- valueTemplate:
- description: |-
- ValueTemplate defines how the property value is expanded.
- * $1 is replaced with the matched key.
- * $2 is replaced with the matched value.
- type: string
- required:
- - nodeFieldRef
- type: object
- name:
- description: Name of the property to set.
- minLength: 1
- type: string
- optional:
- description: Optional values are only set if they have a non-empty
- value
- type: boolean
- value:
- description: Value to set the property to.
- type: string
- valueFrom:
- description: ValueFrom sets the value from an existing resource.
- properties:
- nodeFieldRef:
- description: Select a field of the node. Supports `metadata.name`,
- `metadata.labels['']`, `metadata.annotations['']`.
- minLength: 1
- type: string
- required:
- - nodeFieldRef
- type: object
- required:
- - name
- type: object
- type: array
- x-kubernetes-list-map-keys:
- - name
- x-kubernetes-list-type: map
- storagePools:
- description: StoragePools is a list of storage pools to configure
- on the node.
- items:
- properties:
- filePool:
- description: Configures a file system based storage pool, allocating
- a regular file per volume.
- properties:
- directory:
- description: Directory is the path to the host directory
- used to store volume data.
- type: string
- type: object
- fileThinPool:
- description: Configures a file system based storage pool, allocating
- a sparse file per volume.
- properties:
- directory:
- description: Directory is the path to the host directory
- used to store volume data.
- type: string
- type: object
- lvmPool:
- description: Configures a LVM Volume Group as storage pool.
- properties:
- volumeGroup:
- type: string
- type: object
- lvmThinPool:
- description: Configures a LVM Thin Pool as storage pool.
- properties:
- thinPool:
- description: ThinPool is the name of the thinpool LV (without
- VG prefix).
- type: string
- volumeGroup:
- type: string
- type: object
- name:
- description: Name of the storage pool in linstor.
- minLength: 3
- type: string
- properties:
- description: Properties to set on the storage pool.
- items:
- properties:
- expandFrom:
- description: |-
- ExpandFrom can reference multiple resource fields at once.
- It either sets the property to an aggregate value based on matched resource fields, or expands to multiple
- properties.
- properties:
- delimiter:
- description: Delimiter used to join multiple key and
- value pairs together.
- type: string
- nameTemplate:
- description: |-
- NameTemplate defines how the property key is expanded.
- If set, the template is appended to the defined property name, creating multiple properties instead of one
- aggregate.
- * $1 is replaced with the matched key.
- * $2 is replaced with the matched value.
- type: string
- nodeFieldRef:
- description: Select a field of the node. Supports
- `metadata.name`, `metadata.labels['']`, `metadata.annotations['']`.
- minLength: 1
- type: string
- valueTemplate:
- description: |-
- ValueTemplate defines how the property value is expanded.
- * $1 is replaced with the matched key.
- * $2 is replaced with the matched value.
- type: string
- required:
- - nodeFieldRef
- type: object
- name:
- description: Name of the property to set.
- minLength: 1
- type: string
- optional:
- description: Optional values are only set if they have
- a non-empty value
- type: boolean
- value:
- description: Value to set the property to.
- type: string
- valueFrom:
- description: ValueFrom sets the value from an existing
- resource.
- properties:
- nodeFieldRef:
- description: Select a field of the node. Supports
- `metadata.name`, `metadata.labels['']`, `metadata.annotations['']`.
- minLength: 1
- type: string
- required:
- - nodeFieldRef
- type: object
- required:
- - name
- type: object
- type: array
- x-kubernetes-list-map-keys:
- - name
- x-kubernetes-list-type: map
- source:
- properties:
- hostDevices:
- description: HostDevices is a list of device paths used
- to configure the given pool.
- items:
- type: string
- minItems: 1
- type: array
- type: object
- zfsPool:
- description: Configures a ZFS system based storage pool, allocating
- zvols from the given zpool.
- properties:
- zPool:
- description: ZPool is the name of the ZFS zpool.
- type: string
- type: object
- zfsThinPool:
- description: Configures a ZFS system based storage pool, allocating
- sparse zvols from the given zpool.
- properties:
- zPool:
- description: ZPool is the name of the ZFS zpool.
- type: string
- type: object
- required:
- - name
- type: object
- type: array
- type: object
- status:
- description: LinstorSatelliteConfigurationStatus defines the observed
- state of LinstorSatelliteConfiguration
- properties:
- appliedTo:
- description: AppliedTo lists the LinstorSatellite resource this configuration
- was applied to
- items:
- type: string
- type: array
- conditions:
- description: Current LINSTOR Satellite Config state
- items:
- description: Condition contains details for one aspect of the current
- state of this API Resource.
- properties:
- lastTransitionTime:
- description: |-
- lastTransitionTime is the last time the condition transitioned from one status to another.
- This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
- format: date-time
- type: string
- message:
- description: |-
- message is a human readable message indicating details about the transition.
- This may be an empty string.
- maxLength: 32768
- type: string
- observedGeneration:
- description: |-
- observedGeneration represents the .metadata.generation that the condition was set based upon.
- For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
- with respect to the current state of the instance.
- format: int64
- minimum: 0
- type: integer
- reason:
- description: |-
- reason contains a programmatic identifier indicating the reason for the condition's last transition.
- Producers of specific condition types may define expected values and meanings for this field,
- and whether the values are considered a guaranteed API.
- The value should be a CamelCase string.
- This field may not be empty.
- maxLength: 1024
- minLength: 1
- pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
- type: string
- status:
- description: status of the condition, one of True, False, Unknown.
- enum:
- - "True"
- - "False"
- - Unknown
- type: string
- type:
- description: type of condition in CamelCase or in foo.example.com/CamelCase.
- maxLength: 316
- pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
- type: string
- required:
- - lastTransitionTime
- - message
- - reason
- - status
- - type
- type: object
- type: array
- x-kubernetes-list-map-keys:
- - type
- x-kubernetes-list-type: map
- matched:
- description: Number of configured LinstorSatellite resource.
- format: int64
- type: integer
- type: object
- type: object
- served: true
- storage: true
- subresources:
- status: {}
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
- annotations:
- controller-gen.kubebuilder.io/version: v0.18.0
- name: linstorsatellites.piraeus.io
-spec:
- group: piraeus.io
- names:
- kind: LinstorSatellite
- listKind: LinstorSatelliteList
- plural: linstorsatellites
- singular: linstorsatellite
- scope: Cluster
- versions:
- - additionalPrinterColumns:
- - description: If the LINSTOR Satellite is connected
- jsonPath: .status.conditions[?(@.type=='Available')].status
- name: Connected
- type: string
- - description: If the LINSTOR Satellite is fully configured
- jsonPath: .status.conditions[?(@.type=='Configured')].status
- name: Configured
- type: string
- - description: The Satellite Configurations applied to this Satellite
- jsonPath: .metadata.annotations.piraeus\.io/applied-configurations
- name: Applied Configurations
- priority: 10
- type: string
- - description: The deletion policy of the Satellite
- jsonPath: .spec.deletionPolicy
- name: Deletion Policy
- type: string
- - description: The used capacity on the node
- jsonPath: .status.capacity
- name: Used Capacity
- type: string
- - description: The number of volumes on the node
- jsonPath: .status.numberOfVolumes
- name: Volumes
- type: integer
- - description: The number of snapshots on the node
- jsonPath: .status.numberOfSnapshots
- name: Snapshots
- priority: 10
- type: integer
- - description: The storage providers supported by the node
- jsonPath: .status.storageProviders
- name: Storage Providers
- priority: 10
- type: string
- - description: The device layers supported by the node
- jsonPath: .status.deviceLayers
- name: Device Layers
- priority: 10
- type: string
- - jsonPath: .metadata.creationTimestamp
- name: Age
- type: date
- name: v1
- schema:
- openAPIV3Schema:
- description: LinstorSatellite is the Schema for the linstorsatellites API
- properties:
- apiVersion:
- description: |-
- APIVersion defines the versioned schema of this representation of an object.
- Servers should convert recognized schemas to the latest internal value, and
- may reject unrecognized values.
- More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
- type: string
- kind:
- description: |-
- Kind is a string value representing the REST resource this object represents.
- Servers may infer this from the endpoint the client submits requests to.
- Cannot be updated.
- In CamelCase.
- More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
- type: string
- metadata:
- type: object
- spec:
- description: LinstorSatelliteSpec defines the desired state of LinstorSatellite
- properties:
- clusterRef:
- description: ClusterRef references the LinstorCluster used to create
- this LinstorSatellite.
- properties:
- caReference:
- description: |-
- CAReference configures the CA certificate to use when validating TLS certificates.
- If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate.
- properties:
- key:
- default: ca.crt
- description: |-
- Key to select in the resource.
- Defaults to ca.crt if not specified.
- type: string
- kind:
- default: Secret
- description: Kind of the resource containing the CA Certificate,
- either a ConfigMap or Secret.
- enum:
- - ConfigMap
- - Secret
- type: string
- name:
- description: Name of the resource containing the CA Certificate.
- type: string
- optional:
- description: Optional specifies whether the resource and its
- key must exist.
- type: boolean
- required:
- - name
- type: object
- clientSecretName:
- description: ClientSecretName references the secret used by the
- operator to validate the https endpoint.
- type: string
- externalController:
- description: |-
- ExternalController references an external controller.
- When set, the Operator uses the external cluster to register satellites.
- properties:
- url:
- description: URL of the external controller.
- minLength: 3
- type: string
- required:
- - url
- type: object
- name:
- description: Name of the LinstorCluster resource controlling this
- satellite.
- type: string
- type: object
- deletionPolicy:
- default: Retain
- description: |-
- DeletionPolicy configures the way LinstorSatellite resources are deleted.
-
- A LinstorSatellite may be deleted because:
- * It no longer matches the affinity and node selector of the LinstorCluster resource.
- * The node it references has been removed from Kubernetes.
- * It was manually deleted outside the Operator.
-
- A LinstorSatellite may store the last copy of a volume, in which case it is not desirable to unconditionally remove
- the satellite from the cluster. For this reason, the following deletion policies exist:
-
- * DeletionPolicyEvacuate will start evacuation of the LINSTOR Satellite and wait until it completes before removing the LinstorSatellite object, comparable to the "linstor node evacuate" command.
- * DeletionPolicyRetain will retain the LINSTOR Satellite, keeping it registered in LINSTOR, but removing associated Kubernetes resources.
- * DeletionPolicyDelete will remove the LINSTOR Satellite from the LINSTOR Cluster without prior eviction, comparable to the "linstor node lost" command.
- enum:
- - Evacuate
- - Retain
- - Delete
- type: string
- evacuationStrategy:
- description: EvacuationStrategy configures the evacuation of volumes
- from a Satellite when DeletionPolicy "Evacuate" is used.
- properties:
- attachedVolumeReattachTimeout:
- default: 5m
- description: |-
- AttachedVolumeReattachTimeout configures how long evacuation waits for attached volumes to reattach on
- different nodes. Setting this to 0 disable this evacuation step.
- type: string
- unattachedVolumeAttachTimeout:
- default: 5m
- description: |-
- UnattachedVolumeAttachTimeout configures how long evacuation waits for unattached volumes to attach on
- different nodes. Setting this to 0 disable this evacuation step.
- type: string
- type: object
- internalTLS:
- description: |-
- InternalTLS configures secure communication for the LINSTOR Satellite.
-
- If set, the control traffic between LINSTOR Controller and Satellite will be encrypted using mTLS.
- The Controller will use the client key from `LinstorCluster.spec.internalTLS` when connecting.
- nullable: true
- properties:
- caReference:
- description: |-
- CAReference configures the CA certificate to use when validating TLS certificates.
- If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate.
- properties:
- key:
- default: ca.crt
- description: |-
- Key to select in the resource.
- Defaults to ca.crt if not specified.
- type: string
- kind:
- default: Secret
- description: Kind of the resource containing the CA Certificate,
- either a ConfigMap or Secret.
- enum:
- - ConfigMap
- - Secret
- type: string
- name:
- description: Name of the resource containing the CA Certificate.
- type: string
- optional:
- description: Optional specifies whether the resource and its
- key must exist.
- type: boolean
- required:
- - name
- type: object
- certManager:
- description: |-
- CertManager references a cert-manager Issuer or ClusterIssuer.
- If set, a Certificate resource will be created, provisioning the secret references in SecretName using the
- issuer configured here.
- properties:
- group:
- description: Group of the resource being referred to.
- type: string
- kind:
- description: Kind of the resource being referred to.
- type: string
- name:
- description: Name of the resource being referred to.
- type: string
- required:
- - name
- type: object
- secretName:
- description: SecretName references a secret holding the TLS key
- and certificates.
- type: string
- tlsHandshakeDaemon:
- description: |-
- TLSHandshakeDaemon enables tlshd for establishing TLS sessions for use by DRBD.
-
- If enabled, adds a new sidecar to the LINSTOR Satellite that runs the tlshd handshake daemon.
- The daemon uses the TLS certificate and key to establish secure connections on behalf of DRBD.
- type: boolean
- type: object
- ipFamilies:
- description: |-
- IPFamilies configures the IP Family (IPv4 or IPv6) to use to connect to the LINSTOR Satellite.
-
- If set, the control traffic between LINSTOR Controller and Satellite will use only the given IP Family.
- If not set, the Operator will configure all families found in the Satellites Pods' Status.
- items:
- description: IPFamily represents the IP Family (IPv4 or IPv6).
- enum:
- - IPv4
- - IPv6
- type: string
- type: array
- patches:
- description: |-
- Patches is a list of kustomize patches to apply.
-
- See https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/ for how to create patches.
- items:
- description: Patch represent either a Strategic Merge Patch or a
- JSON patch and its targets.
- properties:
- options:
- description: Options is a list of options for the patch
- properties:
- allowKindChange:
- description: AllowKindChange allows kind changes to the
- resource.
- type: boolean
- allowNameChange:
- description: AllowNameChange allows name changes to the
- resource.
- type: boolean
- type: object
- patch:
- description: Patch is the content of a patch.
- minLength: 1
- type: string
- target:
- description: Target points to the resources that the patch is
- applied to
- properties:
- annotationSelector:
- description: |-
- AnnotationSelector is a string that follows the label selection expression
- https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api
- It matches against the resource annotations.
- type: string
- group:
- type: string
- kind:
- type: string
- labelSelector:
- description: |-
- LabelSelector is a string that follows the label selection expression
- https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api
- It matches against the resource labels.
- type: string
- name:
- description: Name of the resource.
- type: string
- namespace:
- description: Namespace the resource belongs to, if it can
- belong to a namespace.
- type: string
- version:
- type: string
- type: object
- required:
- - patch
- type: object
- type: array
- properties:
- description: Properties is a list of properties to set on the node.
- items:
- properties:
- expandFrom:
- description: |-
- ExpandFrom can reference multiple resource fields at once.
- It either sets the property to an aggregate value based on matched resource fields, or expands to multiple
- properties.
- properties:
- delimiter:
- description: Delimiter used to join multiple key and value
- pairs together.
- type: string
- nameTemplate:
- description: |-
- NameTemplate defines how the property key is expanded.
- If set, the template is appended to the defined property name, creating multiple properties instead of one
- aggregate.
- * $1 is replaced with the matched key.
- * $2 is replaced with the matched value.
- type: string
- nodeFieldRef:
- description: Select a field of the node. Supports `metadata.name`,
- `metadata.labels['']`, `metadata.annotations['']`.
- minLength: 1
- type: string
- valueTemplate:
- description: |-
- ValueTemplate defines how the property value is expanded.
- * $1 is replaced with the matched key.
- * $2 is replaced with the matched value.
- type: string
- required:
- - nodeFieldRef
- type: object
- name:
- description: Name of the property to set.
- minLength: 1
- type: string
- optional:
- description: Optional values are only set if they have a non-empty
- value
- type: boolean
- value:
- description: Value to set the property to.
- type: string
- valueFrom:
- description: ValueFrom sets the value from an existing resource.
- properties:
- nodeFieldRef:
- description: Select a field of the node. Supports `metadata.name`,
- `metadata.labels['']`, `metadata.annotations['']`.
- minLength: 1
- type: string
- required:
- - nodeFieldRef
- type: object
- required:
- - name
- type: object
- type: array
- x-kubernetes-list-map-keys:
- - name
- x-kubernetes-list-type: map
- repository:
- description: Repository used to pull workload images.
- type: string
- storagePools:
- description: StoragePools is a list of storage pools to configure
- on the node.
- items:
- properties:
- filePool:
- description: Configures a file system based storage pool, allocating
- a regular file per volume.
- properties:
- directory:
- description: Directory is the path to the host directory
- used to store volume data.
- type: string
- type: object
- fileThinPool:
- description: Configures a file system based storage pool, allocating
- a sparse file per volume.
- properties:
- directory:
- description: Directory is the path to the host directory
- used to store volume data.
- type: string
- type: object
- lvmPool:
- description: Configures a LVM Volume Group as storage pool.
- properties:
- volumeGroup:
- type: string
- type: object
- lvmThinPool:
- description: Configures a LVM Thin Pool as storage pool.
- properties:
- thinPool:
- description: ThinPool is the name of the thinpool LV (without
- VG prefix).
- type: string
- volumeGroup:
- type: string
- type: object
- name:
- description: Name of the storage pool in linstor.
- minLength: 3
- type: string
- properties:
- description: Properties to set on the storage pool.
- items:
- properties:
- expandFrom:
- description: |-
- ExpandFrom can reference multiple resource fields at once.
- It either sets the property to an aggregate value based on matched resource fields, or expands to multiple
- properties.
- properties:
- delimiter:
- description: Delimiter used to join multiple key and
- value pairs together.
- type: string
- nameTemplate:
- description: |-
- NameTemplate defines how the property key is expanded.
- If set, the template is appended to the defined property name, creating multiple properties instead of one
- aggregate.
- * $1 is replaced with the matched key.
- * $2 is replaced with the matched value.
- type: string
- nodeFieldRef:
- description: Select a field of the node. Supports
- `metadata.name`, `metadata.labels['']`, `metadata.annotations['']`.
- minLength: 1
- type: string
- valueTemplate:
- description: |-
- ValueTemplate defines how the property value is expanded.
- * $1 is replaced with the matched key.
- * $2 is replaced with the matched value.
- type: string
- required:
- - nodeFieldRef
- type: object
- name:
- description: Name of the property to set.
- minLength: 1
- type: string
- optional:
- description: Optional values are only set if they have
- a non-empty value
- type: boolean
- value:
- description: Value to set the property to.
- type: string
- valueFrom:
- description: ValueFrom sets the value from an existing
- resource.
- properties:
- nodeFieldRef:
- description: Select a field of the node. Supports
- `metadata.name`, `metadata.labels['']`, `metadata.annotations['']`.
- minLength: 1
- type: string
- required:
- - nodeFieldRef
- type: object
- required:
- - name
- type: object
- type: array
- x-kubernetes-list-map-keys:
- - name
- x-kubernetes-list-type: map
- source:
- properties:
- hostDevices:
- description: HostDevices is a list of device paths used
- to configure the given pool.
- items:
- type: string
- minItems: 1
- type: array
- type: object
- zfsPool:
- description: Configures a ZFS system based storage pool, allocating
- zvols from the given zpool.
- properties:
- zPool:
- description: ZPool is the name of the ZFS zpool.
- type: string
- type: object
- zfsThinPool:
- description: Configures a ZFS system based storage pool, allocating
- sparse zvols from the given zpool.
- properties:
- zPool:
- description: ZPool is the name of the ZFS zpool.
- type: string
- type: object
- required:
- - name
- type: object
- type: array
- required:
- - clusterRef
- type: object
- status:
- description: LinstorSatelliteStatus defines the observed state of LinstorSatellite
- properties:
- availableCapacityBytes:
- description: The number of bytes in total in all storage pools on
- this Satellite.
- format: int64
- type: integer
- capacity:
- description: Capacity mirrors the information from TotalCapacityBytes
- and FreeCapacityBytes in a human-readable string.
- type: string
- conditions:
- description: Current LINSTOR Satellite state
- items:
- description: Condition contains details for one aspect of the current
- state of this API Resource.
- properties:
- lastTransitionTime:
- description: |-
- lastTransitionTime is the last time the condition transitioned from one status to another.
- This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
- format: date-time
- type: string
- message:
- description: |-
- message is a human readable message indicating details about the transition.
- This may be an empty string.
- maxLength: 32768
- type: string
- observedGeneration:
- description: |-
- observedGeneration represents the .metadata.generation that the condition was set based upon.
- For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
- with respect to the current state of the instance.
- format: int64
- minimum: 0
- type: integer
- reason:
- description: |-
- reason contains a programmatic identifier indicating the reason for the condition's last transition.
- Producers of specific condition types may define expected values and meanings for this field,
- and whether the values are considered a guaranteed API.
- The value should be a CamelCase string.
- This field may not be empty.
- maxLength: 1024
- minLength: 1
- pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
- type: string
- status:
- description: status of the condition, one of True, False, Unknown.
- enum:
- - "True"
- - "False"
- - Unknown
- type: string
- type:
- description: type of condition in CamelCase or in foo.example.com/CamelCase.
- maxLength: 316
- pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
- type: string
- required:
- - lastTransitionTime
- - message
- - reason
- - status
- - type
- type: object
- type: array
- x-kubernetes-list-map-keys:
- - type
- x-kubernetes-list-type: map
- deviceLayers:
- description: DeviceLayers lists the device layers (LUKS, CACHE, etc...)
- this Satellite supports.
- items:
- type: string
- type: array
- freeCapacityBytes:
- description: The number of bytes free in all storage pools on this
- Satellite.
- format: int64
- type: integer
- numberOfSnapshots:
- description: The number of snapshots on this Satellite.
- format: int32
- type: integer
- numberOfVolumes:
- description: The number of volumes on this Satellite.
- format: int32
- type: integer
- storageProviders:
- description: StorageProviders lists the storage providers (LVM, ZFS,
- etc...) this Satellite supports.
- items:
- type: string
- type: array
- type: object
- type: object
- served: true
- storage: true
- subresources:
- status: {}
-{{ end }}
diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/deployment.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/deployment.yaml
index c67501972..0c3209e19 100644
--- a/packages/system/piraeus-operator/charts/piraeus/templates/deployment.yaml
+++ b/packages/system/piraeus-operator/charts/piraeus/templates/deployment.yaml
@@ -3,6 +3,7 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "piraeus-operator.fullname" . }}-controller-manager
+ namespace: {{ .Release.Namespace }}
labels:
{{- include "piraeus-operator.labels" . | nindent 4 }}
spec:
@@ -70,6 +71,9 @@ spec:
- mountPath: /tmp/k8s-webhook-server/serving-certs
name: cert
readOnly: true
+ {{- with .Values.operator.extraVolumeMounts }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
{{- if .Values.kubeRbacProxy.enabled }}
- args:
- --secure-listen-address=0.0.0.0:8443
@@ -91,8 +95,11 @@ spec:
securityContext:
{{- toYaml .Values.kubeRbacProxy.securityContext | nindent 12}}
volumeMounts:
- - mountPath: /etc/tls
- name: cert
+ - mountPath: /etc/tls
+ name: cert
+ {{- with .Values.kubeRbacProxy.extraVolumeMounts }}
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
{{- end }}
securityContext:
runAsNonRoot: true
@@ -100,6 +107,10 @@ spec:
{{ toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "piraeus-operator.serviceAccountName" . }}
+ {{- with .Values.imagePullSecrets }}
+ imagePullSecrets:
+ {{- toYaml . | nindent 8 }}
+ {{- end }}
terminationGracePeriodSeconds: 10
priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }}
nodeSelector:
@@ -113,3 +124,6 @@ spec:
secret:
defaultMode: 420
secretName: {{ include "piraeus-operator.certifcateName" . }}
+ {{- with .Values.extraVolumes }}
+ {{- toYaml . | nindent 6 }}
+ {{- end }}
diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/extra-objects.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/extra-objects.yaml
new file mode 100644
index 000000000..b0ec6fad2
--- /dev/null
+++ b/packages/system/piraeus-operator/charts/piraeus/templates/extra-objects.yaml
@@ -0,0 +1,15 @@
+{{- /* Normalize extraObjects to a list, easier to loop over */ -}}
+{{- $extraObjects := .Values.extraManifests | default (list) -}}
+
+{{- if kindIs "map" $extraObjects -}}
+ {{- $extraObjects = values $extraObjects -}}
+{{- end -}}
+
+{{- range $extraObjects }}
+---
+ {{- if kindIs "map" . }}
+ {{- tpl (toYaml .) $ | nindent 0 }}
+ {{- else if kindIs "string" . }}
+ {{- tpl . $ | nindent 0 }}
+ {{- end }}
+{{- end }}
diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/metrics-service.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/metrics-service.yaml
index d1a24e3ca..49dee196a 100644
--- a/packages/system/piraeus-operator/charts/piraeus/templates/metrics-service.yaml
+++ b/packages/system/piraeus-operator/charts/piraeus/templates/metrics-service.yaml
@@ -2,6 +2,7 @@ apiVersion: v1
kind: Service
metadata:
name: {{ include "piraeus-operator.fullname" . }}-controller-manager-metrics-service
+ namespace: {{ .Release.Namespace }}
labels:
{{- include "piraeus-operator.labels" . | nindent 4 }}
spec:
diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/rbac.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/rbac.yaml
index c532e02d4..56b7d019b 100644
--- a/packages/system/piraeus-operator/charts/piraeus/templates/rbac.yaml
+++ b/packages/system/piraeus-operator/charts/piraeus/templates/rbac.yaml
@@ -15,7 +15,6 @@ rules:
- events
- persistentvolumes
- pods
- - secrets
- serviceaccounts
- services
verbs:
@@ -59,6 +58,13 @@ rules:
- pods/eviction
verbs:
- create
+- apiGroups:
+ - ""
+ resources:
+ - pods/exec
+ verbs:
+ - create
+ - get
- apiGroups:
- apiextensions.k8s.io
resources:
@@ -71,6 +77,12 @@ rules:
- patch
- update
- watch
+- apiGroups:
+ - apiextensions.k8s.io
+ resources:
+ - customresourcedefinitions/status
+ verbs:
+ - patch
- apiGroups:
- apps
resources:
@@ -235,7 +247,6 @@ rules:
- snapshot.storage.k8s.io
resources:
- volumesnapshotclasses
- - volumesnapshots
verbs:
- get
- list
@@ -257,6 +268,15 @@ rules:
verbs:
- patch
- update
+- apiGroups:
+ - snapshot.storage.k8s.io
+ resources:
+ - volumesnapshots
+ verbs:
+ - get
+ - list
+ - update
+ - watch
- apiGroups:
- storage.k8s.io
resources:
@@ -283,6 +303,7 @@ rules:
- storage.k8s.io
resources:
- storageclasses
+ - volumeattributesclasses
verbs:
- get
- list
@@ -306,8 +327,30 @@ rules:
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
+metadata:
+ name: {{ include "piraeus-operator.fullname" . }}-controller-manager
+ namespace: {{ .Release.Namespace }}
+ labels:
+ {{- include "piraeus-operator.labels" . | nindent 4 }}
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - secrets
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
metadata:
name: {{ include "piraeus-operator.fullname" . }}-leader-election
+ namespace: {{ .Release.Namespace }}
labels:
{{- include "piraeus-operator.labels" . | nindent 4 }}
rules:
diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/rolebindings.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/rolebindings.yaml
index 97e53b952..fc5212cc4 100644
--- a/packages/system/piraeus-operator/charts/piraeus/templates/rolebindings.yaml
+++ b/packages/system/piraeus-operator/charts/piraeus/templates/rolebindings.yaml
@@ -17,6 +17,22 @@ subjects:
namespace: {{ .Release.Namespace }}
---
apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ include "piraeus-operator.fullname" . }}-controller-manager
+ namespace: {{ .Release.Namespace }}
+ labels:
+ {{- include "piraeus-operator.labels" . | nindent 4 }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ include "piraeus-operator.fullname" . }}-controller-manager
+subjects:
+- kind: ServiceAccount
+ name: {{ include "piraeus-operator.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: {{ include "piraeus-operator.fullname" . }}-controller-manager
diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/serviceaccount.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/serviceaccount.yaml
index 89315de95..c4997dd24 100644
--- a/packages/system/piraeus-operator/charts/piraeus/templates/serviceaccount.yaml
+++ b/packages/system/piraeus-operator/charts/piraeus/templates/serviceaccount.yaml
@@ -4,6 +4,7 @@ apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "piraeus-operator.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace }}
labels:
{{- include "piraeus-operator.labels" . | nindent 4 }}
{{ end }}
diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/validating-webhook-configuration.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/validating-webhook-configuration.yaml
index 5a6c5c9e6..04d05bfaf 100644
--- a/packages/system/piraeus-operator/charts/piraeus/templates/validating-webhook-configuration.yaml
+++ b/packages/system/piraeus-operator/charts/piraeus/templates/validating-webhook-configuration.yaml
@@ -9,6 +9,7 @@ apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: {{ include "piraeus-operator.fullname" . }}
+ namespace: {{ .Release.Namespace }}
labels:
{{- include "piraeus-operator.labels" . | nindent 4 }}
spec:
@@ -36,6 +37,7 @@ apiVersion: v1
kind: Secret
metadata:
name: {{ include "piraeus-operator.certifcateName" . }}
+ namespace: {{ .Release.Namespace }}
labels:
{{- include "piraeus-operator.labels" . | nindent 4 }}
type: kubernetes.io/tls
diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/webhook-service.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/webhook-service.yaml
index d360269bc..57f8966c9 100644
--- a/packages/system/piraeus-operator/charts/piraeus/templates/webhook-service.yaml
+++ b/packages/system/piraeus-operator/charts/piraeus/templates/webhook-service.yaml
@@ -2,6 +2,7 @@ apiVersion: v1
kind: Service
metadata:
name: {{ include "piraeus-operator.fullname" . }}-webhook-service
+ namespace: {{ .Release.Namespace }}
labels:
{{- include "piraeus-operator.labels" . | nindent 4 }}
spec:
diff --git a/packages/system/piraeus-operator/charts/piraeus/values.yaml b/packages/system/piraeus-operator/charts/piraeus/values.yaml
index bcc4dc9af..ff0d92b5a 100644
--- a/packages/system/piraeus-operator/charts/piraeus/values.yaml
+++ b/packages/system/piraeus-operator/charts/piraeus/values.yaml
@@ -29,6 +29,12 @@ operator:
# cpu: 100m
# memory: 128Mi
+ ## Additional volume mounts
+ extraVolumeMounts: [ ]
+ # - name: clusterapi-kubeconfig
+ # mountPath: /etc/operator
+ # readOnly: true
+
kubeRbacProxy:
enabled: false
image:
@@ -54,6 +60,16 @@ kubeRbacProxy:
# cpu: 100m
# memory: 128Mi
+ ## Additional volume mounts
+ extraVolumeMounts: [ ]
+
+
+## Additional volumes
+extraVolumes: []
+# - name: clusterapi-kubeconfig
+# secret:
+# secretName: super-secret
+
webhook:
timeoutSeconds: 2
failurePolicy: Fail
@@ -111,3 +127,30 @@ imageConfigOverride: []
# tag: my-custom-tag
# Results in the image example.com/piraeus/linstor-csi:my-custom-tag being used.
# See templates/config.yaml for available components.
+
+## Extra manifests to deploy. Can be of type dict or list.
+## If dict, keys are ignored and only values are used.
+## Items contained within extraObjects can be defined as dict or string and are passed through tpl.
+extraManifests: null
+ # - apiVersion: v1
+ # kind: ConfigMap
+ # metadata:
+ # labels:
+ # name: piraeus-extra
+ # data:
+ # extra-data: "value"
+ #
+ # can also be defined as a string, useful for templating field names
+ # - |
+ # apiVersion: v1
+ # kind: Secret
+ # type: Opaque
+ # metadata:
+ # name: super-secret
+ # labels:
+ # {{- range $key, $value := .Values.commonLabels }}
+ # {{ $key }}: {{ $value }}
+ # {{- end }}
+ # data:
+ # plaintext: Zm9vYmFy
+# templated: '{{ print "foobar" | upper | b64enc }}'
From e1ed8c771938e31c9130b06931d649ef85761e49 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Fri, 13 Nov 2026 02:23:32 +0300
Subject: [PATCH 089/114] fix(linstor): update linstor-csi to v1.13.2
piraeus-operator v2.12.1 is released against LINSTOR CSI v1.13.2, and
the release line carries IO QoS limits through the qos.linbit.com/*
StorageClass parameters.
The operator gives the CSI pods their client certificate as mounted
files through LS_USER_CERTIFICATE_FILE and LS_USER_KEY_FILE, instead
of inline values in LS_USER_CERTIFICATE and LS_USER_KEY. The
linstor-wait-until v0.3.1 in this image is built on a golinstor
release that reads only the inline variables, so its init containers
present no client certificate, the LINSTOR API closes the TLS
connection, and they retry on EOF forever. Move to
linstor-wait-until v1.0.1, the version upstream v1.13.2 pins.
The Protocol C override for dual attach is dropped. Since v1.12 the
driver no longer sets allow-two-primaries itself: it asks LINSTOR to
manage dual-primary through make-available, and LINSTOR 1.35 sets
protocol C on the connections that need it. Upstream closed the
equivalent change as superseded for that reason
(piraeusdatastore/linstor-csi#435).
The relocate-after-clone patch is carried forward. Volume IDs became
a struct for consistency groups; Clone and VolFromSnap handle
standalone resources, so relocation keys on the resource name. The
enumer output is regenerated rather than hand-patched.
Upstream now pins grpc v1.84.0, which builds on Go 1.27 and includes
the fix for GO-2026-6443, so the grpc override is no longer needed.
The golden-image clone guard in vm-disk was re-checked: parameter
parsing for allowRemoteVolumeAccess and layerList, and the default
derived from the layer list, are unchanged in this range.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
packages/system/linstor/Makefile | 2 +-
.../linstor/images/linstor-csi/Dockerfile | 11 +-
.../001-relocate-after-clone-restore.diff | 103 +++++++-------
...2-protocol-c-override-for-dual-attach.diff | 132 ------------------
4 files changed, 57 insertions(+), 191 deletions(-)
delete mode 100644 packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff
diff --git a/packages/system/linstor/Makefile b/packages/system/linstor/Makefile
index 74e3f5aef..5e44ebade 100644
--- a/packages/system/linstor/Makefile
+++ b/packages/system/linstor/Makefile
@@ -8,7 +8,7 @@ LINSTOR_VERSION ?= 1.33.3
# The golden-image clone guard in packages/apps/vm-disk/templates/dv.yaml
# reads allowRemoteVolumeAccess and layerList the way this release does and
# has to be re-checked against the new one.
-LINSTOR_CSI_VERSION ?= v1.11.2
+LINSTOR_CSI_VERSION ?= v1.13.2
test:
helm unittest .
diff --git a/packages/system/linstor/images/linstor-csi/Dockerfile b/packages/system/linstor/images/linstor-csi/Dockerfile
index 50e755090..3ff1d154d 100644
--- a/packages/system/linstor/images/linstor-csi/Dockerfile
+++ b/packages/system/linstor/images/linstor-csi/Dockerfile
@@ -1,8 +1,8 @@
FROM --platform=$BUILDPLATFORM golang:1.27@sha256:3680233e3204827fbdc66088528ae6d4b3d034f51d03a99d454f6de034888244 AS builder
ARG GOPROXY
-ARG VERSION=v1.11.2
-ARG LINSTOR_WAIT_UNTIL_VERSION=v0.3.1
+ARG VERSION=v1.13.2
+ARG LINSTOR_WAIT_UNTIL_VERSION=v1.0.1
ARG TARGETARCH
ARG TARGETOS
@@ -15,13 +15,6 @@ RUN git apply /patches/*.diff
RUN for attempt in 1 2 3; do go mod download && break; [ "$attempt" = 3 ] && exit 1; sleep $((attempt * 10)); done
-# Upstream pins grpc v1.81.0, which uses http2.TrailerPrefix; on Go 1.27
-# golang.org/x/net builds http2 on top of net/http and no longer defines it.
-# v1.83.2 also carries the fix for GO-2026-6443, a server panic reachable
-# through the CSI gRPC endpoint.
-RUN go get google.golang.org/grpc@v1.83.2 && \
- go mod tidy
-
# go build never compiles a _test.go, so nothing else runs the tests of the
# packages the patches touch. Each package is tested from its own directory:
# that finds the module it belongs to, and an empty list runs nothing.
diff --git a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff
index 0e2726708..29c209444 100644
--- a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff
+++ b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff
@@ -1,38 +1,37 @@
diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go
-index ac4651d..46d565d 100644
--- a/pkg/client/linstor.go
+++ b/pkg/client/linstor.go
-@@ -462,6 +462,14 @@ func (s *Linstor) Clone(ctx context.Context, vol, src *volume.Info, params *volu
+@@ -498,6 +498,14 @@
return err
}
+ if params.RelocateAfterClone {
+ logger.Debug("relocate resources to optimal nodes")
+
-+ if err := s.relocateResources(ctx, vol.ID, rGroup.Name); err != nil {
++ if err := s.relocateResources(ctx, vol.ResourceName, rGroup.Name); err != nil {
+ logger.WithError(err).Warn("resource relocation failed, volume is still usable")
+ }
+ }
+
logger.Debug("reconcile extra properties")
- err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties})
-@@ -1297,6 +1305,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v
+ err = s.client.ResourceDefinitions.Modify(ctx, vol.ResourceName, lapi.GenericPropsModify{OverrideProps: vol.Properties})
+@@ -1614,6 +1622,14 @@
return err
}
+ if snapParams != nil && snapParams.RelocateAfterRestore {
+ logger.Debug("relocate resources to optimal nodes")
+
-+ if err := s.relocateResources(ctx, vol.ID, rGroup.Name); err != nil {
++ if err := s.relocateResources(ctx, vol.ResourceName, rGroup.Name); err != nil {
+ logger.WithError(err).Warn("resource relocation failed, volume is still usable")
+ }
+ }
+
logger.Debug("reconcile extra properties")
- err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties})
-@@ -1487,9 +1503,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu
+ err = s.client.ResourceDefinitions.Modify(ctx, vol.ResourceName, lapi.GenericPropsModify{OverrideProps: vol.Properties})
+@@ -1818,9 +1834,8 @@
return fmt.Errorf("snapshot '%s' not deployed on any node", snap.Name)
}
@@ -44,7 +43,7 @@ index ac4651d..46d565d 100644
for _, snapNode := range snap.Nodes {
if err := s.NodeAvailable(ctx, snapNode); err != nil {
logger.WithField("selected node candidate", snapNode).WithError(err).Debug("node is not available")
-@@ -1497,13 +1512,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu
+@@ -1828,15 +1843,25 @@
}
if slices.Contains(preferredNodes, snapNode) {
@@ -58,8 +57,8 @@ index ac4651d..46d565d 100644
}
+
+ available = append(available, snapNode)
-+ }
-+
+ }
+
+ // Pick a random node from preferred candidates, falling back to any available node.
+ // Randomization distributes restore load across snapshot nodes, preventing all
+ // clones of the same source from concentrating on a single node.
@@ -71,13 +70,20 @@ index ac4651d..46d565d 100644
+ var selectedNode string
+ if len(candidates) > 0 {
+ selectedNode = candidates[rand.Intn(len(candidates))]
- }
-
++ }
++
if selectedNode == "" {
-@@ -1696,6 +1721,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In
- return nil
- }
-
+ return fmt.Errorf("no node [%s] available for snapshot '%s'", strings.Join(snap.Nodes, ","), snap.Name)
+ }
+@@ -2030,6 +2055,114 @@
+ err = volumeScheduler.Create(ctx, vol.ResourceName, params, topologies)
+ if err != nil {
+ return err
++ }
++
++ return nil
++}
++
+const propertyRelocationTriggered = "Aux/csi-relocation-triggered"
+
+// relocateResources migrates replicas from their current nodes to the nodes that
@@ -181,19 +187,13 @@ index ac4651d..46d565d 100644
+ logger.WithError(err).Warn("migrate-disk failed, skipping this pair")
+ continue
+ }
-+ }
-+
-+ return nil
-+}
-+
- // FindSnapsByID searches the snapshot in the backend
- func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) {
- snapshotId, err := volume.ParseSnapshotId(id)
+ }
+
+ return nil
diff --git a/pkg/volume/parameter.go b/pkg/volume/parameter.go
-index 39acd95..54d1dfb 100644
--- a/pkg/volume/parameter.go
+++ b/pkg/volume/parameter.go
-@@ -50,6 +50,7 @@ const (
+@@ -51,6 +51,7 @@
nfsservicename
nfssquash
nfsrecoveryvolumesize
@@ -201,24 +201,24 @@ index 39acd95..54d1dfb 100644
)
// Parameters configuration for linstor volumes.
-@@ -118,6 +119,8 @@ type Parameters struct {
- // NfsRecoveryVolumeSize sets the volume size (in bytes) of the recovery volume used by the NFS server.
- // Defaults to 300MiB.
- NfsRecoveryVolumeBytes int64
+@@ -122,6 +123,8 @@
+ // IOQoSLimits holds the IO QoS limits parsed from the qos.linbit.com/* parameters. They are stored on
+ // the LINSTOR resource definition and passed to the nri-volume-qos plugin via VolumeAttachment metadata.
+ IOQoSLimits meta.Limits
+ // RelocateAfterClone triggers asynchronous relocation of replicas to optimal nodes after cloning.
+ RelocateAfterClone bool
}
const DefaultDisklessStoragePoolName = "DfltDisklessStorPool"
-@@ -140,6 +143,7 @@ func NewParameters(params map[string]string, topologyPrefix string) (Parameters,
+@@ -144,6 +147,7 @@
NfsServiceName: "linstor-csi-nfs",
NfsSquash: "no_root_squash",
NfsRecoveryVolumeBytes: 300 * 1024 * 1024,
+ RelocateAfterClone: true,
}
- for k, v := range params {
-@@ -287,6 +291,13 @@ func NewParameters(params map[string]string, topologyPrefix string) (Parameters,
+ limits, err := meta.ParseLimits(params)
+@@ -298,6 +302,13 @@
}
p.NfsRecoveryVolumeBytes = s.Value()
@@ -233,10 +233,9 @@ index 39acd95..54d1dfb 100644
}
diff --git a/pkg/volume/paramkey_enumer.go b/pkg/volume/paramkey_enumer.go
-index 5474963..2eb8a7c 100644
--- a/pkg/volume/paramkey_enumer.go
+++ b/pkg/volume/paramkey_enumer.go
-@@ -7,11 +7,11 @@ import (
+@@ -7,11 +7,11 @@
"strings"
)
@@ -251,7 +250,7 @@ index 5474963..2eb8a7c 100644
func (i paramKey) String() string {
if i < 0 || i >= paramKey(len(_paramKeyIndex)-1) {
-@@ -50,9 +50,10 @@ func _paramKeyNoOp() {
+@@ -50,9 +50,10 @@
_ = x[nfsservicename-(23)]
_ = x[nfssquash-(24)]
_ = x[nfsrecoveryvolumesize-(25)]
@@ -262,17 +261,24 @@ index 5474963..2eb8a7c 100644
+var _paramKeyValues = []paramKey{allowremotevolumeaccess, autoplace, clientlist, disklessonremaining, disklessstoragepool, donotplacewithregex, encryption, fsopts, layerlist, mountopts, nodelist, placementcount, placementpolicy, replicasondifferent, replicasonsame, sizekib, storagepool, postmountxfsopts, resourcegroup, usepvcname, overprovision, xreplicasondifferent, nfsconfigtemplate, nfsservicename, nfssquash, nfsrecoveryvolumesize, relocateafterclone}
var _paramKeyNameToValueMap = map[string]paramKey{
- _paramKeyName[0:23]: allowremotevolumeaccess,
-@@ -107,6 +108,8 @@ var _paramKeyNameToValueMap = map[string]paramKey{
+ _paramKeyName[0:23]: allowremotevolumeaccess,
+@@ -81,6 +82,7 @@
+ _paramKeyName[310:324]: nfsservicename,
+ _paramKeyName[324:333]: nfssquash,
+ _paramKeyName[333:354]: nfsrecoveryvolumesize,
++ _paramKeyName[354:372]: relocateafterclone,
+ }
+
+ var _paramKeyLowerNameToValueMap = map[string]paramKey{
+@@ -110,6 +112,7 @@
+ _paramKeyLowerName[310:324]: nfsservicename,
_paramKeyLowerName[324:333]: nfssquash,
- _paramKeyName[333:354]: nfsrecoveryvolumesize,
_paramKeyLowerName[333:354]: nfsrecoveryvolumesize,
-+ _paramKeyName[354:372]: relocateafterclone,
+ _paramKeyLowerName[354:372]: relocateafterclone,
}
var _paramKeyNames = []string{
-@@ -136,6 +139,7 @@ var _paramKeyNames = []string{
+@@ -139,6 +142,7 @@
_paramKeyName[310:324],
_paramKeyName[324:333],
_paramKeyName[333:354],
@@ -281,18 +287,17 @@ index 5474963..2eb8a7c 100644
// paramKeyString retrieves an enum value from the enum constants string name.
diff --git a/pkg/volume/snapshot_params.go b/pkg/volume/snapshot_params.go
-index d167cb8..50b70fb 100644
--- a/pkg/volume/snapshot_params.go
+++ b/pkg/volume/snapshot_params.go
-@@ -35,6 +35,7 @@ type SnapshotParameters struct {
- LinstorTargetUrl string `json:"linstor-target-url,omitempty"`
- LinstorTargetClusterID string `json:"linstor-target-cluster-id,omitempty"`
- LinstorTargetStoragePool string `json:"linstor-target-storage-pool,omitempty"`
-+ RelocateAfterRestore bool `json:"relocate-after-restore,omitempty"`
+@@ -38,6 +38,7 @@
+ LinstorTargetUrl string `json:"linstor-target-url,omitempty"`
+ LinstorTargetClusterID string `json:"linstor-target-cluster-id,omitempty"`
+ LinstorTargetStoragePool string `json:"linstor-target-storage-pool,omitempty"`
++ RelocateAfterRestore bool `json:"relocate-after-restore,omitempty"`
}
func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParameters, error) {
-@@ -91,6 +92,13 @@ func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParamete
+@@ -116,6 +117,13 @@
p.LinstorTargetClusterID = v
case "/linstor-target-storage-pool":
p.LinstorTargetStoragePool = v
diff --git a/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff b/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff
deleted file mode 100644
index 279c636b1..000000000
--- a/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff
+++ /dev/null
@@ -1,132 +0,0 @@
-diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go
-index ac4651d..9d75c79 100644
---- a/pkg/client/linstor.go
-+++ b/pkg/client/linstor.go
-@@ -653,11 +653,35 @@ func (s *Linstor) Attach(ctx context.Context, volId, node string, rwxBlock bool)
- }
-
- if otherResInUse > 0 && rwxBlock {
-- rdPropsModify := lapi.GenericPropsModify{OverrideProps: map[string]string{
-+ rdProps := map[string]string{
- linstor.PropertyAllowTwoPrimaries: "yes",
-- }}
-+ }
-
-- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify)
-+ // DRBD requires Protocol C whenever allow-two-primaries is enabled.
-+ // If the resource is configured with Protocol A or B (typically
-+ // through its resource-group), drbdadm adjust on the satellites
-+ // fails with "Protocol C required" (errno 139) and live migration
-+ // cannot proceed. Override Protocol to C on the resource-definition
-+ // itself so it applies to every connection (including diskless
-+ // peers, e.g. a TieBreaker). The marker lets Detach revert exactly
-+ // the override we installed without disturbing operator-set props.
-+ proto, protoErr := s.getEffectiveDrbdProtocol(ctx, volId)
-+ if protoErr != nil {
-+ s.log.WithError(protoErr).WithField("volume", volId).
-+ Warn("failed to determine effective DRBD protocol; skipping Protocol=C override for dual-attach")
-+ } else if proto == "A" || proto == "B" {
-+ s.log.WithFields(logrus.Fields{
-+ "volume": volId,
-+ "protocol": proto,
-+ }).Info("installing Protocol=C override on resource-definition for dual-attach")
-+
-+ rdProps[linstor.PropertyDrbdNetProtocol] = "C"
-+ rdProps[linstor.PropertyCsiProtocolOverride] = "yes"
-+ }
-+
-+ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{
-+ OverrideProps: rdProps,
-+ })
- if err != nil {
- return "", err
- }
-@@ -774,11 +798,26 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error {
- }
-
- if resInUse == 1 {
-- rdPropsModify := lapi.GenericPropsModify{DeleteProps: []string{
-- linstor.PropertyAllowTwoPrimaries,
-- }}
-+ deleteProps := []string{linstor.PropertyAllowTwoPrimaries}
-+
-+ // Drop the Protocol=C override only if Attach installed it (gated by
-+ // our marker), so an operator-set Protocol property on the
-+ // resource-definition is preserved.
-+ rd, getErr := s.client.ResourceDefinitions.Get(ctx, volId)
-+ if getErr != nil {
-+ log.WithError(getErr).Warn("failed to fetch resource-definition props; skipping Protocol=C override removal")
-+ } else if rd.Props[linstor.PropertyCsiProtocolOverride] != "" {
-+ log.Info("removing Protocol=C override from resource-definition")
-
-- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify)
-+ deleteProps = append(deleteProps,
-+ linstor.PropertyDrbdNetProtocol,
-+ linstor.PropertyCsiProtocolOverride,
-+ )
-+ }
-+
-+ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{
-+ DeleteProps: deleteProps,
-+ })
- if err != nil {
- return nil404(err)
- }
-@@ -805,6 +844,33 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error {
- return nil404(s.client.Resources.Delete(ctx, volId, node))
- }
-
-+// getEffectiveDrbdProtocol returns the DRBD network protocol ("A", "B" or "C")
-+// that LINSTOR will hand to drbdadm for the given resource. Resource-definition
-+// properties take precedence; otherwise the value is inherited from the
-+// resource-group. An empty string is returned when neither level sets the
-+// property (LINSTOR's compiled-in default is "C").
-+func (s *Linstor) getEffectiveDrbdProtocol(ctx context.Context, volId string) (string, error) {
-+ rd, err := s.client.ResourceDefinitions.Get(ctx, volId)
-+ if err != nil {
-+ return "", err
-+ }
-+
-+ if v, ok := rd.Props[linstor.PropertyDrbdNetProtocol]; ok {
-+ return v, nil
-+ }
-+
-+ if rd.ResourceGroupName == "" {
-+ return "", nil
-+ }
-+
-+ rg, err := s.client.ResourceGroups.Get(ctx, rd.ResourceGroupName)
-+ if err != nil {
-+ return "", err
-+ }
-+
-+ return rg.Props[linstor.PropertyDrbdNetProtocol], nil
-+}
-+
- // CapacityBytes returns the amount of free space in the storage pool specified by the params and topology.
- func (s *Linstor) CapacityBytes(ctx context.Context, storagePools []string, overProvision *float64, segments map[string]string) (int64, error) {
- log := s.log.WithField("storage-pools", storagePools).WithField("segments", segments)
-diff --git a/pkg/linstor/const.go b/pkg/linstor/const.go
-index 9a5f79c..c8bc9c3 100644
---- a/pkg/linstor/const.go
-+++ b/pkg/linstor/const.go
-@@ -44,6 +44,19 @@ const (
- // PropertyAllowTwoPrimaries is DRBD option to allow second primary. Mainly used for live-migration.
- PropertyAllowTwoPrimaries = lc.NamespcDrbdNetOptions + "/allow-two-primaries"
-
-+ // PropertyDrbdNetProtocol is the DRBD network replication protocol option
-+ // (A = async, B = semi-sync, C = sync). DRBD requires Protocol C whenever
-+ // allow-two-primaries is enabled, otherwise drbdadm adjust fails with
-+ // "Protocol C required" (errno 139).
-+ PropertyDrbdNetProtocol = lc.NamespcDrbdNetOptions + "/protocol"
-+
-+ // PropertyCsiProtocolOverride marks a resource-connection where this driver
-+ // has installed a temporary Protocol=C override to make a Protocol-A/B
-+ // resource compatible with allow-two-primaries during live migration. The
-+ // marker lets us safely remove only the overrides we set, without touching
-+ // connection-level overrides installed by the operator.
-+ PropertyCsiProtocolOverride = lc.NamespcAuxiliary + "/csi-protocol-override"
-+
- // CreatedForTemporaryDisklessAttach marks a resource as temporary, i.e. it should be removed after it is no longer
- // needed.
- CreatedForTemporaryDisklessAttach = "temporary-diskless-attach"
From c1652a730d77bd3217597a152cdcaafcf1e68970 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Fri, 13 Nov 2026 02:41:08 +0300
Subject: [PATCH 090/114] fix(linstor): update LINSTOR server to v1.35.3
piraeus-operator v2.12.1 ships LINSTOR 1.35.3 as its controller and
satellite version, and the in-repo piraeus-server build has to match it.
LINSTOR 1.34 raised the minimum Java to 21, so the build and runtime
stages move from bookworm to trixie, and the system Gradle used by
debian/rules follows the upstream wrapper to 8.14.2.
Six carried patches are part of upstream by v1.35.3 and are dropped;
the stale-bitmap adjust retry is rebased onto the new DrbdAdm.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
packages/system/linstor/Makefile | 2 +-
.../linstor/images/piraeus-server/Dockerfile | 9 +-
.../images/piraeus-server/patches/README.md | 34 +-
.../patches/allow-toggle-disk-retry.diff | 261 -------
.../patches/fix-duplicate-tcp-ports.diff | 137 ----
.../patches/fix-luks-header-size.diff | 570 ---------------
.../patches/fix-min-io-probe-device-wait.diff | 660 ------------------
.../piraeus-server/patches/native-protoc.diff | 10 +-
.../retry-adjust-after-stale-bitmap.diff | 25 +-
.../patches/retry-secondary-after-mkfs.diff | 104 ---
.../snapshot-restore-drbd-initialized.diff | 67 --
11 files changed, 33 insertions(+), 1846 deletions(-)
delete mode 100644 packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff
delete mode 100644 packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff
delete mode 100644 packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff
delete mode 100644 packages/system/linstor/images/piraeus-server/patches/fix-min-io-probe-device-wait.diff
delete mode 100644 packages/system/linstor/images/piraeus-server/patches/retry-secondary-after-mkfs.diff
delete mode 100644 packages/system/linstor/images/piraeus-server/patches/snapshot-restore-drbd-initialized.diff
diff --git a/packages/system/linstor/Makefile b/packages/system/linstor/Makefile
index 5e44ebade..e58fc8578 100644
--- a/packages/system/linstor/Makefile
+++ b/packages/system/linstor/Makefile
@@ -4,7 +4,7 @@ export NAMESPACE=cozy-$(NAME)
include ../../../hack/common-envs.mk
include ../../../hack/package.mk
-LINSTOR_VERSION ?= 1.33.3
+LINSTOR_VERSION ?= 1.35.3
# The golden-image clone guard in packages/apps/vm-disk/templates/dv.yaml
# reads allowRemoteVolumeAccess and layerList the way this release does and
# has to be re-checked against the new one.
diff --git a/packages/system/linstor/images/piraeus-server/Dockerfile b/packages/system/linstor/images/piraeus-server/Dockerfile
index 2c8a5500a..631303b72 100644
--- a/packages/system/linstor/images/piraeus-server/Dockerfile
+++ b/packages/system/linstor/images/piraeus-server/Dockerfile
@@ -1,9 +1,10 @@
-ARG DISTRO=bookworm@sha256:f37a335e82bca302e955fa39f9dfe28f1be618f016f8a2b56318e5a5111afc26
+# LINSTOR 1.35 builds and runs on Java 21, which bookworm does not ship.
+ARG DISTRO=trixie@sha256:9c6e381421d0f48d1b8133e573f33c185ee16cbf505eda9c4a8a4791b25ea8a6
ARG LINSTOR_VERSION
# ------------------------------------------------------------------------------
# Build linstor-server from source
-FROM --platform=$BUILDPLATFORM debian:bookworm@sha256:f37a335e82bca302e955fa39f9dfe28f1be618f016f8a2b56318e5a5111afc26 AS builder
+FROM --platform=$BUILDPLATFORM debian:trixie@sha256:9c6e381421d0f48d1b8133e573f33c185ee16cbf505eda9c4a8a4791b25ea8a6 AS builder
ARG LINSTOR_VERSION
ARG VERSION=${LINSTOR_VERSION}
@@ -13,10 +14,10 @@ ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update \
&& apt-get -y upgrade \
&& apt-get -y install build-essential git default-jdk-headless python3-all debhelper wget unzip && \
- wget https://services.gradle.org/distributions/gradle-8.9-bin.zip -O /tmp/gradle.zip && \
+ wget https://services.gradle.org/distributions/gradle-8.14.2-bin.zip -O /tmp/gradle.zip && \
unzip -d /opt /tmp/gradle.zip && \
rm /tmp/gradle.zip && \
- ln -s /opt/gradle-8.9/bin/gradle /usr/local/bin/gradle
+ ln -s /opt/gradle-8.14.2/bin/gradle /usr/local/bin/gradle
RUN git clone https://github.com/LINBIT/linstor-server.git /linstor-server
WORKDIR /linstor-server
diff --git a/packages/system/linstor/images/piraeus-server/patches/README.md b/packages/system/linstor/images/piraeus-server/patches/README.md
index d2ded4285..f0b80dc92 100644
--- a/packages/system/linstor/images/piraeus-server/patches/README.md
+++ b/packages/system/linstor/images/piraeus-server/patches/README.md
@@ -1,36 +1,22 @@
# LINSTOR Server Patches
-Custom patches for piraeus-server (linstor-server) v1.33.3.
+Custom patches for piraeus-server (linstor-server) v1.35.3.
-- **allow-toggle-disk-retry.diff** — Backport maintainer implementation of toggle-disk retry/abort
- - Source PR/comment: [#475](https://github.com/LINBIT/linstor-server/pull/475), [maintainer note](https://github.com/LINBIT/linstor-server/pull/475#issuecomment-3949630419)
- - Backported from upstream commit: [`3d97f71c9`](https://github.com/LINBIT/linstor-server/commit/3d97f71c95a493588d3d521c63eac4d846935fb3)
-- **fix-duplicate-tcp-ports.diff** — Preserve DRBD TCP ports during toggle-disk and avoid redundant `ensureStackDataExists()`
- - Source PR/review: [#476](https://github.com/LINBIT/linstor-server/pull/476), [review suggestion](https://github.com/LINBIT/linstor-server/pull/476#discussion_r3007725079)
- - Backported from commits: [`79d6375c5`](https://github.com/kvaps/linstor-server/commit/79d6375c55d6181b35a7b7f0fe8dbdfb86e126cd), [`bcc89902f`](https://github.com/kvaps/linstor-server/commit/bcc89902f4f61ac1589dd07ebb7f5aae1935370d)
-- **fix-luks-header-size.diff** — Account for LUKS2 `--offset`, metadata/keyslots sizing, and device `optimal_io_size`
- - Source PR/comment: [#472](https://github.com/LINBIT/linstor-server/pull/472), [maintainer note](https://github.com/LINBIT/linstor-server/pull/472#issuecomment-3949687603)
- - Backported from commits: [`ccc85fbd2`](https://github.com/LINBIT/linstor-server/commit/ccc85fbd2c65f0b97c52403fa80f1efdb886ec4e), [`71b601554`](https://github.com/LINBIT/linstor-server/commit/71b601554f41bcb50cd5bd06989c5b0d3a814acd)
- - Note: upstream commit [`3d0402a0c`](https://github.com/LINBIT/linstor-server/commit/3d0402a0c25f0a4b57b380321f10e89982f26e7a) is already included in `v1.33.1`
- **fix-k8s-create-already-exists.diff** — Adopt the existing object when a create in the Kubernetes CRD backend returns 409 AlreadyExists. The HTTP client retries a POST after a 5xx or a dropped connection, so a create the apiserver already persisted comes back as 409; a single-object transaction has no rollback entry, the key is dropped from memory while the CR stays, and every later write of that key fails until the controller restarts. Seen as a PV stuck in `Released` because linstor-csi could not set `BalanceResourcesEnabled` on the resource definition.
- Source PR: [#531](https://github.com/LINBIT/linstor-server/pull/531)
- Backported from commit: [`8badf8f05`](https://github.com/lexfrei/linstor-server/commit/8badf8f0526ff3b1b391857e4a046ed46b95a79c)
- **retry-adjust-after-stale-bitmap.diff** — Retry `drbdadm adjust` after detaching a stale local bitmap state
- Source PR: [#491](https://github.com/LINBIT/linstor-server/pull/491)
- Backported from commit: [`51ae50a84`](https://github.com/kvaps/linstor-server/commit/51ae50a84dcb98093f543b819652c750a94d96c9)
-- **retry-secondary-after-mkfs.diff** — Retry `drbdadm secondary` after mkfs when DRBD reports the device as held open by an external probe (Talos block-controller, udev, multipathd, etc.). Without this retry, a transient `Device is held open by someone` aborts resource initialization, leaves the satellite in an intermediate state, and prevents the controller from receiving the final `UpToDate` event — orphan PVs in `Released` then cannot be cleaned up.
- - Related upstream issue: [#268](https://github.com/LINBIT/linstor-server/issues/268)
- - Related upstream issue: [drbd #74](https://github.com/LINBIT/drbd/issues/74) (same EBUSY pattern with multipathd)
-
-- **fix-min-io-probe-device-wait.diff** — Wait for temporary probe devices before reading pool I/O properties; retry failures and fall back to a probe when ZFS volumes have no usable device path.
- - Source PR: [#528](https://github.com/LINBIT/linstor-server/pull/528), fixes [#527](https://github.com/LINBIT/linstor-server/issues/527)
- - Adapted for v1.33.3; inactive LVM volumes retain the existing no-probe behavior.
-
+ - Rebased onto v1.35.3, where `DrbdAdm` gained the `DFLT_WINDOWS_TIMEOUT_MS` constant; `DrbdadmAdjust` still receives the failed `OutputData` through `ExtCmdFailedException`, so its own retry on "Resize not allowed during resync" keeps working on top of this one.
- **native-protoc.diff** — Download the protoc build for the build host's architecture instead of always `linux-x86_64`. The builder stage runs on `$BUILDPLATFORM`, so on a native arm64 builder the x86_64 protoc cannot execute and `./gradlew generateJava` fails with `Syntax error: ")" unexpected`.
- Not proposed upstream yet.
-- **snapshot-restore-drbd-initialized.diff** — Mark the volume definitions of a restored snapshot as DRBD-initialized, unless the restored resource definition carries `DrbdOptions/ForceInitialSyncPermanent`, which keeps the existing path that recreates metadata, as for a backup. Without the flag the initial UpToDate race is still open on a restored volume, and `Linstor/Drbd/InitialUptoDateOn`, copied from the source volume through the snapshot, names whichever node won it for the source. When linstor-csi autoplaces a replica on that node, the empty new volume wins the race, gets mkfs and goes StandAlone against the restored replica, so a pod attached there sees an empty filesystem. With the flag set, nothing wins the race, and an autoplaced replica resyncs from the restored one like any replica added to an existing volume.
- - Backported from upstream commit: [`9d8cdf511`](https://github.com/LINBIT/linstor-server/commit/9d8cdf5118338fc72a34bb99799e452e031ea8f6), first released in v1.34.0; the `CHANGELOG.md` hunk is left out
- - Drop when LINSTOR is upgraded to v1.34.0 or later
- - On ZFS and thin pools the autoplaced replicas still skip the initial sync and rely on the source volume's day0 UUID being in the restored replica's history, upstream included; if it is not, DRBD reports unrelated data and goes StandAlone rather than syncing
- - Fixes [cozystack/cozystack#4862](https://github.com/cozystack/cozystack/issues/4862)
+Dropped on the move from v1.33.3, because v1.35.3 already carries them:
+
+- `allow-toggle-disk-retry.diff` — upstream [`3d97f71c9`](https://github.com/LINBIT/linstor-server/commit/3d97f71c95a493588d3d521c63eac4d846935fb3), v1.34.0
+- `fix-luks-header-size.diff` — upstream [`ccc85fbd2`](https://github.com/LINBIT/linstor-server/commit/ccc85fbd2c65f0b97c52403fa80f1efdb886ec4e) and [`71b601554`](https://github.com/LINBIT/linstor-server/commit/71b601554f41bcb50cd5bd06989c5b0d3a814acd), v1.34.0
+- `snapshot-restore-drbd-initialized.diff` — upstream [`9d8cdf511`](https://github.com/LINBIT/linstor-server/commit/9d8cdf5118338fc72a34bb99799e452e031ea8f6), v1.34.0
+- `fix-duplicate-tcp-ports.diff` — upstream [`521c6d211`](https://github.com/LINBIT/linstor-server/commit/521c6d21121dbbcc26cf6eda723c24a5b34889be) (from [#476](https://github.com/LINBIT/linstor-server/pull/476)), v1.34.0
+- `retry-secondary-after-mkfs.diff` — upstream [`677789c02`](https://github.com/LINBIT/linstor-server/commit/677789c028ac), v1.34.0
+- `fix-min-io-probe-device-wait.diff` — upstream [`6e5557839`](https://github.com/LINBIT/linstor-server/commit/6e5557839c911c7a1db240dcb6fc348383a74266) (from [#528](https://github.com/LINBIT/linstor-server/pull/528)), v1.35.1
diff --git a/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff b/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff
deleted file mode 100644
index 952ba16c5..000000000
--- a/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff
+++ /dev/null
@@ -1,261 +0,0 @@
-diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java
-index d93a18014..a944cb809 100644
---- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java
-+++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java
-@@ -111,6 +111,14 @@ import reactor.util.function.Tuple2;
- @Singleton
- public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionListener
- {
-+ private enum ToggleDiskAction
-+ {
-+ NOOP,
-+ NORMAL,
-+ ABORT,
-+ RETRY
-+ }
-+
- private final AccessContext apiCtx;
- private final ScopeRunner scopeRunner;
- private final BackgroundRunner backgroundRunner;
-@@ -386,69 +394,33 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
- "Toggle Disk on %s/%s %s", nodeNameStr, rscNameStr, removeDisk ? "removing disk" : "adding disk");
-
- Resource rsc = ctrlApiDataLoader.loadRsc(nodeName, rscName, true);
-+ ResourceDefinition rscDfn = rsc.getResourceDefinition();
-
-- if (hasDiskAddRequested(rsc))
-- {
-- throw new ApiRcException(ApiCallRcImpl.simpleEntry(
-- ApiConsts.FAIL_RSC_BUSY,
-- "Addition of disk to resource already requested",
-- true
-- ));
-- }
-- if (hasDiskRemoveRequested(rsc))
-- {
-- throw new ApiRcException(ApiCallRcImpl.simpleEntry(
-- ApiConsts.FAIL_RSC_BUSY,
-- "Removal of disk from resource already requested",
-- true
-- ));
-- }
-+ ToggleDiskAction action = determineToggleDiskAction(rsc, removeDisk);
-+ errorReporter.logDebug("Toggle Disk action: %s", action);
-
-- if (!removeDisk && !ctrlVlmCrtApiHelper.isDiskless(rsc))
-+ switch (action)
- {
-- throw new ApiRcException(ApiCallRcImpl.simpleEntry(
-- ApiConsts.WARN_RSC_ALREADY_HAS_DISK,
-- "Resource already has disk",
-- true
-- ));
-- }
-- if (removeDisk && ctrlVlmCrtApiHelper.isDiskless(rsc))
-- {
-- throw new ApiRcException(ApiCallRcImpl.simpleEntry(
-- ApiConsts.WARN_RSC_ALREADY_DISKLESS,
-- "Resource already diskless",
-- true
-- ));
-+ case NOOP:
-+ return handleNoopAction(rsc, removeDisk);
-+ case RETRY:
-+ return handleRetryAction(rsc, removeDisk, toggleIntoTiebreakerRef, context);
-+ case ABORT:
-+ clearToggleDiskFlags(rsc);
-+ errorReporter.logInfo(
-+ "Aborting previous toggle disk transition, starting new transition to %s",
-+ removeDisk ? "diskless" : "diskful"
-+ );
-+ break;
-+ case NORMAL:
-+ break;
-+ default:
-+ throw new ImplementationError("Unhandled case: " + action);
- }
-
-- ResourceDefinition rscDfn = rsc.getResourceDefinition();
-- AccessContext peerCtx = peerAccCtx.get();
-- if (removeDisk)
-- {
-- // Prevent removal of the last disk
-- int haveDiskCount = countDisksAndIsOnline(rscDfn);
-- if (haveDiskCount <= 1)
-- {
-- throw new ApiRcException(ApiCallRcImpl.simpleEntry(
-- ApiConsts.FAIL_INSUFFICIENT_REPLICA_COUNT,
-- "Cannot remove the disk from the only online resource with a disk",
-- true
-- ));
-- }
-+ validateToggleDiskPreconditions(rsc, removeDisk);
-
-- if (!LayerUtils.hasLayer(getLayerData(peerCtx, rsc), DeviceLayerKind.DRBD))
-- {
-- throw new ApiRcException(ApiCallRcImpl.simpleEntry(
-- ApiConsts.FAIL_INVLD_LAYER_STACK,
-- "Toggle disk is only supported in combination with DRBD",
-- true
-- ));
-- }
-- }
-- else
-- {
-- ensureAllPeersHavePeerSlotLeft(rscDfn);
-- }
-+ AccessContext peerCtx = peerAccCtx.get();
-
- // Save the requested storage pool in the resource properties.
- // This does not cause the storage pool to be used automatically.
-@@ -628,10 +600,10 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
-
- ctrlTransactionHelper.commit();
-
-- String action = removeDisk ? "Removal of disk from" : "Addition of disk to";
-+ String actionStr = removeDisk ? "Removal of disk from" : "Addition of disk to";
- responses.addEntry(ApiCallRcImpl.simpleEntry(
- ApiConsts.MODIFIED,
-- action + " resource '" + rscDfn.getName().displayValue + "' " +
-+ actionStr + " resource '" + rscDfn.getName().displayValue + "' " +
- "on node '" + rsc.getNode().getName().displayValue + "' registered"
- ));
-
-@@ -651,6 +623,40 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
- );
- }
-
-+ private Flux handleNoopAction(Resource rscRef, boolean removeDiskRef)
-+ {
-+ String state = removeDiskRef ? "diskless" : "diskful";
-+ ApiCallRcImpl responses = new ApiCallRcImpl();
-+ responses.addEntry(ApiCallRcImpl.simpleEntry(
-+ ApiConsts.INFO_NOOP,
-+ "Resource '" + rscRef.getResourceDefinition().getName().displayValue + "' on node '" +
-+ rscRef.getNode().getName().displayValue + "' is already " + state
-+ ));
-+ return Flux.just(responses);
-+ }
-+
-+ private Flux handleRetryAction(
-+ Resource rscRef,
-+ boolean removeDisk,
-+ boolean toggleIntoTiebreakerRef,
-+ ResponseContext context
-+ )
-+ {
-+ String direction = removeDisk ? "diskless" : "diskful";
-+ ApiCallRcImpl responses = new ApiCallRcImpl();
-+ NodeName nodeName = rscRef.getNode().getName();
-+ ResourceName rscName = rscRef.getResourceDefinition().getName();
-+ responses.addEntry(ApiCallRcImpl.simpleEntry(
-+ ApiConsts.INFO_NOOP,
-+ "Retrying toggle disk to " + direction + " for resource '" + rscName.displayValue +
-+ "' on node '" + nodeName.displayValue + "'"
-+ ));
-+
-+ return Flux
-+ .just(responses)
-+ .concatWith(updateAndAdjustDisk(nodeName, rscName, removeDisk, toggleIntoTiebreakerRef, context));
-+ }
-+
- private long getVlmDfnSizePrivileged(VolumeDefinition vlmDfnRef)
- {
- try
-@@ -781,6 +787,96 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
- }
- }
-
-+ private ToggleDiskAction determineToggleDiskAction(Resource rsc, boolean removeDisk)
-+ {
-+ boolean isDiskless = ctrlVlmCrtApiHelper.isDiskless(rsc);
-+ boolean diskAddRequested = hasDiskAddRequested(rsc);
-+ boolean diskRemoveRequested = hasDiskRemoveRequested(rsc);
-+
-+ ToggleDiskAction action;
-+ if (isDiskless)
-+ {
-+ if (diskAddRequested)
-+ {
-+ action = removeDisk ? ToggleDiskAction.ABORT : ToggleDiskAction.RETRY;
-+ }
-+ else if (diskRemoveRequested)
-+ {
-+ action = removeDisk ? ToggleDiskAction.RETRY : ToggleDiskAction.ABORT;
-+ }
-+ else
-+ {
-+ action = removeDisk ? ToggleDiskAction.NOOP : ToggleDiskAction.NORMAL;
-+ }
-+ }
-+ else
-+ {
-+ if (diskRemoveRequested)
-+ {
-+ action = removeDisk ? ToggleDiskAction.RETRY : ToggleDiskAction.ABORT;
-+ }
-+ else
-+ {
-+ action = removeDisk ? ToggleDiskAction.NORMAL : ToggleDiskAction.NOOP;
-+ }
-+ }
-+
-+ return action;
-+ }
-+
-+ private void validateToggleDiskPreconditions(Resource rsc, boolean removeDisk)
-+ {
-+ ResourceDefinition rscDfn = rsc.getResourceDefinition();
-+ if (removeDisk)
-+ {
-+ validateDiskRemovalAllowed(rsc, rscDfn);
-+ }
-+ else
-+ {
-+ ensureAllPeersHavePeerSlotLeft(rscDfn);
-+ }
-+ }
-+
-+ private void clearToggleDiskFlags(Resource rsc)
-+ {
-+ try
-+ {
-+ rsc.getStateFlags().disableFlags(
-+ apiCtx,
-+ Resource.Flags.DISK_ADD_REQUESTED,
-+ Resource.Flags.DISK_ADDING,
-+ Resource.Flags.DISK_REMOVE_REQUESTED,
-+ Resource.Flags.DISK_REMOVING
-+ );
-+ }
-+ catch (AccessDeniedException | DatabaseException exc)
-+ {
-+ throw new ImplementationError(exc);
-+ }
-+ }
-+
-+ private void validateDiskRemovalAllowed(Resource rsc, ResourceDefinition rscDfn)
-+ {
-+ int haveDiskCount = countDisksAndIsOnline(rscDfn);
-+ if (haveDiskCount <= 1)
-+ {
-+ throw new ApiRcException(ApiCallRcImpl.simpleEntry(
-+ ApiConsts.FAIL_INSUFFICIENT_REPLICA_COUNT,
-+ "Cannot remove the disk from the only online resource with a disk",
-+ true
-+ ));
-+ }
-+
-+ if (!LayerUtils.hasLayer(getLayerData(peerAccCtx.get(), rsc), DeviceLayerKind.DRBD))
-+ {
-+ throw new ApiRcException(ApiCallRcImpl.simpleEntry(
-+ ApiConsts.FAIL_INVLD_LAYER_STACK,
-+ "Toggle disk is only supported in combination with DRBD",
-+ true
-+ ));
-+ }
-+ }
-+
- /**
- * Although we need to rebuild the layerData as the layerList might have changed, if we do not
- * deactivate (i.e. down) the current resource, we need to make sure that deleting DrbdRscData
diff --git a/packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff b/packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff
deleted file mode 100644
index ed106f98c..000000000
--- a/packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff
+++ /dev/null
@@ -1,137 +0,0 @@
-diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java
-index d93a18014..01cfbbacf 100644
---- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java
-+++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java
-@@ -37,6 +37,7 @@ import com.linbit.linstor.core.objects.StorPool;
- import com.linbit.linstor.core.objects.Volume;
- import com.linbit.linstor.core.objects.VolumeDefinition;
- import com.linbit.linstor.core.objects.utils.MixedStorPoolHelper;
-+import com.linbit.linstor.core.types.TcpPortNumber;
- import com.linbit.linstor.dbdrivers.DatabaseException;
- import com.linbit.linstor.event.EventWaiter;
- import com.linbit.linstor.event.ObjectIdentifier;
-@@ -85,6 +86,7 @@ import java.util.List;
- import java.util.Map;
- import java.util.Map.Entry;
- import java.util.Set;
-+import java.util.TreeSet;
-
- import org.reactivestreams.Publisher;
- import reactor.core.publisher.Flux;
-@@ -575,12 +577,13 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
-
- /*
- * We also have to remove the currently diskless DrbdRscData and free up the node-id as now we must
-- * use the shared resource's node-id
-+ * use the shared resource's node-id. We still need to preserve TCP ports though.
- */
-+ copyDrbdTcpPortsIfExists(rsc, payload);
- }
- else
- {
-- copyDrbdNodeIdIfExists(rsc, payload);
-+ copyDrbdSettings(rsc, payload);
- }
- /*
- * rebuilds the layerdata in case we just removed it..
-@@ -782,10 +785,20 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
- }
-
- /**
-- * Although we need to rebuild the layerData as the layerList might have changed, if we do not
-- * deactivate (i.e. down) the current resource, we need to make sure that deleting DrbdRscData
-- * and recreating a new DrbdRscData ends up with the same node-id as before.
-+ * Copies DRBD settings (node-id and TCP ports) from the existing DrbdRscData into the payload
-+ * before removeLayerData() deletes them. This ensures that recreated DrbdRscData ends up with
-+ * the same node-id and TCP ports as before.
-+ *
-+ * TCP ports must be preserved because if the satellite misses the update (e.g. due to controller
-+ * restart or connectivity issues), it will keep the old ports while peers receive the new ones,
-+ * causing DRBD connections to fail with StandAlone state.
- */
-+ private void copyDrbdSettings(Resource rsc, LayerPayload payload) throws ImplementationError
-+ {
-+ copyDrbdNodeIdIfExists(rsc, payload);
-+ copyDrbdTcpPortsIfExists(rsc, payload);
-+ }
-+
- private void copyDrbdNodeIdIfExists(Resource rsc, LayerPayload payload) throws ImplementationError
- {
- Set> drbdRscDataSet = LayerRscUtils.getRscDataByLayer(
-@@ -804,6 +817,28 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
- }
- }
-
-+ private void copyDrbdTcpPortsIfExists(Resource rsc, LayerPayload payload) throws ImplementationError
-+ {
-+ Set> drbdRscDataSet = LayerRscUtils.getRscDataByLayer(
-+ getLayerData(apiCtx, rsc),
-+ DeviceLayerKind.DRBD
-+ );
-+ if (!drbdRscDataSet.isEmpty())
-+ {
-+ DrbdRscData drbdRscData = (DrbdRscData) drbdRscDataSet.iterator().next();
-+ Collection tcpPorts = drbdRscData.getTcpPortList();
-+ if (tcpPorts != null && !tcpPorts.isEmpty())
-+ {
-+ Set portInts = new TreeSet<>();
-+ for (TcpPortNumber port : tcpPorts)
-+ {
-+ portInts.add(port.value);
-+ }
-+ payload.drbdRsc.tcpPorts = portInts;
-+ }
-+ }
-+ }
-+
- private List removeLayerData(Resource rscRef)
- {
- List layerList;
-@@ -1058,15 +1093,15 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL
- /*
- * We also have to remove the possible meta-children of previous StorageRscData.
- * LayerData will be recreated with ensureStackDataExists.
-- * However, we still need to remember our node-id if we had / have DRBD in the list
-+ * However, we still need to remember our DRBD settings if we had / have DRBD in the list
- */
-- copyDrbdNodeIdIfExists(rsc, payload);
-+ copyDrbdSettings(rsc, payload);
- layerList = removeLayerData(rsc);
- }
- else
- {
- markDiskAdded(rsc);
-- ctrlLayerStackHelper.resetStoragePools(rsc);
-+ ctrlLayerStackHelper.resetStoragePools(rsc, false);
- }
- ctrlLayerStackHelper.ensureStackDataExists(rsc, layerList, payload);
-
-diff --git a/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java b/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java
-index 3538b380c..f9733b6f1 100644
---- a/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java
-+++ b/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java
-@@ -263,6 +263,11 @@ public class CtrlRscLayerDataFactory
- }
-
- public void resetStoragePools(Resource rscRef)
-+ {
-+ resetStoragePools(rscRef, true);
-+ }
-+
-+ public void resetStoragePools(Resource rscRef, boolean callEnsureStackDataExistsRef)
- {
- try
- {
-@@ -276,8 +281,10 @@ public class CtrlRscLayerDataFactory
-
- rscDataToProcess.addAll(rscData.getChildren());
- }
--
-- ensureStackDataExists(rscRef, null, new LayerPayload());
-+ if (callEnsureStackDataExistsRef)
-+ {
-+ ensureStackDataExists(rscRef, null, new LayerPayload());
-+ }
- }
- catch (AccessDeniedException exc)
- {
diff --git a/packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff b/packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff
deleted file mode 100644
index 8611825bc..000000000
--- a/packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff
+++ /dev/null
@@ -1,570 +0,0 @@
-diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java
---- a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java
-+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java
-@@ -2081,7 +2081,6 @@ public abstract class AbsStorageProvider<
- )
- throws IOException, AccessDeniedException
- {
-- final StorPoolName storPoolObjName = storPoolObj.getName();
- if (storDevicePath != null)
- {
- errorReporter.logDebug("updateMinIoSize: Have storDevicePath \"%s\"", storDevicePath);
-@@ -2094,35 +2093,18 @@ public abstract class AbsStorageProvider<
- "updateMinIoSize: Block device path is \"%s\"",
- blockDevicePath.toString()
- );
-- final long minIoSize = BlockSizeInfo.getBlockSize(blockDevicePath);
--
-- boolean updateValue = true;
--
-- final String propKey = StorageConstants.NAMESPACE_INTERNAL + '/' +
-- StorageConstants.BLK_DEV_MIN_IO_SIZE;
-- final Props storPoolProps = storPoolObj.getProps(storDriverAccCtx);
-- final @Nullable String currentPropValue = storPoolProps.getProp(propKey);
-- if (currentPropValue != null)
-- {
-- try
-- {
-- final long currentPropMinIoSize = Long.parseLong(currentPropValue);
-- updateValue = currentPropMinIoSize != minIoSize;
-- }
-- catch (NumberFormatException ignored)
-- {
-- }
-- }
--
-- if (updateValue)
-- {
-- final String propValue = Long.toString(minIoSize);
-- errorReporter.logDebug(
-- "Storage pool \"%s\": Set property \"%s\" = \"%s\"",
-- storPoolObjName.displayValue, propKey, propValue
-- );
-- propsChange.changeStorPoolProp(storPoolObj, propKey, propValue);
-- }
-+ updatePropIfNeeded(
-+ propsChange,
-+ storPoolObj,
-+ StorageConstants.NAMESPACE_INTERNAL + '/' + StorageConstants.BLK_DEV_MIN_IO_SIZE,
-+ Long.toString(BlockSizeInfo.getBlockSize(blockDevicePath))
-+ );
-+ updatePropIfNeeded(
-+ propsChange,
-+ storPoolObj,
-+ StorageConstants.NAMESPACE_INTERNAL + '/' + StorageConstants.BLK_DEV_OPT_IO_SIZE,
-+ Long.toString(BlockSizeInfo.getOptimalIoSize(blockDevicePath))
-+ );
- }
- else
- {
-@@ -2130,6 +2112,28 @@ public abstract class AbsStorageProvider<
- }
- }
-
-+ private void updatePropIfNeeded(
-+ LocalPropsChangePojo propsChangeRef,
-+ StorPool storPoolObjRef,
-+ String propKeyRef,
-+ String propValueRef
-+ )
-+ throws AccessDeniedException
-+ {
-+ final Props storPoolProps = storPoolObjRef.getProps(storDriverAccCtx);
-+ final @Nullable String currentPropValue = storPoolProps.getProp(propKeyRef);
-+ if (currentPropValue == null || !currentPropValue.equals(propValueRef))
-+ {
-+ errorReporter.logDebug(
-+ "Storage pool \"%s\": Set property \"%s\" = \"%s\"",
-+ storPoolObjRef.getName().displayValue,
-+ propKeyRef,
-+ propValueRef
-+ );
-+ propsChangeRef.changeStorPoolProp(storPoolObjRef, propKeyRef, propValueRef);
-+ }
-+ }
-+
- @SuppressWarnings("unchecked")
- @Override
- public void updateAllocatedSize(VlmProviderObject vlmDataRef)
-diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java
---- a/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java
-+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java
-@@ -3,8 +3,11 @@ package com.linbit.linstor.layer.storage.utils;
- import com.linbit.utils.MathUtils;
- import com.linbit.utils.SymbolicLinkResolver;
-
-+import static com.linbit.linstor.layer.storage.BlockSizeConsts.DFLT_OPT_IO_SIZE;
- import static com.linbit.linstor.layer.storage.BlockSizeConsts.DFLT_IO_SIZE;
-+import static com.linbit.linstor.layer.storage.BlockSizeConsts.MAX_OPT_IO_SIZE;
- import static com.linbit.linstor.layer.storage.BlockSizeConsts.MAX_IO_SIZE;
-+import static com.linbit.linstor.layer.storage.BlockSizeConsts.MIN_OPT_IO_SIZE;
- import static com.linbit.linstor.layer.storage.BlockSizeConsts.MIN_IO_SIZE;
-
- import java.io.FileInputStream;
-@@ -13,6 +16,8 @@ import java.nio.file.Path;
-
- public class BlockSizeInfo
- {
-+ private static final int NUMBER_BUFFER_SIZE = 32;
-+
- /**
- * Determines the blocksize, aka minimum I/O size, for the specified backing storage path.
- *
-@@ -51,7 +56,7 @@ public class BlockSizeInfo
- final Path infoSourceName = blockDevice.getFileName();
- final Path infoSource = Path.of("/sys/block", infoSourceName.toString(), "queue/physical_block_size");
-
-- final byte[] data = new byte[32];
-+ final byte[] data = new byte[NUMBER_BUFFER_SIZE];
- try (final FileInputStream fileIn = new FileInputStream(infoSource.toString()))
- {
- final int readCount = fileIn.read(data);
-@@ -75,4 +80,38 @@ public class BlockSizeInfo
- }
- return blockSize;
- }
-+
-+ public static long getOptimalIoSize(final Path storageObj)
-+ {
-+ long optIoSize = DFLT_OPT_IO_SIZE;
-+ try
-+ {
-+ final Path blockDevice = SymbolicLinkResolver.resolveSymLink(storageObj);
-+ final Path infoSourceName = blockDevice.getFileName();
-+ final Path infoSource = Path.of("/sys/block", infoSourceName.toString(), "queue/optimal_io_size");
-+
-+ final byte[] data = new byte[NUMBER_BUFFER_SIZE];
-+ try (final FileInputStream fileIn = new FileInputStream(infoSource.toString()))
-+ {
-+ final int readCount = fileIn.read(data);
-+ if (readCount > 0)
-+ {
-+ String numberStr = new String(data, 0, readCount);
-+ numberStr = numberStr.trim();
-+ try
-+ {
-+ final long unboundedOptIoSize = Long.parseLong(numberStr);
-+ optIoSize = MathUtils.bounds(MIN_OPT_IO_SIZE, unboundedOptIoSize, MAX_OPT_IO_SIZE);
-+ }
-+ catch (NumberFormatException ignored)
-+ {
-+ }
-+ }
-+ }
-+ }
-+ catch (IOException ignored)
-+ {
-+ }
-+ return optIoSize;
-+ }
- }
-diff --git a/server/src/main/java/com/linbit/SizeConv.java b/server/src/main/java/com/linbit/SizeConv.java
---- a/server/src/main/java/com/linbit/SizeConv.java
-+++ b/server/src/main/java/com/linbit/SizeConv.java
-@@ -16,6 +16,7 @@ public class SizeConv
- public enum SizeUnit
- {
- UNIT_B,
-+ UNIT_SECTORS,
- UNIT_KiB,
- UNIT_MiB,
- UNIT_GiB,
-@@ -41,6 +42,9 @@ public class SizeConv
- case UNIT_B:
- factor = FACTOR_B;
- break;
-+ case UNIT_SECTORS:
-+ factor = FACTOR_SECTORS;
-+ break;
- case UNIT_KiB:
- factor = FACTOR_KiB;
- break;
-@@ -111,6 +115,9 @@ public class SizeConv
- case "b":
- unit = SizeUnit.UNIT_B;
- break;
-+ case "s":
-+ unit = SizeUnit.UNIT_SECTORS;
-+ break;
- case "k":
- // fall-through
- case "kb":
-@@ -217,6 +224,9 @@ public class SizeConv
- // Factor 1
- public static final BigInteger FACTOR_B = BigInteger.valueOf(1L);
-
-+ // Factor 512
-+ public static final BigInteger FACTOR_SECTORS = BigInteger.valueOf(512L);
-+
- // Factor 1,024
- // Naming convention exception: SI unit capitalization rules
- @SuppressWarnings("checkstyle:constantname")
-diff --git a/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java b/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java
---- a/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java
-+++ b/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java
-@@ -1,27 +1,62 @@
- package com.linbit.linstor.layer.luks;
-
-+import com.linbit.SizeConv;
-+import com.linbit.SizeConv.SizeUnit;
- import com.linbit.exceptions.InvalidSizeException;
-+import com.linbit.linstor.PriorityProps;
-+import com.linbit.linstor.annotation.Nullable;
-+import com.linbit.linstor.api.ApiConsts;
-+import com.linbit.linstor.core.objects.AbsResource;
-+import com.linbit.linstor.core.objects.AbsVolume;
-+import com.linbit.linstor.core.objects.Resource;
-+import com.linbit.linstor.core.objects.ResourceDefinition;
-+import com.linbit.linstor.core.objects.ResourceGroup;
-+import com.linbit.linstor.core.objects.Snapshot;
-+import com.linbit.linstor.core.objects.SnapshotVolume;
-+import com.linbit.linstor.core.objects.StorPool;
-+import com.linbit.linstor.core.objects.Volume;
-+import com.linbit.linstor.core.objects.VolumeDefinition;
- import com.linbit.linstor.dbdrivers.DatabaseException;
- import com.linbit.linstor.layer.AbsLayerSizeCalculator;
-+import com.linbit.linstor.netcom.Peer;
-+import com.linbit.linstor.propscon.InvalidKeyException;
-+import com.linbit.linstor.propscon.ReadOnlyProps;
- import com.linbit.linstor.security.AccessDeniedException;
-+import com.linbit.linstor.storage.StorageConstants;
- import com.linbit.linstor.storage.data.adapter.luks.LuksVlmData;
- import com.linbit.linstor.storage.interfaces.categories.resource.VlmProviderObject;
- import com.linbit.linstor.storage.kinds.DeviceLayerKind;
- import com.linbit.linstor.storage.kinds.ExtTools;
- import com.linbit.linstor.storage.kinds.ExtToolsInfo;
- import com.linbit.linstor.storage.kinds.ExtToolsInfo.Version;
-+import com.linbit.linstor.utils.layer.LayerVlmUtils;
-+import com.linbit.utils.ShellUtils;
-+import com.linbit.utils.SignedAlign;
-
- import javax.inject.Inject;
- import javax.inject.Singleton;
-
-+import java.util.Iterator;
-+import java.util.LinkedList;
-+import java.util.List;
-+import java.util.Set;
-+import java.util.regex.Matcher;
-+import java.util.regex.Pattern;
-+
- @Singleton
- public class LuksLayerSizeCalculator extends AbsLayerSizeCalculator>
- {
-+ public static final String LUKS2_OPT_METADATA_SIZE = "--luks2-metadata-size";
-+ public static final String LUKS2_OPT_KEYSLOTS_SIZE = "--luks2-keyslots-size";
-+ public static final String LUKS2_OPT_OFFSET = "--offset";
-+ private static final String LUKS2_OPT_ALIGN_PAYLOAD = "--align-payload";
-+ private static final long DFLT_LUKS2_METADATA_SIZE_IN_BYTES = 16L << 10;
-+ private static final long DFLT_ALIGNMENT_1MIB_IN_BYTES = 1L << 20;
-+ private static final long LUKS1_HEADER_SIZE_IN_KIB = 2L << 10;
-+ private static final long LUKS2_HEADER_SIZE_IN_KIB = 16L << 10;
-+ private static final long LUKS2_HEADER_SIZE_IN_BYTES = LUKS2_HEADER_SIZE_IN_KIB << 10;
-
-- // linstor calculates in KiB
-- private static final int MIB = 1024;
-- private static final int LUKS1_HEADER_SIZE = 2 * MIB;
-- private static final int LUKS2_HEADER_SIZE = 16 * MIB;
-+ private static final Pattern PATTERN_SIZE = Pattern.compile("(\\d+)([kKmMgGtT]i?[bB]?|[sS]|)");
-
- @Inject
- public LuksLayerSizeCalculator(AbsLayerSizeCalculatorInit initRef)
-@@ -74,30 +109,266 @@ public class LuksLayerSizeCalculator extends AbsLayerSizeCalculator vlmDataRef)
-- throws AccessDeniedException
-+ throws AccessDeniedException, InvalidSizeException
- {
-- ExtToolsInfo cryptSetupInfo = vlmDataRef.getRscLayerObject()
-+ @Nullable Peer peer = vlmDataRef.getRscLayerObject()
- .getAbsResource()
- .getNode()
-- .getPeer(sysCtx)
-- .getExtToolsManager()
-+ .getPeer(sysCtx);
-+ if (peer == null)
-+ {
-+ throw new InvalidSizeException(
-+ "Could not calculate size of LUKS volume, since cryptsetup's version could not be determined",
-+ null
-+ );
-+ }
-+ ExtToolsInfo cryptSetupInfo = peer.getExtToolsManager()
- .getExtToolInfo(ExtTools.CRYPT_SETUP);
-- long luksHeaderSize;
-+
-+ final long luksHeaderSize;
- if (cryptSetupInfo != null && cryptSetupInfo.isSupported())
- {
- if (cryptSetupInfo.hasVersionOrHigher(new Version(2, 1)))
- {
-- luksHeaderSize = LUKS2_HEADER_SIZE;
-+ luksHeaderSize = calcLuks2HeaderSize(vlmDataRef);
- }
- else
- {
-- luksHeaderSize = LUKS1_HEADER_SIZE;
-+ luksHeaderSize = LUKS1_HEADER_SIZE_IN_KIB;
- }
- }
- else
- {
-- luksHeaderSize = -1;
-+ throw new InvalidSizeException(
-+ "Could not calculate size of LUKS volume, since cryptsetup's version could not be determined",
-+ null
-+ );
- }
- return luksHeaderSize;
- }
-+
-+ private long calcLuks2HeaderSize(VlmProviderObject> vlmDataRef) throws AccessDeniedException
-+ {
-+ PriorityProps prioProps = getPrioProps(vlmDataRef);
-+
-+ final @Nullable String userOptProp = prioProps.getProp(
-+ ApiConsts.KEY_STOR_DRIVER_LUKS_FORMAT_OPTIONS,
-+ ApiConsts.NAMESPC_STORAGE_DRIVER
-+ );
-+ List userOptions = userOptProp != null ?
-+ ShellUtils.shellSplit(userOptProp) :
-+ new LinkedList<>();
-+
-+ final long alignedHeaderSizeInBytes;
-+ final long unalignedHeaderSizeInBytes;
-+ @Nullable Long cryptsetupOffsetInBytes = getLongOptionValue(
-+ userOptions,
-+ LUKS2_OPT_OFFSET,
-+ SizeUnit.UNIT_SECTORS
-+ );
-+ if (cryptsetupOffsetInBytes != null)
-+ {
-+ alignedHeaderSizeInBytes = cryptsetupOffsetInBytes;
-+ }
-+ else
-+ {
-+ @Nullable Long cryptsetupLuks2KeyslotsSize = getLongOptionValue(
-+ userOptions,
-+ LUKS2_OPT_KEYSLOTS_SIZE
-+ );
-+ if (cryptsetupLuks2KeyslotsSize == null)
-+ {
-+ unalignedHeaderSizeInBytes = LUKS2_HEADER_SIZE_IN_BYTES;
-+ }
-+ else
-+ {
-+ @Nullable Long cryptsetupLuks2MetadataSize = getLongOptionValue(
-+ userOptions,
-+ LUKS2_OPT_METADATA_SIZE
-+ );
-+ cryptsetupLuks2MetadataSize = cryptsetupLuks2MetadataSize == null ?
-+ DFLT_LUKS2_METADATA_SIZE_IN_BYTES :
-+ cryptsetupLuks2MetadataSize;
-+
-+ unalignedHeaderSizeInBytes = 2 * cryptsetupLuks2MetadataSize + cryptsetupLuks2KeyslotsSize;
-+ }
-+
-+ long alignment = getAlignment(vlmDataRef, userOptions);
-+ alignedHeaderSizeInBytes = new SignedAlign(alignment).ceiling(unalignedHeaderSizeInBytes);
-+ }
-+ return SizeConv.convert(
-+ alignedHeaderSizeInBytes,
-+ SizeUnit.UNIT_B,
-+ SizeUnit.UNIT_KiB
-+ );
-+ }
-+
-+ private long getAlignment(VlmProviderObject> vlmDataRef, List userOptions) throws AccessDeniedException
-+ {
-+ @Nullable Long cryptsetupAlignPayloadInBytes = getLongOptionValue(
-+ userOptions,
-+ LUKS2_OPT_ALIGN_PAYLOAD,
-+ SizeUnit.UNIT_SECTORS
-+ );
-+
-+ long alignment = DFLT_ALIGNMENT_1MIB_IN_BYTES;
-+ if (cryptsetupAlignPayloadInBytes != null)
-+ {
-+ alignment = cryptsetupAlignPayloadInBytes;
-+ }
-+ else
-+ {
-+ final long maxOptIoSize = getMaxOptIoSize(vlmDataRef);
-+ alignment = Math.max(alignment, maxOptIoSize);
-+ }
-+ return alignment;
-+ }
-+
-+ private long getMaxOptIoSize(VlmProviderObject> vlmDataRef) throws InvalidKeyException, AccessDeniedException
-+ {
-+ long ret = 0;
-+ Set storPoolSet = LayerVlmUtils.getStorPoolSet(vlmDataRef, sysCtx);
-+ for (StorPool sp : storPoolSet)
-+ {
-+ @Nullable String strValue = sp.getProps(sysCtx)
-+ .getProp(
-+ StorageConstants.BLK_DEV_OPT_IO_SIZE,
-+ StorageConstants.NAMESPACE_INTERNAL
-+ );
-+ if (strValue != null)
-+ {
-+ try
-+ {
-+ long parsed = Long.parseLong(strValue);
-+ ret = Math.max(parsed, ret);
-+ }
-+ catch (NumberFormatException ignored)
-+ {
-+ errorReporter.logWarning(
-+ "LuksHeaderSize: Failed to parse '%s' from prop %s. Defaulting to 0 opt_io_size " +
-+ "(no recommendation/hint)",
-+ strValue,
-+ StorageConstants.NAMESPACE_INTERNAL + "/" + StorageConstants.BLK_DEV_OPT_IO_SIZE
-+ );
-+ }
-+ }
-+ }
-+ return ret;
-+ }
-+
-+ private @Nullable Long getLongOptionValue(List userOptionsRef, String optRef)
-+ {
-+ return getLongOptionValue(userOptionsRef, optRef, SizeUnit.UNIT_B);
-+ }
-+
-+ @SuppressWarnings("checkstyle:magicnumber")
-+ private @Nullable Long getLongOptionValue(List userOptionsRef, String optRef, SizeUnit dfltSizeUnit)
-+ {
-+ @Nullable Long ret = null;
-+ @Nullable String val = findLastValue(userOptionsRef, optRef);
-+ if (val != null && !val.isBlank())
-+ {
-+ Matcher matcher = PATTERN_SIZE.matcher(val);
-+ if (matcher.matches())
-+ {
-+ try
-+ {
-+ ret = Long.parseLong(matcher.group(1));
-+ String unit = matcher.group(2);
-+ SizeUnit sizeUnit;
-+ if (unit.isBlank())
-+ {
-+ sizeUnit = dfltSizeUnit;
-+ }
-+ else
-+ {
-+ boolean forcePowerOfTwo = unit.length() == 1 || unit.length() == 3;
-+ sizeUnit = SizeUnit.parse(unit, forcePowerOfTwo);
-+ }
-+
-+ ret = SizeConv.convert(ret, sizeUnit, SizeUnit.UNIT_B);
-+ }
-+ catch (NumberFormatException ignored)
-+ {
-+ errorReporter.logWarning(
-+ "LuksHeaderSize: Failed to parse '%s' from option '%s %s'.",
-+ matcher.group(1),
-+ optRef,
-+ val
-+ );
-+ }
-+ }
-+ }
-+ return ret;
-+ }
-+
-+ private @Nullable String findLastValue(List userOptionsRef, String optRef)
-+ {
-+ Iterator it = userOptionsRef.iterator();
-+ @Nullable String val = null;
-+ while (it.hasNext())
-+ {
-+ String opt = it.next();
-+ if (opt.equals(optRef))
-+ {
-+ if (it.hasNext())
-+ {
-+ val = it.next();
-+ }
-+ else
-+ {
-+ val = null;
-+ }
-+ }
-+ else if (opt.startsWith(optRef + "="))
-+ {
-+ val = opt.substring(optRef.length() + 1);
-+ if (val.isBlank())
-+ {
-+ val = null;
-+ }
-+ }
-+ }
-+ return val;
-+ }
-+
-+ private PriorityProps getPrioProps(VlmProviderObject> vlmDataRef) throws AccessDeniedException
-+ {
-+ final AbsVolume> vlm = vlmDataRef.getVolume();
-+ final AbsResource> rsc = vlm.getAbsResource();
-+ final ResourceDefinition rscDfn = vlm.getResourceDefinition();
-+ final VolumeDefinition vlmDfn = vlm.getVolumeDefinition();
-+ final ResourceGroup rscGrp = rscDfn.getResourceGroup();
-+
-+ final ReadOnlyProps vlmProps;
-+ final ReadOnlyProps rscProps;
-+ if (vlm instanceof Volume)
-+ {
-+ vlmProps = ((Volume) vlm).getProps(sysCtx);
-+ rscProps = ((Resource) rsc).getProps(sysCtx);
-+ }
-+ else
-+ {
-+ vlmProps = ((SnapshotVolume) vlm).getVlmProps(sysCtx);
-+ rscProps = ((Snapshot) rsc).getRscProps(sysCtx);
-+ }
-+
-+ final PriorityProps prioProps = new PriorityProps(
-+ vlmProps,
-+ rscProps
-+ );
-+ for (StorPool storPool : LayerVlmUtils.getStorPoolSet(vlmDataRef, sysCtx))
-+ {
-+ prioProps.addProps(storPool.getProps(sysCtx));
-+ }
-+ prioProps.addProps(
-+ rsc.getNode().getProps(sysCtx),
-+ vlmDfn.getProps(sysCtx),
-+ rscDfn.getProps(sysCtx),
-+ rscGrp.getVolumeGroupProps(sysCtx, vlmDfn.getVolumeNumber()),
-+ rscGrp.getProps(sysCtx),
-+ stltProps
-+ );
-+ return prioProps;
-+ }
- }
-diff --git a/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java b/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java
---- a/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java
-+++ b/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java
-@@ -13,4 +13,9 @@ public class BlockSizeConsts
-
- // Default value for the minimum_io_size value of non-storage layers
- public static final long DFLT_SPECIAL_IO_SIZE = (1L << 12);
-+
-+ // optimal_io_size may be 0 to indicate no recommendation.
-+ public static final long MIN_OPT_IO_SIZE = 0;
-+ public static final long DFLT_OPT_IO_SIZE = 0;
-+ public static final long MAX_OPT_IO_SIZE = Long.MAX_VALUE;
- }
-diff --git a/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java b/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java
---- a/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java
-+++ b/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java
-@@ -11,6 +11,7 @@ public class StorageConstants
- public static final String NAMESPACE_NVME = ApiConsts.NAMESPC_STORAGE_DRIVER + "/NVME";
- public static final String NAMESPACE_INTERNAL = NAMESPACE_STOR_DRIVER + "/internal/";
-
-+ public static final String BLK_DEV_OPT_IO_SIZE = "optIoSize";
- public static final String BLK_DEV_MIN_IO_SIZE = "minIoSize";
- public static final String BLK_DEV_MIN_IO_SIZE_AUTO = "minIoSizeAuto";
- public static final String BLK_DEV_MAX_BIO_SIZE = "maxBioSize";
diff --git a/packages/system/linstor/images/piraeus-server/patches/fix-min-io-probe-device-wait.diff b/packages/system/linstor/images/piraeus-server/patches/fix-min-io-probe-device-wait.diff
deleted file mode 100644
index bb6185b88..000000000
--- a/packages/system/linstor/images/piraeus-server/patches/fix-min-io-probe-device-wait.diff
+++ /dev/null
@@ -1,660 +0,0 @@
-diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java
---- a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java
-+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java
-@@ -163,6 +163,8 @@
- }
-
- private static final long DFLT_WAIT_UNTIL_DEVICE_CREATED_TIMEOUT_IN_MS = 5000;
-+ private static final int PROBE_VLM_ATTEMPTS = 3;
-+ private static final long PROBE_VLM_RETRY_DELAY_IN_MS = 1000;
- protected static final int DFLT_STRIPES = 1;
- public static final long SIZE_OF_NOT_FOUND_STOR_POOL = ApiConsts.VAL_STOR_POOL_SPACE_NOT_FOUND;
-
-@@ -184,6 +186,7 @@
- protected final Set changedStoragePoolStrings = new HashSet<>();
- private final String typeDescr;
- private final FileSystemWatch fsWatch;
-+ private final Object probeVlmLock = new Object();
- protected final DeviceProviderKind kind;
- private final DrbdInvalidateUtils drbdInvalidateUtils;
-
-@@ -2022,61 +2025,126 @@
- final StorPoolName storPoolObjName = storPoolObj.getName();
- errorReporter.logDebug("ENTER updateMinIoSize method: Storage pool \"%s\"", storPoolObjName.displayValue);
- Collection> vlmProviderList = storPoolObj.getVolumes(storDriverAccCtx);
-+ boolean haveInfo = false;
- if (vlmProviderList != null && vlmProviderList.size() >= 1)
- {
- errorReporter.logDebug("updateMinIoSize: Have vlmProviderList with %d items", vlmProviderList.size());
- Iterator> vlmProviderIter = vlmProviderList.iterator();
-- boolean haveInfo = false;
- while (vlmProviderIter.hasNext() && !haveInfo)
- {
- VlmProviderObject vlmProvider = vlmProviderIter.next();
- errorReporter.logDebug("updateMinIoSize: Have vlmProvider");
- final String storDevicePath = vlmProvider.getDevicePath();
-+ if (storDevicePath != null)
-+ {
-+ try
-+ {
-+ updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange);
-+ haveInfo = true;
-+ }
-+ catch (IOException ignored)
-+ {
-+ }
-+ }
-+ }
-+ }
-+ // An inactive shared LVM volume can be owned by another node; do not probe it by creating a volume.
-+ if (!haveInfo && this instanceof ProbeVlmStorageProvider &&
-+ (vlmProviderList == null || vlmProviderList.isEmpty() ||
-+ kind == DeviceProviderKind.ZFS || kind == DeviceProviderKind.ZFS_THIN))
-+ {
-+ updateMinIoSizeByProbeVlm((ProbeVlmStorageProvider) this, storPoolObj, propsChange);
-+ }
-+ errorReporter.logDebug("EXIT updateMinIoSize method");
-+ }
-+
-+ private void updateMinIoSizeByProbeVlm(
-+ ProbeVlmStorageProvider storPrv,
-+ StorPool storPoolObj,
-+ LocalPropsChangePojo propsChange
-+ )
-+ throws AccessDeniedException
-+ {
-+ // Providers use a fixed probe name. Keep creation, probing and cleanup in one critical section.
-+ synchronized (probeVlmLock)
-+ {
-+ boolean haveInfo = false;
-+ boolean interrupted = false;
-+ String lastFailure = "";
-+ int attempts = 0;
-+ while (attempts < PROBE_VLM_ATTEMPTS && !haveInfo && !interrupted)
-+ {
-+ if (attempts > 0)
-+ {
-+ try
-+ {
-+ Thread.sleep(PROBE_VLM_RETRY_DELAY_IN_MS);
-+ }
-+ catch (InterruptedException exc)
-+ {
-+ Thread.currentThread().interrupt();
-+ lastFailure = "interrupted before the next probe attempt";
-+ break;
-+ }
-+ }
-+ attempts++;
-+ boolean cleaned = false;
- try
- {
-- updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange);
-- haveInfo = true;
-- }
-- catch (IOException ignored)
-- {
-- }
-- }
-- }
-- else
-- {
-- errorReporter.logDebug("updateMinIoSize: Don't have vlmProviderList, using temporary volumes");
-- if (this instanceof ProbeVlmStorageProvider)
-- {
-- final ProbeVlmStorageProvider storPrv = (ProbeVlmStorageProvider) this;
-- try
-- {
-- final String storDevicePath = storPrv.createTmpProbeVlm(storPoolObj);
-+ @Nullable String storDevicePath = storPrv.createTmpProbeVlm(storPoolObj);
-+ if (storDevicePath == null)
-+ {
-+ lastFailure = "temporary volume has no device path";
-+ }
-+ else
-+ {
-+ waitUntilDeviceCreated(storDevicePath, getWaitTimeoutAfterCreate(storPoolObj));
-+ updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange);
-+ haveInfo = true;
-+ }
-+ }
-+ catch (StorageException | IOException exc)
-+ {
-+ lastFailure = exc.getMessage();
-+ interrupted = exc.getCause() instanceof InterruptedException;
-+ errorReporter.logDebug("Temporary volume probe attempt %d failed: %s", attempts, lastFailure);
-+ }
-+ finally
-+ {
- try
- {
-- updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange);
-- }
-- catch (IOException ignored)
-- {
-- }
-- }
-- catch (StorageException ignored)
-- {
-- errorReporter.logDebug("updateMinIoSize: Temporary volume creation failed");
-- }
-- finally
-- {
-- try
-- {
- storPrv.deleteTmpProbeVlm(storPoolObj);
-- }
-- catch (StorageException ignored)
-- {
-- errorReporter.logDebug("updateMinIoSize: Temporary volume deletion failed");
-- }
-- }
-- }
-- }
-- errorReporter.logDebug("EXIT updateMinIoSize method");
-+ cleaned = true;
-+ }
-+ catch (StorageException exc)
-+ {
-+ interrupted |= exc.getCause() instanceof InterruptedException;
-+ errorReporter.logWarning(
-+ "Failed to delete temporary probe volume for storage pool '%s': %s",
-+ storPoolObj.getName().displayValue,
-+ exc.getMessage()
-+ );
-+ }
-+ }
-+ if (interrupted)
-+ {
-+ Thread.currentThread().interrupt();
-+ }
-+ if (!cleaned)
-+ {
-+ break;
-+ }
-+ }
-+ if (!haveInfo)
-+ {
-+ errorReporter.logWarning(
-+ "Could not determine block device information for storage pool '%s' after %d probe attempts: %s",
-+ storPoolObj.getName().displayValue,
-+ attempts,
-+ lastFailure
-+ );
-+ }
-+ }
- }
-
- private void updateMinIoSizeByDevice(
-diff --git a/src/test/java/com/linbit/fsevent/ProbeVolumeDeviceWaitTest.java b/src/test/java/com/linbit/fsevent/ProbeVolumeDeviceWaitTest.java
-new file mode 100644
---- /dev/null
-+++ b/src/test/java/com/linbit/fsevent/ProbeVolumeDeviceWaitTest.java
-@@ -0,0 +1,467 @@
-+package com.linbit.fsevent;
-+
-+import com.linbit.Platform;
-+import com.linbit.linstor.core.identifier.StorPoolName;
-+import com.linbit.linstor.core.objects.Resource;
-+import com.linbit.linstor.core.objects.StorPool;
-+import com.linbit.linstor.core.pojos.LocalPropsChangePojo;
-+import com.linbit.linstor.layer.storage.AbsStorageProvider.AbsStorageProviderInit;
-+import com.linbit.linstor.layer.storage.BlockSizeConsts;
-+import com.linbit.linstor.layer.storage.lvm.LvmProvider;
-+import com.linbit.linstor.layer.storage.zfs.ZfsThinProvider;
-+import com.linbit.linstor.logging.ErrorReporter;
-+import com.linbit.linstor.propscon.Props;
-+import com.linbit.linstor.storage.StorageConstants;
-+import com.linbit.linstor.storage.StorageException;
-+import com.linbit.linstor.storage.interfaces.categories.resource.VlmProviderObject;
-+
-+import java.io.IOException;
-+import java.nio.file.Files;
-+import java.nio.file.Path;
-+import java.util.Collections;
-+import java.util.List;
-+import java.util.Map;
-+import java.util.concurrent.Callable;
-+import java.util.concurrent.CountDownLatch;
-+import java.util.concurrent.ExecutorService;
-+import java.util.concurrent.Executors;
-+import java.util.concurrent.Future;
-+import java.util.concurrent.ScheduledExecutorService;
-+import java.util.concurrent.ScheduledFuture;
-+import java.util.concurrent.TimeUnit;
-+import java.util.concurrent.atomic.AtomicInteger;
-+
-+import org.junit.After;
-+import org.junit.Before;
-+import org.junit.Rule;
-+import org.junit.Test;
-+import org.junit.rules.TemporaryFolder;
-+import org.junit.rules.Timeout;
-+
-+import static org.junit.Assert.assertEquals;
-+import static org.junit.Assert.assertFalse;
-+import static org.junit.Assert.assertTrue;
-+import static org.junit.Assume.assumeTrue;
-+import static org.mockito.Mockito.doReturn;
-+import static org.mockito.Mockito.mock;
-+import static org.mockito.Mockito.mockingDetails;
-+import static org.mockito.Mockito.when;
-+
-+public class ProbeVolumeDeviceWaitTest
-+{
-+ @Rule
-+ public final TemporaryFolder directory = new TemporaryFolder();
-+
-+ @Rule
-+ public final Timeout timeout = Timeout.seconds(15);
-+
-+ private FileSystemWatch watch;
-+ private Path device;
-+ private Path link;
-+ private StorPool pool;
-+ private ProbeProvider provider;
-+ private ErrorReporter reporter;
-+ private ScheduledExecutorService executor;
-+
-+ @Before
-+ public void setUp() throws Exception
-+ {
-+ assumeTrue(Platform.isLinux());
-+ device = directory.newFile("zd-test").toPath();
-+ link = directory.getRoot().toPath().resolve("probe");
-+ executor = Executors.newSingleThreadScheduledExecutor();
-+ reporter = mock(ErrorReporter.class);
-+ watch = new FileSystemWatch(reporter);
-+ watch.start();
-+ pool = mock(StorPool.class);
-+ when(pool.getName()).thenReturn(new StorPoolName("data"));
-+ when(pool.getVolumes(null)).thenReturn(Collections.emptyList());
-+ when(pool.getProps(null)).thenReturn(mock(Props.class));
-+ provider = new ProbeProvider(providerInit());
-+ }
-+
-+ @After
-+ public void tearDown() throws Exception
-+ {
-+ if (executor != null)
-+ {
-+ executor.shutdownNow();
-+ assertTrue(executor.awaitTermination(2, TimeUnit.SECONDS));
-+ }
-+ if (watch != null)
-+ {
-+ watch.shutdown(false);
-+ watch.awaitShutdown(2000);
-+ watch.cancelAllWatchKeys();
-+ }
-+ }
-+
-+ @Test
-+ public void delayedDevicePublishesPropertiesBeforeCleanup() throws Exception
-+ {
-+ provider.delayMillis = 50;
-+ assertPublished(probe());
-+ assertCleaned(1);
-+ assertEquals(0, warningCount());
-+ }
-+
-+ @Test
-+ public void existingProbeDevicePublishesProperties() throws Exception
-+ {
-+ assertPublished(probe());
-+ assertCleaned(1);
-+ }
-+
-+ @Test
-+ public void missingDeviceTimesOutAndCleansEveryAttempt() throws Exception
-+ {
-+ provider.delayMillis = -1;
-+ provider.timeoutMillis = 20;
-+ assertTrue(probe().isEmpty());
-+ assertCleaned(3);
-+ assertEquals(1, warningCount());
-+ }
-+
-+ @Test
-+ public void failedCreationStillCleansEveryAttempt() throws Exception
-+ {
-+ provider.failCreateTimes = Integer.MAX_VALUE;
-+ assertTrue(probe().isEmpty());
-+ assertCleaned(3);
-+ assertEquals(1, warningCount());
-+ }
-+
-+ @Test
-+ public void transientCreationFailureIsRetried() throws Exception
-+ {
-+ provider.failCreateTimes = 1;
-+ provider.delayMillis = 50;
-+ assertPublished(probe());
-+ assertCleaned(2);
-+ assertEquals(0, warningCount());
-+ }
-+
-+ @Test
-+ public void nullProbePathDoesNotPublishProperties() throws Exception
-+ {
-+ provider.nullPath = true;
-+ assertTrue(probe().isEmpty());
-+ assertCleaned(3);
-+ assertEquals(1, warningCount());
-+ }
-+
-+ @Test
-+ public void nullFirstPathDoesNotSkipUsableVolume() throws Exception
-+ {
-+ Files.createSymbolicLink(link, device);
-+ doReturn(List.of(volume(null), volume(link))).when(pool).getVolumes(null);
-+ assertPublished(probe());
-+ assertEquals(0, provider.createCalls);
-+ assertEquals(0, provider.deleteCalls);
-+ }
-+
-+ @Test
-+ public void missingFirstPathDoesNotSkipUsableVolume() throws Exception
-+ {
-+ Files.createSymbolicLink(link, device);
-+ doReturn(List.of(volume(link.resolveSibling("missing")), volume(link))).when(pool).getVolumes(null);
-+ assertPublished(probe());
-+ assertEquals(0, provider.createCalls);
-+ assertEquals(0, provider.deleteCalls);
-+ }
-+
-+ @Test
-+ public void allNullPathsFallBackToProbe() throws Exception
-+ {
-+ doReturn(List.of(volume(null))).when(pool).getVolumes(null);
-+ provider.delayMillis = 50;
-+ assertPublished(probe());
-+ assertCleaned(1);
-+ }
-+
-+ @Test
-+ public void allMissingPathsFallBackToProbe() throws Exception
-+ {
-+ doReturn(List.of(
-+ volume(link.resolveSibling("missing-a")), volume(link.resolveSibling("missing-b"))
-+ )).when(pool).getVolumes(null);
-+ provider.delayMillis = 50;
-+ assertPublished(probe());
-+ assertCleaned(1);
-+ }
-+
-+ @Test
-+ public void cleanupFailureAfterSuccessIsReported() throws Exception
-+ {
-+ provider.failDelete = true;
-+ assertPublished(probe());
-+ assertEquals(1, provider.createCalls);
-+ assertEquals(1, provider.deleteCalls);
-+ assertEquals(1, warningCount());
-+ }
-+
-+ @Test
-+ public void cleanupFailurePreventsAnotherCreationAttempt() throws Exception
-+ {
-+ provider.delayMillis = -1;
-+ provider.timeoutMillis = 20;
-+ provider.failDelete = true;
-+ assertTrue(probe().isEmpty());
-+ assertEquals(1, provider.createCalls);
-+ assertEquals(1, provider.deleteCalls);
-+ assertEquals(2, warningCount());
-+ }
-+
-+ @Test
-+ public void interruptedDeviceWaitCleansUpAndPreservesInterrupt() throws Exception
-+ {
-+ provider.delayMillis = -1;
-+ Thread callingThread = Thread.currentThread();
-+ executor.schedule(callingThread::interrupt, 50, TimeUnit.MILLISECONDS);
-+ try
-+ {
-+ assertTrue(probe().isEmpty());
-+ assertTrue(callingThread.isInterrupted());
-+ assertCleaned(1);
-+ assertEquals(1, warningCount());
-+ }
-+ finally
-+ {
-+ Thread.interrupted();
-+ }
-+ }
-+
-+ @Test
-+ public void interruptedRetryDelayDoesNotCreateAnotherProbe() throws Exception
-+ {
-+ provider.failCreateTimes = Integer.MAX_VALUE;
-+ Thread callingThread = Thread.currentThread();
-+ executor.schedule(callingThread::interrupt, 100, TimeUnit.MILLISECONDS);
-+ try
-+ {
-+ assertTrue(probe().isEmpty());
-+ assertTrue(callingThread.isInterrupted());
-+ assertCleaned(1);
-+ assertEquals(1, warningCount());
-+ }
-+ finally
-+ {
-+ Thread.interrupted();
-+ }
-+ }
-+
-+ @Test
-+ public void interruptedCleanupPreservesInterruptAndStopsRetries() throws Exception
-+ {
-+ provider.failCreateTimes = Integer.MAX_VALUE;
-+ provider.interruptDelete = true;
-+ try
-+ {
-+ assertTrue(probe().isEmpty());
-+ assertTrue(Thread.currentThread().isInterrupted());
-+ assertEquals(1, provider.createCalls);
-+ assertEquals(1, provider.deleteCalls);
-+ }
-+ finally
-+ {
-+ Thread.interrupted();
-+ }
-+ }
-+
-+ @Test
-+ public void concurrentProbesDoNotShareTheTemporaryVolume() throws Exception
-+ {
-+ provider.creationDelayMillis = 200;
-+ ExecutorService callers = Executors.newFixedThreadPool(2);
-+ CountDownLatch ready = new CountDownLatch(2);
-+ CountDownLatch start = new CountDownLatch(1);
-+ Callable call = () ->
-+ {
-+ ready.countDown();
-+ start.await();
-+ return probe();
-+ };
-+ try
-+ {
-+ Future first = callers.submit(call);
-+ Future second = callers.submit(call);
-+ assertTrue(ready.await(2, TimeUnit.SECONDS));
-+ start.countDown();
-+ assertFalse(first.get(10, TimeUnit.SECONDS).isEmpty());
-+ assertFalse(second.get(10, TimeUnit.SECONDS).isEmpty());
-+ assertEquals(1, provider.peakActiveProbes.get());
-+ assertCleaned(2);
-+ }
-+ finally
-+ {
-+ callers.shutdownNow();
-+ assertTrue(callers.awaitTermination(2, TimeUnit.SECONDS));
-+ }
-+ }
-+
-+ @Test
-+ public void inactiveLvmVolumesDoNotCreateProbe() throws Exception
-+ {
-+ doReturn(List.of(volume(null))).when(pool).getVolumes(null);
-+ LvmProvider lvm = new LvmProvider(providerInit())
-+ {
-+ @Override
-+ public String createTmpProbeVlm(StorPool ignored)
-+ {
-+ throw new AssertionError("Inactive shared LVM volumes must not trigger a probe volume");
-+ }
-+
-+ @Override
-+ public void deleteTmpProbeVlm(StorPool ignored)
-+ {
-+ throw new AssertionError("No probe volume was created");
-+ }
-+ };
-+ LocalPropsChangePojo changes = new LocalPropsChangePojo();
-+ lvm.updateMinIoSize(pool, changes);
-+ assertTrue(changes.isEmpty());
-+ }
-+
-+ private AbsStorageProviderInit providerInit()
-+ {
-+ return new AbsStorageProviderInit(
-+ reporter, null, null, null, null, null, null, null, null, null, watch, null, null, null
-+ );
-+ }
-+
-+ private LocalPropsChangePojo probe() throws Exception
-+ {
-+ LocalPropsChangePojo changes = new LocalPropsChangePojo();
-+ provider.updateMinIoSize(pool, changes);
-+ return changes;
-+ }
-+
-+ private void assertPublished(LocalPropsChangePojo changesRef)
-+ {
-+ String namespace = StorageConstants.NAMESPACE_INTERNAL + '/';
-+ assertEquals(Map.of(
-+ namespace + StorageConstants.BLK_DEV_MIN_IO_SIZE, Long.toString(BlockSizeConsts.DFLT_IO_SIZE),
-+ namespace + StorageConstants.BLK_DEV_OPT_IO_SIZE, Long.toString(BlockSizeConsts.DFLT_OPT_IO_SIZE)
-+ ), changesRef.changedStorPoolProps.get(pool.getName()));
-+ }
-+
-+ private void assertCleaned(int attemptsRef)
-+ {
-+ assertEquals(attemptsRef, provider.createCalls);
-+ assertEquals(attemptsRef, provider.deleteCalls);
-+ assertFalse(Files.exists(link));
-+ }
-+
-+ @SuppressWarnings("unchecked")
-+ private VlmProviderObject volume(Path pathRef)
-+ {
-+ VlmProviderObject volume = mock(VlmProviderObject.class);
-+ when(volume.getDevicePath()).thenReturn(pathRef == null ? null : pathRef.toString());
-+ return volume;
-+ }
-+
-+ private long warningCount()
-+ {
-+ return mockingDetails(reporter).getInvocations().stream()
-+ .filter(invocation -> invocation.getMethod().getName().equals("logWarning"))
-+ .count();
-+ }
-+
-+ private class ProbeProvider extends ZfsThinProvider
-+ {
-+ private long delayMillis;
-+ private long creationDelayMillis;
-+ private long timeoutMillis = 4000;
-+ private int failCreateTimes;
-+ private boolean nullPath;
-+ private boolean failDelete;
-+ private boolean interruptDelete;
-+ private int createCalls;
-+ private int deleteCalls;
-+ private ScheduledFuture> create;
-+ private final AtomicInteger activeProbes = new AtomicInteger();
-+ private final AtomicInteger peakActiveProbes = new AtomicInteger();
-+
-+ ProbeProvider(AbsStorageProviderInit initRef)
-+ {
-+ super(initRef);
-+ }
-+
-+ @Override
-+ protected long getWaitTimeoutAfterCreate(StorPool ignored)
-+ {
-+ return timeoutMillis;
-+ }
-+
-+ @Override
-+ public String createTmpProbeVlm(StorPool ignored) throws StorageException
-+ {
-+ peakActiveProbes.accumulateAndGet(activeProbes.incrementAndGet(), Math::max);
-+ createCalls++;
-+ if (creationDelayMillis > 0)
-+ {
-+ try
-+ {
-+ Thread.sleep(creationDelayMillis);
-+ }
-+ catch (InterruptedException exc)
-+ {
-+ throw new StorageException("Test probe creation interrupted", exc);
-+ }
-+ }
-+ if (failCreateTimes > 0)
-+ {
-+ failCreateTimes--;
-+ throw new StorageException("Probe creation failed");
-+ }
-+ if (nullPath)
-+ {
-+ return null;
-+ }
-+ if (delayMillis == 0)
-+ {
-+ try
-+ {
-+ Files.createSymbolicLink(link, device);
-+ }
-+ catch (IOException exc)
-+ {
-+ throw new StorageException("Test device creation failed", exc);
-+ }
-+ }
-+ else if (delayMillis > 0)
-+ {
-+ create = executor.schedule(() -> Files.createSymbolicLink(link, device),
-+ delayMillis, TimeUnit.MILLISECONDS);
-+ }
-+ return link.toString();
-+ }
-+
-+ @Override
-+ public void deleteTmpProbeVlm(StorPool ignored) throws StorageException
-+ {
-+ deleteCalls++;
-+ if (interruptDelete)
-+ {
-+ throw new StorageException("Probe cleanup interrupted", new InterruptedException());
-+ }
-+ if (failDelete)
-+ {
-+ throw new StorageException("Probe cleanup failed");
-+ }
-+ try
-+ {
-+ if (create != null && !create.cancel(false))
-+ {
-+ create.get();
-+ }
-+ Files.deleteIfExists(link);
-+ activeProbes.decrementAndGet();
-+ }
-+ catch (Exception exc)
-+ {
-+ throw new StorageException("Test device cleanup failed", exc);
-+ }
-+ }
-+ }
-+}
diff --git a/packages/system/linstor/images/piraeus-server/patches/native-protoc.diff b/packages/system/linstor/images/piraeus-server/patches/native-protoc.diff
index f3e601698..fa8596177 100644
--- a/packages/system/linstor/images/piraeus-server/patches/native-protoc.diff
+++ b/packages/system/linstor/images/piraeus-server/patches/native-protoc.diff
@@ -1,8 +1,8 @@
diff --git a/build.gradle b/build.gradle
-index 75b7f64..d2dfec9 100644
+index 9df6c840cdc0..9894a436712c 100644
--- a/build.gradle
+++ b/build.gradle
-@@ -37,6 +37,9 @@ configurations {
+@@ -34,6 +34,9 @@ configurations {
}
def protobufVersion = '31.1'
@@ -12,7 +12,7 @@ index 75b7f64..d2dfec9 100644
def protoc = file("${projectDir}/tools/protoc-" + protobufVersion + '/bin/protoc')
def logbackConfig = "logback.xml"
def jvmOpts = ["-Xms32M"]
-@@ -577,7 +580,7 @@ tasks.register('buildTestCommands', Exec) {
+@@ -650,7 +653,7 @@ tasks.register('buildTestCommands', Exec) {
tasks.register('unzipProtoc', Copy) {
dependsOn(downloadProtoc)
@@ -21,7 +21,7 @@ index 75b7f64..d2dfec9 100644
def outputDir = file("tools/protoc-" + protobufVersion)
from zipTree(zipFile)
-@@ -591,7 +594,7 @@ tasks.register('unzipProtoc', Copy) {
+@@ -664,7 +667,7 @@ tasks.register('unzipProtoc', Copy) {
tasks.register('downloadProtoc') {
outputs.dir "tools"
@@ -30,7 +30,7 @@ index 75b7f64..d2dfec9 100644
outputs.file protozip
doLast {
-@@ -599,7 +602,7 @@ tasks.register('downloadProtoc') {
+@@ -672,7 +675,7 @@ tasks.register('downloadProtoc') {
mkdir "tools"
println "downloading protoc..."
new URL('https://github.com/google/protobuf/releases/download/v'
diff --git a/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff b/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff
index 009591674..e5e0ec013 100644
--- a/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff
+++ b/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff
@@ -1,28 +1,27 @@
diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
-index 5627d1be8..ece191292 100644
+index 0a19f875b702..d44e19b70d88 100644
--- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
+++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
-@@ -42,6 +42,9 @@ import java.util.Arrays;
- import java.util.List;
+@@ -45,6 +45,8 @@ import java.util.List;
+ import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.TimeUnit;
-+import java.nio.charset.StandardCharsets;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Singleton
-@@ -56,6 +58,9 @@ public class DrbdAdm
-
- public static final int WAIT_CONNECT_RES_TIME = 10;
- private static final long DOWN_WAIT_TIMEOUT_SEC = 5;
+@@ -68,6 +70,9 @@ public class DrbdAdm
+ private static final long SECONDARY_RETRY_INITIAL_BACKOFF_MS = 250L;
+ private static final long SECONDARY_RETRY_MAX_BACKOFF_MS = 2_000L;
+ private static final String DEVICE_HELD_OPEN_ERR_MSG = "Device is held open by someone";
+ private static final long FORCE_DETACH_RETRY_WAIT_MS = 250;
+ private static final String BITMAP_LEAK_ERR_MSG = "already has a bitmap, this should not happen";
+ private static final Pattern BITMAP_LEAK_MINOR_PATTERN = Pattern.compile("\\bminor\\s+(\\d+)\\b");
private final ExtCmdFactory extCmdFactory;
- private final AccessContext sysCtx;
-@@ -131,8 +136,38 @@ public class DrbdAdm
+ private final StltConfigAccessor stltCfgAccessor;
+@@ -143,8 +148,38 @@ public class DrbdAdm
// command.add(resName);
command.add(drbdRscData.getSuffixedResourceName());
// execute(Arrays.asList("drbdsetup", "show", drbdRscData.getSuffixedResourceName()));
@@ -35,7 +34,7 @@ index 5627d1be8..ece191292 100644
+ ExtCmd extCmd = extCmdFactory.create();
+ if (Platform.isWindows())
+ {
-+ extCmd.setTimeout(TimeoutType.WAIT, 5 * 60 * 1000);
++ extCmd.setTimeout(TimeoutType.WAIT, DFLT_WINDOWS_TIMEOUT_MS);
+ }
+
+ OutputData outputData = extCmd.pipeExec(ProcessBuilder.Redirect.from(nullDevice), commandArr);
@@ -63,8 +62,8 @@ index 5627d1be8..ece191292 100644
drbdRscData.setAdjustRequired(false);
}
-@@ -805,6 +840,75 @@ public class DrbdAdm
- }
+@@ -955,6 +990,75 @@ public class DrbdAdm
+ return ret;
}
+ static boolean isBitmapLeakOnAttach(OutputData outputData)
diff --git a/packages/system/linstor/images/piraeus-server/patches/retry-secondary-after-mkfs.diff b/packages/system/linstor/images/piraeus-server/patches/retry-secondary-after-mkfs.diff
deleted file mode 100644
index f213e9037..000000000
--- a/packages/system/linstor/images/piraeus-server/patches/retry-secondary-after-mkfs.diff
+++ /dev/null
@@ -1,104 +0,0 @@
-diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
-index 71c1df6..f91903b 100644
---- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
-+++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java
-@@ -62,6 +62,10 @@ public class DrbdAdm
- private static final long FORCE_DETACH_RETRY_WAIT_MS = 250;
- private static final String BITMAP_LEAK_ERR_MSG = "already has a bitmap, this should not happen";
- private static final Pattern BITMAP_LEAK_MINOR_PATTERN = Pattern.compile("\\bminor\\s+(\\d+)\\b");
-+ private static final int SECONDARY_RETRY_MAX_ATTEMPTS = 7;
-+ private static final long SECONDARY_RETRY_INITIAL_BACKOFF_MS = 250L;
-+ private static final long SECONDARY_RETRY_MAX_BACKOFF_MS = 2000L;
-+ private static final String DEVICE_HELD_OPEN_ERR_MSG = "Device is held open by someone";
-
- private final ExtCmdFactory extCmdFactory;
- private final AccessContext sysCtx;
-@@ -945,6 +949,79 @@ public class DrbdAdm
- return true;
- }
-
-+ /**
-+ * Switches a resource to secondary mode after mkfs, retrying when DRBD reports the device as held
-+ * open by another process. On Talos and other systems with active block-device probing
-+ * (machined/udev), the kernel uevent for a freshly-promoted device may cause an external probe
-+ * to open the device just before LINSTOR demotes it back to Secondary, producing
-+ * "Device is held open by someone" (errno -12) and aborting resource initialization.
-+ * Probes typically release the fd within milliseconds, so a short retry with exponential backoff
-+ * is sufficient to win the race without changing the contract of {@link #secondary}.
-+ */
-+ public void secondaryAfterMkfs(DrbdRscData drbdRscData) throws ExtCmdFailedException
-+ {
-+ String[] commandArr = new String[] {
-+ DRBDADM_UTIL, "-vvv", "secondary", drbdRscData.getSuffixedResourceName()
-+ };
-+
-+ File nullDevice = new File(Platform.nullDevice());
-+
-+ OutputData lastOutput = null;
-+ for (int attempt = 1; attempt <= SECONDARY_RETRY_MAX_ATTEMPTS; attempt++)
-+ {
-+ try
-+ {
-+ ExtCmd extCmd = extCmdFactory.create();
-+ if (Platform.isWindows())
-+ {
-+ extCmd.setTimeout(TimeoutType.WAIT, 5 * 60 * 1000);
-+ }
-+
-+ lastOutput = extCmd.pipeExec(ProcessBuilder.Redirect.from(nullDevice), commandArr);
-+ if (lastOutput.exitCode == 0)
-+ {
-+ return;
-+ }
-+ if (!isDeviceHeldOpenError(lastOutput) || attempt == SECONDARY_RETRY_MAX_ATTEMPTS)
-+ {
-+ throw new ExtCmdFailedException(commandArr, lastOutput);
-+ }
-+ }
-+ catch (ChildProcessTimeoutException timeoutExc)
-+ {
-+ throw new ExtCmdFailedException(commandArr, timeoutExc);
-+ }
-+ catch (IOException ioExc)
-+ {
-+ throw new ExtCmdFailedException(commandArr, ioExc);
-+ }
-+
-+ long sleepMs = Math.min(
-+ SECONDARY_RETRY_INITIAL_BACKOFF_MS * (1L << (attempt - 1)),
-+ SECONDARY_RETRY_MAX_BACKOFF_MS
-+ );
-+ try
-+ {
-+ Thread.sleep(sleepMs);
-+ }
-+ catch (InterruptedException ie)
-+ {
-+ Thread.currentThread().interrupt();
-+ throw new ExtCmdFailedException(commandArr, lastOutput);
-+ }
-+ }
-+ }
-+
-+ static boolean isDeviceHeldOpenError(OutputData outputData)
-+ {
-+ if (outputData == null || outputData.stderrData == null)
-+ {
-+ return false;
-+ }
-+ String stderr = new String(outputData.stderrData, StandardCharsets.UTF_8);
-+ return stderr.contains(DEVICE_HELD_OPEN_ERR_MSG);
-+ }
-+
- public static class DrbdPrimary implements AutoCloseable
- {
- private final DrbdAdm drbdAdm;
-@@ -963,7 +1040,7 @@ public class DrbdAdm
- @Override
- public void close() throws ExtCmdFailedException
- {
-- drbdAdm.secondary(drbdRscData);
-+ drbdAdm.secondaryAfterMkfs(drbdRscData);
- }
- }
- }
diff --git a/packages/system/linstor/images/piraeus-server/patches/snapshot-restore-drbd-initialized.diff b/packages/system/linstor/images/piraeus-server/patches/snapshot-restore-drbd-initialized.diff
deleted file mode 100644
index 7c72aab69..000000000
--- a/packages/system/linstor/images/piraeus-server/patches/snapshot-restore-drbd-initialized.diff
+++ /dev/null
@@ -1,67 +0,0 @@
-diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java
-index a639c27cc..4dabc1a83 100644
---- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java
-+++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java
-@@ -434,26 +434,39 @@ public class CtrlSnapshotRestoreApiCallHandler
- }
- }
-
-- /*
-- * When a snapshot was received as a backup, it is marked to re-create its local metadata. Before the
-- * snapshot was sent the new DRBD_INITIALIZED flag and the VlmDfn property InitialUpToDateOn were
-+ /**
-+ * Sets the {@value InternalApiConsts#KEY_LINSTOR_DRBD_INITIAL_UPTODATE_ON} property to the nodeName
-+ * of the first Snapshot (which has to have a disk) of the given SnapshotDefinition.
-+ *
-+ * If this SnapshotDefinition is a backup, we also unset the {@link VolumeDefinition.Flags#DRBD_INITIALIZED}
-+ * flag. Otherwise we ensure the same flag is set.
-+ *
-+ * Backup path: When a snapshot was received as a backup, it is marked to re-create its local metadata.
-+ * Before the snapshot was sent the new DRBD_INITIALIZED flag and the VlmDfn property InitialUpToDateOn were
- * cleared. If DRBD_INITIALZED would be set, no peer would even read the property (on purpose).
- * Without the property, no peer would think of itself as a winner, thus the DRBD resource would never
- * become UpToDate.
- * To avoid this, this method sets the InitialUpToDateOn property to let the first resource win the
-- * race, so it can become UpToDate and therefore also pull the other peers eventually into UpToDate state.
-+ * race, so it can become UpToDate and therefore also pull the other peers eventually into UpToDate state.
- *
-- * If the given SnapDfn had also non-backup snapshots, those should be able to keep their metadata.
-+ * If the given SnapDfn had also non-backup snapshots, those should be able to keep their metadata.
- * In that case we do not have to do anything. If however all snapshots are descendants from backups
- * we must pick one and decide the UpToDate race upfront so that the winner can set its local DRBD
-- * as UpToDate, which also triggers the controller to set the DRBD_INITIALZED flag eventually.
-+ * as UpToDate, which also triggers the controller to set the DRBD_INITIALZED flag eventually.
-+ *
-+ * Non-Backup path: When a regular snapshot (i.e. not from a backup) is restored, we can or even should
-+ * assume that the backing snapshot LVs have correct metadata which do not need be be initialized. When
-+ * a SnapshotVolumeDefinition is created it unfortunately does not properly store the VolumeDefinition's flags,
-+ * therefore the restored VolumeDefinition will also not have the DRBD_INITIALIZED flag set, although it
-+ * refers to a device with healthy and UpToDate metadata. To fix this issue we simply set the flag in this case.
- */
- private void checkDrbdInitializedState(ResourceDefinition rscDfnRef) throws AccessDeniedException
- {
- AccessContext accCtx = peerAccCtx.get();
-- if (DrbdLayerUtils.isForceInitialSyncSet(accCtx, rscDfnRef) && rscDfnRef.getDiskfulCount(accCtx) > 0)
-+ if (rscDfnRef.getDiskfulCount(accCtx) > 0)
- {
- String winnerNodeName = rscDfnRef.getDiskfulResources(accCtx).get(0).getNode().getName().value;
-+ boolean uninitializeDrbd = DrbdLayerUtils.isForceInitialSyncSet(accCtx, rscDfnRef);
- Iterator vlmDfnIt = rscDfnRef.iterateVolumeDfn(accCtx);
- while (vlmDfnIt.hasNext())
- {
-@@ -462,7 +475,14 @@ public class CtrlSnapshotRestoreApiCallHandler
- {
- vlmDfn.getProps(accCtx)
- .setProp(InternalApiConsts.KEY_LINSTOR_DRBD_INITIAL_UPTODATE_ON, winnerNodeName);
-- vlmDfn.getFlags().disableFlags(accCtx, VolumeDefinition.Flags.DRBD_INITIALIZED);
-+ if (uninitializeDrbd)
-+ {
-+ vlmDfn.getFlags().disableFlags(accCtx, VolumeDefinition.Flags.DRBD_INITIALIZED);
-+ }
-+ else
-+ {
-+ vlmDfn.getFlags().enableFlags(accCtx, VolumeDefinition.Flags.DRBD_INITIALIZED);
-+ }
- }
- catch (InvalidKeyException | InvalidValueException exc)
- {
From 9e861c117840cbc4f3886bc7f53dbdd0fb692427 Mon Sep 17 00:00:00 2001
From: Mattia Eleuteri
Date: Fri, 13 Nov 2026 09:19:25 +0100
Subject: [PATCH 091/114] fix(seaweedfs): stop the master crashing on the
crowded volume map
SeaweedFS 4.45 updates the crowded-volume map of a volume layout under
the read lock, while Assign reads that map under the same read lock to
count writable volumes. Under a steady write load the Go runtime aborts
the master with "concurrent map read and map write", a fatal error that
recover cannot catch, and the S3 gateway answers 500 until a master is
elected again. Taking the write lock for the update, as every other
mutation of the map already does, removes the race.
Upstream fixed it in seaweedfs/seaweedfs#11216, released in 4.46. The
patch carries that change and its regression test unchanged, so the
build runs the test and the patch stops applying, and is dropped, once
the vendored release carries the fix.
Signed-off-by: Mattia Eleuteri
Assisted-by: LLM
---
.../seaweedfs/images/seaweedfs/Dockerfile | 9 ++
.../patches/volume-layout-crowded-lock.diff | 87 +++++++++++++++++++
2 files changed, 96 insertions(+)
create mode 100644 packages/system/seaweedfs/images/seaweedfs/patches/volume-layout-crowded-lock.diff
diff --git a/packages/system/seaweedfs/images/seaweedfs/Dockerfile b/packages/system/seaweedfs/images/seaweedfs/Dockerfile
index 8f42f3ea2..d52ee7329 100644
--- a/packages/system/seaweedfs/images/seaweedfs/Dockerfile
+++ b/packages/system/seaweedfs/images/seaweedfs/Dockerfile
@@ -16,6 +16,15 @@
# seaweedfs/seaweedfs#11674 after 4.48; drop the patch on the first vendored
# release that carries it.
#
+# patches/volume-layout-crowded-lock.diff makes SetVolumeCrowded take the write
+# lock of the volume layout. Upstream 4.45 mutates the crowded map under the
+# read lock, while Assign reads it under the same read lock through
+# GetWritableVolumeCount, so under a steady stream of writes the master dies on
+# a fatal "concurrent map read and map write", which recover cannot catch, and
+# S3 answers 500 until it is back. Backport of seaweedfs/seaweedfs#11216,
+# released in 4.46; drop the patch on the first vendored release that carries
+# it.
+#
# `make update` re-pins VERSION from the vendored chart, SOURCE_SHA256 from the
# source archive of that tag, UPSTREAM_DIGEST from the upstream image of that
# tag and the builder from upstream's docker/Dockerfile.go_build, and checks
diff --git a/packages/system/seaweedfs/images/seaweedfs/patches/volume-layout-crowded-lock.diff b/packages/system/seaweedfs/images/seaweedfs/patches/volume-layout-crowded-lock.diff
new file mode 100644
index 000000000..12e09ca67
--- /dev/null
+++ b/packages/system/seaweedfs/images/seaweedfs/patches/volume-layout-crowded-lock.diff
@@ -0,0 +1,87 @@
+diff --git a/weed/topology/crowded_writable_test.go b/weed/topology/crowded_writable_test.go
+--- a/weed/topology/crowded_writable_test.go
++++ b/weed/topology/crowded_writable_test.go
+@@ -1,6 +1,7 @@
+ package topology
+
+ import (
++ "sync"
+ "testing"
+ "time"
+
+@@ -73,3 +74,52 @@ func TestCrowdedCountStillMatchesWritables(t *testing.T) {
+ t.Errorf("writable=%d crowded=%d, want 1 and 1", writable, crowded)
+ }
+ }
++
++// SetVolumeCrowded mutates the crowded map while GetWritableVolumeCount reads
++// it on the Assign hot path. Under RLock both could run at once and the Go
++// runtime aborts with "concurrent map read and map write". Run with -race to
++// reproduce the original bug; the write lock makes this pass.
++func TestSetVolumeCrowdedNoRaceWithGetWritableVolumeCount(t *testing.T) {
++ rp, _ := super_block.NewReplicaPlacementFromString("000")
++ vl := NewVolumeLayout(rp, needle.EMPTY_TTL, types.HardDriveType, 10000, false)
++ // GetWritableVolumeCount iterates vl.writables, so give it something to
++ // read while the crowded map is being mutated.
++ vl.writables = []needle.VolumeId{1, 2, 3}
++
++ var wg sync.WaitGroup
++ stop := make(chan struct{})
++ for range 8 {
++ wg.Add(1)
++ go func() {
++ defer wg.Done()
++ for {
++ select {
++ case <-stop:
++ return
++ default:
++ vl.SetVolumeCrowded(needle.VolumeId(2))
++ }
++ }
++ }()
++ }
++ for range 8 {
++ wg.Add(1)
++ go func() {
++ defer wg.Done()
++ for {
++ select {
++ case <-stop:
++ return
++ default:
++ vl.GetWritableVolumeCount()
++ }
++ }
++ }()
++ }
++
++ // The race detector needs only a brief window to flag a concurrent
++ // map read/write; let the goroutines hammer the lock briefly.
++ time.Sleep(50 * time.Millisecond)
++ close(stop)
++ wg.Wait()
++}
+diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go
+--- a/weed/topology/volume_layout.go
++++ b/weed/topology/volume_layout.go
+@@ -1019,11 +1019,14 @@ func (vl *VolumeLayout) setVolumeCrowded(vid needle.VolumeId) {
+ }
+
+ func (vl *VolumeLayout) SetVolumeCrowded(vid needle.VolumeId) {
+- // since delete is guarded by accessLock.Lock(),
+- // and is always called in sequential order,
+- // RLock() should be safe enough
+- vl.accessLock.RLock()
+- defer vl.accessLock.RUnlock()
++ // setVolumeCrowded mutates the crowded map, and GetWritableVolumeCount
++ // reads it under RLock on the Assign hot path. Two concurrent RLock
++ // holders with one writing the map triggers a fatal
++ // "concurrent map read and map write". Take the write lock: this path
++ // is a low-frequency single consumer driven by the crowded-volume
++ // event loop, and every other mutation of crowded already holds Lock().
++ vl.accessLock.Lock()
++ defer vl.accessLock.Unlock()
+
+ vl.setVolumeCrowded(vid)
+ }
From 0f16bf9c7336c03f3236ad68528106964d02c2c0 Mon Sep 17 00:00:00 2001
From: Mattia Eleuteri
Date: Fri, 13 Nov 2026 09:30:50 +0100
Subject: [PATCH 092/114] feat(redis): add maxmemory, eviction policy and
snapshot toggle
A Redis used purely as a cache had no durable way to bound its memory
or stop writing RDB snapshots: the chart only disabled snapshots
together with the persistent volume, and a CONFIG SET is lost when the
pod restarts. Snapshots and replica full syncs then filled the volume
and left the instance crashlooping.
maxmemory, maxmemory-policy and save are always rendered with an
explicit value, because the operator applies customConfig with CONFIG
SET on every reconcile and never resets a directive whose line
disappears; leaving the defaults implicit would make reverting a
setting a no-op until the next pod restart. maxmemory is refused above
90% of the memory request, the threshold above which the operator
silently skips the line and leaves Redis unbounded.
Assisted-by: LLM
Signed-off-by: Mattia Eleuteri
---
api/apps/v1alpha1/redis/types.go | 11 ++
.../v1alpha1/redis/zz_generated.deepcopy.go | 1 +
packages/apps/redis/README.md | 13 +-
.../apps/redis/templates/redisfailover.yaml | 25 ++++
.../apps/redis/tests/cache_config_test.yaml | 118 ++++++++++++++++++
.../apps/redis/tests/min_replicas_test.yaml | 6 +-
packages/apps/redis/values.schema.json | 33 +++++
packages/apps/redis/values.yaml | 18 +++
packages/system/redis-rd/cozyrds/redis.yaml | 4 +-
9 files changed, 219 insertions(+), 10 deletions(-)
create mode 100644 packages/apps/redis/tests/cache_config_test.yaml
diff --git a/api/apps/v1alpha1/redis/types.go b/api/apps/v1alpha1/redis/types.go
index c2363749c..03ff0175d 100644
--- a/api/apps/v1alpha1/redis/types.go
+++ b/api/apps/v1alpha1/redis/types.go
@@ -51,6 +51,14 @@ type ConfigSpec struct {
// Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.
// +kubebuilder:validation:Minimum=0
MinReplicasMaxLag *int `json:"minReplicasMaxLag,omitempty"`
+ // Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers.
+ Maxmemory resource.Quantity `json:"maxmemory,omitempty"`
+ // Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.
+ // +kubebuilder:default:="noeviction"
+ MaxmemoryPolicy MaxmemoryPolicy `json:"maxmemoryPolicy"`
+ // Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty.
+ // +kubebuilder:default:=true
+ Snapshots bool `json:"snapshots"`
// Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.
// +kubebuilder:default:={}
Sentinel Sentinel `json:"sentinel,omitempty"`
@@ -80,6 +88,9 @@ type TLS struct {
// +kubebuilder:validation:Enum="no";"optional";"yes"
type AuthClients string
+// +kubebuilder:validation:Enum="noeviction";"allkeys-lru";"allkeys-lfu";"allkeys-random";"volatile-lru";"volatile-lfu";"volatile-random";"volatile-ttl"
+type MaxmemoryPolicy string
+
// +kubebuilder:validation:Enum="t1.nano";"t1.micro";"t1.small";"t1.medium";"t1.large";"t1.xlarge";"t1.2xlarge";"t1.4xlarge";"c1.nano";"c1.micro";"c1.small";"c1.medium";"c1.large";"c1.xlarge";"c1.2xlarge";"c1.4xlarge";"s1.nano";"s1.micro";"s1.small";"s1.medium";"s1.large";"s1.xlarge";"s1.2xlarge";"s1.4xlarge";"u1.nano";"u1.micro";"u1.small";"u1.medium";"u1.large";"u1.xlarge";"u1.2xlarge";"u1.4xlarge";"m1.nano";"m1.micro";"m1.small";"m1.medium";"m1.large";"m1.xlarge";"m1.2xlarge";"m1.4xlarge";"nano";"micro";"small";"medium";"large";"xlarge";"2xlarge"
type ResourcesPreset string
diff --git a/api/apps/v1alpha1/redis/zz_generated.deepcopy.go b/api/apps/v1alpha1/redis/zz_generated.deepcopy.go
index 489c28e85..86437ea58 100644
--- a/api/apps/v1alpha1/redis/zz_generated.deepcopy.go
+++ b/api/apps/v1alpha1/redis/zz_generated.deepcopy.go
@@ -66,6 +66,7 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) {
*out = new(int)
**out = **in
}
+ out.Maxmemory = in.Maxmemory.DeepCopy()
in.Sentinel.DeepCopyInto(&out.Sentinel)
}
diff --git a/packages/apps/redis/README.md b/packages/apps/redis/README.md
index 84cb04930..4044ce951 100644
--- a/packages/apps/redis/README.md
+++ b/packages/apps/redis/README.md
@@ -39,11 +39,14 @@ Service utilizes the freshworks-oss Redis Operator (a maintained fork of the arc
### Application-specific parameters
-| Name | Description | Type | Value |
-| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | ------ |
-| `authEnabled` | Enable password generation. | `bool` | `true` |
-| `minReplicasToWrite` | Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value. | `*int` | `null` |
-| `minReplicasMaxLag` | Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it. | `*int` | `null` |
+| Name | Description | Type | Value |
+| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ------------ |
+| `authEnabled` | Enable password generation. | `bool` | `true` |
+| `minReplicasToWrite` | Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value. | `*int` | `null` |
+| `minReplicasMaxLag` | Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it. | `*int` | `null` |
+| `maxmemory` | Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers. | `quantity` | `""` |
+| `maxmemoryPolicy` | Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset. | `string` | `noeviction` |
+| `snapshots` | Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty. | `bool` | `true` |
### Sentinel parameters
diff --git a/packages/apps/redis/templates/redisfailover.yaml b/packages/apps/redis/templates/redisfailover.yaml
index 05a96e3cc..ba36e3f6c 100644
--- a/packages/apps/redis/templates/redisfailover.yaml
+++ b/packages/apps/redis/templates/redisfailover.yaml
@@ -116,8 +116,33 @@ spec:
- loglevel notice
{{- if not .Values.size }}
- appendonly no
+ {{- end }}
+ {{- /* The operator applies these lines with CONFIG SET on every reconcile and never
+ resets a line that disappears, so each one is always rendered with an explicit
+ value: switching back to the default must reach the running pods too. The
+ schedule is the Redis 7 and 8 default. */}}
+ {{- if and .Values.size .Values.snapshots }}
+ - save 3600 1 300 100 60 10000
+ {{- else }}
- save ""
{{- end }}
+ {{- $maxmemory := 0 }}
+ {{- if not (kindIs "invalid" .Values.maxmemory) }}
+ {{- $maxmemory = include "cozy-lib.resources.toFloat" (.Values.maxmemory | toString) | float64 | int64 }}
+ {{- $redisResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.resourcesPreset .Values.resources $) | fromYaml }}
+ {{- $memoryRequest := include "cozy-lib.resources.toFloat" (dig "requests" "memory" "0" $redisResources | toString) | float64 }}
+ {{- /* The operator compares maxmemory with 90% of the pod's memory request
+ (reservedPodMemoryPercent) and silently skips the line when it is larger. */}}
+ {{- $allowed := mulf $memoryRequest 0.9 | int64 }}
+ {{- if lt $maxmemory 0 }}
+ {{- fail (printf "maxmemory (%v) must not be negative" .Values.maxmemory) }}
+ {{- end }}
+ {{- if gt $maxmemory $allowed }}
+ {{- fail (printf "maxmemory (%v, %d bytes) must be at most 90%% of the Redis memory request (%d bytes): the operator ignores a larger value" .Values.maxmemory $maxmemory $allowed) }}
+ {{- end }}
+ {{- end }}
+ - maxmemory {{ $maxmemory }}
+ - maxmemory-policy {{ .Values.maxmemoryPolicy | default "noeviction" }}
{{- /* A master with `replicas` pods has replicas-1 replicas, so a requirement of
`replicas` or more can never be met and the master would refuse every write. */}}
{{- if and (not (kindIs "invalid" .Values.minReplicasToWrite)) (ge (.Values.minReplicasToWrite | int) (.Values.replicas | int)) }}
diff --git a/packages/apps/redis/tests/cache_config_test.yaml b/packages/apps/redis/tests/cache_config_test.yaml
new file mode 100644
index 000000000..557b5d3d4
--- /dev/null
+++ b/packages/apps/redis/tests/cache_config_test.yaml
@@ -0,0 +1,118 @@
+suite: Redis cache tuning (maxmemory, eviction, snapshots)
+
+# The operator applies customConfig with CONFIG SET and never resets a line
+# that disappears, so every directive here is rendered with an explicit value
+# even at its default: reverting a setting has to reach the running pods.
+
+release:
+ name: my-redis
+ namespace: tenant-test
+
+set:
+ _cluster:
+ cluster-domain: cluster.local
+ _namespace:
+ host: example.com
+
+templates:
+ - templates/redisfailover.yaml
+
+tests:
+ - it: "renders the Redis defaults explicitly"
+ documentSelector:
+ path: kind
+ value: RedisFailover
+ asserts:
+ - contains:
+ path: spec.redis.customConfig
+ content: save 3600 1 300 100 60 10000
+ - contains:
+ path: spec.redis.customConfig
+ content: maxmemory 0
+ - contains:
+ path: spec.redis.customConfig
+ content: maxmemory-policy noeviction
+ - notContains:
+ path: spec.redis.customConfig
+ content: appendonly no
+
+ - it: "disables snapshots while keeping the volume"
+ set:
+ snapshots: false
+ documentSelector:
+ path: kind
+ value: RedisFailover
+ asserts:
+ - contains:
+ path: spec.redis.customConfig
+ content: save ""
+ - notContains:
+ path: spec.redis.customConfig
+ content: save 3600 1 300 100 60 10000
+ - equal:
+ path: spec.redis.storage.persistentVolumeClaim.spec.resources.requests.storage
+ value: 1Gi
+
+ - it: "keeps snapshots off without a volume even when enabled"
+ set:
+ size: null
+ snapshots: true
+ documentSelector:
+ path: kind
+ value: RedisFailover
+ asserts:
+ - contains:
+ path: spec.redis.customConfig
+ content: save ""
+ - contains:
+ path: spec.redis.customConfig
+ content: appendonly no
+ - notExists:
+ path: spec.redis.storage
+
+ - it: "renders maxmemory in bytes with the chosen policy"
+ set:
+ resourcesPreset: t1.large
+ maxmemory: 1Gi
+ maxmemoryPolicy: allkeys-lru
+ documentSelector:
+ path: kind
+ value: RedisFailover
+ asserts:
+ - contains:
+ path: spec.redis.customConfig
+ content: maxmemory 1073741824
+ - contains:
+ path: spec.redis.customConfig
+ content: maxmemory-policy allkeys-lru
+
+ - it: "sizes the maxmemory ceiling from explicit resources"
+ set:
+ resources:
+ memory: 1Gi
+ maxmemory: 900Mi
+ documentSelector:
+ path: kind
+ value: RedisFailover
+ asserts:
+ - contains:
+ path: spec.redis.customConfig
+ content: maxmemory 943718400
+
+ # The operator compares maxmemory with 90% of the pod's memory request and
+ # silently skips a larger line, which would leave Redis unbounded.
+ - it: "refuses maxmemory above 90% of the memory request"
+ set:
+ resources:
+ memory: 1Gi
+ maxmemory: 1000Mi
+ asserts:
+ - failedTemplate:
+ errorPattern: must be at most 90% of the Redis memory request
+
+ - it: "refuses a negative maxmemory"
+ set:
+ maxmemory: -1Mi
+ asserts:
+ - failedTemplate:
+ errorPattern: must not be negative
diff --git a/packages/apps/redis/tests/min_replicas_test.yaml b/packages/apps/redis/tests/min_replicas_test.yaml
index f0de83e86..13217884d 100644
--- a/packages/apps/redis/tests/min_replicas_test.yaml
+++ b/packages/apps/redis/tests/min_replicas_test.yaml
@@ -1,7 +1,7 @@
suite: Redis min-replicas write safety
-# Both settings are opt-in: unset keeps the Redis defaults, so existing
-# instances render the same customConfig as before.
+# Both settings are opt-in: unset keeps the Redis defaults, so neither
+# directive is rendered.
release:
name: my-redis
@@ -30,7 +30,7 @@ tests:
pattern: ^tcp-keepalive 0$
- lengthEqual:
path: spec.redis.customConfig
- count: 2
+ count: 5
- it: "renders min-replicas-to-write when set"
set:
diff --git a/packages/apps/redis/values.schema.json b/packages/apps/redis/values.schema.json
index 1a301615f..b3b143320 100644
--- a/packages/apps/redis/values.schema.json
+++ b/packages/apps/redis/values.schema.json
@@ -171,6 +171,39 @@
"type": "integer",
"minimum": 0
},
+ "maxmemory": {
+ "description": "Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers.",
+ "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$",
+ "anyOf": [
+ {
+ "type": "integer"
+ },
+ {
+ "type": "string"
+ }
+ ],
+ "x-kubernetes-int-or-string": true
+ },
+ "maxmemoryPolicy": {
+ "description": "Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.",
+ "type": "string",
+ "default": "noeviction",
+ "enum": [
+ "noeviction",
+ "allkeys-lru",
+ "allkeys-lfu",
+ "allkeys-random",
+ "volatile-lru",
+ "volatile-lfu",
+ "volatile-random",
+ "volatile-ttl"
+ ]
+ },
+ "snapshots": {
+ "description": "Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty.",
+ "type": "boolean",
+ "default": true
+ },
"sentinel": {
"description": "Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.",
"type": "object",
diff --git a/packages/apps/redis/values.yaml b/packages/apps/redis/values.yaml
index 7917ca34d..0f8380ae7 100644
--- a/packages/apps/redis/values.yaml
+++ b/packages/apps/redis/values.yaml
@@ -119,6 +119,24 @@ authEnabled: true
## @param {*int} [minReplicasMaxLag] - Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.
## @minimum 0
+## @param {quantity} [maxmemory] - Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers.
+
+## @enum {string} MaxmemoryPolicy - Values accepted by the Redis `maxmemory-policy` directive.
+## @value noeviction
+## @value allkeys-lru
+## @value allkeys-lfu
+## @value allkeys-random
+## @value volatile-lru
+## @value volatile-lfu
+## @value volatile-random
+## @value volatile-ttl
+
+## @param {MaxmemoryPolicy} maxmemoryPolicy="noeviction" - Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.
+maxmemoryPolicy: "noeviction"
+
+## @param {bool} snapshots - Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty.
+snapshots: true
+
##
## @section Sentinel parameters
##
diff --git a/packages/system/redis-rd/cozyrds/redis.yaml b/packages/system/redis-rd/cozyrds/redis.yaml
index cfcaeb01d..02046b97e 100644
--- a/packages/system/redis-rd/cozyrds/redis.yaml
+++ b/packages/system/redis-rd/cozyrds/redis.yaml
@@ -8,7 +8,7 @@ spec:
plural: redises
singular: redis
openAPISchema: |-
- {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}
+ {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"maxmemory":{"description":"Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"maxmemoryPolicy":{"description":"Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.","type":"string","default":"noeviction","enum":["noeviction","allkeys-lru","allkeys-lfu","allkeys-random","volatile-lru","volatile-lfu","volatile-random","volatile-ttl"]},"snapshots":{"description":"Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty.","type":"boolean","default":true},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}
release:
prefix: redis-
labels:
@@ -25,7 +25,7 @@ spec:
tags:
- cache
icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODNfMzIxMykiLz4KPHBhdGggZD0iTTEyMC4xNDkgOTUuNTQ5MUMxMTQuNTg2IDk4LjQ0ODUgODUuNzcwOSAxMTAuMjk2IDc5LjYzNjMgMTEzLjQ5NUM3My41MDE2IDExNi42OTMgNzAuMDkzNyAxMTYuNjYyIDY1LjI0NzIgMTE0LjM0NkM2MC40MDEyIDExMi4wMjkgMjkuNzM2NCA5OS42NDIzIDI0LjIxMjUgOTcuMDAxOUMyMS40NTE5IDk1LjY4MjcgMjAgOTQuNTY4NyAyMCA5My41MTY2VjgyLjk4MDlDMjAgODIuOTgwOSA1OS45MjIgNzQuMjkwMSA2Ni4zNjY5IDcxLjk3NzhDNzIuODExNSA2OS42NjU2IDc1LjA0NzYgNjkuNTgyMSA4MC41MzIgNzEuNTkxQzg2LjAxNzMgNzMuNjAwOCAxMTguODEyIDc5LjUxNzYgMTI0LjIzMyA4MS41MDI5TDEyNC4yMyA5MS44ODk2QzEyNC4yMzEgOTIuOTMxMSAxMjIuOTggOTQuMDczNiAxMjAuMTQ5IDk1LjU0OTFaIiBmaWxsPSIjOTEyNjI2Ii8+CjxwYXRoIGQ9Ik0xMjAuMTQ3IDg1LjA3NTJDMTE0LjU4NSA4Ny45NzM0IDg1Ljc3IDk5LjgyMTggNzkuNjM1NCAxMDMuMDJDNzMuNTAxMSAxMDYuMjE5IDcwLjA5MzIgMTA2LjE4NyA2NS4yNDcyIDEwMy44NzFDNjAuNDAwNyAxMDEuNTU1IDI5LjczNzEgODkuMTY2OCAyNC4yMTM2IDg2LjUyOEMxOC42OTAxIDgzLjg4NzYgMTguNTc0NCA4Mi4wNzA0IDI0LjAwMDMgNzkuOTQ1OEMyOS40MjYxIDc3LjgyMDUgNTkuOTIxNSA2NS44NTYxIDY2LjM2NzIgNjMuNTQzOEM3Mi44MTE4IDYxLjIzMjQgNzUuMDQ3NSA2MS4xNDgxIDgwLjUzMTkgNjMuMTU3OEM4Ni4wMTY4IDY1LjE2NjggMTE0LjY2IDc2LjU2NzYgMTIwLjA3OSA3OC41NTI1QzEyNS41MDEgODAuNTM5OSAxMjUuNzA5IDgyLjE3NjMgMTIwLjE0NyA4NS4wNzUyWiIgZmlsbD0iI0M2MzAyQiIvPgo8cGF0aCBkPSJNMTIwLjE0OSA3OC41MDJDMTE0LjU4NiA4MS40MDE4IDg1Ljc3MDkgOTMuMjQ5MyA3OS42MzYzIDk2LjQ0ODhDNzMuNTAxNiA5OS42NDYyIDcwLjA5MzcgOTkuNjE1MiA2NS4yNDcyIDk3LjI5ODVDNjAuNDAwOCA5NC45ODMgMjkuNzM2NCA4Mi41OTUyIDI0LjIxMjUgNzkuOTU0N0MyMS40NTE5IDc4LjYzNTUgMjAgNzcuNTIzMiAyMCA3Ni40NzA3VjY1LjkzMzhDMjAgNjUuOTMzOCA1OS45MjIgNTcuMjQzNCA2Ni4zNjY5IDU0LjkzMTFDNzIuODExNSA1Mi42MTg5IDc1LjA0NzYgNTIuNTM1IDgwLjUzMiA1NC41NDQzQzg2LjAxNzcgNTYuNTUzNiAxMTguODEzIDYyLjQ2OTMgMTI0LjIzMyA2NC40NTVMMTI0LjIzIDc0Ljg0MjhDMTI0LjIzMSA3NS44ODQgMTIyLjk4IDc3LjAyNjQgMTIwLjE0OSA3OC41MDJaIiBmaWxsPSIjOTEyNjI2Ii8+CjxwYXRoIGQ9Ik0xMjAuMTQ3IDY4LjAyODJDMTE0LjU4NSA3MC45MjcxIDg1Ljc3IDgyLjc3NDcgNzkuNjM1NCA4NS45NzM3QzczLjUwMTEgODkuMTcxNiA3MC4wOTMyIDg5LjE0MDIgNjUuMjQ3MiA4Ni44MjM1QzYwLjQwMDcgODQuNTA4NCAyOS43MzcxIDcyLjEyMDEgMjQuMjEzNiA2OS40ODA5QzE4LjY5MDEgNjYuODQxMyAxOC41NzQ0IDY1LjAyMzcgMjQuMDAwMyA2Mi44OTg0QzI5LjQyNjEgNjAuNzc0MiA1OS45MjE5IDQ4LjgwOSA2Ni4zNjcyIDQ2LjQ5NzJDNzIuODExOCA0NC4xODUzIDc1LjA0NzUgNDQuMTAxNCA4MC41MzE5IDQ2LjExMDhDODYuMDE2OCA0OC4xMTk3IDExNC42NiA1OS41MTk3IDEyMC4wNzkgNjEuNTA1NUMxMjUuNTAxIDYzLjQ5MjQgMTI1LjcwOSA2NS4xMjkyIDEyMC4xNDcgNjguMDI4MloiIGZpbGw9IiNDNjMwMkIiLz4KPHBhdGggZD0iTTEyMC4xNDkgNjAuODIyNEMxMTQuNTg2IDYzLjcyMTQgODUuNzcwOSA3NS41Njk4IDc5LjYzNjMgNzguNzY5MkM3My41MDE2IDgxLjk2NzEgNzAuMDkzNyA4MS45MzU3IDY1LjI0NzIgNzkuNjE5QzYwLjQwMDggNzcuMzAzNSAyOS43MzY0IDY0LjkxNTIgMjQuMjEyNSA2Mi4yNzZDMjEuNDUxOSA2MC45NTU2IDIwIDU5Ljg0MjggMjAgNTguNzkxNVY0OC4yNTQyQzIwIDQ4LjI1NDIgNTkuOTIyIDM5LjU2NDIgNjYuMzY2OSAzNy4yNTI0QzcyLjgxMTUgMzQuOTM5NyA3NS4wNDc2IDM0Ljg1NjcgODAuNTMyIDM2Ljg2NTZDODYuMDE3NyAzOC44NzQ5IDExOC44MTMgNDQuNzkwNSAxMjQuMjMzIDQ2Ljc3NjNMMTI0LjIzIDU3LjE2MzdDMTI0LjIzMSA1OC4yMDQgMTIyLjk4IDU5LjM0NjUgMTIwLjE0OSA2MC44MjI0WiIgZmlsbD0iIzkxMjYyNiIvPgo8cGF0aCBkPSJNMTIwLjE0NyA1MC4zNDlDMTE0LjU4NSA1My4yNDc5IDg1Ljc2OTggNjUuMDk2MyA3OS42MzUyIDY4LjI5NDFDNzMuNTAwOSA3MS40OTIgNzAuMDkzIDcxLjQ2MDYgNjUuMjQ2OSA2OS4xNDUxQzYwLjQwMDkgNjYuODI4MyAyOS43MzY5IDU0LjQ0MDkgMjQuMjEzOCA1MS44MDEzQzE4LjY4OTkgNDkuMTYyMSAxOC41NzQ2IDQ3LjM0NDEgMjQgNDUuMjE5MkMyOS40MjU5IDQzLjA5NDYgNTkuOTIxNyAzMS4xMzEgNjYuMzY3IDI4LjgxODRDNzIuODExNiAyNi41MDYxIDc1LjA0NzMgMjYuNDIzIDgwLjUzMTcgMjguNDMyNEM4Ni4wMTY2IDMwLjQ0MTcgMTE0LjY1OSA0MS44NDE4IDEyMC4wNzkgNDMuODI3NUMxMjUuNTAxIDQ1LjgxMjggMTI1LjcwOSA0Ny40NSAxMjAuMTQ3IDUwLjM0OVoiIGZpbGw9IiNDNjMwMkIiLz4KPHBhdGggZD0iTTg0Ljg1NDEgNDAuMDk5NEw3NS44OTI2IDQxLjAyOThMNzMuODg2NSA0NS44NTdMNzAuNjQ2MyA0MC40NzAzTDYwLjI5ODMgMzkuNTQwNEw2OC4wMTk3IDM2Ljc1NThMNjUuNzAzIDMyLjQ4MTRMNzIuOTMyMSAzNS4zMDg4TDc5Ljc0NzEgMzMuMDc3NUw3Ny45MDUyIDM3LjQ5NzJMODQuODU0MSA0MC4wOTk0Wk03My4zNTE1IDYzLjUxODRMNTYuNjI2NiA1Ni41ODE2TDgwLjU5MiA1Mi45MDI5TDczLjM1MTUgNjMuNTE4NFpNNTAuMTYzNyA0Mi43ODI2QzU3LjIzODEgNDIuNzgyNiA2Mi45NzMgNDUuMDA1NyA2Mi45NzMgNDcuNzQ3NUM2Mi45NzMgNTAuNDkwMSA1Ny4yMzgxIDUyLjcxMjggNTAuMTYzNyA1Mi43MTI4QzQzLjA4OTMgNTIuNzEyOCAzNy4zNTQ1IDUwLjQ4OTcgMzcuMzU0NSA0Ny43NDc1QzM3LjM1NDUgNDUuMDA1NyA0My4wODkzIDQyLjc4MjYgNTAuMTYzNyA0Mi43ODI2WiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTk1LjQ0MzQgNDEuNDE3NEwxMDkuNjI3IDQ3LjAyMjRMOTUuNDU1NiA1Mi42MjJMOTUuNDQzNCA0MS40MTc0WiIgZmlsbD0iIzYyMUIxQyIvPgo8cGF0aCBkPSJNNzkuNzUyOSA0Ny42MjYxTDk1LjQ0NDkgNDEuNDE4OUw5NS40NTcxIDUyLjYyMzZMOTMuOTE4NCA1My4yMjU0TDc5Ljc1MjkgNDcuNjI2MVoiIGZpbGw9IiM5QTI5MjgiLz4KPGRlZnM+CjxsaW5lYXJHcmFkaWVudCBpZD0icGFpbnQwX2xpbmVhcl82ODNfMzIxMyIgeDE9IjE4OSIgeTE9IjIxMC41IiB4Mj0iMCIgeTI9IjAiIGdyYWRpZW50VW5pdHM9InVzZXJTcGFjZU9uVXNlIj4KPHN0b3Agc3RvcC1jb2xvcj0iI0E4MDAwMCIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiNGRkNGQ0YiLz4KPC9saW5lYXJHcmFkaWVudD4KPC9kZWZzPgo8L3N2Zz4K
- keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "tls"], ["spec", "authEnabled"], ["spec", "sentinel"]]
+ keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "tls"], ["spec", "authEnabled"], ["spec", "maxmemoryPolicy"], ["spec", "snapshots"], ["spec", "sentinel"]]
secrets:
# Backstop for the label-only include below, which matches on the label
# alone with no name constraint. These two Secrets are the ones whose
From 4c3a4b42ccc70343895237cce4de69fa6b7ea16b Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Thu, 12 Nov 2026 23:27:04 +0300
Subject: [PATCH 093/114] style: clear unused code and style findings reported
by staticcheck
Remove helpers and constants nothing references, escape format
characters in test literals, and take the simplifications staticcheck
suggests where they read the same. Error strings no longer start with a
capital: they keep their meaning, with the kind or component name moved
out of first position. Callers match errors by type, not text; the one
test that matches text, for Kafka, keeps the phrase it looks for.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
cmd/backport-batch/injection_test.go | 6 ++---
cmd/backport-batch/onto_test.go | 4 +--
cmd/backport-batch/policy_test.go | 4 +--
cmd/cozypkg/cmd/add.go | 2 +-
cmd/cozystack-operator/main.go | 2 +-
.../altinitystrategy_controller.go | 2 +-
.../etcdstrategy_controller.go | 4 +--
.../kafkastrategy_controller.go | 6 ++---
.../rabbitmqstrategy_controller.go | 2 +-
internal/controller/kubeovnplunger/metrics.go | 2 +-
internal/controller/workload_controller.go | 5 ----
internal/fluxinstall/install.go | 2 +-
internal/lineagecontrollerwebhook/webhook.go | 2 +-
internal/migrationcontroller/inventory.go | 4 +--
internal/operator/package_reconciler_test.go | 9 -------
internal/telemetry/collector.go | 4 +--
internal/telemetry/operator_collector.go | 22 +++++++--------
pkg/apiserver/apiserver.go | 6 ++---
pkg/cmd/server/openapi.go | 4 +--
pkg/ovnstatus/ovnstatus.go | 27 -------------------
pkg/ovnstatus/ovnstatus_test.go | 4 +--
pkg/reservation/evaluator.go | 6 ++---
22 files changed, 43 insertions(+), 86 deletions(-)
diff --git a/cmd/backport-batch/injection_test.go b/cmd/backport-batch/injection_test.go
index f1d12b004..a1f5abbe6 100644
--- a/cmd/backport-batch/injection_test.go
+++ b/cmd/backport-batch/injection_test.go
@@ -142,11 +142,11 @@ func TestOptionInjection(t *testing.T) {
// no bidi override reorders it and no zero-width character hides in it.
func TestDeferredBlockCleansFileNames(t *testing.T) {
l, _ := parseLine("release-1.6")
- files := []string{"app/txt.exe", "app/zerowidth.yaml"}
+ files := []string{"app/\u202etxt.exe", "app/zero\u200bwidth.yaml"}
conflict := pick{v: &verdict{o: &original{number: 5}}, files: files, failed: strings.Repeat("a", 40)}
waiting := pick{v: &verdict{o: &original{number: 6}}, files: files}
block := deferredBlock(l, []pick{conflict}, nil, []pick{waiting})
- if strings.ContainsAny(block, "") {
+ if strings.ContainsAny(block, "\u202e\u200b") {
t.Errorf("the block carries format characters: %q", block)
}
want := []string{"app/txt.exe", "app/zerowidth.yaml"}
@@ -154,7 +154,7 @@ func TestDeferredBlockCleansFileNames(t *testing.T) {
if err != nil || !slices.Equal(got[5].Files, want) || !slices.Equal(got[6].Files, want) {
t.Errorf("parseDeferred = %+v, %v; want both originals with %q", got, err, want)
}
- if files[0] != "app/txt.exe" {
+ if files[0] != "app/\u202etxt.exe" {
t.Errorf("deferredBlock rewrote the pick's own file names: %q", files)
}
}
diff --git a/cmd/backport-batch/onto_test.go b/cmd/backport-batch/onto_test.go
index ee4d52d33..994059235 100644
--- a/cmd/backport-batch/onto_test.go
+++ b/cmd/backport-batch/onto_test.go
@@ -41,12 +41,12 @@ func TestOntoContinues(t *testing.T) {
f.lines()
head := pushed(t, f)
f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"})
- code, stdout, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head, "--date", "2026-10-29")
+ code, _, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head, "--date", "2026-10-29")
if code != 2 || !strings.Contains(stderr, "not the head of backport-batch/release-1.1-20261029") {
t.Errorf("--onto with another date = %d (%s), want a refusal", code, stderr)
}
f.now = f.now.AddDate(0, 0, 3)
- code, stdout, stderr = f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head)
+ code, stdout, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head)
if code != 0 {
t.Fatalf("run --onto = %d\n%s", code, stderr)
}
diff --git a/cmd/backport-batch/policy_test.go b/cmd/backport-batch/policy_test.go
index 920c18a0b..2cd7d20f3 100644
--- a/cmd/backport-batch/policy_test.go
+++ b/cmd/backport-batch/policy_test.go
@@ -176,7 +176,7 @@ func TestPolicyPageMatchesCode(t *testing.T) {
t.Errorf("the page's block gives %d released lines, want 2 written out", pol.released)
}
- rule2 := regexp.MustCompile("(?m)^2\\. Its title type is (.*)$").FindStringSubmatch(page)
+ rule2 := regexp.MustCompile(`(?m)^2\. Its title type is (.*)$`).FindStringSubmatch(page)
if rule2 == nil {
t.Fatal("the page has no rule 2")
}
@@ -187,7 +187,7 @@ func TestPolicyPageMatchesCode(t *testing.T) {
if !slices.Equal(types, takenTypes) {
t.Errorf("rule 2 on the page takes %v, the code %v", types, takenTypes)
}
- never := regexp.MustCompile("(?m)^### What waits for the next minor\\n\\n(.*)$").FindStringSubmatch(page)
+ never := regexp.MustCompile(`(?m)^### What waits for the next minor\n\n(.*)$`).FindStringSubmatch(page)
if never == nil {
t.Fatal("the page has no list of what never goes into a patch release")
}
diff --git a/cmd/cozypkg/cmd/add.go b/cmd/cozypkg/cmd/add.go
index d882ce287..8c5e22026 100644
--- a/cmd/cozypkg/cmd/add.go
+++ b/cmd/cozypkg/cmd/add.go
@@ -390,7 +390,7 @@ func createPackageFromFile(ctx context.Context, k8sClient client.Client, filePat
}
}
- return fmt.Errorf("Package %s not found in file", packageName)
+ return fmt.Errorf("no Package %s in file", packageName)
}
func installPackage(ctx context.Context, k8sClient client.Client, packageSourceName string) error {
diff --git a/cmd/cozystack-operator/main.go b/cmd/cozystack-operator/main.go
index e11290aed..e1a2aa1de 100644
--- a/cmd/cozystack-operator/main.go
+++ b/cmd/cozystack-operator/main.go
@@ -663,7 +663,7 @@ func validateGitRef(refMap map[string]string) error {
return fmt.Errorf("commit SHA should be at least 7 characters, got: %s", commit)
}
for _, c := range commit {
- if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) {
+ if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') {
return fmt.Errorf("commit SHA should be a hexadecimal string, got: %s", commit)
}
}
diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go
index f0c597583..4314c3dbf 100644
--- a/internal/backupcontroller/altinitystrategy_controller.go
+++ b/internal/backupcontroller/altinitystrategy_controller.go
@@ -631,7 +631,7 @@ func (r *BackupJobReconciler) createAltinityBackupArtifact(
// used name meets the earlier run's Backup here. Only one recording
// this run's archive was created by an earlier reconcile of this run.
if got := existing.Spec.DriverMetadata[altinityArchiveMetaKey]; got != archive {
- return nil, fmt.Errorf("Backup %s/%s already exists and records archive %q, not %q that this run wrote",
+ return nil, fmt.Errorf("existing Backup %s/%s records archive %q, not %q that this run wrote",
backup.Namespace, backup.Name, got, archive)
}
return existing, nil
diff --git a/internal/backupcontroller/etcdstrategy_controller.go b/internal/backupcontroller/etcdstrategy_controller.go
index 2f2f2732c..4f530ca2f 100644
--- a/internal/backupcontroller/etcdstrategy_controller.go
+++ b/internal/backupcontroller/etcdstrategy_controller.go
@@ -134,14 +134,14 @@ var etcdClusterGVR = schema.GroupVersionResource{
// the wrong CRD via the typed client.
func validateEtcdApplicationRef(ref corev1.TypedLocalObjectReference) error {
if ref.Kind != etcdAppKind {
- return fmt.Errorf("Etcd strategy supports applicationRef.kind=%q, got %q", etcdAppKind, ref.Kind)
+ return fmt.Errorf("the Etcd strategy supports applicationRef.kind=%q, got %q", etcdAppKind, ref.Kind)
}
apiGroup := ""
if ref.APIGroup != nil {
apiGroup = *ref.APIGroup
}
if apiGroup != "" && apiGroup != etcdapp.GroupName {
- return fmt.Errorf("Etcd strategy supports applicationRef.apiGroup=%q, got %q", etcdapp.GroupName, apiGroup)
+ return fmt.Errorf("the Etcd strategy supports applicationRef.apiGroup=%q, got %q", etcdapp.GroupName, apiGroup)
}
return nil
}
diff --git a/internal/backupcontroller/kafkastrategy_controller.go b/internal/backupcontroller/kafkastrategy_controller.go
index 70f594d65..f7c34ce53 100644
--- a/internal/backupcontroller/kafkastrategy_controller.go
+++ b/internal/backupcontroller/kafkastrategy_controller.go
@@ -104,10 +104,10 @@ func kafkaClusterName(appName string) string {
// apps.cozystack.io/Kafka application.
func validateKafkaApplicationRef(ref corev1.TypedLocalObjectReference) error {
if ref.Kind != kafkaApplicationKind {
- return fmt.Errorf("Kafka strategy only supports applicationRef.kind=%q, got %q", kafkaApplicationKind, ref.Kind)
+ return fmt.Errorf("the Kafka strategy only supports applicationRef.kind=%q, got %q", kafkaApplicationKind, ref.Kind)
}
if ref.APIGroup != nil && *ref.APIGroup != "" && *ref.APIGroup != backupsv1alpha1.DefaultApplicationAPIGroup {
- return fmt.Errorf("Kafka strategy only supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup)
+ return fmt.Errorf("the Kafka strategy only supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup)
}
return nil
}
@@ -528,7 +528,7 @@ func (r *BackupJobReconciler) createKafkaBackupArtifact(
// RabbitMQ driver's guard).
if existing.Spec.ApplicationRef.Kind != backup.Spec.ApplicationRef.Kind ||
existing.Spec.ApplicationRef.Name != backup.Spec.ApplicationRef.Name {
- return nil, fmt.Errorf("Backup %s/%s already exists for a different application (%s/%s, not %s/%s)",
+ return nil, fmt.Errorf("existing Backup %s/%s belongs to a different application (%s/%s, not %s/%s)",
backup.Namespace, backup.Name,
existing.Spec.ApplicationRef.Kind, existing.Spec.ApplicationRef.Name,
backup.Spec.ApplicationRef.Kind, backup.Spec.ApplicationRef.Name)
diff --git a/internal/backupcontroller/rabbitmqstrategy_controller.go b/internal/backupcontroller/rabbitmqstrategy_controller.go
index adfe11d83..4c3d9a55c 100644
--- a/internal/backupcontroller/rabbitmqstrategy_controller.go
+++ b/internal/backupcontroller/rabbitmqstrategy_controller.go
@@ -513,7 +513,7 @@ func (r *BackupJobReconciler) createRabbitmqBackupArtifact(
// this run's success while describing something else.
if existing.Spec.ApplicationRef.Kind != backup.Spec.ApplicationRef.Kind ||
existing.Spec.ApplicationRef.Name != backup.Spec.ApplicationRef.Name {
- return nil, fmt.Errorf("Backup %s/%s already exists for a different application (%s/%s, not %s/%s)",
+ return nil, fmt.Errorf("existing Backup %s/%s belongs to a different application (%s/%s, not %s/%s)",
backup.Namespace, backup.Name,
existing.Spec.ApplicationRef.Kind, existing.Spec.ApplicationRef.Name,
backup.Spec.ApplicationRef.Kind, backup.Spec.ApplicationRef.Name)
diff --git a/internal/controller/kubeovnplunger/metrics.go b/internal/controller/kubeovnplunger/metrics.go
index 1330db814..523467fcd 100644
--- a/internal/controller/kubeovnplunger/metrics.go
+++ b/internal/controller/kubeovnplunger/metrics.go
@@ -219,7 +219,7 @@ func (r *KubeOVNPlunger) WriteClusterMetrics(db string, snaps []ovnstatus.Health
r.metrics.allAgree.WithLabelValues(db, cid).Set(b2f(ecv.AllAgree))
r.metrics.membersExpected.WithLabelValues(db, cid).Set(float64(expectedReplicas))
r.metrics.membersObserved.WithLabelValues(db, cid).Set(float64(ecv.MembersCount))
- r.metrics.ipsExpected.WithLabelValues(db, cid).Set(float64(len(ecv.ConsensusResult.TruthView.Members))) // optional; or len(hints.ExpectedIPs)
+ r.metrics.ipsExpected.WithLabelValues(db, cid).Set(float64(len(ecv.TruthView.Members))) // optional; or len(hints.ExpectedIPs)
r.metrics.ipsObserved.WithLabelValues(db, cid).Set(float64(ecv.DistinctIPCount))
r.metrics.excessMembers.WithLabelValues(db, cid).Set(float64(ecv.ExpectedExcess))
r.metrics.missingMembers.WithLabelValues(db, cid).Set(float64(ecv.ExpectedShortfall))
diff --git a/internal/controller/workload_controller.go b/internal/controller/workload_controller.go
index d0d9e7430..052358a05 100644
--- a/internal/controller/workload_controller.go
+++ b/internal/controller/workload_controller.go
@@ -3,7 +3,6 @@ package controller
import (
"context"
"strings"
- "time"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -15,10 +14,6 @@ import (
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
)
-const (
- deletionRequeueDelay = 30 * time.Second
-)
-
// WorkloadMonitorReconciler reconciles a WorkloadMonitor object
type WorkloadReconciler struct {
client.Client
diff --git a/internal/fluxinstall/install.go b/internal/fluxinstall/install.go
index d16d7d854..042bdfd5e 100644
--- a/internal/fluxinstall/install.go
+++ b/internal/fluxinstall/install.go
@@ -171,7 +171,7 @@ func extractNamespace(objects []*unstructured.Unstructured) (string, error) {
if obj.GetKind() == "Namespace" {
namespace := obj.GetName()
if namespace == "" {
- return "", fmt.Errorf("Namespace object has no name")
+ return "", fmt.Errorf("the Namespace object has no name")
}
return namespace, nil
}
diff --git a/internal/lineagecontrollerwebhook/webhook.go b/internal/lineagecontrollerwebhook/webhook.go
index ea8b0d01b..7b29e1f76 100644
--- a/internal/lineagecontrollerwebhook/webhook.go
+++ b/internal/lineagecontrollerwebhook/webhook.go
@@ -176,7 +176,7 @@ func (h *LineageControllerWebhook) computeLabels(ctx context.Context, obj *unstr
return s
}
s = s[:63]
- for b := s[62]; !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9')); s = s[:len(s)-1] {
+ for b := s[62]; (b < 'a' || b > 'z') && (b < 'A' || b > 'Z') && (b < '0' || b > '9'); s = s[:len(s)-1] {
b = s[len(s)-1]
}
return s
diff --git a/internal/migrationcontroller/inventory.go b/internal/migrationcontroller/inventory.go
index d56bec2da..52a5e4cd8 100644
--- a/internal/migrationcontroller/inventory.go
+++ b/internal/migrationcontroller/inventory.go
@@ -146,7 +146,7 @@ func (c *inventoryClient) VMs(ctx context.Context, providerUID string) ([]invent
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("Forklift inventory returned %s listing VMs", resp.Status)
+ return nil, fmt.Errorf("listing VMs: the Forklift inventory returned %s", resp.Status)
}
var vms []inventoryVMRef
@@ -195,7 +195,7 @@ func (c *inventoryClient) VM(ctx context.Context, providerUID, vmID string) (*in
case http.StatusNotFound:
return nil, ¬FoundError{vmID: vmID}
default:
- return nil, fmt.Errorf("Forklift inventory returned %s for VM %q", resp.Status, vmID)
+ return nil, fmt.Errorf("VM %q: the Forklift inventory returned %s", vmID, resp.Status)
}
vm := &inventoryVM{}
diff --git a/internal/operator/package_reconciler_test.go b/internal/operator/package_reconciler_test.go
index c9923e7dc..fd8a15fe5 100644
--- a/internal/operator/package_reconciler_test.go
+++ b/internal/operator/package_reconciler_test.go
@@ -790,15 +790,6 @@ func scanRecordingClient(scheme *runtime.Scheme, listed *[]string, writes *int,
}).Build()
}
-func listedKind(listed []string, kind string) bool {
- for _, l := range listed {
- if strings.Contains(l, kind) {
- return true
- }
- }
- return false
-}
-
// The comparison guarding that write has to be semantic rather than structural, or a
// LimitRange written in a different but equivalent notation would re-apply on every
// reconcile — invisible in a test that only looked at the resulting object, since every one
diff --git a/internal/telemetry/collector.go b/internal/telemetry/collector.go
index f4bb8dd88..a948c7495 100644
--- a/internal/telemetry/collector.go
+++ b/internal/telemetry/collector.go
@@ -118,11 +118,11 @@ func (c *Collector) collect(ctx context.Context) {
// Write application count metrics
for kind, count := range appKindCounts {
- metrics.WriteString(fmt.Sprintf(
+ fmt.Fprintf(&metrics,
"cozy_application_count{kind=\"%s\"} %d\n",
kind,
count,
- ))
+ )
}
// Send metrics only if there's something to send
diff --git a/internal/telemetry/operator_collector.go b/internal/telemetry/operator_collector.go
index 8b4ec3028..2c0e1b9bb 100644
--- a/internal/telemetry/operator_collector.go
+++ b/internal/telemetry/operator_collector.go
@@ -133,11 +133,11 @@ func (c *OperatorCollector) collect(ctx context.Context) {
var metrics strings.Builder
// Add cluster info metric
- metrics.WriteString(fmt.Sprintf(
+ fmt.Fprintf(&metrics,
"cozy_cluster_info{cozystack_version=\"%s\",kubernetes_version=\"%s\"} 1\n",
version.Version,
k8sVersion.GitVersion,
- ))
+ )
// Collect node metrics grouped by OS and kernel
nodeOSCount := make(map[string]map[string]int) // os -> kernel -> count
@@ -153,12 +153,12 @@ func (c *OperatorCollector) collect(ctx context.Context) {
for osKey, kernels := range nodeOSCount {
for kernel, count := range kernels {
- metrics.WriteString(fmt.Sprintf(
+ fmt.Fprintf(&metrics,
"cozy_nodes_count{os=\"%s\",kernel=\"%s\"} %d\n",
osKey,
kernel,
count,
- ))
+ )
}
}
@@ -174,11 +174,11 @@ func (c *OperatorCollector) collect(ctx context.Context) {
}
for resourceName, total := range capacityTotals {
- metrics.WriteString(fmt.Sprintf(
+ fmt.Fprintf(&metrics,
"cozy_cluster_capacity{resource=\"%s\"} %d\n",
resourceName,
total,
- ))
+ )
}
// Collect LoadBalancer services metrics
@@ -192,7 +192,7 @@ func (c *OperatorCollector) collect(ctx context.Context) {
lbCount++
}
}
- metrics.WriteString(fmt.Sprintf("cozy_loadbalancers_count %d\n", lbCount))
+ fmt.Fprintf(&metrics, "cozy_loadbalancers_count %d\n", lbCount)
}
// Collect PV metrics grouped by driver and size
@@ -224,12 +224,12 @@ func (c *OperatorCollector) collect(ctx context.Context) {
for size, drivers := range pvMetrics {
for driver, count := range drivers {
- metrics.WriteString(fmt.Sprintf(
+ fmt.Fprintf(&metrics,
"cozy_pvs_count{driver=\"%s\",size=\"%s\"} %d\n",
driver,
size,
count,
- ))
+ )
}
}
}
@@ -244,11 +244,11 @@ func (c *OperatorCollector) collect(ctx context.Context) {
if variant == "" {
variant = "default"
}
- metrics.WriteString(fmt.Sprintf(
+ fmt.Fprintf(&metrics,
"cozy_package_info{name=\"%s\",variant=\"%s\"} 1\n",
pkg.Name,
variant,
- ))
+ )
}
}
diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go
index c3555e46c..06f024bc6 100644
--- a/pkg/apiserver/apiserver.go
+++ b/pkg/apiserver/apiserver.go
@@ -78,7 +78,7 @@ func init() {
// Register HelmRelease types.
if err := helmv2.AddToScheme(mgrScheme); err != nil {
- panic(fmt.Errorf("Failed to add HelmRelease types to scheme: %w", err))
+ panic(fmt.Errorf("failed to add HelmRelease types to scheme: %w", err))
}
// Register the in-tree CiliumNetworkPolicy mirror backing SecurityGroup.
@@ -87,10 +87,10 @@ func init() {
}
if err := corev1.AddToScheme(mgrScheme); err != nil {
- panic(fmt.Errorf("Failed to add core types to scheme: %w", err))
+ panic(fmt.Errorf("failed to add core types to scheme: %w", err))
}
if err := rbacv1.AddToScheme(mgrScheme); err != nil {
- panic(fmt.Errorf("Failed to add RBAC types to scheme: %w", err))
+ panic(fmt.Errorf("failed to add RBAC types to scheme: %w", err))
}
// Register Cozystack types for WorkloadMonitor queries.
diff --git a/pkg/cmd/server/openapi.go b/pkg/cmd/server/openapi.go
index 90af89cd0..d0cbe3289 100644
--- a/pkg/cmd/server/openapi.go
+++ b/pkg/cmd/server/openapi.go
@@ -319,7 +319,7 @@ func BuildPostProcessV3(kindSchemas map[string]string) func(*spec3.OpenAPI) (*sp
base, ok1 := doc.Components.Schemas[baseRef]
list, ok2 := doc.Components.Schemas[baseListRef]
stat, ok3 := doc.Components.Schemas[baseStatusRef]
- if !(ok1 && ok2 && ok3) {
+ if !ok1 || !ok2 || !ok3 {
return doc, nil // not the apps GV — nothing to patch
}
@@ -463,7 +463,7 @@ func BuildPostProcessV2(kindSchemas map[string]string) func(*spec.Swagger) (*spe
base, ok1 := defs[baseRef]
list, ok2 := defs[baseListRef]
stat, ok3 := defs[baseStatusRef]
- if !(ok1 && ok2 && ok3) {
+ if !ok1 || !ok2 || !ok3 {
return sw, nil // not the apps GV — nothing to patch
}
diff --git a/pkg/ovnstatus/ovnstatus.go b/pkg/ovnstatus/ovnstatus.go
index e6d682e51..b50c3f7ed 100644
--- a/pkg/ovnstatus/ovnstatus.go
+++ b/pkg/ovnstatus/ovnstatus.go
@@ -277,33 +277,6 @@ func defaultRunner(ctx context.Context, bin string, args ...string) (string, err
return out, nil
}
-/************** Helpers **************/
-
-func parseClusterStatusJSON(out string) (ClusterStatus, bool) {
- var cs ClusterStatus
- if json.Unmarshal([]byte(out), &cs) == nil && len(cs.Servers) > 0 {
- return cs, true
- }
- var wrap struct {
- Data ClusterStatus `json:"data"`
- }
- if json.Unmarshal([]byte(out), &wrap) == nil && len(wrap.Data.Servers) > 0 {
- return wrap.Data, true
- }
- return ClusterStatus{}, false
-}
-
-func portOf(db DB) string {
- switch db {
- case DBNorthbound:
- return "6641"
- case DBSouthbound:
- return "6642"
- default:
- return "0"
- }
-}
-
/************** Minimal text fallback for "Servers" **************/
// Accepts variants like:
diff --git a/pkg/ovnstatus/ovnstatus_test.go b/pkg/ovnstatus/ovnstatus_test.go
index 8dde66b3a..8c44f5413 100644
--- a/pkg/ovnstatus/ovnstatus_test.go
+++ b/pkg/ovnstatus/ovnstatus_test.go
@@ -28,9 +28,7 @@ var expectedServersBlock = `` +
func TestExtractServersBlock(t *testing.T) {
if actual := extractServersBlock(testStdout); actual != expectedServersBlock {
- fmt.Println([]byte(actual))
- fmt.Println([]byte(expectedServersBlock))
- t.Errorf("error extracting servers block from following string:\n%s\nexpected:\n%s\ngot:\n%s\n", testStdout, expectedServersBlock, actual)
+ t.Errorf("error extracting servers block from following string:\n%s\nexpected:\n%q\ngot:\n%q\n", testStdout, expectedServersBlock, actual)
}
}
diff --git a/pkg/reservation/evaluator.go b/pkg/reservation/evaluator.go
index 6927b0bf2..40d5c2ced 100644
--- a/pkg/reservation/evaluator.go
+++ b/pkg/reservation/evaluator.go
@@ -284,9 +284,9 @@ func mulQuantity(q, n ref.Val) ref.Val {
if !ok {
return types.MaybeNoSuchOverloadErr(n)
}
- out := qq.Quantity.DeepCopy()
+ out := qq.DeepCopy()
if !out.Mul(int64(nn)) {
- return types.NewErr("quantity %s * %d overflows", qq.Quantity.String(), int64(nn))
+ return types.NewErr("quantity %s * %d overflows", qq.String(), int64(nn))
}
return apiservercel.Quantity{Quantity: &out}
}
@@ -294,7 +294,7 @@ func mulQuantity(q, n ref.Val) ref.Val {
func toQuantity(v ref.Val) (resource.Quantity, error) {
switch t := v.(type) {
case apiservercel.Quantity:
- return t.Quantity.DeepCopy(), nil
+ return t.DeepCopy(), nil
case types.Int:
return *resource.NewQuantity(int64(t), resource.DecimalSI), nil
case types.Uint:
From 238e85577e63211c8124603ae87d4cab01931098 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Thu, 12 Nov 2026 23:28:51 +0300
Subject: [PATCH 094/114] refactor(lineage): resolve a HelmRelease's
application in a helper
The walk used a loop that always ended in its first pass to get
"break to the fallback" control flow. A function returning whether it
resolved the application says the same without the loop. A test pins
that each failure to resolve still falls back to the HelmRelease's own
label and that a resolved one ends the walk.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
pkg/lineage/helmrelease_test.go | 70 +++++++++++++++++++++++++++++++++
pkg/lineage/lineage.go | 59 ++++++++++++++-------------
2 files changed, 99 insertions(+), 30 deletions(-)
create mode 100644 pkg/lineage/helmrelease_test.go
diff --git a/pkg/lineage/helmrelease_test.go b/pkg/lineage/helmrelease_test.go
new file mode 100644
index 000000000..c8ef2676e
--- /dev/null
+++ b/pkg/lineage/helmrelease_test.go
@@ -0,0 +1,70 @@
+package lineage
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "testing"
+
+ helmv2 "github.com/fluxcd/helm-controller/api/v2"
+ "k8s.io/apimachinery/pkg/api/meta"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/runtime"
+ dynamicfake "k8s.io/client-go/dynamic/fake"
+)
+
+// namedAppMapper maps a HelmRelease to the Kubernetes application it names;
+// a HelmRelease missing from the map fails to map.
+type namedAppMapper map[string]string
+
+func (m namedAppMapper) Map(hr *helmv2.HelmRelease) (string, string, string, error) {
+ app, ok := m[hr.Name]
+ if !ok {
+ return "", "", "", fmt.Errorf("no application for HelmRelease %s", hr.Name)
+ }
+ return "apps.cozystack.io/v1alpha1", "Kubernetes", strings.TrimSuffix(hr.Name, app), nil
+}
+
+// A HelmRelease that cannot be resolved to its application hands the walk to
+// the HelmRelease its label names; one that resolves ends the walk there.
+func TestWalkHelmReleaseFallsBackToItsLabel(t *testing.T) {
+ parent := map[string]string{HRLabel: "kubernetes-b"}
+ unconvertible := capkObject(HRAPIVersion, HRKind, "kubernetes-a", parent, nil)
+ unconvertible.Object["spec"] = "not an object"
+ for _, tc := range []struct {
+ name string
+ hr *unstructured.Unstructured
+ mapped namedAppMapper
+ want []string
+ }{
+ {name: "mapped", hr: capkObject(HRAPIVersion, HRKind, "kubernetes-a", parent, nil), mapped: namedAppMapper{"kubernetes-a": "a", "kubernetes-b": "b"}, want: []string{"a"}},
+ {name: "unmappable", hr: capkObject(HRAPIVersion, HRKind, "kubernetes-a", parent, nil), mapped: namedAppMapper{"kubernetes-b": "b"}, want: []string{"b"}},
+ {name: "application missing", hr: capkObject(HRAPIVersion, HRKind, "kubernetes-a", parent, nil), mapped: namedAppMapper{"kubernetes-a": "absent", "kubernetes-b": "b"}, want: []string{"b"}},
+ {name: "unconvertible", hr: unconvertible, mapped: namedAppMapper{"kubernetes-a": "a", "kubernetes-b": "b"}, want: []string{"b"}},
+ {name: "unmappable without a label", hr: capkObject(HRAPIVersion, HRKind, "kubernetes-a", nil, nil), mapped: namedAppMapper{"kubernetes-b": "b"}, want: nil},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ objs := []runtime.Object{
+ capkObject("apps.cozystack.io/v1alpha1", "Kubernetes", "a", nil, nil),
+ capkObject("apps.cozystack.io/v1alpha1", "Kubernetes", "b", nil, nil),
+ capkObject(HRAPIVersion, HRKind, "kubernetes-b", nil, nil),
+ }
+ mapper := meta.NewDefaultRESTMapper(nil)
+ for _, o := range objs {
+ mapper.Add(o.GetObjectKind().GroupVersionKind(), meta.RESTScopeNamespace)
+ }
+ dyn := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), objs...)
+ var got []string
+ for _, id := range WalkOwnershipGraph(context.Background(), dyn, mapper, tc.mapped, tc.hr) {
+ got = append(got, id.Kind+"/"+id.Name)
+ }
+ var want []string
+ for _, name := range tc.want {
+ want = append(want, "Kubernetes/"+name)
+ }
+ if strings.Join(got, ",") != strings.Join(want, ",") {
+ t.Fatalf("WalkOwnershipGraph(%s) = %v, want %v", tc.name, got, want)
+ }
+ })
+ }
+}
diff --git a/pkg/lineage/lineage.go b/pkg/lineage/lineage.go
index 1546df647..8d8614b68 100644
--- a/pkg/lineage/lineage.go
+++ b/pkg/lineage/lineage.go
@@ -113,36 +113,8 @@ func WalkOwnershipGraph(
return
}
- // I want "if err1 != nil go to next block, if err2 != nil, go to next block, etc semantics",
- // like an early return from a function, but if all checks succeed, I don't want to do the rest
- // of the function, so it's a `for { if err { break } if othererr { break } if allgood { return }
- for {
- if obj.GetAPIVersion() != HRAPIVersion || obj.GetKind() != HRKind {
- break
- }
- hr := helmReleaseFromUnstructured(obj)
- if hr == nil {
- break
- }
- a, k, p, err := appMapper.Map(hr)
- if err != nil {
- l.Error(err, "failed to map HelmRelease to app")
- break
- }
- ownerObj, err := getUnstructuredObject(ctx, client, mapper, state.cache, a, k, obj.GetNamespace(), strings.TrimPrefix(obj.GetName(), p))
- if err != nil {
- l.Error(err, "couldn't get unstructured object", "APIVersion", a, "Kind", k, "Name", strings.TrimPrefix(obj.GetName(), p))
- break
- }
- // successfully mapped a HelmRelease to a custom app, no need to continue
- out = append(out,
- ObjectID{
- APIVersion: ownerObj.GetAPIVersion(),
- Kind: ownerObj.GetKind(),
- Namespace: ownerObj.GetNamespace(),
- Name: ownerObj.GetName(),
- },
- )
+ if app, ok := helmReleaseApp(ctx, client, mapper, appMapper, state.cache, obj); ok {
+ out = append(out, app)
return
}
@@ -160,6 +132,33 @@ func WalkOwnershipGraph(
return
}
+// helmReleaseApp resolves a HelmRelease to the application it deploys. When
+// it cannot, the walk goes on through the HelmRelease's own label instead.
+func helmReleaseApp(ctx context.Context, client dynamic.Interface, mapper meta.RESTMapper, appMapper AppMapper, cache *ObjectCache, obj *unstructured.Unstructured) (ObjectID, bool) {
+ hr := helmReleaseFromUnstructured(obj)
+ if hr == nil {
+ return ObjectID{}, false
+ }
+ l := log.FromContext(ctx)
+ a, k, p, err := appMapper.Map(hr)
+ if err != nil {
+ l.Error(err, "failed to map HelmRelease to app")
+ return ObjectID{}, false
+ }
+ name := strings.TrimPrefix(obj.GetName(), p)
+ app, err := getUnstructuredObject(ctx, client, mapper, cache, a, k, obj.GetNamespace(), name)
+ if err != nil {
+ l.Error(err, "couldn't get unstructured object", "APIVersion", a, "Kind", k, "Name", name)
+ return ObjectID{}, false
+ }
+ return ObjectID{
+ APIVersion: app.GetAPIVersion(),
+ Kind: app.GetKind(),
+ Namespace: app.GetNamespace(),
+ Name: app.GetName(),
+ }, true
+}
+
// kubevirtMachineName returns the KubevirtMachine a CAPK-created
// VirtualMachine belongs to. It only follows a KubevirtMachine in the VM's own
// namespace: CAPK can also run against an external infrastructure cluster,
From 04fa87ccd1edf6dfec048717a3bd1ee7d3ab1a1d Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Thu, 12 Nov 2026 23:30:02 +0300
Subject: [PATCH 095/114] style(backups): clear staticcheck findings in
overlapping controllers
Drop a field-manager constant nothing uses in the MariaDB driver, switch
on the MongoDB operator's state as a tagged switch, and reword the error
strings of the MongoDB driver and the workload monitor so they do not
start with a capital. The errors keep their meaning and nothing matches
their text.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
.../backupcontroller/mariadbstrategy_controller.go | 2 --
.../backupcontroller/psmdbstrategy_controller.go | 14 +++++++-------
internal/controller/workloadmonitor_controller.go | 4 ++--
3 files changed, 9 insertions(+), 11 deletions(-)
diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go
index 9fd6780db..452fff071 100644
--- a/internal/backupcontroller/mariadbstrategy_controller.go
+++ b/internal/backupcontroller/mariadbstrategy_controller.go
@@ -30,8 +30,6 @@ import (
// ---------------------------------------------------------------------------
const (
- mariadbFieldManager = "cozystack-mariadb-backup-driver"
-
// mariadb-operator's MariaDB ApplicationDefinition (release.prefix). The
// cozystack mariadb-rd renders the HelmRelease with releaseName =
// "mariadb-" + appName (see packages/system/mariadb-rd/cozyrds/mariadb.yaml,
diff --git a/internal/backupcontroller/psmdbstrategy_controller.go b/internal/backupcontroller/psmdbstrategy_controller.go
index 9caa218a7..ccf3e7735 100644
--- a/internal/backupcontroller/psmdbstrategy_controller.go
+++ b/internal/backupcontroller/psmdbstrategy_controller.go
@@ -528,8 +528,8 @@ func (r *BackupJobReconciler) reconcileMongoDB(ctx context.Context, j *backupsv1
}
}
- switch state := mdbBackup.Status.State; {
- case state == psmdbtypes.StateReady:
+ switch state := mdbBackup.Status.State; state {
+ case psmdbtypes.StateReady:
// Backup completed successfully. Materialise the Cozystack artifact and
// finalise the BackupJob. Idempotent: reuse the existing artifact if a
// previous reconcile created it and then raced on the status update.
@@ -570,7 +570,7 @@ func (r *BackupJobReconciler) reconcileMongoDB(ctx context.Context, j *backupsv1
}
return ctrl.Result{}, nil
- case state == psmdbtypes.StateError || state == psmdbtypes.StateRejected:
+ case psmdbtypes.StateError, psmdbtypes.StateRejected:
message := mdbBackup.Status.Error
// The residual half of the injection race: on the useSystemBucket flow the
// operator can latch state=error because it resolved the injected storage
@@ -1740,8 +1740,8 @@ func (r *RestoreJobReconciler) reconcileMongoDBRestore(ctx context.Context, rest
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure psmdb.percona.com/PerconaServerMongoDBRestore: %v", err))
}
- switch state := mdbRestore.Status.State; {
- case state == psmdbtypes.StateReady:
+ switch state := mdbRestore.Status.State; state {
+ case psmdbtypes.StateReady:
now := metav1.Now()
restoreJob.Status.CompletedAt = &now
restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
@@ -1756,7 +1756,7 @@ func (r *RestoreJobReconciler) reconcileMongoDBRestore(ctx context.Context, rest
}
return ctrl.Result{}, nil
- case state == psmdbtypes.StateError || state == psmdbtypes.StateRejected:
+ case psmdbtypes.StateError, psmdbtypes.StateRejected:
message := mdbRestore.Status.Error
if message == "" {
message = fmt.Sprintf("psmdb.percona.com/PerconaServerMongoDBRestore %s/%s reported state=%s", mdbRestore.Namespace, mdbRestore.Name, state)
@@ -1855,7 +1855,7 @@ func (r *RestoreJobReconciler) resolveMongoDBBackupSource(ctx context.Context, b
}
if destination == "" {
return nil, fmt.Errorf(
- "Backup has no restorable destination (the operator PerconaServerMongoDBBackup was reaped and no snapshot destination is persisted); re-run the BackupJob")
+ "no restorable destination for the Backup (the operator PerconaServerMongoDBBackup was reaped and no snapshot destination is persisted); re-run the BackupJob")
}
src := &psmdbtypes.BackupSource{
Type: psmdbtypes.BackupTypeLogical,
diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go
index 984ab7b21..217fd0bbf 100644
--- a/internal/controller/workloadmonitor_controller.go
+++ b/internal/controller/workloadmonitor_controller.go
@@ -528,7 +528,7 @@ func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, p
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("Prometheus returned status %d", resp.StatusCode)
+ return nil, fmt.Errorf("unexpected status %d from Prometheus", resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
@@ -549,7 +549,7 @@ func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, p
return nil, fmt.Errorf("parsing Prometheus response: %w", err)
}
if promResp.Status != "success" {
- return nil, fmt.Errorf("Prometheus query finished with status %q", promResp.Status)
+ return nil, fmt.Errorf("the Prometheus query finished with status %q", promResp.Status)
}
for _, r := range promResp.Data.Result {
From 2cda7d692e1bb2343ca6b71fc2a1335eefc2aade Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Thu, 12 Nov 2026 23:30:02 +0300
Subject: [PATCH 096/114] test(backport): drop an unused fixture helper
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
cmd/backport-batch/fixture_test.go | 9 ---------
1 file changed, 9 deletions(-)
diff --git a/cmd/backport-batch/fixture_test.go b/cmd/backport-batch/fixture_test.go
index ac1307c46..08aa5f2ac 100644
--- a/cmd/backport-batch/fixture_test.go
+++ b/cmd/backport-batch/fixture_test.go
@@ -522,15 +522,6 @@ func (f *fixture) batchPR(n int, base, branch, body string) pullRequest {
return pr
}
-// botBatchPR is batchPR opened by the bot the scheduled batch runs as.
-func (f *fixture) botBatchPR(gh *fakeGH, n int, base, branch, body string) pullRequest {
- f.t.Helper()
- gh.bot = defaultBatchBot
- pr := f.batchPR(n, base, branch, body)
- pr.User.Login, pr.User.Type, pr.AuthorAssociation = defaultBatchBot, "Bot", "CONTRIBUTOR"
- return pr
-}
-
// mergeBatch merges the open batch into release-1.1 with a merge commit, as
// GitHub does, and deletes its branch, as the repository's
// delete_branch_on_merge does. The pull request stays, merged, with its body
From d308e9e5e59aea5be1a41952fef9f8c3035845c5 Mon Sep 17 00:00:00 2001
From: Mattia Eleuteri
Date: Fri, 13 Nov 2026 09:43:05 +0100
Subject: [PATCH 097/114] fix(redis): refuse sub-byte maxmemory and apply the
policy first
A quantity such as 512m is 0.512 bytes; the integer cast truncated it,
or -512m, to 0, which Redis reads as no limit, so the bound a tenant
set could vanish without an error. The operator also sends the
directives in list order, and lowering maxmemory below the dataset
evicts at once, so the policy has to be set before the bound.
Turning snapshots off does not remove an RDB already on the volume,
which Redis still loads on restart; the parameter no longer says the
data is lost.
Assisted-by: LLM
Signed-off-by: Mattia Eleuteri
---
api/apps/v1alpha1/redis/types.go | 2 +-
packages/apps/redis/README.md | 2 +-
.../apps/redis/templates/redisfailover.yaml | 18 ++++++++---
.../apps/redis/tests/cache_config_test.yaml | 32 +++++++++++++++++++
packages/apps/redis/values.schema.json | 2 +-
packages/apps/redis/values.yaml | 2 +-
packages/system/redis-rd/cozyrds/redis.yaml | 2 +-
7 files changed, 50 insertions(+), 10 deletions(-)
diff --git a/api/apps/v1alpha1/redis/types.go b/api/apps/v1alpha1/redis/types.go
index 03ff0175d..41cf03c14 100644
--- a/api/apps/v1alpha1/redis/types.go
+++ b/api/apps/v1alpha1/redis/types.go
@@ -56,7 +56,7 @@ type ConfigSpec struct {
// Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.
// +kubebuilder:default:="noeviction"
MaxmemoryPolicy MaxmemoryPolicy `json:"maxmemoryPolicy"`
- // Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty.
+ // Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty.
// +kubebuilder:default:=true
Snapshots bool `json:"snapshots"`
// Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.
diff --git a/packages/apps/redis/README.md b/packages/apps/redis/README.md
index 4044ce951..be03bbea3 100644
--- a/packages/apps/redis/README.md
+++ b/packages/apps/redis/README.md
@@ -46,7 +46,7 @@ Service utilizes the freshworks-oss Redis Operator (a maintained fork of the arc
| `minReplicasMaxLag` | Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it. | `*int` | `null` |
| `maxmemory` | Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers. | `quantity` | `""` |
| `maxmemoryPolicy` | Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset. | `string` | `noeviction` |
-| `snapshots` | Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty. | `bool` | `true` |
+| `snapshots` | Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty. | `bool` | `true` |
### Sentinel parameters
diff --git a/packages/apps/redis/templates/redisfailover.yaml b/packages/apps/redis/templates/redisfailover.yaml
index ba36e3f6c..330a2d517 100644
--- a/packages/apps/redis/templates/redisfailover.yaml
+++ b/packages/apps/redis/templates/redisfailover.yaml
@@ -128,21 +128,29 @@ spec:
{{- end }}
{{- $maxmemory := 0 }}
{{- if not (kindIs "invalid" .Values.maxmemory) }}
- {{- $maxmemory = include "cozy-lib.resources.toFloat" (.Values.maxmemory | toString) | float64 | int64 }}
+ {{- $maxmemoryBytes := include "cozy-lib.resources.toFloat" (.Values.maxmemory | toString) | float64 }}
+ {{- /* Checked before the int64 cast, which would truncate a sub-byte quantity such
+ as 512m (0.512 bytes) or -512m to 0, and Redis reads 0 as no limit. */}}
+ {{- if lt $maxmemoryBytes 0.0 }}
+ {{- fail (printf "maxmemory (%v) must not be negative" .Values.maxmemory) }}
+ {{- end }}
+ {{- if ne (floor $maxmemoryBytes) $maxmemoryBytes }}
+ {{- fail (printf "maxmemory (%v) must be a whole number of bytes, e.g. 512Mi rather than 512m (millibytes)" .Values.maxmemory) }}
+ {{- end }}
+ {{- $maxmemory = $maxmemoryBytes | int64 }}
{{- $redisResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.resourcesPreset .Values.resources $) | fromYaml }}
{{- $memoryRequest := include "cozy-lib.resources.toFloat" (dig "requests" "memory" "0" $redisResources | toString) | float64 }}
{{- /* The operator compares maxmemory with 90% of the pod's memory request
(reservedPodMemoryPercent) and silently skips the line when it is larger. */}}
{{- $allowed := mulf $memoryRequest 0.9 | int64 }}
- {{- if lt $maxmemory 0 }}
- {{- fail (printf "maxmemory (%v) must not be negative" .Values.maxmemory) }}
- {{- end }}
{{- if gt $maxmemory $allowed }}
{{- fail (printf "maxmemory (%v, %d bytes) must be at most 90%% of the Redis memory request (%d bytes): the operator ignores a larger value" .Values.maxmemory $maxmemory $allowed) }}
{{- end }}
{{- end }}
- - maxmemory {{ $maxmemory }}
+ {{- /* The operator sends these in list order, and lowering maxmemory below the
+ dataset evicts at once, so the policy has to be in place first. */}}
- maxmemory-policy {{ .Values.maxmemoryPolicy | default "noeviction" }}
+ - maxmemory {{ $maxmemory }}
{{- /* A master with `replicas` pods has replicas-1 replicas, so a requirement of
`replicas` or more can never be met and the master would refuse every write. */}}
{{- if and (not (kindIs "invalid" .Values.minReplicasToWrite)) (ge (.Values.minReplicasToWrite | int) (.Values.replicas | int)) }}
diff --git a/packages/apps/redis/tests/cache_config_test.yaml b/packages/apps/redis/tests/cache_config_test.yaml
index 557b5d3d4..1b618bcac 100644
--- a/packages/apps/redis/tests/cache_config_test.yaml
+++ b/packages/apps/redis/tests/cache_config_test.yaml
@@ -116,3 +116,35 @@ tests:
asserts:
- failedTemplate:
errorPattern: must not be negative
+
+ # A sub-byte quantity would truncate to maxmemory 0, which Redis reads as no
+ # limit, so the bound the tenant asked for would silently disappear.
+ - it: "refuses a maxmemory that is not a whole number of bytes"
+ set:
+ maxmemory: 512m
+ asserts:
+ - failedTemplate:
+ errorPattern: must be a whole number of bytes
+
+ - it: "refuses a negative sub-byte maxmemory"
+ set:
+ maxmemory: -512m
+ asserts:
+ - failedTemplate:
+ errorPattern: must not be negative
+
+ - it: "applies the eviction policy before the memory bound"
+ set:
+ resourcesPreset: t1.large
+ maxmemory: 1Gi
+ maxmemoryPolicy: allkeys-lru
+ documentSelector:
+ path: kind
+ value: RedisFailover
+ asserts:
+ - equal:
+ path: spec.redis.customConfig[3]
+ value: maxmemory-policy allkeys-lru
+ - equal:
+ path: spec.redis.customConfig[4]
+ value: maxmemory 1073741824
diff --git a/packages/apps/redis/values.schema.json b/packages/apps/redis/values.schema.json
index b3b143320..183fb6a53 100644
--- a/packages/apps/redis/values.schema.json
+++ b/packages/apps/redis/values.schema.json
@@ -200,7 +200,7 @@
]
},
"snapshots": {
- "description": "Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty.",
+ "description": "Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty.",
"type": "boolean",
"default": true
},
diff --git a/packages/apps/redis/values.yaml b/packages/apps/redis/values.yaml
index 0f8380ae7..ed14c4f46 100644
--- a/packages/apps/redis/values.yaml
+++ b/packages/apps/redis/values.yaml
@@ -134,7 +134,7 @@ authEnabled: true
## @param {MaxmemoryPolicy} maxmemoryPolicy="noeviction" - Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.
maxmemoryPolicy: "noeviction"
-## @param {bool} snapshots - Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty.
+## @param {bool} snapshots - Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty.
snapshots: true
##
diff --git a/packages/system/redis-rd/cozyrds/redis.yaml b/packages/system/redis-rd/cozyrds/redis.yaml
index 02046b97e..8c3045b0f 100644
--- a/packages/system/redis-rd/cozyrds/redis.yaml
+++ b/packages/system/redis-rd/cozyrds/redis.yaml
@@ -8,7 +8,7 @@ spec:
plural: redises
singular: redis
openAPISchema: |-
- {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"maxmemory":{"description":"Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"maxmemoryPolicy":{"description":"Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.","type":"string","default":"noeviction","enum":["noeviction","allkeys-lru","allkeys-lfu","allkeys-random","volatile-lru","volatile-lfu","volatile-random","volatile-ttl"]},"snapshots":{"description":"Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty.","type":"boolean","default":true},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}
+ {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"maxmemory":{"description":"Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"maxmemoryPolicy":{"description":"Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.","type":"string","default":"noeviction","enum":["noeviction","allkeys-lru","allkeys-lfu","allkeys-random","volatile-lru","volatile-lfu","volatile-random","volatile-ttl"]},"snapshots":{"description":"Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty.","type":"boolean","default":true},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}
release:
prefix: redis-
labels:
From 05818aa3b221a670d94c332f461ae3bb046a5de3 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Thu, 12 Nov 2026 23:34:44 +0300
Subject: [PATCH 098/114] fix(cli): fail cozypkg and check-readiness on
unwritable output
For the commands this change touches (cozypkg list, the validation
report of validate, push and tap, tap's summary, and check-readiness),
a write error the process observes is a failure: the command exits
non-zero and says whether the action it reports on already happened.
Progress lines printed before the decisive step stay best-effort. A
closed pipe ends the process with SIGPIPE, as for other Unix tools.
`cozypkg list` and the validation report dropped write errors, and the
list deferred its tabwriter flush, so a closed or full stdout still
reported success with nothing printed. The list now writes to the
command's writer and returns the flush error, and the report returns
its write error. When validation refuses a push or a tap, or fails a
validate run, the refusal stays the reported reason and the report's
write error is joined to it. A push whose report cannot be written
stops before flux runs and says "nothing was pushed"; a tap whose
summary cannot be written fails with "tap applied, but writing its
summary failed".
check-readiness fails --help, --version and its readiness result the
same way, in one-shot, --wait and --watch mode, with exit 1 as for a
result that is not ready, which its help now lists among the exit
statuses along with SIGPIPE for a closed pipe; the watch loop stops at
the first frame it cannot write, and a timed-out wait still ends with
its verdict.
push writes its "Pushing" line before it runs flux and stops with
"nothing was pushed" when that line cannot be written, with validation
skipped or after the report.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
cmd/check-readiness/main.go | 55 ++++--
cmd/check-readiness/main_test.go | 68 +++++++
cmd/cozypkg/cmd/list.go | 48 +++--
cmd/cozypkg/cmd/output_test.go | 185 ++++++++++++++++++
cmd/cozypkg/cmd/push.go | 12 +-
cmd/cozypkg/cmd/tap.go | 34 ++--
cmd/cozypkg/cmd/validate.go | 17 +-
test/check-readiness/main_test.go | 79 ++++++++
.../testdata/golden/help-long.stdout | 4 +
.../testdata/golden/help-short.stdout | 4 +
.../testdata/golden/unknown-arg.stderr | 4 +
.../golden/watch-nonnumeric-interval.stderr | 4 +
12 files changed, 458 insertions(+), 56 deletions(-)
create mode 100644 cmd/check-readiness/main_test.go
create mode 100644 cmd/cozypkg/cmd/output_test.go
diff --git a/cmd/check-readiness/main.go b/cmd/check-readiness/main.go
index 50fb8aaf8..eb13f62e1 100644
--- a/cmd/check-readiness/main.go
+++ b/cmd/check-readiness/main.go
@@ -22,6 +22,7 @@ package main
import (
"fmt"
+ "io"
"os"
"os/exec"
"strconv"
@@ -58,6 +59,10 @@ Options:
--no-color Disable color output (auto-disabled on non-TTY)
-h, --help Show this help
+Exit status: 0 when everything is ready, 1 when something is not ready or a
+write to stdout fails, 2 on invalid arguments. A closed pipe ends the process
+with SIGPIPE, like other Unix tools.
+
`
// entry is a resource catalog item.
@@ -181,10 +186,16 @@ func run(args []string) int {
cfg.coreOnly = true
i++
case "-h", "--help":
- fmt.Fprint(os.Stdout, usageText)
+ if _, err := fmt.Fprint(os.Stdout, usageText); err != nil {
+ fmt.Fprintf(os.Stderr, "writing usage: %v\n", err)
+ return 1
+ }
return 0
case "--version":
- fmt.Fprintln(os.Stdout, Version)
+ if _, err := fmt.Fprintln(os.Stdout, Version); err != nil {
+ fmt.Fprintf(os.Stderr, "writing version: %v\n", err)
+ return 1
+ }
return 0
default:
fmt.Fprintf(os.Stderr, "Unknown argument: %s\n", a)
@@ -213,14 +224,14 @@ func run(args []string) int {
cfg.timeoutS = sec
if cfg.waitMode {
- return cfg.runWait()
+ return cfg.runWait(os.Stdout)
}
if cfg.watch {
- cfg.runWatch()
- return 0
+ return cfg.runWatch(os.Stdout)
+ }
+ if writeOutput(os.Stdout, cfg.runOnce()) != nil {
+ return 1
}
- out := cfg.runOnce()
- fmt.Print(out)
if cfg.foundIssues {
return 1
}
@@ -281,18 +292,31 @@ func parseTimeout(raw string) (int, bool) {
return 0, false
}
-func (cfg *config) runWait() int {
+// writeOutput prints a result. A result its reader never got fails the run,
+// whatever the resources' state, since the exit code alone would vouch for it.
+func writeOutput(w io.Writer, parts ...string) error {
+ for _, part := range parts {
+ if _, err := io.WriteString(w, part); err != nil {
+ fmt.Fprintf(os.Stderr, "writing output: %v\n", err)
+ return err
+ }
+ }
+ return nil
+}
+
+func (cfg *config) runWait(w io.Writer) int {
start := time.Now()
for {
output := cfg.runOnce()
elapsed := int(time.Since(start).Seconds())
if strings.Contains(output, "All resources are ready.") {
- fmt.Print(output)
- fmt.Printf("%sReady after %ds.%s\n", cfg.green, elapsed, cfg.reset)
+ if writeOutput(w, output, fmt.Sprintf("%sReady after %ds.%s\n", cfg.green, elapsed, cfg.reset)) != nil {
+ return 1
+ }
return 0
}
if elapsed >= cfg.timeoutS {
- fmt.Print(output)
+ _ = writeOutput(w, output)
fmt.Fprintf(os.Stderr, "%s%sTimeout after %ds — resources still not ready.%s\n", cfg.red, cfg.bold, cfg.timeoutS, cfg.reset)
return 1
}
@@ -300,13 +324,14 @@ func (cfg *config) runWait() int {
}
}
-func (cfg *config) runWatch() {
+func (cfg *config) runWatch(w io.Writer) int {
for {
output := cfg.runOnce()
// Smooth clear is cosmetic and untested; just reprint.
- fmt.Printf("%sLast updated: %s (refreshing every %ds, Ctrl+C to stop)%s\n", cfg.bold, time.Now().Format(time.UnixDate), cfg.interval, cfg.reset)
- fmt.Println()
- fmt.Print(output)
+ header := fmt.Sprintf("%sLast updated: %s (refreshing every %ds, Ctrl+C to stop)%s\n\n", cfg.bold, time.Now().Format(time.UnixDate), cfg.interval, cfg.reset)
+ if writeOutput(w, header, output) != nil {
+ return 1
+ }
time.Sleep(time.Duration(cfg.interval) * time.Second)
}
}
diff --git a/cmd/check-readiness/main_test.go b/cmd/check-readiness/main_test.go
new file mode 100644
index 000000000..9afddeedf
--- /dev/null
+++ b/cmd/check-readiness/main_test.go
@@ -0,0 +1,68 @@
+/*
+Copyright 2026 The Cozystack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "io"
+ "os"
+ "strings"
+ "testing"
+)
+
+func TestHelpAndVersionSucceedOnWritableStdout(t *testing.T) {
+ r, w, err := os.Pipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = r.Close() })
+ stdout := os.Stdout
+ os.Stdout = w
+ t.Cleanup(func() { os.Stdout = stdout })
+ read := make(chan string, 1)
+ go func() {
+ b, _ := io.ReadAll(r)
+ read <- string(b)
+ }()
+
+ for _, arg := range []string{"--help", "--version"} {
+ if code := run([]string{arg}); code != 0 {
+ t.Errorf("run(%s) = %d, want 0", arg, code)
+ }
+ }
+ _ = w.Close()
+ if out := <-read; !strings.Contains(out, "Usage:") || !strings.Contains(out, Version) {
+ t.Errorf("stdout = %q, want the usage text and the version", out)
+ }
+}
+
+func TestHelpAndVersionFailOnUnwritableStdout(t *testing.T) {
+ r, w, err := os.Pipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = r.Close()
+ _ = w.Close()
+ stdout := os.Stdout
+ os.Stdout = w
+ t.Cleanup(func() { os.Stdout = stdout })
+
+ for _, arg := range []string{"--help", "--version"} {
+ if code := run([]string{arg}); code == 0 {
+ t.Errorf("run(%s) with an unwritable stdout = 0, want a failure", arg)
+ }
+ }
+}
diff --git a/cmd/cozypkg/cmd/list.go b/cmd/cozypkg/cmd/list.go
index 6f7375026..4b08a602b 100644
--- a/cmd/cozypkg/cmd/list.go
+++ b/cmd/cozypkg/cmd/list.go
@@ -19,7 +19,7 @@ package cmd
import (
"context"
"fmt"
- "os"
+ "io"
"strings"
"text/tabwriter"
@@ -78,24 +78,25 @@ Use --components flag to show components on separate lines.`,
}
if listCmdFlags.installed {
- return listPackages(ctx, k8sClient, listCmdFlags.components)
+ return listPackages(ctx, k8sClient, cmd.OutOrStdout(), listCmdFlags.components)
}
- return listPackageSources(ctx, k8sClient, listCmdFlags.components)
+ return listPackageSources(ctx, k8sClient, cmd.OutOrStdout(), listCmdFlags.components)
},
}
-func listPackageSources(ctx context.Context, k8sClient client.Client, showComponents bool) error {
+func listPackageSources(ctx context.Context, k8sClient client.Client, out io.Writer, showComponents bool) error {
var psList cozyv1alpha1.PackageSourceList
if err := k8sClient.List(ctx, &psList); err != nil {
return fmt.Errorf("failed to list PackageSources: %w", err)
}
// Use tabwriter for better column alignment
- w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
- defer w.Flush()
+ w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0)
// Print header
- fmt.Fprintln(w, "NAME\tVARIANTS\tREADY\tSTATUS")
+ if _, err := fmt.Fprintln(w, "NAME\tVARIANTS\tREADY\tSTATUS"); err != nil {
+ return err
+ }
// Print rows
for _, ps := range psList.Items {
@@ -123,24 +124,26 @@ func listPackageSources(ctx context.Context, k8sClient client.Client, showCompon
}
}
- fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", ps.Name, variantsStr, ready, status)
+ if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", ps.Name, variantsStr, ready, status); err != nil {
+ return err
+ }
// Show components if requested
if showComponents {
for _, variant := range ps.Spec.Variants {
for _, component := range variant.Components {
- fmt.Fprintf(w, " %s\t%s\t\t\n",
- fmt.Sprintf("%s.%s", ps.Name, component.Name),
- variant.Name)
+ if _, err := fmt.Fprintf(w, " %s.%s\t%s\t\t\n", ps.Name, component.Name, variant.Name); err != nil {
+ return err
+ }
}
}
}
}
- return nil
+ return w.Flush()
}
-func listPackages(ctx context.Context, k8sClient client.Client, showComponents bool) error {
+func listPackages(ctx context.Context, k8sClient client.Client, out io.Writer, showComponents bool) error {
var pkgList cozyv1alpha1.PackageList
if err := k8sClient.List(ctx, &pkgList); err != nil {
return fmt.Errorf("failed to list Packages: %w", err)
@@ -160,11 +163,12 @@ func listPackages(ctx context.Context, k8sClient client.Client, showComponents b
}
// Use tabwriter for better column alignment
- w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
- defer w.Flush()
+ w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0)
// Print header
- fmt.Fprintln(w, "NAME\tVARIANT\tREADY\tSTATUS")
+ if _, err := fmt.Fprintln(w, "NAME\tVARIANT\tREADY\tSTATUS"); err != nil {
+ return err
+ }
// Print rows
for _, pkg := range pkgList.Items {
@@ -187,7 +191,9 @@ func listPackages(ctx context.Context, k8sClient client.Client, showComponents b
}
}
- fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", pkg.Name, variant, ready, status)
+ if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", pkg.Name, variant, ready, status); err != nil {
+ return err
+ }
// Show components if requested
if showComponents {
@@ -197,9 +203,9 @@ func listPackages(ctx context.Context, k8sClient client.Client, showComponents b
for _, v := range ps.Spec.Variants {
if v.Name == variant {
for _, component := range v.Components {
- fmt.Fprintf(w, " %s\t%s\t\t\n",
- fmt.Sprintf("%s.%s", pkg.Name, component.Name),
- variant)
+ if _, err := fmt.Fprintf(w, " %s.%s\t%s\t\t\n", pkg.Name, component.Name, variant); err != nil {
+ return err
+ }
}
break
}
@@ -208,7 +214,7 @@ func listPackages(ctx context.Context, k8sClient client.Client, showComponents b
}
}
- return nil
+ return w.Flush()
}
func init() {
diff --git a/cmd/cozypkg/cmd/output_test.go b/cmd/cozypkg/cmd/output_test.go
new file mode 100644
index 000000000..9eed21639
--- /dev/null
+++ b/cmd/cozypkg/cmd/output_test.go
@@ -0,0 +1,185 @@
+/*
+Copyright 2026 The Cozystack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
+ "github.com/spf13/cobra"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+type failingWriter struct{}
+
+func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("broken pipe") }
+
+func TestPrintReportReturnsWriteError(t *testing.T) {
+ r := &Report{Findings: []Finding{{Severity: SeverityError, Code: "X", Message: "m"}}}
+ if err := printReport(r, failingWriter{}); err == nil {
+ t.Fatal("printReport to an unwritable writer returned nil")
+ }
+}
+
+// The table is buffered until the tabwriter flushes, so a failing reader shows
+// up only as a Flush error, which list has to return.
+func TestListReturnsWriteError(t *testing.T) {
+ s := runtime.NewScheme()
+ utilruntime.Must(cozyv1alpha1.AddToScheme(s))
+ ps := &cozyv1alpha1.PackageSource{ObjectMeta: metav1.ObjectMeta{Name: "acme.app"}}
+ pkg := &cozyv1alpha1.Package{ObjectMeta: metav1.ObjectMeta{Name: "acme.app"}}
+ cl := fake.NewClientBuilder().WithScheme(s).WithObjects(ps, pkg).Build()
+
+ for name, list := range map[string]func(io.Writer) error{
+ "sources": func(w io.Writer) error { return listPackageSources(context.Background(), cl, w, true) },
+ "packages": func(w io.Writer) error { return listPackages(context.Background(), cl, w, true) },
+ } {
+ t.Run(name, func(t *testing.T) {
+ if err := list(failingWriter{}); err == nil {
+ t.Fatal("list to an unwritable writer returned nil")
+ }
+ var out strings.Builder
+ if err := list(&out); err != nil || !strings.Contains(out.String(), "acme.app") {
+ t.Fatalf("list = %v, output %q; want the row", err, out.String())
+ }
+ })
+ }
+}
+
+// A command that cannot print its validation report still fails, and a
+// refusal keeps validation as its reason with the write error joined to it.
+func TestValidationCommandsReportUnwritableReport(t *testing.T) {
+ fakeFlux := t.TempDir()
+ if err := os.WriteFile(filepath.Join(fakeFlux, "flux"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("PATH", fakeFlux+string(os.PathListSeparator)+os.Getenv("PATH"))
+
+ emptyRepo := func(t *testing.T) string {
+ root := t.TempDir()
+ if err := os.Mkdir(filepath.Join(root, "packages"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ return root
+ }
+ run := func(c *cobra.Command, args ...string) error {
+ c.SetOut(failingWriter{})
+ t.Cleanup(func() { c.SetOut(nil) })
+ return c.RunE(c, args)
+ }
+ push := func(root string) error {
+ pushCmdFlags.path = root
+ t.Cleanup(func() { pushCmdFlags.path = "." })
+ return run(pushCmd, "oci://registry.example/acme/apps:v1")
+ }
+
+ for _, tc := range []struct {
+ name string
+ run func(t *testing.T) error
+ refusal string
+ }{
+ {name: "validate, valid", run: func(t *testing.T) error { return run(validateCmd, validRepo(t)) }},
+ {name: "validate, invalid", run: func(t *testing.T) error { return run(validateCmd, emptyRepo(t)) }, refusal: "validation failed"},
+ {name: "push, valid", run: func(t *testing.T) error { return push(validRepo(t)) }, refusal: "nothing was pushed"},
+ {name: "push, invalid", run: func(t *testing.T) error { return push(emptyRepo(t)) }, refusal: "refusing to push"},
+ {name: "tap, invalid", run: func(t *testing.T) error { return run(tapCmd, "oci://registry.example/acme/apps:v1") }, refusal: "refusing to tap"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ err := tc.run(t)
+ if err == nil || !strings.Contains(err.Error(), "broken pipe") {
+ t.Fatalf("err = %v, want the write error", err)
+ }
+ if tc.refusal != "" && !strings.Contains(err.Error(), tc.refusal) {
+ t.Errorf("err = %v, want the refusal %q kept", err, tc.refusal)
+ }
+ })
+ }
+}
+
+// A tap that applied is reported as applied even when its summary is lost,
+// and the lost summary still fails the command.
+func TestTapSummaryReportsUnwritableOutput(t *testing.T) {
+ err := writeTapSummary(failingWriter{}, "oci://registry.example/acme/apps:v1", "tap-acme", []string{"acme.app"})
+ if err == nil || !strings.Contains(err.Error(), "tap applied") || !strings.Contains(err.Error(), "broken pipe") {
+ t.Fatalf("err = %v, want the write error with the tap reported as applied", err)
+ }
+ var out strings.Builder
+ if err := writeTapSummary(&out, "oci://registry.example/acme/apps:v1", "tap-acme", []string{"acme.app"}); err != nil || !strings.Contains(out.String(), "cozypkg add acme.app") {
+ t.Fatalf("writeTapSummary = %v, output %q", err, out.String())
+ }
+}
+
+// A clean report has no finding lines, so its summary is its only write and
+// alone decides the result.
+func TestPrintReportReturnsSummaryWriteError(t *testing.T) {
+ if err := printReport(&Report{}, failingWriter{}); err == nil {
+ t.Fatal("printReport of a clean report to an unwritable writer returned nil")
+ }
+}
+
+// push runs flux only after its status line was written, with validation
+// skipped and after a report that was written.
+func TestPushStopsBeforeFluxWhenItsStatusIsLost(t *testing.T) {
+ fakeFlux := t.TempDir()
+ ran := filepath.Join(t.TempDir(), "flux-ran")
+ if err := os.WriteFile(filepath.Join(fakeFlux, "flux"), []byte("#!/bin/sh\ntouch "+ran+"\n"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("PATH", fakeFlux+string(os.PathListSeparator)+os.Getenv("PATH"))
+ for _, skip := range []bool{true, false} {
+ t.Run(fmt.Sprintf("skip-validate=%v", skip), func(t *testing.T) {
+ pushCmdFlags.path, pushCmdFlags.skipValidate = validRepo(t), skip
+ pushCmdFlags.source, pushCmdFlags.revision = "https://example.com/acme.git", "v1"
+ t.Cleanup(func() {
+ pushCmdFlags.path, pushCmdFlags.skipValidate = ".", false
+ pushCmdFlags.source, pushCmdFlags.revision = "", ""
+ })
+ pushCmd.SetOut(&failOn{want: "Pushing"})
+ t.Cleanup(func() { pushCmd.SetOut(nil) })
+ err := pushCmd.RunE(pushCmd, []string{"oci://registry.example/acme/apps:v1"})
+ if err == nil || !strings.Contains(err.Error(), "nothing was pushed") {
+ t.Fatalf("push losing its status = %v, want nothing pushed", err)
+ }
+ if _, statErr := os.Stat(ran); statErr == nil {
+ t.Fatal("flux ran after the status line was lost")
+ }
+ })
+ }
+}
+
+// failOn fails every write that contains want, and takes every other one.
+type failOn struct {
+ want string
+ buf strings.Builder
+}
+
+func (w *failOn) Write(p []byte) (int, error) {
+ if strings.Contains(string(p), w.want) {
+ return 0, errors.New("broken pipe")
+ }
+ return w.buf.Write(p)
+}
diff --git a/cmd/cozypkg/cmd/push.go b/cmd/cozypkg/cmd/push.go
index 3f8e65cac..f60004034 100644
--- a/cmd/cozypkg/cmd/push.go
+++ b/cmd/cozypkg/cmd/push.go
@@ -17,6 +17,7 @@ limitations under the License.
package cmd
import (
+ "errors"
"fmt"
"os"
"os/exec"
@@ -106,10 +107,13 @@ published. Source URL and revision are derived from git when not given.`,
if err != nil {
return err
}
- printReport(rep, cmd.OutOrStdout())
+ printErr := printReport(rep, cmd.OutOrStdout())
if rep.HasErrors() {
errs, _, _ := rep.Counts()
- return fmt.Errorf("refusing to push: validation failed with %d error(s) (use --skip-validate to override)", errs)
+ return errors.Join(fmt.Errorf("refusing to push: validation failed with %d error(s) (use --skip-validate to override)", errs), printErr)
+ }
+ if printErr != nil {
+ return fmt.Errorf("nothing was pushed: writing the validation report failed: %w", printErr)
}
}
@@ -129,7 +133,9 @@ published. Source URL and revision are derived from git when not given.`,
}
fluxArgs := buildFluxPushArgs(ref, packagesDir, source, revision, pushCmdFlags.reproducible)
- fmt.Fprintf(cmd.OutOrStdout(), "Pushing %s (source=%s revision=%s)\n", ref, source, revision)
+ if _, err := fmt.Fprintf(cmd.OutOrStdout(), "Pushing %s (source=%s revision=%s)\n", ref, source, revision); err != nil {
+ return fmt.Errorf("nothing was pushed: writing the push status failed: %w", err)
+ }
flux := exec.Command("flux", fluxArgs...)
flux.Stdout = cmd.OutOrStdout()
flux.Stderr = cmd.ErrOrStderr()
diff --git a/cmd/cozypkg/cmd/tap.go b/cmd/cozypkg/cmd/tap.go
index e04edebd7..2d5f7bb9b 100644
--- a/cmd/cozypkg/cmd/tap.go
+++ b/cmd/cozypkg/cmd/tap.go
@@ -18,6 +18,7 @@ package cmd
import (
"context"
+ "errors"
"fmt"
"io"
"regexp"
@@ -402,9 +403,8 @@ runs its charts in your management cluster: tap only sources you trust.`,
return err
}
if rep.HasErrors() {
- printReport(rep, cmd.OutOrStdout())
errs, _, _ := rep.Counts()
- return fmt.Errorf("refusing to tap: artifact failed validation with %d error(s) (use --skip-validate to override)", errs)
+ return errors.Join(fmt.Errorf("refusing to tap: artifact failed validation with %d error(s) (use --skip-validate to override)", errs), printReport(rep, cmd.OutOrStdout()))
}
}
@@ -487,19 +487,29 @@ runs its charts in your management cluster: tap only sources you trust.`,
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "warning: could not reset materialization state on OCIRepository/%s: %v\n", srcName, err)
}
- fmt.Fprintf(cmd.OutOrStdout(), "Tapped %s\n OCIRepository/%s\n", fullRef, srcName)
- for _, n := range names {
- fmt.Fprintf(cmd.OutOrStdout(), " PackageSource/%s\n", n)
- }
- _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Tapping installs nothing. Install an app with:\n")
- for _, n := range names {
- fmt.Fprintf(cmd.OutOrStdout(), " cozypkg add %s\n", n)
- }
- _, _ = fmt.Fprintf(cmd.OutOrStdout(), "A privileged install component is not installed until you confirm it with 'cozypkg add --allow-privileged'.\n")
- return nil
+ return writeTapSummary(cmd.OutOrStdout(), fullRef, srcName, names)
},
}
+// writeTapSummary reports what a tap created. The tap is applied by then, so a
+// summary that cannot be written fails the command while saying so.
+func writeTapSummary(w io.Writer, fullRef, srcName string, names []string) error {
+ var b strings.Builder
+ fmt.Fprintf(&b, "Tapped %s\n OCIRepository/%s\n", fullRef, srcName)
+ for _, n := range names {
+ fmt.Fprintf(&b, " PackageSource/%s\n", n)
+ }
+ b.WriteString("Tapping installs nothing. Install an app with:\n")
+ for _, n := range names {
+ fmt.Fprintf(&b, " cozypkg add %s\n", n)
+ }
+ b.WriteString("A privileged install component is not installed until you confirm it with 'cozypkg add --allow-privileged'.\n")
+ if _, err := io.WriteString(w, b.String()); err != nil {
+ return fmt.Errorf("tap applied, but writing its summary failed: %w", err)
+ }
+ return nil
+}
+
var untapCmd = &cobra.Command{
Use: "untap ",
Short: "Remove a tapped repository",
diff --git a/cmd/cozypkg/cmd/validate.go b/cmd/cozypkg/cmd/validate.go
index 809f0f246..f781cfd8b 100644
--- a/cmd/cozypkg/cmd/validate.go
+++ b/cmd/cozypkg/cmd/validate.go
@@ -21,6 +21,7 @@ import (
"bytes"
"context"
"encoding/json"
+ "errors"
"fmt"
"io"
"io/fs"
@@ -535,16 +536,19 @@ func pullOCIArtifact(ref string) (string, func(), error) {
return dir, cleanup, nil
}
-func printReport(r *Report, w io.Writer) {
+func printReport(r *Report, w io.Writer) error {
for _, f := range r.Findings {
loc := f.Location
if loc == "" {
loc = "-"
}
- fmt.Fprintf(w, "[%s] %s (%s): %s\n", f.Severity, loc, f.Code, f.Message)
+ if _, err := fmt.Fprintf(w, "[%s] %s (%s): %s\n", f.Severity, loc, f.Code, f.Message); err != nil {
+ return err
+ }
}
errs, warns, infos := r.Counts()
- fmt.Fprintf(w, "\nSummary: %d error(s), %d warning(s), %d info\n", errs, warns, infos)
+ _, err := fmt.Fprintf(w, "\nSummary: %d error(s), %d warning(s), %d info\n", errs, warns, infos)
+ return err
}
var validateCmdFlags struct {
@@ -663,10 +667,13 @@ runs "helm lint" on every component chart.`,
if err != nil {
return err
}
- printReport(rep, cmd.OutOrStdout())
+ printErr := printReport(rep, cmd.OutOrStdout())
if rep.HasErrors() {
errs, _, _ := rep.Counts()
- return fmt.Errorf("validation failed: %d error(s)", errs)
+ return errors.Join(fmt.Errorf("validation failed: %d error(s)", errs), printErr)
+ }
+ if printErr != nil {
+ return fmt.Errorf("writing the validation report failed: %w", printErr)
}
return nil
},
diff --git a/test/check-readiness/main_test.go b/test/check-readiness/main_test.go
index 622340c7f..01e6c1505 100644
--- a/test/check-readiness/main_test.go
+++ b/test/check-readiness/main_test.go
@@ -13,6 +13,7 @@ package checkreadiness_test
import (
"bytes"
+ "context"
"errors"
"flag"
"fmt"
@@ -23,6 +24,7 @@ import (
"runtime"
"strings"
"testing"
+ "time"
)
var update = flag.Bool("update", false, "regenerate golden files")
@@ -335,3 +337,80 @@ func repoRoot(t *testing.T) string {
}
return root
}
+
+// A result that never reaches stdout fails the run, whatever the cluster's
+// state, and the watch loop stops instead of refreshing into the void. stdout
+// is a read-only descriptor, so every write fails without a SIGPIPE.
+func TestUnwritableStdoutFails(t *testing.T) {
+ root := repoRoot(t)
+ fakeBin := filepath.Join(root, "test", "check-readiness", "fakekubectl")
+ base := filepath.Join(root, "test", "check-readiness", "testdata", "fixtures", "_base")
+ for _, tc := range []struct {
+ name string
+ args []string
+ }{
+ {name: "once", args: []string{"--no-color"}},
+ {name: "wait", args: []string{"--wait", "--no-color"}},
+ {name: "watch", args: []string{"-w", "1", "--no-color"}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ stdout, err := os.Open(os.DevNull)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = stdout.Close() })
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ t.Cleanup(cancel)
+ argv := subjectArgv(tc.args)
+ cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
+ cmd.Env = testEnv(fakeBin, base, base, filepath.Join(t.TempDir(), "kubectl.log"))
+ var stderr bytes.Buffer
+ cmd.Stdout, cmd.Stderr = stdout, &stderr
+ runErr := cmd.Run()
+ if ctx.Err() != nil {
+ t.Fatalf("still running after its output failed\nstderr:\n%s", stderr.String())
+ }
+ if code := exitCode(t, runErr); code != 1 {
+ t.Errorf("exit code = %d, want 1\nstderr:\n%s", code, stderr.String())
+ }
+ if !strings.Contains(stderr.String(), "writing output") {
+ t.Errorf("stderr does not say the output was lost:\n%s", stderr.String())
+ }
+ })
+ }
+}
+
+// A wait that times out ends with its verdict: the table comes first and the
+// timeout line last, also when the table could not be written.
+func TestWaitTimeoutVerdictComesLast(t *testing.T) {
+ root := repoRoot(t)
+ fakeBin := filepath.Join(root, "test", "check-readiness", "fakekubectl")
+ fixtures := filepath.Join(root, "test", "check-readiness", "testdata", "fixtures")
+ args := []string{"-w", "1", "--wait", "--timeout", "1s", "--no-color"}
+ for _, unwritable := range []bool{false, true} {
+ t.Run(fmt.Sprintf("unwritable=%v", unwritable), func(t *testing.T) {
+ cmd := scriptCommand(args)
+ cmd.Env = testEnv(fakeBin, filepath.Join(fixtures, "helmrelease-not-ready"), filepath.Join(fixtures, "_base"), filepath.Join(t.TempDir(), "kubectl.log"))
+ var combined bytes.Buffer
+ cmd.Stdout, cmd.Stderr = &combined, &combined
+ if unwritable {
+ stdout, err := os.Open(os.DevNull)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = stdout.Close() })
+ cmd.Stdout = stdout
+ }
+ if code := exitCode(t, cmd.Run()); code != 1 {
+ t.Errorf("exit code = %d, want 1\n%s", code, combined.String())
+ }
+ lines := strings.Split(strings.TrimRight(combined.String(), "\n"), "\n")
+ if last := lines[len(lines)-1]; !strings.HasPrefix(last, "Timeout after") {
+ t.Errorf("last line = %q, want the timeout verdict\n%s", last, combined.String())
+ }
+ if unwritable && !strings.Contains(combined.String(), "writing output") {
+ t.Errorf("stderr does not say the table was lost:\n%s", combined.String())
+ }
+ })
+ }
+}
diff --git a/test/check-readiness/testdata/golden/help-long.stdout b/test/check-readiness/testdata/golden/help-long.stdout
index f1f289e08..428f135aa 100644
--- a/test/check-readiness/testdata/golden/help-long.stdout
+++ b/test/check-readiness/testdata/golden/help-long.stdout
@@ -21,3 +21,7 @@ Options:
--no-color Disable color output (auto-disabled on non-TTY)
-h, --help Show this help
+Exit status: 0 when everything is ready, 1 when something is not ready or a
+write to stdout fails, 2 on invalid arguments. A closed pipe ends the process
+with SIGPIPE, like other Unix tools.
+
diff --git a/test/check-readiness/testdata/golden/help-short.stdout b/test/check-readiness/testdata/golden/help-short.stdout
index f1f289e08..428f135aa 100644
--- a/test/check-readiness/testdata/golden/help-short.stdout
+++ b/test/check-readiness/testdata/golden/help-short.stdout
@@ -21,3 +21,7 @@ Options:
--no-color Disable color output (auto-disabled on non-TTY)
-h, --help Show this help
+Exit status: 0 when everything is ready, 1 when something is not ready or a
+write to stdout fails, 2 on invalid arguments. A closed pipe ends the process
+with SIGPIPE, like other Unix tools.
+
diff --git a/test/check-readiness/testdata/golden/unknown-arg.stderr b/test/check-readiness/testdata/golden/unknown-arg.stderr
index d6ec7c0db..ca652d9ea 100644
--- a/test/check-readiness/testdata/golden/unknown-arg.stderr
+++ b/test/check-readiness/testdata/golden/unknown-arg.stderr
@@ -22,3 +22,7 @@ Options:
--no-color Disable color output (auto-disabled on non-TTY)
-h, --help Show this help
+Exit status: 0 when everything is ready, 1 when something is not ready or a
+write to stdout fails, 2 on invalid arguments. A closed pipe ends the process
+with SIGPIPE, like other Unix tools.
+
diff --git a/test/check-readiness/testdata/golden/watch-nonnumeric-interval.stderr b/test/check-readiness/testdata/golden/watch-nonnumeric-interval.stderr
index 3cb2c43bb..cc23fc457 100644
--- a/test/check-readiness/testdata/golden/watch-nonnumeric-interval.stderr
+++ b/test/check-readiness/testdata/golden/watch-nonnumeric-interval.stderr
@@ -22,3 +22,7 @@ Options:
--no-color Disable color output (auto-disabled on non-TTY)
-h, --help Show this help
+Exit status: 0 when everything is ready, 1 when something is not ready or a
+write to stdout fails, 2 on invalid arguments. A closed pipe ends the process
+with SIGPIPE, like other Unix tools.
+
From f99e511eea92e311eaf68234f1d8e16cdcfb07d4 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Thu, 12 Nov 2026 23:42:56 +0300
Subject: [PATCH 099/114] fix(backport): fail a run whose output could not be
written
Every report, check and verify result goes to stdout, and every write
error there was dropped, so a run whose output never reached its
reader still exited 0. The terminal writer now keeps the first stdout
error, and the run fails with it unless it already failed. Only stdout
is tracked; a write error on stderr is still discarded.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
cmd/backport-batch/README.md | 2 ++
cmd/backport-batch/assemble.go | 24 ++++++-------
cmd/backport-batch/check.go | 16 ++++-----
cmd/backport-batch/main.go | 36 +++++++++++++++-----
cmd/backport-batch/notify.go | 14 ++++----
cmd/backport-batch/output_test.go | 56 +++++++++++++++++++++++++++++++
cmd/backport-batch/report.go | 30 ++++++++---------
cmd/backport-batch/verify.go | 10 +++---
8 files changed, 133 insertions(+), 55 deletions(-)
create mode 100644 cmd/backport-batch/output_test.go
diff --git a/cmd/backport-batch/README.md b/cmd/backport-batch/README.md
index 81afd0ef6..4919035c0 100644
--- a/cmd/backport-batch/README.md
+++ b/cmd/backport-batch/README.md
@@ -27,6 +27,8 @@ Requires `git` and an authenticated `gh` on PATH, and a remote called `origin`.
Build the binary wherever an exit code matters: `go run` turns any non-zero exit into its own `1`, so under it a check that could not be completed looks the same as one that found something missing.
+A command whose output on stdout could not be written exits 2 where it would have exited 0, and says on stderr that the output was lost; anything the run did before that, a push or a published pull request included, has still happened.
+
### run
Switches the current working tree to `backport-batch/-`, created at `origin/`, and picks onto it one original at a time, in the order they merged into `main`, as one unit each on the branch's first-parent line. A pull request merged into `main` with a merge commit `M` goes on as its own commits, `git rev-list --reverse --topo-order --no-merges M^1..M^2`, each picked with `git cherry-pick -x --signoff`, so each keeps its author, message and sign-offs and gains the `(cherry picked from commit …)` line; the merges that brought `main` into its branch are left out, since `main` already has what they brought. A merge commit then joins the picks to the batch: its first parent is the batch before the original, its second the last pick, and its message is `M`'s with a paragraph of its own, `Backported-from: M`. A pull request squash-merged into `main` goes on as one pick. When any commit of an original conflicts, the pick is aborted, the branch goes back to where it was before that original, and the original is recorded with the commit that failed and its files; the run goes on with the next one. A commit the branch already carries is skipped as empty, and an original none of whose commits changes anything is reported as carried. The tree has to be clean. A local branch of that name is reset to where the run starts; when that drops commits only it has, the run first prints its tip and the `git branch` command that restores them, and so does `--defer`. Signing follows your git configuration; hooks are off for the picks and the merge, so each commit carries its message as git writes it.
diff --git a/cmd/backport-batch/assemble.go b/cmd/backport-batch/assemble.go
index 3c025517e..a0aab98db 100644
--- a/cmd/backport-batch/assemble.go
+++ b/cmd/backport-batch/assemble.go
@@ -153,7 +153,7 @@ func (a *app) run(args []string) int {
}
// The scheduled job has half an hour for every line; this says where it
// goes.
- fmt.Fprintf(a.stderr, "prepared %s in %s, %s of it judging partial units and what originals build on\n", l.branch(), time.Since(started).Round(time.Millisecond), b.detecting.Round(time.Millisecond))
+ _, _ = fmt.Fprintf(a.stderr, "prepared %s in %s, %s of it judging partial units and what originals build on\n", l.branch(), time.Since(started).Round(time.Millisecond), b.detecting.Round(time.Millisecond))
if drops != nil {
return a.redrop(b, drops, reason, *undropArg != "", *push)
}
@@ -449,20 +449,20 @@ func (a *app) finish(b *batch, r *report, branch string, date time.Time, push, r
}
body, left := r.body(template)
p.body = body
- fmt.Fprint(a.stdout, body)
+ _, _ = fmt.Fprint(a.stdout, body)
skipped := 0
for _, items := range r.skipped {
skipped += len(items)
}
for _, rule := range left {
for _, v := range r.skipped[rule] {
- fmt.Fprintf(a.stderr, "skipped by %s: %s %s (%s)\n", ruleName(rule), v.o.url(), terminalSafe(v.o.title), terminalSafe(v.reason))
+ _, _ = fmt.Fprintf(a.stderr, "skipped by %s: %s %s (%s)\n", ruleName(rule), v.o.url(), terminalSafe(v.o.title), terminalSafe(v.reason))
}
}
for _, p := range r.waiting {
- fmt.Fprintf(a.stderr, "waiting for a maintainer: %s %s changes %s\n", p.v.o.url(), terminalSafe(p.v.o.title), terminalSafe(strings.Join(p.files, ", ")))
+ _, _ = fmt.Fprintf(a.stderr, "waiting for a maintainer: %s %s changes %s\n", p.v.o.url(), terminalSafe(p.v.o.title), terminalSafe(strings.Join(p.files, ", ")))
}
- fmt.Fprintf(a.stderr, "%s at %.12s: %d on the branch, %d deferred, %d waiting for a maintainer, %d skipped\ntitle: %s\nreport: %s\n",
+ _, _ = fmt.Fprintf(a.stderr, "%s at %.12s: %d on the branch, %d deferred, %d waiting for a maintainer, %d skipped\ntitle: %s\nreport: %s\n",
branch, head, len(r.onBranch), len(r.deferred)+len(r.held), len(r.waiting), skipped, p.title, doc.ID)
if !push {
return 0
@@ -471,8 +471,8 @@ func (a *app) finish(b *batch, r *report, branch string, date time.Time, push, r
// Picking nothing is not a failure of the run; check still reports
// whatever the line lacks. Merged and closed batches are never
// written, so there is nowhere to keep what the run found.
- fmt.Fprintf(a.stderr, "%s carries nothing beyond %s; nothing to push\n", branch, b.line.branch())
- fmt.Fprintf(a.stderr, "no batch pull request is open on %s; this run's classification is not retained\n", b.line.branch())
+ _, _ = fmt.Fprintf(a.stderr, "%s carries nothing beyond %s; nothing to push\n", branch, b.line.branch())
+ _, _ = fmt.Fprintf(a.stderr, "no batch pull request is open on %s; this run's classification is not retained\n", b.line.branch())
return 0
}
if err := a.publish(b, p, rebuilt); err != nil {
@@ -537,7 +537,7 @@ func (a *app) warnReset(branch, start string) error {
if err != nil || kept {
return err
}
- fmt.Fprintf(a.stderr, "resetting the local %s from %s, dropping commits %.12s does not carry; git branch %s-%.12s %s restores them\n", branch, old, start, branch, old, old)
+ _, _ = fmt.Fprintf(a.stderr, "resetting the local %s from %s, dropping commits %.12s does not carry; git branch %s-%.12s %s restores them\n", branch, old, start, branch, old, old)
return nil
}
@@ -733,7 +733,7 @@ func (a *app) publish(b *batch, p publication, rebuilt bool) error {
if err != nil {
return err
}
- fmt.Fprintf(a.stderr, "opened draft %s\n", url)
+ _, _ = fmt.Fprintf(a.stderr, "opened draft %s\n", url)
if err := a.postReport(n, p.report); err != nil {
return fmt.Errorf("posting report %.12s on #%d: %w; the body keeps it, and the next run --push posts it", p.report.doc.ID, n, err)
}
@@ -745,7 +745,7 @@ func (a *app) publish(b *batch, p publication, rebuilt bool) error {
if err := a.gh.editPull(b.pr.Number, p.title, name); err != nil {
return err
}
- fmt.Fprintf(a.stderr, "updated %s\n", pullURL(b.pr))
+ _, _ = fmt.Fprintf(a.stderr, "updated %s\n", pullURL(b.pr))
return nil
}
@@ -861,7 +861,7 @@ func (a *app) finishClose(l line, url string, c closing) error {
if err := a.deleteBranch(l, c); err != nil {
return err
}
- fmt.Fprintf(a.stderr, "closed %s: without what is deferred it carries nothing beyond %s; the next batch tries them again\n", url, l.branch())
+ _, _ = fmt.Fprintf(a.stderr, "closed %s: without what is deferred it carries nothing beyond %s; the next batch tries them again\n", url, l.branch())
return nil
}
@@ -933,7 +933,7 @@ func (a *app) resumeClose(b *batch, push, changing bool) (bool, int) {
if err := a.deleteBranch(b.line, c); err != nil {
return false, a.fail(err)
}
- fmt.Fprintf(a.stderr, "deleted %s, which the close of #%d left\n", c.Branch, c.PR)
+ _, _ = fmt.Fprintf(a.stderr, "deleted %s, which the close of #%d left\n", c.Branch, c.PR)
return false, a.finished(c, changing)
}
diff --git a/cmd/backport-batch/check.go b/cmd/backport-batch/check.go
index d8acf4e37..2101d251f 100644
--- a/cmd/backport-batch/check.go
+++ b/cmd/backport-batch/check.go
@@ -73,14 +73,14 @@ func (a *app) check(args []string) int {
case d.kind == outcomeWarning:
a.warn("%s", d.text)
case d.kind == outcomeNote:
- fmt.Fprintf(a.stderr, "note: %s\n", d.text)
+ _, _ = fmt.Fprintf(a.stderr, "note: %s\n", d.text)
}
}
if len(missing)+len(failing) == 0 {
if *cut {
- fmt.Fprintf(a.stdout, "%s: no change the policy takes is missing before the cut\n", l.branch())
+ _, _ = fmt.Fprintf(a.stdout, "%s: no change the policy takes is missing before the cut\n", l.branch())
} else {
- fmt.Fprintf(a.stdout, "%s: every change the policy takes is on the line or on the batch\n", l.branch())
+ _, _ = fmt.Fprintf(a.stdout, "%s: every change the policy takes is on the line or on the batch\n", l.branch())
}
return 0
}
@@ -89,13 +89,13 @@ func (a *app) check(args []string) int {
where = "nor on batch pull request #" + strconv.Itoa(b.pr.Number)
}
if len(missing) > 0 {
- fmt.Fprintf(a.stdout, "%s: %d changes the policy takes are not on the line, %s:\n", l.branch(), len(missing), where)
+ _, _ = fmt.Fprintf(a.stdout, "%s: %d changes the policy takes are not on the line, %s:\n", l.branch(), len(missing), where)
for _, text := range missing {
- fmt.Fprintf(a.stdout, " %s\n", text)
+ _, _ = fmt.Fprintf(a.stdout, " %s\n", text)
}
}
for _, text := range failing {
- fmt.Fprintf(a.stdout, "%s: %s\n", l.branch(), text)
+ _, _ = fmt.Fprintf(a.stdout, "%s: %s\n", l.branch(), text)
}
return 1
}
@@ -326,10 +326,10 @@ func (a *app) fold(args []string) int {
return a.fail(err)
}
if why != "" {
- fmt.Fprintf(a.stdout, "refused: %s\n", why)
+ _, _ = fmt.Fprintf(a.stdout, "refused: %s\n", why)
return 1
}
- fmt.Fprintln(a.stdout, "ok")
+ _, _ = fmt.Fprintln(a.stdout, "ok")
return 0
}
diff --git a/cmd/backport-batch/main.go b/cmd/backport-batch/main.go
index 23b357dff..19102e57f 100644
--- a/cmd/backport-batch/main.go
+++ b/cmd/backport-batch/main.go
@@ -111,6 +111,9 @@ type app struct {
policyRef string
// tool is the commit backport-batch was built from, for the record.
tool string
+ // unwritten is the first error writing to stdout, which fails the run:
+ // what it printed never reached its reader.
+ unwritten error
}
func main() {
@@ -119,10 +122,21 @@ func main() {
}
func (a *app) main(args []string) int {
- a.stdout, a.stderr = textWriter{a.stdout}, textWriter{a.stderr}
+ a.stdout, a.stderr = textWriter{w: a.stdout, failed: &a.unwritten}, textWriter{w: a.stderr}
+ code := a.command(args)
+ if a.unwritten != nil {
+ _, _ = fmt.Fprintf(a.stderr, "backport-batch: output was lost: %v; anything the run did, such as a push or a publish, still happened\n", a.unwritten)
+ if code == 0 {
+ code = 2
+ }
+ }
+ return code
+}
+
+func (a *app) command(args []string) int {
a.gh = &permCache{github: a.gh, perms: map[string]string{}}
if len(args) == 0 {
- fmt.Fprint(a.stderr, usageText)
+ _, _ = fmt.Fprint(a.stderr, usageText)
return 2
}
if bot := a.gh.batchBot(); bot != "" && !botLoginRE.MatchString(bot) {
@@ -144,20 +158,26 @@ func (a *app) main(args []string) int {
case "lines":
return a.lines(args[1:])
case "-h", "--help", "help":
- fmt.Fprint(a.stdout, usageText)
+ _, _ = fmt.Fprint(a.stdout, usageText)
return 0
}
- fmt.Fprintf(a.stderr, "unknown command %q\n\n%s", args[0], usageText)
+ _, _ = fmt.Fprintf(a.stderr, "unknown command %q\n\n%s", args[0], usageText)
return 2
}
// textWriter keeps output fit for a terminal. Titles, file names and the
// messages git relays come from other people, and a control sequence among
// them would move the cursor or rewrite what was printed before.
-type textWriter struct{ w io.Writer }
+type textWriter struct {
+ w io.Writer
+ failed *error
+}
func (t textWriter) Write(p []byte) (int, error) {
if _, err := io.WriteString(t.w, clean(string(p), true)); err != nil {
+ if t.failed != nil && *t.failed == nil {
+ *t.failed = err
+ }
return 0, err
}
return len(p), nil
@@ -166,7 +186,7 @@ func (t textWriter) Write(p []byte) (int, error) {
func (a *app) flags(name string) *flag.FlagSet {
fs := flag.NewFlagSet(name, flag.ContinueOnError)
fs.SetOutput(a.stderr)
- fs.Usage = func() { fmt.Fprint(a.stderr, usageText) }
+ fs.Usage = func() { _, _ = fmt.Fprint(a.stderr, usageText) }
return fs
}
@@ -189,12 +209,12 @@ func parse(fs *flag.FlagSet, args []string) error {
}
func (a *app) fail(err error) int {
- fmt.Fprintf(a.stderr, "backport-batch: %v\n", err)
+ _, _ = fmt.Fprintf(a.stderr, "backport-batch: %v\n", err)
return 2
}
func (a *app) warn(format string, args ...any) {
- fmt.Fprintf(a.stderr, "warning: "+format+"\n", args...)
+ _, _ = fmt.Fprintf(a.stderr, "warning: "+format+"\n", args...)
}
// verdict is what the policy says about one original for the line.
diff --git a/cmd/backport-batch/notify.go b/cmd/backport-batch/notify.go
index fe301a2ff..3460e287d 100644
--- a/cmd/backport-batch/notify.go
+++ b/cmd/backport-batch/notify.go
@@ -135,11 +135,11 @@ func (a *app) notify(args []string) int {
stalled := days >= stalledAfter
var text []string
if stalled {
- fmt.Fprintf(a.stdout, "%s: batch pull request #%d has been open since %s, %d days\n", l.branch(), pr.Number, opened.Format(time.DateOnly), days)
+ _, _ = fmt.Fprintf(a.stdout, "%s: batch pull request #%d has been open since %s, %d days\n", l.branch(), pr.Number, opened.Format(time.DateOnly), days)
text = append(text, fmt.Sprintf("This batch has been open since %s, %d days, through at least one scheduled run. Merge it, or close it and the next run starts a new one; until then each run adds what the rules take to it.", opened.Format(time.DateOnly), days))
}
if len(partial) > 0 {
- fmt.Fprintf(a.stdout, "%s: %d units on #%d carry only part of their originals\n", l.branch(), len(partial), pr.Number)
+ _, _ = fmt.Fprintf(a.stdout, "%s: %d units on #%d carry only part of their originals\n", l.branch(), len(partial), pr.Number)
var b strings.Builder
b.WriteString("These are on the batch but carry only part of their originals on this line; take each out with `backport-batch run --defer` before the cut, and port what applies as a pull request of its own:\n")
for _, n := range partial {
@@ -148,7 +148,7 @@ func (a *app) notify(args []string) int {
text = append(text, b.String())
}
if len(conflicts) > 0 {
- fmt.Fprintf(a.stdout, "%s: %d deferred conflicts on #%d wait for a maintainer\n", l.branch(), count(conflicts), pr.Number)
+ _, _ = fmt.Fprintf(a.stdout, "%s: %d deferred conflicts on #%d wait for a maintainer\n", l.branch(), count(conflicts), pr.Number)
var b strings.Builder
fmt.Fprintf(&b, "These conflicts wait for a maintainer, as [resolving a conflict](%sresolve-conflict.md) describes; unresolved, each goes to the next batch, and so does what waits on one:\n", tasksURL)
for _, n := range conflicts {
@@ -158,9 +158,9 @@ func (a *app) notify(args []string) int {
}
if len(waiting) > 0 {
if count(waiting) == len(waiting) {
- fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s and wait for a maintainer\n", l.branch(), len(waiting), pr.Number, workflowsDir)
+ _, _ = fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s and wait for a maintainer\n", l.branch(), len(waiting), pr.Number, workflowsDir)
} else {
- fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s or wait on one that does\n", l.branch(), len(waiting), pr.Number, workflowsDir)
+ _, _ = fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s or wait on one that does\n", l.branch(), len(waiting), pr.Number, workflowsDir)
}
var b strings.Builder
fmt.Fprintf(&b, "These change `%s`, which the batch's token may not push, and wait for a maintainer to bring them over, as [changes to workflows](%s#changes-to-workflows) describes, or wait on one that does:\n", workflowsDir, policyURL)
@@ -170,7 +170,7 @@ func (a *app) notify(args []string) int {
text = append(text, b.String())
}
if len(parks) > 0 {
- fmt.Fprintf(a.stdout, "%s: %d parked originals on #%d, named once\n", l.branch(), len(parks), pr.Number)
+ _, _ = fmt.Fprintf(a.stdout, "%s: %d parked originals on #%d, named once\n", l.branch(), len(parks), pr.Number)
var b strings.Builder
fmt.Fprintf(&b, "These cannot land on this line whole, so the batch parked them; each is named here once. A maintainer ports what applies as a pull request of its own, as [fixes that exist only on a release line](%s#fixes-that-exist-only-on-a-release-line) describes, or drops the original with `backport-batch run --drop`:\n", policyURL)
for _, p := range parks {
@@ -354,7 +354,7 @@ func (a *app) lines(args []string) int {
return a.fail(err)
}
for _, l := range slices.Backward(lines) {
- fmt.Fprintln(a.stdout, l.branch())
+ _, _ = fmt.Fprintln(a.stdout, l.branch())
}
return 0
}
diff --git a/cmd/backport-batch/output_test.go b/cmd/backport-batch/output_test.go
new file mode 100644
index 000000000..0362f44b7
--- /dev/null
+++ b/cmd/backport-batch/output_test.go
@@ -0,0 +1,56 @@
+/*
+Copyright 2026 The Cozystack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "errors"
+ "strings"
+ "testing"
+ "time"
+)
+
+type failingWriter struct{}
+
+func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("broken pipe") }
+
+// Output that never reached its reader fails the command, and stderr says why.
+func TestUnwritableOutputFailsTheCommand(t *testing.T) {
+ var stderr strings.Builder
+ a := &app{gh: &fakeGH{}, stdout: failingWriter{}, stderr: &stderr}
+ if code := a.main([]string{"help"}); code != 2 {
+ t.Fatalf("help with an unwritable stdout = %d, want 2", code)
+ }
+ if !strings.Contains(stderr.String(), "broken pipe") {
+ t.Errorf("stderr does not name the write failure:\n%s", stderr.String())
+ }
+}
+
+// A command that already failed keeps its own exit code when its output is
+// lost as well: check still says something is missing.
+func TestUnwritableOutputKeepsTheCommandsFailure(t *testing.T) {
+ f := newFixture(t)
+ f.lines()
+ f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"})
+ var stderr strings.Builder
+ a := &app{gh: &fakeGH{remoteHead: f.remoteHead}, stdout: failingWriter{}, stderr: &stderr, now: func() time.Time { return f.now }, tool: f.tool}
+ if code := a.main([]string{"check", "--line", "release-1.1"}); code != 1 {
+ t.Fatalf("check with #6 missing and an unwritable stdout = %d, want 1\n%s", code, stderr.String())
+ }
+ if !strings.Contains(stderr.String(), "broken pipe") {
+ t.Errorf("stderr does not name the write failure:\n%s", stderr.String())
+ }
+}
diff --git a/cmd/backport-batch/report.go b/cmd/backport-batch/report.go
index d1ae4a4b5..ee072e332 100644
--- a/cmd/backport-batch/report.go
+++ b/cmd/backport-batch/report.go
@@ -708,7 +708,7 @@ func (a *app) report(args []string) int {
return a.fail(fmt.Errorf("batch pull request #%d keeps no report %.12s", pr.Number, *id))
}
if *asJSON {
- fmt.Fprintln(a.stdout, s.raw)
+ _, _ = fmt.Fprintln(a.stdout, s.raw)
return 0
}
a.printReport(&pr, s, inBody, inComment)
@@ -721,29 +721,29 @@ func (a *app) report(args []string) int {
func (a *app) printReport(pr *pullRequest, s storedReport, inBody *storedReport, inComment bool) {
d, b := s.doc, s.doc.Built
safe := terminalSafe
- fmt.Fprintf(a.stdout, "report %s for %s, made %s by a %s run", safe(d.ID), safe(d.Line), safe(b.At), safe(b.Options.Mode))
+ _, _ = fmt.Fprintf(a.stdout, "report %s for %s, made %s by a %s run", safe(d.ID), safe(d.Line), safe(b.At), safe(b.Options.Mode))
if b.Options.NoWorkflowFiles {
- fmt.Fprint(a.stdout, " with --no-workflow-files")
+ _, _ = fmt.Fprint(a.stdout, " with --no-workflow-files")
}
- fmt.Fprintln(a.stdout)
- fmt.Fprintf(a.stdout, " main %s, line %s, newer line %s, policy %s\n batch %s, tool %s, run %s\n", safe(b.Main), safe(b.Line), safe(or(b.Newer, "none")), safe(b.Policy), safe(b.Batch), safe(or(b.Tool, "unknown")), safe(or(b.Run, "none")))
+ _, _ = fmt.Fprintln(a.stdout)
+ _, _ = fmt.Fprintf(a.stdout, " main %s, line %s, newer line %s, policy %s\n batch %s, tool %s, run %s\n", safe(b.Main), safe(b.Line), safe(or(b.Newer, "none")), safe(b.Policy), safe(b.Batch), safe(or(b.Tool, "unknown")), safe(or(b.Run, "none")))
current := "none"
if inBody != nil {
current = safe(inBody.doc.ID)
}
switch {
case inBody != nil && inBody.doc.ID == d.ID:
- fmt.Fprintf(a.stdout, " #%d's body carries this report\n", pr.Number)
+ _, _ = fmt.Fprintf(a.stdout, " #%d's body carries this report\n", pr.Number)
case pr.Head.SHA == b.Batch:
- fmt.Fprintf(a.stdout, " #%d's head is %.12s, the batch this report describes, but its body carries report %.12s\n", pr.Number, safe(b.Batch), current)
+ _, _ = fmt.Fprintf(a.stdout, " #%d's head is %.12s, the batch this report describes, but its body carries report %.12s\n", pr.Number, safe(b.Batch), current)
default:
- fmt.Fprintf(a.stdout, " publication unconfirmed: #%d's head is %.12s, not %.12s, and its body carries report %.12s\n", pr.Number, safe(pr.Head.SHA), safe(b.Batch), current)
+ _, _ = fmt.Fprintf(a.stdout, " publication unconfirmed: #%d's head is %.12s, not %.12s, and its body carries report %.12s\n", pr.Number, safe(pr.Head.SHA), safe(b.Batch), current)
}
if !inComment {
if prState(pr) == "open" {
- fmt.Fprintln(a.stdout, " comment not posted; the next backport-batch run --push posts it")
+ _, _ = fmt.Fprintln(a.stdout, " comment not posted; the next backport-batch run --push posts it")
} else {
- fmt.Fprintf(a.stdout, " comment not posted; kept in the body only, since nothing is written to a %s pull request\n", prState(pr))
+ _, _ = fmt.Fprintf(a.stdout, " comment not posted; kept in the body only, since nothing is written to a %s pull request\n", prState(pr))
}
}
type line struct {
@@ -802,22 +802,22 @@ func (a *app) printReport(pr *pullRequest, s storedReport, inBody *storedReport,
}
slices.SortFunc(lines, func(x, y line) int { return cmp.Compare(x.n, y.n) })
for _, l := range lines {
- fmt.Fprintf(a.stdout, "#%d %s\n", l.n, terminalSafe(l.text))
+ _, _ = fmt.Fprintf(a.stdout, "#%d %s\n", l.n, terminalSafe(l.text))
}
overlay := func(name string, list []int) {
if len(list) > 0 {
- fmt.Fprintf(a.stdout, "%s: %s\n", name, strings.Trim(fmt.Sprint(list), "[]"))
+ _, _ = fmt.Fprintf(a.stdout, "%s: %s\n", name, strings.Trim(fmt.Sprint(list), "[]"))
}
}
overlay("stale, still on the batch", d.Overlays.Stale)
overlay("only part of it applies, still on the batch", d.Overlays.Partial)
for _, e := range d.Overlays.Dropped {
- fmt.Fprintf(a.stdout, "dropped: #%d, merged as %.12s, on #%d: %s\n", e.Original, terminalSafe(e.Merge), e.On, terminalSafe(e.Reason))
+ _, _ = fmt.Fprintf(a.stdout, "dropped: #%d, merged as %.12s, on #%d: %s\n", e.Original, terminalSafe(e.Merge), e.On, terminalSafe(e.Reason))
}
overlay("changed in the pick", d.Overlays.Changed)
for _, n := range slices.Sorted(maps.Keys(d.Diag)) {
for _, ln := range d.Diag[n] {
- fmt.Fprintf(a.stdout, "#%s: %s\n", terminalSafe(n), terminalSafe(ln))
+ _, _ = fmt.Fprintf(a.stdout, "#%s: %s\n", terminalSafe(n), terminalSafe(ln))
}
}
if len(d.Other) > 0 {
@@ -825,7 +825,7 @@ func (a *app) printReport(pr *pullRequest, s storedReport, inBody *storedReport,
for i, sha := range d.Other {
other[i] = safe(sha)
}
- fmt.Fprintf(a.stdout, "commits on main that merged no pull request: %s\n", strings.Join(other, ", "))
+ _, _ = fmt.Fprintf(a.stdout, "commits on main that merged no pull request: %s\n", strings.Join(other, ", "))
}
}
diff --git a/cmd/backport-batch/verify.go b/cmd/backport-batch/verify.go
index 0691df641..03b0e8955 100644
--- a/cmd/backport-batch/verify.go
+++ b/cmd/backport-batch/verify.go
@@ -66,7 +66,7 @@ func (a *app) verify(args []string) int {
return a.fail(err)
}
if pr == nil {
- fmt.Fprintf(a.stdout, "%s: no batch pull request is open; nothing to verify\n", l.branch())
+ _, _ = fmt.Fprintf(a.stdout, "%s: no batch pull request is open; nothing to verify\n", l.branch())
return 0
}
if head, err = batchHead(pr); err != nil {
@@ -102,7 +102,7 @@ func (a *app) verify(args []string) int {
}
if why != "" {
refused = append(refused, why)
- fmt.Fprintf(a.stdout, "refused %.12s: %s\n", u.commit, why)
+ _, _ = fmt.Fprintf(a.stdout, "refused %.12s: %s\n", u.commit, why)
continue
}
seen[v] = u.commit
@@ -110,13 +110,13 @@ func (a *app) verify(args []string) int {
if !u.merge() {
n = 1
}
- fmt.Fprintf(a.stdout, "ok %.12s #%d, %d %s\n", u.commit, v.o.number, n, plural(n, "commit"))
+ _, _ = fmt.Fprintf(a.stdout, "ok %.12s #%d, %d %s\n", u.commit, v.o.number, n, plural(n, "commit"))
}
if len(refused) > 0 {
- fmt.Fprintf(a.stdout, "%s: %d of %d units at %.12s are not originals in the batch's shape\n", l.branch(), len(refused), len(units), head)
+ _, _ = fmt.Fprintf(a.stdout, "%s: %d of %d units at %.12s are not originals in the batch's shape\n", l.branch(), len(refused), len(units), head)
return 1
}
- fmt.Fprintf(a.stdout, "%s: %d %s at %.12s, each one original in the batch's shape\n", l.branch(), len(units), plural(len(units), "unit"), head)
+ _, _ = fmt.Fprintf(a.stdout, "%s: %d %s at %.12s, each one original in the batch's shape\n", l.branch(), len(units), plural(len(units), "unit"), head)
return 0
}
From 20ce6c691e50f03e30b31c012d3ba3dd63c62fa8 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Fri, 13 Nov 2026 00:14:38 +0300
Subject: [PATCH 100/114] fix(sse): close a client's channel once when the
server shuts down
Shutdown closed every open client's channel, and the client's handler
closed the same channel again when its request ended, so any stream
still open at shutdown panicked with a close of a closed channel.
Stopping a client is now idempotent.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
internal/sse/server.go | 11 ++++-
internal/sse/server_test.go | 98 +++++++++++++++++++++++++++++++++++++
2 files changed, 107 insertions(+), 2 deletions(-)
create mode 100644 internal/sse/server_test.go
diff --git a/internal/sse/server.go b/internal/sse/server.go
index 0c55032ec..f65817fb1 100644
--- a/internal/sse/server.go
+++ b/internal/sse/server.go
@@ -60,6 +60,7 @@ type Server struct {
type client struct {
ch chan string
closeCh chan struct{}
+ closeOnce sync.Once
flusher http.Flusher
w http.ResponseWriter
req *http.Request
@@ -122,7 +123,7 @@ func (s *Server) ListenAndServe() error {
func (s *Server) Shutdown(ctx context.Context) error {
s.clientsMu.Lock()
for c := range s.clients {
- close(c.closeCh)
+ c.stop()
}
s.clientsMu.Unlock()
return s.http.Shutdown(ctx)
@@ -206,12 +207,18 @@ func (s *Server) handleStream(w http.ResponseWriter, r *http.Request) {
<-r.Context().Done()
// Unregister client
- close(c.closeCh)
+ c.stop()
s.clientsMu.Lock()
delete(s.clients, c)
s.clientsMu.Unlock()
}
+// stop ends the client's pump. Shutdown and the client's own handler both
+// stop it, in either order.
+func (c *client) stop() {
+ c.closeOnce.Do(func() { close(c.closeCh) })
+}
+
func (c *client) pump() {
t := time.NewTicker(c.heartbeat)
defer t.Stop()
diff --git a/internal/sse/server_test.go b/internal/sse/server_test.go
new file mode 100644
index 000000000..be7c21e34
--- /dev/null
+++ b/internal/sse/server_test.go
@@ -0,0 +1,98 @@
+/*
+Copyright 2026 The Cozystack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sse
+
+import (
+ "context"
+ "errors"
+ "io"
+ "log"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+// streamWriter accepts the first okWrites writes and fails every one after.
+type streamWriter struct {
+ mu sync.Mutex
+ header http.Header
+ okWrites int
+ body strings.Builder
+}
+
+func (w *streamWriter) Header() http.Header { return w.header }
+func (w *streamWriter) WriteHeader(int) {}
+func (w *streamWriter) Flush() {}
+
+func (w *streamWriter) Write(p []byte) (int, error) {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ if w.okWrites == 0 {
+ return 0, errors.New("connection reset")
+ }
+ w.okWrites--
+ return w.body.Write(p)
+}
+
+func (w *streamWriter) String() string {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ return w.body.String()
+}
+
+func (s *Server) snapshotClients() []*client {
+ s.clientsMu.RLock()
+ defer s.clientsMu.RUnlock()
+ var out []*client
+ for c := range s.clients {
+ out = append(out, c)
+ }
+ return out
+}
+
+func newTestServer(heartbeat time.Duration) *Server {
+ return New(Options{Heartbeat: heartbeat, Logger: log.New(io.Discard, "", 0)})
+}
+
+// A stream still open when the server shuts down ends when its client
+// disconnects, without closing the client a second time.
+func TestShutdownThenDisconnect(t *testing.T) {
+ s := newTestServer(time.Hour)
+ ctx, cancel := context.WithCancel(context.Background())
+ r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx)
+ done := make(chan struct{})
+ go func() {
+ s.handleStream(&streamWriter{header: http.Header{}, okWrites: 100}, r)
+ close(done)
+ }()
+ deadline := time.Now().Add(time.Second)
+ for len(s.snapshotClients()) == 0 && time.Now().Before(deadline) {
+ time.Sleep(time.Millisecond)
+ }
+ if err := s.Shutdown(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ cancel()
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Fatal("the stream did not end after its client disconnected")
+ }
+}
From 41ad82842a395b238d98889f76378b7e611b1fe2 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Fri, 13 Nov 2026 00:15:00 +0300
Subject: [PATCH 101/114] fix(sse): drop a client as soon as a write to it
fails
Write errors on the event stream were ignored, so a client whose
connection broke kept its handler, pump and registration until the
request context ended, and every event was written into the dead
connection meanwhile. A failed write now ends the stream and
unregisters the client, and the handler also returns once its pump
ends, so Shutdown no longer leaves streams open.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
internal/sse/server.go | 52 +++++++++++++-------
internal/sse/server_test.go | 97 ++++++++++++++++++++++++++++++++++---
2 files changed, 126 insertions(+), 23 deletions(-)
diff --git a/internal/sse/server.go b/internal/sse/server.go
index f65817fb1..166b01f49 100644
--- a/internal/sse/server.go
+++ b/internal/sse/server.go
@@ -187,8 +187,16 @@ func (s *Server) handleStream(w http.ResponseWriter, r *http.Request) {
s.clients[c] = struct{}{}
s.clientsMu.Unlock()
+ defer func() {
+ s.clientsMu.Lock()
+ delete(s.clients, c)
+ s.clientsMu.Unlock()
+ }()
+
// Initial comment to open the stream for some proxies
- fmt.Fprintf(w, ": connected %s\n\n", time.Now().Format(time.RFC3339))
+ if _, err := fmt.Fprintf(w, ": connected %s\n\n", time.Now().Format(time.RFC3339)); err != nil {
+ return
+ }
flusher.Flush()
// Send latest if any
@@ -196,21 +204,24 @@ func (s *Server) handleStream(w http.ResponseWriter, r *http.Request) {
latest := s.latest
s.latestMu.RUnlock()
if latest != "" {
- writeSSE(w, latest)
+ if err := writeSSE(w, latest); err != nil {
+ return
+ }
flusher.Flush()
}
- // Start pump
- go c.pump()
-
- // Block until client disconnects
- <-r.Context().Done()
-
- // Unregister client
+ // A write the client no longer takes ends the pump; the handler then
+ // gives the client up without waiting for the request to end.
+ pumped := make(chan struct{})
+ go func() {
+ c.pump()
+ close(pumped)
+ }()
+ select {
+ case <-r.Context().Done():
+ case <-pumped:
+ }
c.stop()
- s.clientsMu.Lock()
- delete(s.clients, c)
- s.clientsMu.Unlock()
}
// stop ends the client's pump. Shutdown and the client's own handler both
@@ -227,23 +238,30 @@ func (c *client) pump() {
case <-c.closeCh:
return
case msg := <-c.ch:
- writeSSE(c.w, msg)
+ if err := writeSSE(c.w, msg); err != nil {
+ return
+ }
c.flusher.Flush()
case <-t.C:
// heartbeat comment (keeps connections alive through proxies)
- fmt.Fprint(c.w, ": hb\n\n")
+ if _, err := fmt.Fprint(c.w, ": hb\n\n"); err != nil {
+ return
+ }
c.flusher.Flush()
}
}
}
-func writeSSE(w http.ResponseWriter, msg string) {
+func writeSSE(w http.ResponseWriter, msg string) error {
// Split on lines; each needs its own "data:" field per the SSE spec
lines := strings.SplitSeq(strings.TrimRight(msg, "\n"), "\n")
for ln := range lines {
- fmt.Fprintf(w, "data: %s\n", ln)
+ if _, err := fmt.Fprintf(w, "data: %s\n", ln); err != nil {
+ return err
+ }
}
- fmt.Fprint(w, "\n")
+ _, err := fmt.Fprint(w, "\n")
+ return err
}
// Minimal index page with live updates
diff --git a/internal/sse/server_test.go b/internal/sse/server_test.go
index be7c21e34..5ae5ad966 100644
--- a/internal/sse/server_test.go
+++ b/internal/sse/server_test.go
@@ -57,6 +57,37 @@ func (w *streamWriter) String() string {
return w.body.String()
}
+// serve runs the stream handler for a client that never disconnects and
+// reports whether the handler gave the client up within a second.
+func serve(t *testing.T, s *Server, w *streamWriter, publish string) bool {
+ t.Helper()
+ ctx, cancel := context.WithCancel(context.Background())
+ t.Cleanup(cancel)
+ r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx)
+ done := make(chan struct{})
+ go func() {
+ s.handleStream(w, r)
+ close(done)
+ }()
+ if publish != "" {
+ deadline := time.Now().Add(time.Second)
+ for len(s.snapshotClients()) == 0 && time.Now().Before(deadline) {
+ time.Sleep(time.Millisecond)
+ }
+ // Onto the client's queue, which only the pump reads, so the failing
+ // write is the pump's and not the handler's write of latest.
+ for _, c := range s.snapshotClients() {
+ c.ch <- publish
+ }
+ }
+ select {
+ case <-done:
+ return len(s.snapshotClients()) == 0
+ case <-time.After(time.Second):
+ return false
+ }
+}
+
func (s *Server) snapshotClients() []*client {
s.clientsMu.RLock()
defer s.clientsMu.RUnlock()
@@ -71,15 +102,70 @@ func newTestServer(heartbeat time.Duration) *Server {
return New(Options{Heartbeat: heartbeat, Logger: log.New(io.Discard, "", 0)})
}
-// A stream still open when the server shuts down ends when its client
-// disconnects, without closing the client a second time.
-func TestShutdownThenDisconnect(t *testing.T) {
+// A client whose connection stops taking writes is dropped at once, at
+// whichever write first fails, instead of holding its handler until the
+// request context ends.
+func TestStreamDropsClientOnWriteFailure(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ okWrites int
+ latest string
+ publish string
+ heartbeat time.Duration
+ }{
+ {name: "connect comment", okWrites: 0, heartbeat: time.Hour},
+ {name: "latest payload", okWrites: 1, latest: "state", heartbeat: time.Hour},
+ {name: "published event", okWrites: 1, publish: "event", heartbeat: time.Hour},
+ {name: "heartbeat", okWrites: 1, heartbeat: 10 * time.Millisecond},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ s := newTestServer(tc.heartbeat)
+ if tc.latest != "" {
+ s.Publish(tc.latest)
+ }
+ if !serve(t, s, &streamWriter{header: http.Header{}, okWrites: tc.okWrites}, tc.publish) {
+ t.Fatal("the handler kept a client whose writes fail")
+ }
+ })
+ }
+}
+
+// Each line of a payload is its own data field, and the event ends with an
+// empty line.
+func TestStreamFramesMultilinePayload(t *testing.T) {
s := newTestServer(time.Hour)
+ s.Publish("one\ntwo\n")
+ w := &streamWriter{header: http.Header{}, okWrites: 100}
ctx, cancel := context.WithCancel(context.Background())
r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx)
done := make(chan struct{})
go func() {
- s.handleStream(&streamWriter{header: http.Header{}, okWrites: 100}, r)
+ s.handleStream(w, r)
+ close(done)
+ }()
+ deadline := time.Now().Add(time.Second)
+ for !strings.Contains(w.String(), "data: two\n\n") && time.Now().Before(deadline) {
+ time.Sleep(time.Millisecond)
+ }
+ cancel()
+ <-done
+ if got := w.String(); !strings.Contains(got, "data: one\ndata: two\n\n") {
+ t.Fatalf("stream = %q, want both lines framed as one event", got)
+ }
+ if len(s.snapshotClients()) != 0 {
+ t.Fatal("a disconnected client stayed registered")
+ }
+}
+
+// Shutdown ends every open stream, and the handler that then returns does
+// not close the client a second time.
+func TestShutdownEndsOpenStreams(t *testing.T) {
+ s := newTestServer(time.Hour)
+ w := &streamWriter{header: http.Header{}, okWrites: 100}
+ r := httptest.NewRequest(http.MethodGet, "/stream", nil)
+ done := make(chan struct{})
+ go func() {
+ s.handleStream(w, r)
close(done)
}()
deadline := time.Now().Add(time.Second)
@@ -89,10 +175,9 @@ func TestShutdownThenDisconnect(t *testing.T) {
if err := s.Shutdown(context.Background()); err != nil {
t.Fatal(err)
}
- cancel()
select {
case <-done:
case <-time.After(time.Second):
- t.Fatal("the stream did not end after its client disconnected")
+ t.Fatal("Shutdown left the stream open")
}
}
From bd998d9653e1f1b4ab7045599044ccaa744314a8 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Thu, 12 Nov 2026 23:46:11 +0300
Subject: [PATCH 102/114] fix(kube-ovn): fail the plunger when its status
server cannot start
The status stream was started in a bare goroutine whose error was
dropped, and the runnable's registration error was discarded too, so a
port that could not be bound left the plunger running with no status
endpoint and nothing in its logs. The server now runs inside its
runnable: a listen error fails the manager, as a metrics server bind
failure does, and stopping the manager is not reported as an error.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
.../kubeovnplunger/kubeovn_plunger.go | 30 ++++++++---
.../kubeovnplunger/status_server_test.go | 51 +++++++++++++++++++
2 files changed, 75 insertions(+), 6 deletions(-)
create mode 100644 internal/controller/kubeovnplunger/status_server_test.go
diff --git a/internal/controller/kubeovnplunger/kubeovn_plunger.go b/internal/controller/kubeovnplunger/kubeovn_plunger.go
index a45fba073..30aa42c00 100644
--- a/internal/controller/kubeovnplunger/kubeovn_plunger.go
+++ b/internal/controller/kubeovnplunger/kubeovn_plunger.go
@@ -3,8 +3,10 @@ package kubeovnplunger
import (
"bytes"
"context"
+ "errors"
"fmt"
"io"
+ "net/http"
"strings"
"time"
@@ -153,12 +155,9 @@ func (r *KubeOVNPlunger) SetupWithManager(mgr ctrl.Manager, kubeOVNNamespace, ap
Complete(r); err != nil {
return err
}
- _ = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
- go srv.ListenAndServe()
- <-ctx.Done()
- _ = srv.Shutdown(context.Background())
- return nil
- }))
+ if err := mgr.Add(serveStatus(srv)); err != nil {
+ return err
+ }
return mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
ticker := time.NewTicker(rescanInterval)
defer ticker.Stop()
@@ -180,6 +179,25 @@ func (r *KubeOVNPlunger) SetupWithManager(mgr ctrl.Manager, kubeOVNNamespace, ap
}))
}
+// serveStatus runs the status stream for as long as the manager does. A server
+// that cannot start fails the manager, as the metrics server does.
+func serveStatus(s *sse.Server) manager.RunnableFunc {
+ return func(ctx context.Context) error {
+ served := make(chan error, 1)
+ go func() { served <- s.ListenAndServe() }()
+ select {
+ case err := <-served:
+ return err
+ case <-ctx.Done():
+ }
+ _ = s.Shutdown(context.Background())
+ if err := <-served; !errors.Is(err, http.ErrServerClosed) {
+ return err
+ }
+ return nil
+ }
+}
+
type ExecOptions struct {
Namespace string
Pod string
diff --git a/internal/controller/kubeovnplunger/status_server_test.go b/internal/controller/kubeovnplunger/status_server_test.go
new file mode 100644
index 000000000..c894ecf74
--- /dev/null
+++ b/internal/controller/kubeovnplunger/status_server_test.go
@@ -0,0 +1,51 @@
+package kubeovnplunger
+
+import (
+ "context"
+ "io"
+ "log"
+ "net"
+ "testing"
+ "time"
+
+ "github.com/cozystack/cozystack/internal/sse"
+)
+
+func runStatusServer(ctx context.Context, t *testing.T, addr string) (bool, error) {
+ t.Helper()
+ ctx, cancel := context.WithCancel(ctx)
+ t.Cleanup(cancel)
+ s := sse.New(sse.Options{Addr: addr, Logger: log.New(io.Discard, "", 0)})
+ done := make(chan error, 1)
+ go func() { done <- serveStatus(s)(ctx) }()
+ select {
+ case err := <-done:
+ return true, err
+ case <-time.After(2 * time.Second):
+ return false, nil
+ }
+}
+
+// A status server that cannot listen fails its runnable, and with it the
+// manager, rather than leaving the plunger running without it.
+func TestStatusServerReportsListenFailure(t *testing.T) {
+ taken, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = taken.Close() })
+ returned, err := runStatusServer(context.Background(), t, taken.Addr().String())
+ if !returned || err == nil {
+ t.Fatalf("serveStatus on an occupied address = %v (returned %v), want an error", err, returned)
+ }
+}
+
+// Stopping the manager stops the server, and that is not an error.
+func TestStatusServerStopsWithoutError(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ time.AfterFunc(100*time.Millisecond, cancel)
+ returned, err := runStatusServer(ctx, t, "127.0.0.1:0")
+ if !returned || err != nil {
+ t.Fatalf("serveStatus after cancel = %v (returned %v), want nil", err, returned)
+ }
+}
From 87b794a9925e2f9c4813f5da6425f8344f408c03 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Thu, 12 Nov 2026 23:53:11 +0300
Subject: [PATCH 103/114] chore: handle errors dropped in cleanup paths and
tests
The drain tests now assert that RunThenDrain returns the run's nil
error, the idle-tail test fails on any error other than the stream
ending or being cut, and a test fixture file fails the test when it
cannot be closed.
Removing a temporary directory and closing a response body that has
been read are best-effort: a failure leaves nothing for the caller to
act on and does not change any result, so those errors are discarded
explicitly. The same holds for the stub servers' response writes, whose
failure surfaces on the client side of the test.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
.../cnpg_artifact_size_test.go | 4 ++--
.../mariadb_artifact_size_test.go | 2 +-
.../fluxplunger/drain_outcome_test.go | 18 ++++++++++++------
internal/crdinstall/install.go | 2 +-
internal/fluxinstall/install.go | 2 +-
internal/observabilityproxy/server_test.go | 8 +++++++-
internal/telemetry/collector.go | 2 +-
internal/telemetry/operator_collector.go | 2 +-
pkg/cmd/server/watch_initial_events_test.go | 2 +-
pkg/reservation/instancetype_test.go | 4 +++-
10 files changed, 30 insertions(+), 16 deletions(-)
diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go
index bdf9c5d7b..4afef7194 100644
--- a/internal/backupcontroller/cnpg_artifact_size_test.go
+++ b/internal/backupcontroller/cnpg_artifact_size_test.go
@@ -222,13 +222,13 @@ func cnpgListStub(prefix string, seen func(*http.Request)) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "application/xml")
if _, ok := req.URL.Query()["location"]; ok {
- fmt.Fprint(w, ` `)
+ _, _ = fmt.Fprint(w, ` `)
return
}
if seen != nil {
seen(req)
}
- fmt.Fprintf(w, `
+ _, _ = fmt.Fprintf(w, `
bkt %[1]s 2 1000 false
%[1]sbackup.info 7 "a" 2026-10-21T10:01:00.000Z
diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go
index 9d649c15c..ed4a2b7bc 100644
--- a/internal/backupcontroller/mariadb_artifact_size_test.go
+++ b/internal/backupcontroller/mariadb_artifact_size_test.go
@@ -464,7 +464,7 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) {
gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization")
gotDelimiter = req.URL.Query().Get("delimiter")
w.Header().Set("Content-Type", "application/xml")
- fmt.Fprintf(w, `
+ _, _ = fmt.Fprintf(w, `
bkt tenant/db/ 1 1000 false
%s 31337 "x" 2026-10-21T10:01:00.000Z
diff --git a/internal/controller/fluxplunger/drain_outcome_test.go b/internal/controller/fluxplunger/drain_outcome_test.go
index cb74e0177..152091dec 100644
--- a/internal/controller/fluxplunger/drain_outcome_test.go
+++ b/internal/controller/fluxplunger/drain_outcome_test.go
@@ -65,12 +65,14 @@ func (s captureSink) WithName(string) logr.LogSink { return s }
func TestRunThenDrain_FencesLeftBehindAreAnErrorNamingThem(t *testing.T) {
logger, entries := newCaptureLogger()
ctx := log.IntoContext(context.Background(), logger)
- RunThenDrain(ctx,
+ if err := RunThenDrain(ctx,
func(context.Context) error { return nil },
func(context.Context) (int, error) {
return 1, errors.Join(context.DeadlineExceeded, errors.New("still fenced: tenant-root/vm-disk-0"))
},
- time.Minute)
+ time.Minute); err != nil {
+ t.Fatalf("RunThenDrain = %v, want the run's nil error", err)
+ }
for _, e := range *entries {
if e.err != nil && strings.Contains(e.err.Error(), "tenant-root/vm-disk-0") {
return
@@ -84,10 +86,12 @@ func TestRunThenDrain_FencesLeftBehindAreAnErrorNamingThem(t *testing.T) {
func TestRunThenDrain_AnUnansweredListingIsNotACountOfMinusOne(t *testing.T) {
logger, entries := newCaptureLogger()
ctx := log.IntoContext(context.Background(), logger)
- RunThenDrain(ctx,
+ if err := RunThenDrain(ctx,
func(context.Context) error { return nil },
func(context.Context) (int, error) { return -1, context.DeadlineExceeded },
- time.Minute)
+ time.Minute); err != nil {
+ t.Fatalf("RunThenDrain = %v, want the run's nil error", err)
+ }
found := false
for _, e := range *entries {
for i := 0; i+1 < len(e.kv); i += 2 {
@@ -108,10 +112,12 @@ func TestRunThenDrain_AnUnansweredListingIsNotACountOfMinusOne(t *testing.T) {
func TestRunThenDrain_ACompleteDrainSaysSo(t *testing.T) {
logger, entries := newCaptureLogger()
ctx := log.IntoContext(context.Background(), logger)
- RunThenDrain(ctx,
+ if err := RunThenDrain(ctx,
func(context.Context) error { return nil },
func(context.Context) (int, error) { return 0, nil },
- time.Minute)
+ time.Minute); err != nil {
+ t.Fatalf("RunThenDrain = %v, want the run's nil error", err)
+ }
for _, e := range *entries {
if e.err == nil && strings.Contains(e.msg, "Released every uninstall fence") {
return
diff --git a/internal/crdinstall/install.go b/internal/crdinstall/install.go
index 5760beb6b..12706060b 100644
--- a/internal/crdinstall/install.go
+++ b/internal/crdinstall/install.go
@@ -40,7 +40,7 @@ func Install(ctx context.Context, k8sClient client.Client, writeEmbeddedManifest
if err != nil {
return fmt.Errorf("failed to create temp directory: %w", err)
}
- defer os.RemoveAll(tmpDir)
+ defer func() { _ = os.RemoveAll(tmpDir) }()
manifestsDir := filepath.Join(tmpDir, "manifests")
if err := os.MkdirAll(manifestsDir, 0755); err != nil {
diff --git a/internal/fluxinstall/install.go b/internal/fluxinstall/install.go
index 042bdfd5e..63c977803 100644
--- a/internal/fluxinstall/install.go
+++ b/internal/fluxinstall/install.go
@@ -41,7 +41,7 @@ func Install(ctx context.Context, k8sClient client.Client, writeEmbeddedManifest
if err != nil {
return fmt.Errorf("failed to create temp directory: %w", err)
}
- defer os.RemoveAll(tmpDir)
+ defer func() { _ = os.RemoveAll(tmpDir) }()
// Extract embedded manifests (generated by cozypkg)
manifestsDir := filepath.Join(tmpDir, "manifests")
diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go
index df3e23bc4..2117b1551 100644
--- a/internal/observabilityproxy/server_test.go
+++ b/internal/observabilityproxy/server_test.go
@@ -18,6 +18,7 @@ package observabilityproxy
import (
"bufio"
+ "errors"
"io"
"net/http"
"net/http/httptest"
@@ -413,7 +414,12 @@ func TestServerTailCapsAndIdle(t *testing.T) {
closed <- err
}()
select {
- case <-closed:
+ case err := <-closed:
+ // The server cuts an idle tail off mid-stream rather than ending the
+ // response, so the client reads an unexpected EOF; a clean end is fine too.
+ if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) {
+ t.Fatalf("idle tail ended with %v, want the stream to end or be cut", err)
+ }
case <-time.After(3 * time.Second):
t.Fatal("idle tail not closed")
}
diff --git a/internal/telemetry/collector.go b/internal/telemetry/collector.go
index a948c7495..636bbc85a 100644
--- a/internal/telemetry/collector.go
+++ b/internal/telemetry/collector.go
@@ -147,7 +147,7 @@ func (c *Collector) sendMetrics(clusterID, metrics string) error {
if err != nil {
return fmt.Errorf("failed to send request: %w", err)
}
- defer resp.Body.Close()
+ defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
diff --git a/internal/telemetry/operator_collector.go b/internal/telemetry/operator_collector.go
index 2c0e1b9bb..9f2f9fcbe 100644
--- a/internal/telemetry/operator_collector.go
+++ b/internal/telemetry/operator_collector.go
@@ -272,7 +272,7 @@ func (c *OperatorCollector) sendMetrics(clusterID, metrics string) error {
if err != nil {
return fmt.Errorf("failed to send request: %w", err)
}
- defer resp.Body.Close()
+ defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
diff --git a/pkg/cmd/server/watch_initial_events_test.go b/pkg/cmd/server/watch_initial_events_test.go
index 331417545..75d43c3e1 100644
--- a/pkg/cmd/server/watch_initial_events_test.go
+++ b/pkg/cmd/server/watch_initial_events_test.go
@@ -104,7 +104,7 @@ func watchStream(t *testing.T, ts *httptest.Server, cli client.Client, query str
if err != nil {
t.Fatalf("watch: %v", err)
}
- defer resp.Body.Close()
+ defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
t.Fatalf("watch: status %d", resp.StatusCode)
}
diff --git a/pkg/reservation/instancetype_test.go b/pkg/reservation/instancetype_test.go
index 0d072b548..f837af8e3 100644
--- a/pkg/reservation/instancetype_test.go
+++ b/pkg/reservation/instancetype_test.go
@@ -56,7 +56,9 @@ func shippedInstancetypes(t *testing.T) []map[string]any {
out = append(out, obj)
}
}
- fh.Close()
+ if err := fh.Close(); err != nil {
+ t.Fatalf("%s: %v", f, err)
+ }
}
return out
}
From 7a481c47656c041ef2369ef1ca798367c0713629 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Thu, 12 Nov 2026 23:53:12 +0300
Subject: [PATCH 104/114] chore(workloadmonitor): discard best-effort close
errors explicitly
The Prometheus response body is closed after it has been read, and the
test stubs' writes fail on the client side of the test, so neither
error changes a result.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
internal/controller/workloadmonitor_controller.go | 2 +-
internal/controller/workloadmonitor_controller_test.go | 10 +++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go
index 217fd0bbf..b8b0bac13 100644
--- a/internal/controller/workloadmonitor_controller.go
+++ b/internal/controller/workloadmonitor_controller.go
@@ -525,7 +525,7 @@ func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, p
if err != nil {
return nil, fmt.Errorf("querying Prometheus: %w", err)
}
- defer resp.Body.Close()
+ defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status %d from Prometheus", resp.StatusCode)
diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go
index 353066c4b..87f9dd269 100644
--- a/internal/controller/workloadmonitor_controller_test.go
+++ b/internal/controller/workloadmonitor_controller_test.go
@@ -870,7 +870,7 @@ func TestReconcileNoBucketClaimSkips(t *testing.T) {
func TestQueryAllBucketMetrics(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[
+ _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[
{"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"bucket-aaa"},"value":[1713000000,"10485864"]},
{"metric":{"__name__":"SeaweedFS_s3_bucket_physical_size_bytes","bucket":"bucket-aaa"},"value":[1713000000,"20971728"]},
{"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"bucket-bbb"},"value":[1713000000,"0"]}
@@ -909,7 +909,7 @@ func TestQueryAllBucketMetrics(t *testing.T) {
func TestQueryAllBucketMetricsEmpty(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
+ _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
}))
defer srv.Close()
@@ -953,7 +953,7 @@ func TestQueryAllBucketMetricsPathPrefixAndUserinfo(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
gotUser, gotPass, gotOK = r.BasicAuth()
- fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
+ _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
}))
defer srv.Close()
@@ -1219,7 +1219,7 @@ func TestReconcileBucketMetricsFromEndpointOverride(t *testing.T) {
if r.URL.Path != "/vm/select/0/prometheus/api/v1/query" {
t.Errorf("unexpected query path %q", r.URL.Path)
}
- fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[
+ _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[
{"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"cosi-abc123"},"value":[1713000000,"4096"]}
]}}`)
}))
@@ -1331,7 +1331,7 @@ func TestReconcileRetainsLastKnownSizesOnQueryFailure(t *testing.T) {
func TestReconcileRetainsLastKnownSizesWhenBucketMissingFromResult(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
+ _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
}))
defer srv.Close()
From 50959ccca20fa212c836cc2ccccf7583f1bc279a Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Thu, 12 Nov 2026 23:53:20 +0300
Subject: [PATCH 105/114] chore(backport): discard temp-file cleanup errors
explicitly
A temporary file or directory that cannot be removed, or closed after
its write already failed, leaves nothing to act on: the run's result is
the same, and the primary error is the one returned. The test that
unsets an environment variable now fails if it cannot.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
cmd/backport-batch/apisurface.go | 2 +-
cmd/backport-batch/assemble.go | 4 ++--
cmd/backport-batch/effects.go | 2 +-
cmd/backport-batch/notify.go | 6 +++---
cmd/backport-batch/report.go | 8 ++++----
cmd/backport-batch/report_test.go | 4 +++-
cmd/backport-batch/shape.go | 4 ++--
7 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/cmd/backport-batch/apisurface.go b/cmd/backport-batch/apisurface.go
index 706e4be62..6966bce74 100644
--- a/cmd/backport-batch/apisurface.go
+++ b/cmd/backport-batch/apisurface.go
@@ -99,7 +99,7 @@ func apiSnapshot(commit string) (apigate.Snapshot, error) {
if err != nil {
return nil, err
}
- defer os.RemoveAll(dir)
+ defer func() { _ = os.RemoveAll(dir) }()
for name, blob := range paths {
file := filepath.Join(dir, filepath.FromSlash(name))
if err := os.MkdirAll(filepath.Dir(file), 0o700); err != nil {
diff --git a/cmd/backport-batch/assemble.go b/cmd/backport-batch/assemble.go
index a0aab98db..2ebca3483 100644
--- a/cmd/backport-batch/assemble.go
+++ b/cmd/backport-batch/assemble.go
@@ -727,7 +727,7 @@ func (a *app) publish(b *batch, p publication, rebuilt bool) error {
if err != nil {
return err
}
- defer os.Remove(name)
+ defer func() { _ = os.Remove(name) }()
if b.pr == nil {
n, url, err := a.gh.createPull(b.line.branch(), p.branch, p.title, name)
if err != nil {
@@ -828,7 +828,7 @@ func (a *app) closeBatch(b *batch, p publication) error {
if err != nil {
return err
}
- defer os.Remove(name)
+ defer func() { _ = os.Remove(name) }()
if err := a.stillOpen(c.PR); err != nil {
return err
}
diff --git a/cmd/backport-batch/effects.go b/cmd/backport-batch/effects.go
index 350f9a961..6388c42ae 100644
--- a/cmd/backport-batch/effects.go
+++ b/cmd/backport-batch/effects.go
@@ -367,7 +367,7 @@ func (b *batch) alreadyApplied(pre, cur, post string) (applied, unsupported bool
if err != nil {
return false, false, err
}
- defer os.RemoveAll(dir)
+ defer func() { _ = os.RemoveAll(dir) }()
names := map[string]string{}
for _, f := range []struct{ name, oid string }{{"cur", cur}, {"pre", pre}, {"post", post}} {
names[f.name] = filepath.Join(dir, f.name)
diff --git a/cmd/backport-batch/notify.go b/cmd/backport-batch/notify.go
index 3460e287d..e9ee23234 100644
--- a/cmd/backport-batch/notify.go
+++ b/cmd/backport-batch/notify.go
@@ -229,7 +229,7 @@ func (a *app) announced(pr *pullRequest, rec *lineRecord, parks []parkedEntry) e
if err != nil {
return err
}
- defer os.Remove(name)
+ defer func() { _ = os.Remove(name) }()
return a.gh.editBody(pr.Number, name)
}
@@ -309,9 +309,9 @@ func (a *app) comment(pr *pullRequest, mention string, today time.Time, text []s
if err != nil {
return a.fail(err)
}
- defer os.Remove(f.Name())
+ defer func() { _ = os.Remove(f.Name()) }()
if _, err := f.WriteString(b.String()); err != nil {
- f.Close()
+ _ = f.Close()
return a.fail(err)
}
if err := f.Close(); err != nil {
diff --git a/cmd/backport-batch/report.go b/cmd/backport-batch/report.go
index ee072e332..fbfc4d569 100644
--- a/cmd/backport-batch/report.go
+++ b/cmd/backport-batch/report.go
@@ -850,7 +850,7 @@ func (a *app) postReport(n int, s storedReport) error {
if err != nil {
return err
}
- defer os.Remove(name)
+ defer func() { _ = os.Remove(name) }()
return a.gh.createComment(n, name)
}
@@ -873,12 +873,12 @@ func writeTemp(pattern, content string) (string, error) {
return "", err
}
if _, err := f.WriteString(content); err != nil {
- f.Close()
- os.Remove(f.Name())
+ _ = f.Close()
+ _ = os.Remove(f.Name())
return "", err
}
if err := f.Close(); err != nil {
- os.Remove(f.Name())
+ _ = os.Remove(f.Name())
return "", err
}
return f.Name(), nil
diff --git a/cmd/backport-batch/report_test.go b/cmd/backport-batch/report_test.go
index a496f68e0..754b599bb 100644
--- a/cmd/backport-batch/report_test.go
+++ b/cmd/backport-batch/report_test.go
@@ -224,7 +224,9 @@ func TestReportPartition(t *testing.T) {
}
t.Setenv("GIT_COMMITTER_DATE", "2026-01-01T00:00:00Z")
f.pr(4, "fix(app): four, before the start", "", map[string]string{"app/four.txt": "4\n"})
- os.Unsetenv("GIT_COMMITTER_DATE")
+ if err := os.Unsetenv("GIT_COMMITTER_DATE"); err != nil {
+ t.Fatal(err)
+ }
f.setPolicy("release-1.0: 2026-06-01\n")
f.onLine("release-1.0", "fix(app): a line-only eleven", map[string]string{"app/eleven.txt": "line\n"})
pr(5, "fix(app): five", map[string]string{"app/five.txt": "5\n"}, true)
diff --git a/cmd/backport-batch/shape.go b/cmd/backport-batch/shape.go
index ee0a6940d..31d2da501 100644
--- a/cmd/backport-batch/shape.go
+++ b/cmd/backport-batch/shape.go
@@ -121,9 +121,9 @@ func commitMerge(base, tip, message string) (string, error) {
if err != nil {
return "", err
}
- defer os.Remove(f.Name())
+ defer func() { _ = os.Remove(f.Name()) }()
if _, err := f.WriteString(message); err != nil {
- f.Close()
+ _ = f.Close()
return "", err
}
if err := f.Close(); err != nil {
From 939a1ec44b940566cc1267c5b43ffee4f6d32007 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Fri, 13 Nov 2026 00:23:41 +0300
Subject: [PATCH 106/114] refactor: read managed fields through FieldsV1
accessors
Direct access to FieldsV1.Raw is deprecated; GetRawBytes returns the
same bytes and is nil-safe, and NewFieldsV1 builds the test fixtures.
The write-options test still covers a raw-only orphanDependents delete
option, now decoded from its wire form, and asserts that the converted
options keep it.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
internal/controller/fluxplunger/flux_plunger.go | 5 +++--
.../controller/fluxplunger/flux_plunger_test.go | 4 ++--
pkg/registry/write_options_test.go | 16 +++++++++++++---
3 files changed, 18 insertions(+), 7 deletions(-)
diff --git a/internal/controller/fluxplunger/flux_plunger.go b/internal/controller/fluxplunger/flux_plunger.go
index d53ec3eef..22775837b 100644
--- a/internal/controller/fluxplunger/flux_plunger.go
+++ b/internal/controller/fluxplunger/flux_plunger.go
@@ -486,11 +486,12 @@ func ownsSuspensionManagedFields(hr *helmv2.HelmRelease) bool {
// fieldsManageSuspend reports whether a managed-fields set includes spec.suspend,
// encoded by the apiserver as {"f:spec":{"f:suspend":{}}}.
func fieldsManageSuspend(fs *metav1.FieldsV1) bool {
- if fs == nil || len(fs.Raw) == 0 {
+ raw := fs.GetRawBytes()
+ if len(raw) == 0 {
return false
}
var top map[string]json.RawMessage
- if err := json.Unmarshal(fs.Raw, &top); err != nil {
+ if err := json.Unmarshal(raw, &top); err != nil {
return false
}
specRaw, ok := top["f:spec"]
diff --git a/internal/controller/fluxplunger/flux_plunger_test.go b/internal/controller/fluxplunger/flux_plunger_test.go
index 365250621..fc1f8adad 100644
--- a/internal/controller/fluxplunger/flux_plunger_test.go
+++ b/internal/controller/fluxplunger/flux_plunger_test.go
@@ -83,7 +83,7 @@ func suspendManagedFields(manager string, op metav1.ManagedFieldsOperationType)
Operation: op,
APIVersion: helmv2.GroupVersion.String(),
FieldsType: "FieldsV1",
- FieldsV1: &metav1.FieldsV1{Raw: []byte(`{"f:spec":{"f:suspend":{}}}`)},
+ FieldsV1: metav1.NewFieldsV1(`{"f:spec":{"f:suspend":{}}}`),
}}
}
@@ -286,7 +286,7 @@ func TestOwnsSuspensionManagedFields(t *testing.T) {
{"no managed fields", nil, false},
{"plunger apply of other fields", []metav1.ManagedFieldsEntry{{
Manager: suspendFieldManager, Operation: metav1.ManagedFieldsOperationApply,
- FieldsType: "FieldsV1", FieldsV1: &metav1.FieldsV1{Raw: []byte(`{"f:spec":{"f:chart":{}}}`)},
+ FieldsType: "FieldsV1", FieldsV1: metav1.NewFieldsV1(`{"f:spec":{"f:chart":{}}}`),
}}, false},
}
for _, tc := range cases {
diff --git a/pkg/registry/write_options_test.go b/pkg/registry/write_options_test.go
index 488d7a51a..8d6b375f9 100644
--- a/pkg/registry/write_options_test.go
+++ b/pkg/registry/write_options_test.go
@@ -17,7 +17,9 @@ limitations under the License.
package registry
import (
+ "encoding/json"
"reflect"
+ "strings"
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -26,7 +28,6 @@ import (
func TestClientWriteOptionsPreserveAPISemantics(t *testing.T) {
grace := int64(7)
- orphan := true
propagation := metav1.DeletePropagationForeground
uid := types.UID("expected-uid")
rv := "42"
@@ -74,11 +75,20 @@ func TestClientWriteOptionsPreserveAPISemantics(t *testing.T) {
t.Fatalf("conversion mutated API options: create=%#v update=%#v delete=%#v", create, update, deleteOptions)
}
- orphanDelete := &metav1.DeleteOptions{OrphanDependents: &orphan, DryRun: []string{metav1.DryRunAll}}
+ // orphanDependents is deprecated in Go but still accepted on the wire,
+ // which is how a raw-only option reaches the registry.
+ orphanDelete := &metav1.DeleteOptions{}
+ if err := json.Unmarshal([]byte(`{"orphanDependents":true,"dryRun":["All"]}`), orphanDelete); err != nil {
+ t.Fatal(err)
+ }
orphanExpected := orphanDelete.DeepCopy()
- if got := ClientDeleteOptions(orphanDelete).AsDeleteOptions(); !reflect.DeepEqual(got, orphanExpected) {
+ got := ClientDeleteOptions(orphanDelete).AsDeleteOptions()
+ if !reflect.DeepEqual(got, orphanExpected) {
t.Fatalf("raw-only delete option lost: got %#v, want %#v", got, orphanExpected)
}
+ if wire, err := json.Marshal(got); err != nil || !strings.Contains(string(wire), `"orphanDependents":true`) {
+ t.Fatalf("converted delete options = %s (%v), want orphanDependents kept", wire, err)
+ }
if !reflect.DeepEqual(orphanDelete, orphanExpected) {
t.Fatalf("raw-only delete conversion mutated input: got %#v, want %#v", orphanDelete, orphanExpected)
}
From 648928f27299118af5d2d1ee1b559cce007e3a18 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Fri, 13 Nov 2026 00:23:43 +0300
Subject: [PATCH 107/114] refactor(workloadmonitor): use k8s.io/utils/ptr
instead of pointer
The pointer package is deprecated; its Bool and BoolPtr are aliases of
ptr.To, so the values written are unchanged.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
internal/controller/workloadmonitor_controller.go | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go
index b8b0bac13..120ba2533 100644
--- a/internal/controller/workloadmonitor_controller.go
+++ b/internal/controller/workloadmonitor_controller.go
@@ -28,7 +28,7 @@ import (
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/retry"
"k8s.io/client-go/util/workqueue"
- "k8s.io/utils/pointer"
+ "k8s.io/utils/ptr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -423,8 +423,8 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) {
Name: monitor.GetName(),
UID: monitor.GetUID(),
// Set Controller to false to avoid conflict as multiple controllers are not allowed
- Controller: pointer.BoolPtr(false),
- BlockOwnerDeletion: pointer.BoolPtr(true),
+ Controller: ptr.To(false),
+ BlockOwnerDeletion: ptr.To(true),
}
owners = append(owners, newOwnerRef)
}
@@ -1174,9 +1174,9 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
// Default to operational = true, but check MinReplicas if set.
// Use fresh.Spec to avoid making decisions based on a stale cached copy
// when the spec was updated between the initial read and this retry.
- fresh.Status.Operational = pointer.Bool(true)
+ fresh.Status.Operational = ptr.To(true)
if fresh.Spec.MinReplicas != nil && availableReplicas < *fresh.Spec.MinReplicas {
- fresh.Status.Operational = pointer.Bool(false)
+ fresh.Status.Operational = ptr.To(false)
}
// Only the DataVolume check writes Message and Reason, so the stored ones
// are the last DataVolume verdict: kept while no reader is installed yet, so a
@@ -1189,7 +1189,7 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
}
}
if fresh.Status.Message != "" {
- fresh.Status.Operational = pointer.Bool(false)
+ fresh.Status.Operational = ptr.To(false)
}
if equality.Semantic.DeepEqual(stored, &fresh.Status) {
return nil
From 86da6c7a5bbfd11aa5a0f5c61481ddfb475e6c63 Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Fri, 13 Nov 2026 00:26:47 +0300
Subject: [PATCH 108/114] style(lineage-webhook): prefix sentinel error names
with Err
Go names sentinel error variables Err...; the package is internal and
the webhook is their only user.
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin