From 89028b455ef2fbf4772672c2fe6342150efc3200 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 18 Oct 2026 23:16:45 +0300
Subject: [PATCH 001/346] fix(monitoring): route InfoInhibitor to the blackhole
receiver
InfoInhibitor exists to be the source side of an Alertmanager inhibit
rule and, as its own description says, to go to a null receiver. The
platform Alertmanager config has no inhibit rule, and the rule has
carried severity major since it was imported alongside Alerta, so
Alerta shows it one level below critical. Alerta's Telegram and Slack
plugins forward it by default, and a notify_email label inherited from
the info alert gets it emailed too.
It fires whenever a rule labelled severity="info" is active with no
warning or critical alert in the same namespace and cluster. The
platform's own rules no longer use info, but rules it does not write
still do. CDI generates CDIStorageProfilesIncomplete, KubeVirt
generates KubeVirtVMGuestMemoryAvailableLow, and operators add their
own. Each of them turns into a major InfoInhibitor in Alerta.
none is the severity upstream kube-prometheus gives this rule, and the
platform config already sends severity="none" to the blackhole ahead
of every route that delivers. The severity contract refuses none
because such an alert is never delivered, so InfoInhibitor is named as
the one alert for which that is the point. Tests hold both ends, so
neither the label nor the route order can drift back to delivering it.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
hack/alert-severity-contract.bats | 89 +++++++++++++++++++
.../alerts/general.rules.yaml | 2 +-
.../tests/alertmanager_blackhole_test.yaml | 45 ++++++++++
3 files changed, 135 insertions(+), 1 deletion(-)
create mode 100644 packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
diff --git a/hack/alert-severity-contract.bats b/hack/alert-severity-contract.bats
index e1315eb96..44f197000 100644
--- a/hack/alert-severity-contract.bats
+++ b/hack/alert-severity-contract.bats
@@ -9,6 +9,10 @@ load test_helper
# scope: they are re-vendored by `make update`, so a bad severity there is
# fixed upstream or in patches/, not by editing the label.
ALERT_SEVERITY_ALLOWED=" security critical major minor warning indeterminate informational normal ok cleared debug trace unknown "
+# The alerts that exist only to be dropped, the one place `none` is right.
+# InfoInhibitor is the source side of an Alertmanager inhibit rule, which the
+# platform config does not have, so delivering it at any severity is noise.
+ALERT_SEVERITY_BLACKHOLED=" InfoInhibitor "
helm_actions_stripped() {
awk '
@@ -104,6 +108,14 @@ alert_severity_contract() {
break
fi
done
+ if [ "$severity" = "none" ]; then
+ for blackholed in $ALERT_SEVERITY_BLACKHOLED; do
+ if [ "$blackholed" = "$alert" ]; then
+ accepted=1
+ break
+ fi
+ done
+ fi
if [ "$accepted" -eq 0 ]; then
echo "$file: $alert -> '$severity'" >&2
bad=$((bad + 1))
@@ -128,6 +140,83 @@ alert_severity_contract() {
alert_severity_contract
}
+@test "InfoInhibitor is labelled for the blackhole route" {
+ rows=$(mktemp) || return 1
+ if ! alert_severity_rows > "$rows"; then
+ rm -f "$rows"
+ return 1
+ fi
+ tab=$(printf '\t')
+ found=$(grep -c "${tab}InfoInhibitor${tab}" "$rows" || true)
+ delivered=$(grep "${tab}InfoInhibitor${tab}" "$rows" | grep -vc "${tab}none\$" || true)
+ rm -f "$rows"
+ [ "$found" -gt 0 ]
+ [ "$delivered" -eq 0 ]
+}
+
+@test "severity none passes only on an alert that exists to be dropped" {
+ fixture=$(mktemp -d)
+ mkdir -p "$fixture/packages/system/example/alerts"
+ printf '%s\n' \
+ 'apiVersion: operator.victoriametrics.com/v1beta1' \
+ 'kind: VMRule' \
+ 'spec:' \
+ ' groups:' \
+ ' - name: general' \
+ ' rules:' \
+ ' - alert: InfoInhibitor' \
+ ' labels:' \
+ ' severity: none' \
+ > "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
+ git -C "$fixture" init -q
+ git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
+
+ inhibitor_succeeded=0
+ ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || inhibitor_succeeded=$?
+
+ printf '%s\n' \
+ 'apiVersion: operator.victoriametrics.com/v1beta1' \
+ 'kind: VMRule' \
+ 'spec:' \
+ ' groups:' \
+ ' - name: silent' \
+ ' rules:' \
+ ' - alert: SilentAlert' \
+ ' labels:' \
+ ' severity: none' \
+ > "$fixture/packages/system/example/alerts/01-silent.yaml"
+ git -C "$fixture" add packages/system/example/alerts/01-silent.yaml
+
+ silent_succeeded=0
+ ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || silent_succeeded=$?
+ rm -rf "$fixture"
+ [ "$inhibitor_succeeded" -eq 0 ]
+ [ "$silent_succeeded" -ne 0 ]
+}
+
+@test "an alert that exists to be dropped still fails on a refused severity" {
+ fixture=$(mktemp -d)
+ mkdir -p "$fixture/packages/system/example/alerts"
+ printf '%s\n' \
+ 'apiVersion: operator.victoriametrics.com/v1beta1' \
+ 'kind: VMRule' \
+ 'spec:' \
+ ' groups:' \
+ ' - name: general' \
+ ' rules:' \
+ ' - alert: InfoInhibitor' \
+ ' labels:' \
+ ' severity: info' \
+ > "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
+ git -C "$fixture" init -q
+ git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
+
+ contract_succeeded=0
+ ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$?
+ rm -rf "$fixture"
+ [ "$contract_succeeded" -ne 0 ]
+}
+
@test "a parser failure cannot leave the contract green" {
fixture=$(mktemp -d)
mkdir -p "$fixture/packages/system/example/alerts"
diff --git a/packages/system/monitoring-agents/alerts/general.rules.yaml b/packages/system/monitoring-agents/alerts/general.rules.yaml
index 924870c40..45dd6100f 100644
--- a/packages/system/monitoring-agents/alerts/general.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/general.rules.yaml
@@ -52,6 +52,6 @@ spec:
!= "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} ==
1
labels:
- severity: major
+ severity: none
exported_instance: global
service: general.rules
diff --git a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
new file mode 100644
index 000000000..e987bd405
--- /dev/null
+++ b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
@@ -0,0 +1,45 @@
+suite: Alertmanager blackhole route
+templates:
+ - templates/alerta/alerta.yaml
+
+release:
+ name: monitoring
+ namespace: tenant-root
+
+set:
+ _cluster:
+ root-host: example.org
+ _namespace:
+ host: tenant-root.example.org
+ ingress: tenant-root
+
+# Rules labelled severity="none", InfoInhibitor among them, exist to be
+# dropped, and Alerta answers that severity with a 500. Only the heartbeat
+# route may come ahead of the blackhole, so no delivering route claims them.
+tests:
+ - it: drops severity none ahead of every delivering route
+ asserts:
+ - documentSelector:
+ path: metadata.labels.app
+ value: alertmanager
+ matchRegex:
+ path: stringData["alertmanager.yaml"]
+ pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'
+
+ - it: keeps severity none out of email even when its alert name is listed
+ set:
+ alerta:
+ alerts:
+ email:
+ to: "oncall@example.org"
+ from: "alerts@example.org"
+ smarthost: "smtp.example.org:465"
+ alertnames:
+ - InfoInhibitor
+ asserts:
+ - documentSelector:
+ path: metadata.labels.app
+ value: alertmanager
+ matchRegex:
+ path: stringData["alertmanager.yaml"]
+ pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'
From 8d82348e85fb9588ad213d763ece81f7b88e84a8 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Mon, 19 Oct 2026 16:04:58 +0300
Subject: [PATCH 002/346] fix(vm-disk): refuse a disk smaller than the golden
image it clones
CDI checks a clone against its source before it creates the claim. When
the golden image's PVC is in Block mode and requests more than the disk,
it records CloneValidationFailed and leaves the DataVolume pending for
good. The DataVolume spec is immutable, so raising storage on the VMDisk
afterwards changes nothing, and the resize hook finds no claim to patch.
Every image in vm-default-images is 20Gi on replicated, where CDI picks
Block mode for the LINSTOR driver, while the chart defaults storage to
5Gi, so a disk created from an image at the default size never comes up.
The chart now refuses such a disk at render time and names both sizes,
so the user sets storage to the golden's size before any DataVolume
exists. It compares what validateClone compares in CDI v1.66.1, the
golden PVC's request against the disk's request raised to the target
StorageProfile's minimumSupportedPvcSize. A Filesystem golden is not
compared, because CDI sizes that clone from the image inside it or with
a filesystem overhead, and neither is a golden whose PVC does not exist
yet. An existing disk is exempt, as for the other golden checks.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
packages/apps/vm-disk/templates/dv.yaml | 20 ++
.../vm-disk/tests/dv_golden_size_test.yaml | 252 ++++++++++++++++++
2 files changed, 272 insertions(+)
create mode 100644 packages/apps/vm-disk/tests/dv_golden_size_test.yaml
diff --git a/packages/apps/vm-disk/templates/dv.yaml b/packages/apps/vm-disk/templates/dv.yaml
index 09831fc79..9bec73f5f 100644
--- a/packages/apps/vm-disk/templates/dv.yaml
+++ b/packages/apps/vm-disk/templates/dv.yaml
@@ -146,6 +146,26 @@
{{- fail (printf "source.image %q: golden image %q lives in StorageClass %q, and a clone into StorageClass %q of the same provisioner gets no host-assisted fallback from CDI. LINSTOR creates the clone next to the source data, and StorageClass %q does not allow remote volume access from every node, so the clone may never bind. Set storageClass to %q, or ask the platform administrator to set cloneStrategyOverride to copy in the kubevirt-cdi package." .Values.source.image.name $desiredSource.pvc.name $goldenSC $targetSC $targetSC $goldenSC) }}
{{- end }}
{{- end }}
+ {{- /* CDI v1.66.1 refuses to clone a Block source PVC into a smaller
+ request before it creates the claim (validateClone in
+ pkg/controller/datavolume/pvc-clone-controller.go), and the
+ DataVolume then waits forever. It raises the request to the target
+ StorageProfile's minimumSupportedPvcSize before comparing. A
+ Filesystem source is not compared, as CDI sizes that clone from
+ the image inside it or with a filesystem overhead the chart does
+ not reproduce. */}}
+ {{- $goldenSize := dig "spec" "resources" "requests" "storage" "" ($goldenPVC | default dict) | toString }}
+ {{- if and .Values.storage $goldenSize (eq (dig "spec" "volumeMode" "" ($goldenPVC | default dict) | toString) "Block") }}
+ {{- $goldenBytes := include "cozy-lib.resources.toFloat" $goldenSize | float64 }}
+ {{- $diskBytes := include "cozy-lib.resources.toFloat" (.Values.storage | toString) | float64 }}
+ {{- if and (lt $diskBytes $goldenBytes) $targetSC }}
+ {{- $minimum := dig "metadata" "annotations" "cdi.kubevirt.io/minimumSupportedPvcSize" "" (lookup "cdi.kubevirt.io/v1beta1" "StorageProfile" "" $targetSC | default dict) | toString }}
+ {{- $diskBytes = maxf $diskBytes (include "cozy-lib.resources.toFloat" $minimum | float64) }}
+ {{- end }}
+ {{- if lt $diskBytes $goldenBytes }}
+ {{- fail (printf "storage %s is smaller than golden image %q (%s), so CDI refuses to clone it and the disk would stay pending forever. Set storage to %s or more." (.Values.storage | toString) $desiredSource.pvc.name $goldenSize $goldenSize) }}
+ {{- end }}
+ {{- end }}
{{- end }}
{{- else if hasKey .Values.source "http" }}
{{- $desiredSource = dict "http" (dict "url" (required "A valid .Values.source.http.url entry required!" .Values.source.http.url)) }}
diff --git a/packages/apps/vm-disk/tests/dv_golden_size_test.yaml b/packages/apps/vm-disk/tests/dv_golden_size_test.yaml
new file mode 100644
index 000000000..6a68874c7
--- /dev/null
+++ b/packages/apps/vm-disk/tests/dv_golden_size_test.yaml
@@ -0,0 +1,252 @@
+suite: vm-disk refuses a disk smaller than the golden image it clones
+release:
+ name: test-vm-disk
+ namespace: tenant-test
+templates:
+ - templates/dv.yaml
+tests:
+ - it: refuses a new disk at the chart default size from a larger Block golden
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: &scheme
+ "cdi.kubevirt.io/v1beta1/DataVolume":
+ gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: datavolumes}
+ namespaced: true
+ "cdi.kubevirt.io/v1beta1/StorageProfile":
+ gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: storageprofiles}
+ namespaced: false
+ "v1/Namespace":
+ gvr: {group: "", version: v1, resource: namespaces}
+ namespaced: false
+ "v1/PersistentVolumeClaim":
+ gvr: {group: "", version: v1, resource: persistentvolumeclaims}
+ namespaced: true
+ "storage.k8s.io/v1/StorageClass":
+ gvr: {group: storage.k8s.io, version: v1, resource: storageclasses}
+ namespaced: false
+ objects:
+ - &namespace
+ kind: Namespace
+ apiVersion: v1
+ metadata: {name: tenant-test}
+ - &goldenPVC
+ kind: PersistentVolumeClaim
+ apiVersion: v1
+ metadata: {name: vm-default-images-ubuntu, namespace: cozy-public}
+ spec:
+ storageClassName: replicated
+ volumeMode: Block
+ resources: {requests: {storage: 20Gi}}
+ asserts:
+ - failedTemplate:
+ errorPattern: 'storage 5Gi is smaller than golden image "vm-default-images-ubuntu" \(20Gi\), so CDI refuses to clone it and the disk would stay pending forever\. Set storage to 20Gi or more\.'
+
+ - it: clones into a disk the size of the golden
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 20Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - equal:
+ path: spec.storage.resources.requests.storage
+ value: 20Gi
+
+ - it: clones into a disk larger than the golden
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 25Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - equal:
+ path: spec.storage.resources.requests.storage
+ value: 25Gi
+
+ - it: compares bytes, so 21G is smaller than a 20Gi golden
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 21G
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - failedTemplate:
+ errorPattern: 'storage 21G is smaller than golden image "vm-default-images-ubuntu" \(20Gi\)'
+
+ - it: compares bytes, so 20480Mi is the size of a 20Gi golden
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 20480Mi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - equal:
+ path: spec.storage.resources.requests.storage
+ value: 20480Mi
+
+ - it: clones when the target StorageProfile's minimum PVC size covers the golden
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ - kind: StorageProfile
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata:
+ name: replicated
+ annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 20Gi}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: refuses when the target StorageProfile's minimum PVC size is still below the golden
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ - kind: StorageProfile
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata:
+ name: replicated
+ annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 10Gi}
+ asserts:
+ - failedTemplate:
+ errorPattern: 'storage 5Gi is smaller than golden image "vm-default-images-ubuntu" \(20Gi\)'
+
+ - it: reads the minimum PVC size of the default class for an empty storageClass
+ set:
+ source: {image: {name: ubuntu}}
+ storageClass: ""
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ - kind: StorageClass
+ apiVersion: storage.k8s.io/v1
+ metadata:
+ name: replicated
+ annotations: {storageclass.kubernetes.io/is-default-class: "true"}
+ provisioner: linstor.csi.linbit.com
+ - kind: StorageProfile
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata:
+ name: replicated
+ annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 20Gi}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: does not compare sizes for a Filesystem golden
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - kind: PersistentVolumeClaim
+ apiVersion: v1
+ metadata: {name: vm-default-images-ubuntu, namespace: cozy-public}
+ spec:
+ storageClassName: replicated
+ volumeMode: Filesystem
+ resources: {requests: {storage: 20Gi}}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: renders without a cluster to look the golden up in
+ set:
+ source: {image: {name: ubuntu}}
+ asserts:
+ - equal:
+ path: spec.storage.resources.requests.storage
+ value: 5Gi
+
+ - it: clones a golden whose PVC does not exist yet
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - kind: DataVolume
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata: {name: vm-default-images-ubuntu, namespace: cozy-public}
+ status: {phase: ImportScheduled}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: leaves a disk without a size to CDI, which takes the golden's
+ set:
+ source: {image: {name: ubuntu}}
+ storage: null
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: leaves an existing disk alone
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 25Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ - kind: DataVolume
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata:
+ name: test-vm-disk
+ namespace: tenant-test
+ annotations:
+ vm-disk.cozystack.io/source: '{"image":{"name":"ubuntu"}}'
+ vm-disk.cozystack.io/storage-class: replicated
+ spec:
+ source: {pvc: {name: vm-default-images-ubuntu, namespace: cozy-public}}
+ storage: {resources: {requests: {storage: 10Gi}}, storageClassName: replicated}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: does not compare sizes for an http source
+ set:
+ source: {http: {url: "https://example.com/disk.img"}}
+ storage: 1Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: does not compare sizes for an upload source
+ set:
+ source: {upload: {}}
+ storage: 1Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - notFailedTemplate: {}
From f1f30b3071254e3640292b4eb483b04f5d40d77a Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Tue, 20 Oct 2026 12:05:51 +0300
Subject: [PATCH 003/346] fix(etcd): require an even spread of members over
nodes by default
The default spread of the etcd members was ScheduleAnyway, which only
adds to the score of a node, and other scores can outweigh it. A new
etcd on a three-node cluster got its first two members on one node.
On a node-local StorageClass a member cannot leave its node, so losing
that node takes two of the three members and the quorum.
DoNotSchedule with maxSkew 1 still lets an etcd with more members than
nodes start, because the skew is counted between the nodes that exist.
One node takes all three members and two nodes take two and one.
nodeTaintsPolicy is Ignore, so a cordoned node still counts and a
member waits for it instead of landing next to another member.
The operator copies the constraints into an EtcdMember when it creates
it and builds the Pod from that copy, so members that exist keep the
old setting and only members created later get the new one.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../extra/etcd/templates/etcd-cluster.yaml | 3 +-
.../etcd/tests/topology-spread_test.yaml | 46 ++++++++++++++-----
2 files changed, 37 insertions(+), 12 deletions(-)
diff --git a/packages/extra/etcd/templates/etcd-cluster.yaml b/packages/extra/etcd/templates/etcd-cluster.yaml
index 2824ce371..67dd34bd9 100644
--- a/packages/extra/etcd/templates/etcd-cluster.yaml
+++ b/packages/extra/etcd/templates/etcd-cluster.yaml
@@ -69,7 +69,8 @@ spec:
topologySpreadConstraints:
- maxSkew: 1
topologyKey: "kubernetes.io/hostname"
- whenUnsatisfiable: ScheduleAnyway
+ whenUnsatisfiable: DoNotSchedule
+ nodeTaintsPolicy: Ignore
labelSelector:
matchLabels:
app.kubernetes.io/instance: etcd
diff --git a/packages/extra/etcd/tests/topology-spread_test.yaml b/packages/extra/etcd/tests/topology-spread_test.yaml
index 4b96fc310..11bb92243 100644
--- a/packages/extra/etcd/tests/topology-spread_test.yaml
+++ b/packages/extra/etcd/tests/topology-spread_test.yaml
@@ -8,7 +8,7 @@ templates:
- templates/etcd-cluster.yaml
tests:
- - it: uses a soft per-host spread by default
+ - it: uses a strict per-host spread by default
set:
_cluster: {}
documentSelector:
@@ -16,17 +16,41 @@ tests:
value: EtcdCluster
asserts:
- equal:
- path: spec.topologySpreadConstraints[0].maxSkew
- value: 1
+ path: spec.topologySpreadConstraints[0]
+ value:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: DoNotSchedule
+ nodeTaintsPolicy: Ignore
+ labelSelector:
+ matchLabels:
+ app.kubernetes.io/instance: etcd
+ - lengthEqual:
+ path: spec.topologySpreadConstraints
+ count: 1
+
+ - it: renders a soft cluster-wide spread policy as written
+ set:
+ _cluster:
+ scheduling:
+ globalAppTopologySpreadConstraints: |
+ topologySpreadConstraints:
+ - maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+ documentSelector:
+ path: kind
+ value: EtcdCluster
+ asserts:
- equal:
- path: spec.topologySpreadConstraints[0].topologyKey
- value: kubernetes.io/hostname
- - equal:
- path: spec.topologySpreadConstraints[0].whenUnsatisfiable
- value: ScheduleAnyway
- - equal:
- path: spec.topologySpreadConstraints[0].labelSelector.matchLabels["app.kubernetes.io/instance"]
- value: etcd
+ path: spec.topologySpreadConstraints[0]
+ value:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+ labelSelector:
+ matchLabels:
+ app.kubernetes.io/instance: etcd
- lengthEqual:
path: spec.topologySpreadConstraints
count: 1
From 9f6fdd77c6ca964e0adcc387e50993487f2e594e Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sat, 24 Oct 2026 15:52:37 +0300
Subject: [PATCH 004/346] docs(etcd): update the affinity notes for the hard
member spread
The affinity description and the comment above the affinity block
explained the soft default by a member that would stay Pending on its
node-local volume. Now that the spread is DoNotSchedule by default, the
spread keeps such a member Pending as well, once its node holds another
member and some other node holds none. Both texts now explain the soft
default by what is left, that a required rule forbids two members on
one node even when every node has one, and state the node-local case
for both rules.
The description also says that the operator copies affinity into each
EtcdMember when it creates it, so a recreated Pod keeps the rule its
member was created with. README, values.schema.json and the
ApplicationDefinition are regenerated from it.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
packages/extra/etcd/README.md | 20 +++++++++----------
.../extra/etcd/templates/etcd-cluster.yaml | 14 ++++++++-----
packages/extra/etcd/values.schema.json | 2 +-
packages/extra/etcd/values.yaml | 2 +-
packages/system/etcd-rd/cozyrds/etcd.yaml | 2 +-
5 files changed, 22 insertions(+), 18 deletions(-)
diff --git a/packages/extra/etcd/README.md b/packages/extra/etcd/README.md
index e96c63832..91341ff32 100644
--- a/packages/extra/etcd/README.md
+++ b/packages/extra/etcd/README.md
@@ -34,14 +34,14 @@ coordinates).
### Common parameters
-| Name | Description | Type | Value |
-| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------- |
-| `version` | etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image. | `string` | `3.6.11` |
-| `size` | Persistent Volume size. | `quantity` | `4Gi` |
-| `storageClass` | StorageClass used to store the data. | `string` | `""` |
-| `replicas` | Number of etcd replicas. | `int` | `3` |
-| `resources` | Resource configuration for etcd. | `object` | `{}` |
-| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` |
-| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` |
-| `affinity` | Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively. | `object` | `{}` |
+| Name | Description | Type | Value |
+| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------- |
+| `version` | etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image. | `string` | `3.6.11` |
+| `size` | Persistent Volume size. | `quantity` | `4Gi` |
+| `storageClass` | StorageClass used to store the data. | `string` | `""` |
+| `replicas` | Number of etcd replicas. | `int` | `3` |
+| `resources` | Resource configuration for etcd. | `object` | `{}` |
+| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` |
+| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` |
+| `affinity` | Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones. | `object` | `{}` |
diff --git a/packages/extra/etcd/templates/etcd-cluster.yaml b/packages/extra/etcd/templates/etcd-cluster.yaml
index 67dd34bd9..3676f53cb 100644
--- a/packages/extra/etcd/templates/etcd-cluster.yaml
+++ b/packages/extra/etcd/templates/etcd-cluster.yaml
@@ -79,11 +79,15 @@ spec:
affinity:
{{- toYaml . | nindent 4 }}
{{- else }}
- # Soft anti-affinity: spread etcd members across nodes on a best-effort
- # basis so a single node failure is unlikely to take out more than one
- # member. Preferred (not required) because a hard rule would leave a member
- # Pending when replicas exceed schedulable nodes, or when node-local storage
- # pins a member to a node. Override via .Values.affinity for a hard rule.
+ # Soft anti-affinity, a preference on top of the spread constraint above,
+ # which by default is the hard rule that keeps two members off one node
+ # while another node has none. Preferred, not required, because a required
+ # rule also forbids two members on one node when every node already has
+ # one, so an etcd with more members than schedulable nodes could not start
+ # all of them. With node-local storage a recreated member Pod can only
+ # return to its own node, and the default spread keeps it Pending there
+ # while that node holds another member and some other node holds none.
+ # Override via .Values.affinity for a hard rule.
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
diff --git a/packages/extra/etcd/values.schema.json b/packages/extra/etcd/values.schema.json
index b438ca9d7..c33219f4f 100644
--- a/packages/extra/etcd/values.schema.json
+++ b/packages/extra/etcd/values.schema.json
@@ -67,7 +67,7 @@
}
},
"affinity": {
- "description": "Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.",
+ "description": "Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.",
"type": "object",
"default": {},
"x-kubernetes-preserve-unknown-fields": true
diff --git a/packages/extra/etcd/values.yaml b/packages/extra/etcd/values.yaml
index 61b994d7c..c5be94909 100644
--- a/packages/extra/etcd/values.yaml
+++ b/packages/extra/etcd/values.yaml
@@ -23,5 +23,5 @@ resources:
cpu: 1000m
memory: 512Mi
-## @param {object} [affinity] - Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.
+## @param {object} [affinity] - Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.
affinity: {}
diff --git a/packages/system/etcd-rd/cozyrds/etcd.yaml b/packages/system/etcd-rd/cozyrds/etcd.yaml
index 90ed8a80a..1ce88e8ae 100644
--- a/packages/system/etcd-rd/cozyrds/etcd.yaml
+++ b/packages/system/etcd-rd/cozyrds/etcd.yaml
@@ -8,7 +8,7 @@ spec:
plural: etcds
singular: etcd
openAPISchema: |-
- {"title":"Chart Values","type":"object","properties":{"version":{"description":"etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image.","type":"string","default":"3.6.11"},"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"affinity":{"description":"Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}
+ {"title":"Chart Values","type":"object","properties":{"version":{"description":"etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image.","type":"string","default":"3.6.11"},"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"affinity":{"description":"Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}
release:
prefix: ""
labels:
From dc94660a44d2fa09165223723f5f9a5bd794f896 Mon Sep 17 00:00:00 2001
From: Mattia Eleuteri
Date: Sat, 24 Oct 2026 17:05:20 +0200
Subject: [PATCH 005/346] fix(postgres): render the declared databases in a
stable order
keys returns a map's keys in Go's randomized iteration order, so a
release with several databases rendered DEFINED_DBS in a different
order on each upgrade. The init-script Secret, and the checksum the
init Job carries, then changed on upgrades that left the databases
untouched, which makes a no-op upgrade look like a real change in the
release history.
Assisted-by: LLM
Signed-off-by: Mattia Eleuteri
---
.../apps/postgres/templates/init-script.yaml | 2 +-
.../tests/init_script_deterministic_test.yaml | 37 +++++++++++++++++++
2 files changed, 38 insertions(+), 1 deletion(-)
create mode 100644 packages/apps/postgres/tests/init_script_deterministic_test.yaml
diff --git a/packages/apps/postgres/templates/init-script.yaml b/packages/apps/postgres/templates/init-script.yaml
index e78279a2d..395267ff0 100644
--- a/packages/apps/postgres/templates/init-script.yaml
+++ b/packages/apps/postgres/templates/init-script.yaml
@@ -337,7 +337,7 @@ stringData:
RESTORE_MARK={{ with .Values.bootstrap.newServerName }}$(echo {{ b64enc . }} | base64 -d){{ end }}
ADOPTED_MARK=$(psql -v ON_ERROR_STOP=1 -t -A --dbname postgres -c "SELECT current_setting('cozystack.adopted_restore', true)")
MANAGED_DBS=$(psql -v ON_ERROR_STOP=1 -t -A -c "SELECT datname FROM pg_database d JOIN pg_shdescription s ON d.oid = s.objoid WHERE s.description = 'database managed by helm'")
- DEFINED_DBS="{{ join " " (keys .Values.databases) }}"
+ DEFINED_DBS="{{ join " " (keys .Values.databases | sortAlpha) }}"
DELETE_DBS=$(for db in $MANAGED_DBS; do case " $DEFINED_DBS " in *" $db "*) :;; *) echo $db;; esac; done)
if [ -n "$RESTORE_MARK" ] && [ "$ADOPTED_MARK" != "$RESTORE_MARK" ]; then
diff --git a/packages/apps/postgres/tests/init_script_deterministic_test.yaml b/packages/apps/postgres/tests/init_script_deterministic_test.yaml
new file mode 100644
index 000000000..25d0facdd
--- /dev/null
+++ b/packages/apps/postgres/tests/init_script_deterministic_test.yaml
@@ -0,0 +1,37 @@
+suite: init-script renders the same for the same values
+
+# keys returns a map's keys in Go's randomized iteration order, so a release with
+# several databases rendered DEFINED_DBS in a different order on each upgrade. The
+# init-script Secret then changed on upgrades that changed nothing in the
+# databases, and so did the checksum the init Job carries.
+
+templates:
+ - templates/init-script.yaml
+
+release:
+ name: pg-test
+ namespace: tenant-test
+
+set:
+ _cluster:
+ scheduling: {}
+
+tests:
+ - it: lists the declared databases in a stable order
+ set:
+ databases:
+ hotel: {}
+ alpha: {}
+ golf: {}
+ charlie: {}
+ foxtrot: {}
+ bravo: {}
+ echo: {}
+ delta: {}
+ documentSelector:
+ path: metadata.name
+ value: pg-test-init-script
+ asserts:
+ - matchRegex:
+ path: stringData["init.sh"]
+ pattern: 'DEFINED_DBS="alpha bravo charlie delta echo foxtrot golf hotel"'
From 00c11c7a745db9b57d419631c3c88b7ac8c274cb Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:50 +0300
Subject: [PATCH 006/346] test(monitoring): check the listed alert still
reaches email after the blackhole
The case lists InfoInhibitor in email.alertnames but only asserted the
route order up to the blackhole, so it would still pass if the email
route for a listed name stopped being rendered.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../monitoring/tests/alertmanager_blackhole_test.yaml | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
index e987bd405..d4e366180 100644
--- a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
+++ b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
@@ -43,3 +43,9 @@ tests:
matchRegex:
path: stringData["alertmanager.yaml"]
pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'
+ - documentSelector:
+ path: metadata.labels.app
+ value: alertmanager
+ matchRegex:
+ path: stringData["alertmanager.yaml"]
+ pattern: '(?s)\n - receiver: blackhole\n matchers:\n - .severity="none".\n.*\n - receiver: email\n group_wait: 1m\n group_interval: 5m\n repeat_interval: .12h.\n matchers:\n - .alertname=~"\^\(InfoInhibitor\)\$".\n'
From 7bda2bdbc07dd941724e5ea12e51e8c808313b1b Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:30:14 +0300
Subject: [PATCH 007/346] fix(monitoring): select the vmalert Service the
operator creates
The VMServiceScrape selected app.kubernetes.io/instance=vmalert, while the
chart names the VMAlert vmalert- and the
operator puts that name into the instance label of its Service. The scrape
matched nothing, so the metrics of vmalert itself were never collected.
The instance stays in the selector so the scrape keeps pointing at the one
VMAlert this release renders.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../templates/vm/vmalert-scrape.yaml | 4 +-
.../monitoring/tests/vmalert_scrape_test.yaml | 38 +++++++++++++++++++
2 files changed, 41 insertions(+), 1 deletion(-)
create mode 100644 packages/system/monitoring/tests/vmalert_scrape_test.yaml
diff --git a/packages/system/monitoring/templates/vm/vmalert-scrape.yaml b/packages/system/monitoring/templates/vm/vmalert-scrape.yaml
index ee406a275..b467d5564 100644
--- a/packages/system/monitoring/templates/vm/vmalert-scrape.yaml
+++ b/packages/system/monitoring/templates/vm/vmalert-scrape.yaml
@@ -1,3 +1,4 @@
+{{- with first .Values.metricsStorages }}
---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
@@ -13,5 +14,6 @@ spec:
selector:
matchLabels:
app.kubernetes.io/component: monitoring
- app.kubernetes.io/instance: vmalert
+ app.kubernetes.io/instance: vmalert-{{ .name }}
app.kubernetes.io/name: vmalert
+{{- end }}
diff --git a/packages/system/monitoring/tests/vmalert_scrape_test.yaml b/packages/system/monitoring/tests/vmalert_scrape_test.yaml
new file mode 100644
index 000000000..1dfb9ec3e
--- /dev/null
+++ b/packages/system/monitoring/tests/vmalert_scrape_test.yaml
@@ -0,0 +1,38 @@
+suite: vmalert metrics are scraped
+templates:
+ - templates/vm/vmalert-scrape.yaml
+
+release:
+ name: monitoring
+ namespace: tenant-root
+
+set:
+ _cluster:
+ root-host: example.org
+ _namespace:
+ host: tenant-root.example.org
+ ingress: tenant-root
+ metricsStorages:
+ - name: shortterm
+ retentionPeriod: "3d"
+ deduplicationInterval: "15s"
+ - name: longterm
+ retentionPeriod: "14d"
+ deduplicationInterval: "5m"
+
+tests:
+ - it: selects the Service of the VMAlert the chart renders
+ asserts:
+ - equal:
+ path: spec.selector.matchLabels
+ value:
+ app.kubernetes.io/component: monitoring
+ app.kubernetes.io/instance: vmalert-shortterm
+ app.kubernetes.io/name: vmalert
+
+ - it: renders nothing without metrics storages, like the VMAlert itself
+ set:
+ metricsStorages: []
+ asserts:
+ - hasDocuments:
+ count: 0
From 34766ff0b50af0801f2c94c1435dd744211bbc23 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 17:06:52 +0300
Subject: [PATCH 008/346] fix(vertical-pod-autoscaler): run two admission
controllers spread over nodes
With one admission controller and failurePolicy Ignore, a pod created
while that controller is being rescheduled is admitted without its VPA
recommendation, and with updateMode Initial it keeps the defaults for
the rest of its life. On a v1.6.5 cluster a vmstorage pod created during
a node drain got the operator defaults, a 1500Mi limit against a 3.4Gi
recommendation, and was OOMKilled weeks later.
A second replica on another node and a PodDisruptionBudget keep one
controller serving through a drain. The controller still deletes and
recreates its webhook configuration on start, so a restart leaves only
that short gap instead of the whole reschedule.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../tests/vpa_test.yaml | 29 +++++++++++++++++++
.../vertical-pod-autoscaler/values.yaml | 12 ++++++++
2 files changed, 41 insertions(+)
diff --git a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
index 6b8c82675..1d2011330 100644
--- a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
+++ b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
@@ -242,3 +242,32 @@ tests:
count: 0
+
+ - it: runs two admission controllers spread over nodes
+ template: charts/vertical-pod-autoscaler/templates/admission-controller/deployment.yaml
+ asserts:
+ - equal:
+ path: spec.replicas
+ value: 2
+ - equal:
+ path: spec.template.spec.topologySpreadConstraints[0].topologyKey
+ value: kubernetes.io/hostname
+ - equal:
+ path: spec.template.spec.topologySpreadConstraints[0].labelSelector.matchLabels
+ value:
+ app.kubernetes.io/name: vertical-pod-autoscaler
+ app.kubernetes.io/instance: vpa
+ app.kubernetes.io/component: admission-controller
+
+ - it: keeps one admission controller up through a voluntary disruption
+ template: charts/vertical-pod-autoscaler/templates/admission-controller/pdb.yaml
+ asserts:
+ - equal:
+ path: spec.minAvailable
+ value: 1
+ - equal:
+ path: spec.selector.matchLabels
+ value:
+ app.kubernetes.io/name: vertical-pod-autoscaler
+ app.kubernetes.io/instance: vpa
+ app.kubernetes.io/component: admission-controller
diff --git a/packages/system/vertical-pod-autoscaler/values.yaml b/packages/system/vertical-pod-autoscaler/values.yaml
index 46678903e..456b99912 100644
--- a/packages/system/vertical-pod-autoscaler/values.yaml
+++ b/packages/system/vertical-pod-autoscaler/values.yaml
@@ -37,6 +37,18 @@ vertical-pod-autoscaler:
prometheus-cadvisor-job-name: cadvisor
admissionController:
+ replicaCount: 2
+ pdb:
+ create: true
+ topologySpreadConstraints:
+ - maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+ labelSelector:
+ matchLabels:
+ app.kubernetes.io/name: vertical-pod-autoscaler
+ app.kubernetes.io/instance: "{{ .Release.Name }}"
+ app.kubernetes.io/component: admission-controller
resources:
limits:
memory: 256Mi
From 61289c721011af0ca4a61eb278ae2664ae7cd8e7 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 17:09:16 +0300
Subject: [PATCH 009/346] fix(etcd): size the defrag job for the client it runs
The hourly defrag pod runs the etcd-defrag client, which only sends
defragment calls to the members, yet it requested 200m CPU and 256Mi
from the tenant quota. On a sandbox tenant whose quota was mostly taken
by its own etcd and SeaweedFS, the job could not create a pod and the
defrag never ran. The client peaked at 5.1 MiB working set over a day
on a v1.6.5 cluster.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
packages/extra/etcd/templates/etcd-defrag.yaml | 8 ++++----
packages/extra/etcd/tests/defrag_test.yaml | 13 +++++++++++++
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/packages/extra/etcd/templates/etcd-defrag.yaml b/packages/extra/etcd/templates/etcd-defrag.yaml
index 52cc63518..c1eefa0db 100644
--- a/packages/extra/etcd/templates/etcd-defrag.yaml
+++ b/packages/extra/etcd/templates/etcd-defrag.yaml
@@ -29,11 +29,11 @@ spec:
drop: ["ALL"]
resources:
requests:
- cpu: 200m
- memory: 256Mi
+ cpu: 10m
+ memory: 32Mi
limits:
- cpu: 500m
- memory: 512Mi
+ cpu: 100m
+ memory: 64Mi
args:
# The v1alpha2 operator names the headless Service after the
# cluster ({{ $.Release.Name }}) and does not use StatefulSet
diff --git a/packages/extra/etcd/tests/defrag_test.yaml b/packages/extra/etcd/tests/defrag_test.yaml
index 8d74e1a6b..b3acc3401 100644
--- a/packages/extra/etcd/tests/defrag_test.yaml
+++ b/packages/extra/etcd/tests/defrag_test.yaml
@@ -41,3 +41,16 @@ tests:
documentSelector:
path: kind
value: EtcdCluster
+
+ - it: asks the tenant quota only for what the defrag client uses
+ template: templates/etcd-defrag.yaml
+ asserts:
+ - equal:
+ path: spec.jobTemplate.spec.template.spec.containers[0].resources
+ value:
+ requests:
+ cpu: 10m
+ memory: 32Mi
+ limits:
+ cpu: 100m
+ memory: 64Mi
From a7ec05f9eb759e0c1025cf4bf95631df338a85c0 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 17:15:04 +0300
Subject: [PATCH 010/346] fix(monitoring-agents): make a silent Watchdog an
alarm in Alerta
Alerta turns the Watchdog into a heartbeat and stores its severity in the
heartbeat attributes. The alerta heartbeats --alert job of the alerta-web
image takes the severity of a stale heartbeat from those attributes ahead
of its HEARTBEAT_SEVERITY default, so with severity ok the HeartbeatFail
it raises is a normal alert and nothing says the alerting pipeline
stopped. Without a timeout label the heartbeat also took a day to go
stale, while Alertmanager refreshes it every minute.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../alerts/general.rules.yaml | 3 +-
.../tests/watchdog_heartbeat_test.yaml | 29 +++++++++++++++++++
2 files changed, 31 insertions(+), 1 deletion(-)
create mode 100644 packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml
diff --git a/packages/system/monitoring-agents/alerts/general.rules.yaml b/packages/system/monitoring-agents/alerts/general.rules.yaml
index 924870c40..0ac83fe72 100644
--- a/packages/system/monitoring-agents/alerts/general.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/general.rules.yaml
@@ -33,7 +33,8 @@ spec:
is working properly.
expr: vector(1)
labels:
- severity: ok
+ severity: major
+ timeout: "300"
exported_instance: global
service: general.rules
event: Heartbeat
diff --git a/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml b/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml
new file mode 100644
index 000000000..fe630a80c
--- /dev/null
+++ b/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml
@@ -0,0 +1,29 @@
+suite: Watchdog heartbeat raises an alarm in Alerta when it stops
+templates:
+ - templates/alerts.yaml
+
+release:
+ name: monitoring-agents
+ namespace: cozy-monitoring
+
+tests:
+ - it: gives the heartbeat a severity Alerta reports as an alarm
+ documentSelector:
+ path: metadata.name
+ value: alerts-general.rules
+ asserts:
+ - equal:
+ path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.severity
+ value: major
+
+ - it: lets the heartbeat go stale within minutes, not a day
+ documentSelector:
+ path: metadata.name
+ value: alerts-general.rules
+ asserts:
+ - equal:
+ path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.timeout
+ value: "300"
+ - equal:
+ path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.event
+ value: Heartbeat
From 70e1e8b1454db65222a0b3c0952650fa69239fdc Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 20:33:46 +0300
Subject: [PATCH 011/346] test(vertical-pod-autoscaler): check maxSkew and
whenUnsatisfiable of the spread
The test only looked at the topology key and the label selector, so a
change to maxSkew or to whenUnsatisfiable went unnoticed.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
index 1d2011330..0ff338b48 100644
--- a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
+++ b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
@@ -252,6 +252,12 @@ tests:
- equal:
path: spec.template.spec.topologySpreadConstraints[0].topologyKey
value: kubernetes.io/hostname
+ - equal:
+ path: spec.template.spec.topologySpreadConstraints[0].maxSkew
+ value: 1
+ - equal:
+ path: spec.template.spec.topologySpreadConstraints[0].whenUnsatisfiable
+ value: ScheduleAnyway
- equal:
path: spec.template.spec.topologySpreadConstraints[0].labelSelector.matchLabels
value:
From c70f83b375caeaf609884420ecc0ca769184bd74 Mon Sep 17 00:00:00 2001
From: Mattia Eleuteri
Date: Mon, 26 Oct 2026 22:11:09 +0100
Subject: [PATCH 012/346] fix(mariadb-operator): derive the cluster name from
the platform domain
The package pinned clusterName to cozy.local, so on a cluster installed
with any other networking.clusterDomain the operator resolved its own
MariaDB pods as ..svc.cozy.local and got NXDOMAIN. Replication
between replicas was never configured, and User, Database and Grant
resources failed to reconcile, with the operator only retrying at a
verbose log level.
clusterName is a vendored subchart value, out of reach of the _cluster
lookup that package templates use, so the platform now carries it under
the subchart key in the shared values Secret, the same route KEDA and
the OpenSearch operator already take. The literal in the package stays
as the default for a render without that Secret.
Assisted-by: LLM
Signed-off-by: Mattia Eleuteri
---
packages/core/platform/templates/apps.yaml | 8 ++++
...ps_mariadb_operator_cluster_name_test.yaml | 33 ++++++++++++++
.../tests/cluster_name_test.yaml | 43 +++++++++++++++++++
packages/system/mariadb-operator/values.yaml | 2 +
4 files changed, 86 insertions(+)
create mode 100644 packages/core/platform/tests/apps_mariadb_operator_cluster_name_test.yaml
create mode 100644 packages/system/mariadb-operator/tests/cluster_name_test.yaml
diff --git a/packages/core/platform/templates/apps.yaml b/packages/core/platform/templates/apps.yaml
index 030cfe8b2..4a0f0ae89 100644
--- a/packages/core/platform/templates/apps.yaml
+++ b/packages/core/platform/templates/apps.yaml
@@ -33,6 +33,14 @@ stringData:
opensearch-operator:
manager:
dnsBase: {{ .Values.networking.clusterDomain | quote }}
+ {{- /*
+ The operator resolves its own MariaDB pods, and the webhook certificate
+ gets its SAN, from CLUSTER_NAME, which the vendored subchart reads from
+ clusterName. Same subchart-key route as above; a package rendered
+ without this Secret falls back to the literal in its values.yaml.
+ */}}
+ mariadb-operator:
+ clusterName: {{ .Values.networking.clusterDomain | quote }}
_cluster:
root-host: {{ $rootHost | quote }}
bundle-name: {{ .Values.bundles.system.variant | quote }}
diff --git a/packages/core/platform/tests/apps_mariadb_operator_cluster_name_test.yaml b/packages/core/platform/tests/apps_mariadb_operator_cluster_name_test.yaml
new file mode 100644
index 000000000..6cd9426ba
--- /dev/null
+++ b/packages/core/platform/tests/apps_mariadb_operator_cluster_name_test.yaml
@@ -0,0 +1,33 @@
+suite: apps.yaml networking.clusterDomain reaches the mariadb-operator cluster name
+
+templates:
+ - templates/apps.yaml
+
+release:
+ name: cozy-platform
+ namespace: cozy-system
+
+# The operator dials ..svc. to set up replication and
+# to reconcile User, Database and Grant, and the vendored subchart fills
+# CLUSTER_NAME from clusterName. A cluster whose domain is not cozy.local got
+# NXDOMAIN on every such lookup while the literal was pinned in the package.
+# Every pattern anchors clusterName to its mariadb-operator parent, so a sibling
+# block carrying the same domain cannot satisfy the assertion on its own.
+
+tests:
+ - it: carries the cozy.local default under the subchart key
+ asserts:
+ - matchRegex:
+ path: stringData["values.yaml"]
+ pattern: '(?m)^mariadb-operator:\n clusterName: "cozy\.local"$'
+
+ - it: tracks a non-default networking.clusterDomain
+ set:
+ networking.clusterDomain: example.test
+ asserts:
+ - matchRegex:
+ path: stringData["values.yaml"]
+ pattern: '(?m)^mariadb-operator:\n clusterName: "example\.test"$'
+ - notMatchRegex:
+ path: stringData["values.yaml"]
+ pattern: '(?m)^mariadb-operator:\n clusterName: "cozy\.local"$'
diff --git a/packages/system/mariadb-operator/tests/cluster_name_test.yaml b/packages/system/mariadb-operator/tests/cluster_name_test.yaml
new file mode 100644
index 000000000..45d762727
--- /dev/null
+++ b/packages/system/mariadb-operator/tests/cluster_name_test.yaml
@@ -0,0 +1,43 @@
+suite: mariadb-operator package resolves its pods in the platform cluster domain
+release:
+ name: mariadb-operator
+ namespace: cozy-mariadb-operator
+# core/platform hands the cluster domain to this package as
+# mariadb-operator.clusterName in the shared values Secret. Guard that the
+# vendored chart still threads that key into what the operator and the webhook
+# certificate use, so a re-vendor that renames it goes red here.
+tests:
+ - it: sets the operator CLUSTER_NAME from the injected domain
+ templates:
+ - charts/mariadb-operator/templates/operator/deployment.yaml
+ set:
+ mariadb-operator.clusterName: example.test
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: CLUSTER_NAME
+ value: example.test
+
+ - it: issues the webhook certificate for the injected domain
+ templates:
+ - charts/mariadb-operator/templates/webhook/certificate.yaml
+ documentSelector:
+ path: kind
+ value: Certificate
+ set:
+ mariadb-operator.clusterName: example.test
+ asserts:
+ - contains:
+ path: spec.dnsNames
+ content: mariadb-operator-webhook.cozy-mariadb-operator.svc.example.test
+
+ - it: keeps cozy.local as the standalone default
+ templates:
+ - charts/mariadb-operator/templates/operator/deployment.yaml
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: CLUSTER_NAME
+ value: cozy.local
diff --git a/packages/system/mariadb-operator/values.yaml b/packages/system/mariadb-operator/values.yaml
index 2728c084e..285975440 100644
--- a/packages/system/mariadb-operator/values.yaml
+++ b/packages/system/mariadb-operator/values.yaml
@@ -1,4 +1,6 @@
mariadb-operator:
+ # Fallback only: core/platform injects networking.clusterDomain into this key
+ # through the shared cozystack-values Secret (see its apps.yaml).
clusterName: cozy.local
metrics:
enabled: true
From 93faa7a8620da8d00a783f40ceb9090ad1e5c265 Mon Sep 17 00:00:00 2001
From: Mattia Eleuteri
Date: Mon, 26 Oct 2026 22:18:50 +0100
Subject: [PATCH 013/346] fix(lineage, kubernetes): resolve CAPK node VMs to
their cluster
The Cluster API KubeVirt provider creates a node's VirtualMachine
without an ownerReference and links it to its KubevirtMachine through
the capk.cluster.x-k8s.io/kubevirt-machine-{name,namespace} labels
only. The ownership walk therefore stopped at the VM, found no
ancestor, and the lineage webhook left every object below it unmanaged:
the virt-launcher pods of worker nodes never got
internal.cozystack.io/managed-by-cozystack, while still carrying the
application lineage labels copied from the machine template.
Anything that relies on the webhook having resolved those pods misses
them. The securitygroup-controller caches only managed pods, so a
SecurityGroup attached to a Kubernetes application labelled the
control-plane pods and never the nodes.
Follow the CAPK labels from the VM to its KubevirtMachine, whose
ownerReferences reach the CAPI Cluster and from there the cluster's
HelmRelease. Only a KubevirtMachine in the VM's own namespace is
followed: CAPK can target an external infrastructure cluster, and the
walk must never attribute an object across namespaces.
Signed-off-by: Mattia Eleuteri
Assisted-by: LLM
---
pkg/lineage/capk_test.go | 101 +++++++++++++++++++++++++++++++++++++++
pkg/lineage/lineage.go | 38 +++++++++++++++
2 files changed, 139 insertions(+)
create mode 100644 pkg/lineage/capk_test.go
diff --git a/pkg/lineage/capk_test.go b/pkg/lineage/capk_test.go
new file mode 100644
index 000000000..606647eed
--- /dev/null
+++ b/pkg/lineage/capk_test.go
@@ -0,0 +1,101 @@
+package lineage
+
+import (
+ "context"
+ "testing"
+
+ helmv2 "github.com/fluxcd/helm-controller/api/v2"
+ "k8s.io/apimachinery/pkg/api/meta"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ dynamicfake "k8s.io/client-go/dynamic/fake"
+)
+
+const capkNamespace = "tenant-demo"
+
+// kubernetesAppMapper maps the cluster's HelmRelease to its Kubernetes
+// application, as the webhook's ApplicationDefinition mapper does.
+type kubernetesAppMapper struct{}
+
+func (kubernetesAppMapper) Map(*helmv2.HelmRelease) (string, string, string, error) {
+ return "apps.cozystack.io/v1alpha1", "Kubernetes", "kubernetes-", nil
+}
+
+func capkObject(apiVersion, kind, name string, labels map[string]string, owner *unstructured.Unstructured) *unstructured.Unstructured {
+ o := &unstructured.Unstructured{}
+ o.SetAPIVersion(apiVersion)
+ o.SetKind(kind)
+ o.SetNamespace(capkNamespace)
+ o.SetName(name)
+ o.SetLabels(labels)
+ if owner != nil {
+ o.SetOwnerReferences([]metav1.OwnerReference{{APIVersion: owner.GetAPIVersion(), Kind: owner.GetKind(), Name: owner.GetName()}})
+ }
+ return o
+}
+
+// capkNodeChain builds the objects behind the virt-launcher pod of a
+// Kubernetes worker node, as observed on a live cluster: the pod is owned by
+// its VMI and the VMI by its VM, but CAPK leaves the VM without an
+// ownerReference and points at its KubevirtMachine through labels only. From
+// the KubevirtMachine up, ownerReferences run to the CAPI Cluster, which the
+// cluster's HelmRelease labels.
+func capkNodeChain(vmLabels map[string]string) (*unstructured.Unstructured, []runtime.Object, meta.RESTMapper) {
+ app := capkObject("apps.cozystack.io/v1alpha1", "Kubernetes", "cluster-a", nil, nil)
+ hr := capkObject(HRAPIVersion, HRKind, "kubernetes-cluster-a", nil, nil)
+ cluster := capkObject("cluster.x-k8s.io/v1beta1", "Cluster", "kubernetes-cluster-a", map[string]string{HRLabel: "kubernetes-cluster-a"}, nil)
+ md := capkObject("cluster.x-k8s.io/v1beta1", "MachineDeployment", "kubernetes-cluster-a-md0", nil, cluster)
+ ms := capkObject("cluster.x-k8s.io/v1beta1", "MachineSet", "kubernetes-cluster-a-md0-x", nil, md)
+ machine := capkObject("cluster.x-k8s.io/v1beta1", "Machine", "kubernetes-cluster-a-md0-x-y", nil, ms)
+ kvm := capkObject(KubevirtMachineAPIVersion, KubevirtMachineKind, "kubernetes-cluster-a-md0-x-y", nil, machine)
+ vm := capkObject(VMAPIVersion, VMKind, "kubernetes-cluster-a-md0-x-y", vmLabels, nil)
+ vmi := capkObject("kubevirt.io/v1", "VirtualMachineInstance", "kubernetes-cluster-a-md0-x-y", nil, vm)
+ pod := capkObject("v1", "Pod", "virt-launcher-kubernetes-cluster-a-md0-x-y-abcde", nil, vmi)
+
+ objs := []runtime.Object{app, hr, cluster, md, ms, machine, kvm, vm, vmi}
+ mapper := meta.NewDefaultRESTMapper(nil)
+ for _, o := range objs {
+ mapper.Add(o.GetObjectKind().GroupVersionKind(), meta.RESTScopeNamespace)
+ }
+ mapper.Add(schema.GroupVersionKind{Version: "v1", Kind: "Pod"}, meta.RESTScopeNamespace)
+ return pod, objs, mapper
+}
+
+func walkNodePod(t *testing.T, vmLabels map[string]string) []ObjectID {
+ t.Helper()
+ pod, objs, mapper := capkNodeChain(vmLabels)
+ dyn := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), objs...)
+ return WalkOwnershipGraph(context.Background(), dyn, mapper, kubernetesAppMapper{}, pod)
+}
+
+func TestWalkResolvesCAPKNodeVMToKubernetesApp(t *testing.T) {
+ got := walkNodePod(t, map[string]string{
+ KubevirtMachineNameLabel: "kubernetes-cluster-a-md0-x-y",
+ KubevirtMachineNamespaceLabel: capkNamespace,
+ })
+ want := ObjectID{APIVersion: "apps.cozystack.io/v1alpha1", Kind: "Kubernetes", Namespace: capkNamespace, Name: "cluster-a"}
+ if len(got) != 1 || got[0] != want {
+ t.Fatalf("WalkOwnershipGraph(node pod) = %+v, want [%+v]", got, want)
+ }
+}
+
+func TestWalkIgnoresKubevirtMachineOfAnotherNamespace(t *testing.T) {
+ // A VM pointing at a KubevirtMachine outside its own namespace (an external
+ // infrastructure cluster, or a forged label) must not be resolved through it.
+ got := walkNodePod(t, map[string]string{
+ KubevirtMachineNameLabel: "kubernetes-cluster-a-md0-x-y",
+ KubevirtMachineNamespaceLabel: "tenant-other",
+ })
+ if len(got) != 0 {
+ t.Fatalf("WalkOwnershipGraph(node pod) = %+v, want no ancestors", got)
+ }
+}
+
+func TestWalkStopsAtUnlabelledVM(t *testing.T) {
+ // A standalone VM with no CAPK labels and no ownerReference has no ancestors.
+ if got := walkNodePod(t, nil); len(got) != 0 {
+ t.Fatalf("WalkOwnershipGraph(pod of standalone VM) = %+v, want no ancestors", got)
+ }
+}
diff --git a/pkg/lineage/lineage.go b/pkg/lineage/lineage.go
index e1f050052..1546df647 100644
--- a/pkg/lineage/lineage.go
+++ b/pkg/lineage/lineage.go
@@ -20,6 +20,16 @@ const (
HRAPIVersion = "helm.toolkit.fluxcd.io/v2"
HRKind = "HelmRelease"
HRLabel = "helm.toolkit.fluxcd.io/name"
+
+ // KubeVirt VirtualMachines created by the Cluster API KubeVirt provider
+ // (CAPK) for cluster nodes carry no ownerReference: CAPK links them to
+ // their KubevirtMachine through these labels only.
+ VMAPIVersion = "kubevirt.io/v1"
+ VMKind = "VirtualMachine"
+ KubevirtMachineAPIVersion = "infrastructure.cluster.x-k8s.io/v1alpha1"
+ KubevirtMachineKind = "KubevirtMachine"
+ KubevirtMachineNameLabel = "capk.cluster.x-k8s.io/kubevirt-machine-name"
+ KubevirtMachineNamespaceLabel = "capk.cluster.x-k8s.io/kubevirt-machine-namespace"
)
// AppMapper maps HelmRelease to application metadata.
@@ -91,6 +101,18 @@ func WalkOwnershipGraph(
return
}
+ // A node VM of a Kubernetes cluster has no ownerReference, so without this
+ // step the walk would stop at it and never reach the cluster's application.
+ if name, ok := kubevirtMachineName(obj); ok {
+ ownerObj, err := getUnstructuredObject(ctx, client, mapper, state.cache, KubevirtMachineAPIVersion, KubevirtMachineKind, obj.GetNamespace(), name)
+ if err != nil {
+ l.Error(err, "couldn't get KubevirtMachine of VirtualMachine", "namespace", obj.GetNamespace(), "virtualMachine", obj.GetName(), "kubevirtMachine", name)
+ return
+ }
+ out = append(out, WalkOwnershipGraph(ctx, client, mapper, appMapper, ownerObj, state)...)
+ return
+ }
+
// I want "if err1 != nil go to next block, if err2 != nil, go to next block, etc semantics",
// like an early return from a function, but if all checks succeed, I don't want to do the rest
// of the function, so it's a `for { if err { break } if othererr { break } if allgood { return }
@@ -138,6 +160,22 @@ func WalkOwnershipGraph(
return
}
+// kubevirtMachineName returns the KubevirtMachine a CAPK-created
+// VirtualMachine belongs to. It only follows a KubevirtMachine in the VM's own
+// namespace: CAPK can also run against an external infrastructure cluster,
+// where the KubevirtMachine lives elsewhere and is not ours to resolve.
+func kubevirtMachineName(obj *unstructured.Unstructured) (string, bool) {
+ if obj.GetAPIVersion() != VMAPIVersion || obj.GetKind() != VMKind {
+ return "", false
+ }
+ labels := obj.GetLabels()
+ name := labels[KubevirtMachineNameLabel]
+ if name == "" || labels[KubevirtMachineNamespaceLabel] != obj.GetNamespace() {
+ return "", false
+ }
+ return name, true
+}
+
func parseWalkMemory(memory []interface{}) (*walkState, error) {
switch len(memory) {
case 0:
From dde255b7af102f32ea67cfcf0dcd67b15855d8cd Mon Sep 17 00:00:00 2001
From: Mattia Eleuteri
Date: Tue, 27 Oct 2026 08:44:43 +0100
Subject: [PATCH 014/346] fix(securitygroups): project a membership-only group
to a policy Cilium accepts
A SecurityGroup with attachments and no rules still needs its backing
CiliumNetworkPolicy, which carries the attachments the controller
reconciles. Its spec held only an empty ingress list: the CRD anyOf
accepts that, but the agent's api.Rule.Sanitize counts entries, so
Cilium 1.19 marks the policy Valid=False.
Write one empty ingress rule, which allows nothing, and disable default
deny in both directions so the rule does not isolate the member pods.
The policy is then valid and still changes nothing in the datapath. The
reverse projection recognises that exact spec, enableDefaultDeny
included, so the group still reads back with no rules while a
tenant-authored empty ingress rule keeps its default deny and stays
visible.
Fixes #5129
Assisted-by: LLM
Signed-off-by: Mattia Eleuteri
---
pkg/apis/sdn/DESIGN.md | 16 ++-
pkg/registry/sdn/securitygroup/cilium.go | 53 ++++---
pkg/registry/sdn/securitygroup/rest.go | 61 ++++++--
pkg/registry/sdn/securitygroup/rest_test.go | 145 +++++++++++++-------
4 files changed, 197 insertions(+), 78 deletions(-)
diff --git a/pkg/apis/sdn/DESIGN.md b/pkg/apis/sdn/DESIGN.md
index e7fa62f95..017da9d9a 100644
--- a/pkg/apis/sdn/DESIGN.md
+++ b/pkg/apis/sdn/DESIGN.md
@@ -130,7 +130,21 @@ spec:
The securitygroup-controller stamps `securitygroup.sdn.cozystack.io/db: ""` onto the pods of the attached `Postgres/db` application, which is what makes the policy's `endpointSelector` select them.
-A **membership-only SecurityGroup** — one with `attachments` but no `ingress`/`egress` rules — still projects to a backing CiliumNetworkPolicy, because the controller reconciles the policy (not the SecurityGroup) and the attachments live on it. The CiliumNetworkPolicy CRD requires at least one rule section to be present (its `spec` `anyOf` is `ingress | ingressDeny | egress | egressDeny`), so the projection always emits the `ingress` section, an empty list `ingress: []` when the group has no ingress rules. An empty list is the CRD's documented no-op ("if omitted or empty, this rule does not apply at ingress") and `enableDefaultDeny` defaults to `false` for a direction with no rules, so the policy is schema-valid yet datapath-inert: it neither allows nor denies anything, consistent with §7. The empty section is hidden from the SecurityGroup view on read, so a membership-only group round-trips with no `ingress`/`egress`.
+A **membership-only SecurityGroup** — one with `attachments` but no `ingress`/`egress` rules — still projects to a backing CiliumNetworkPolicy, because the controller reconciles the policy (not the SecurityGroup) and the attachments live on it. Cilium rejects a policy none of whose rule sections has an entry: the CRD `anyOf` (`ingress | ingressDeny | egress | egressDeny`) is satisfied by an empty list, but the agent's `api.Rule.Sanitize` counts entries and marks such a policy `Valid=False`. The projection therefore writes one empty ingress rule, which allows nothing, together with `enableDefaultDeny: {ingress: false, egress: false}`, so that the rule does not put the member pods into default-deny either:
+
+```yaml
+spec:
+ endpointSelector:
+ matchLabels:
+ securitygroup.sdn.cozystack.io/db: ""
+ ingress:
+ - {}
+ enableDefaultDeny:
+ ingress: false
+ egress: false
+```
+
+The policy is valid yet datapath-inert: it neither allows nor denies anything, consistent with §7. The override depends on the Cilium agent's `enable-non-default-deny-policies` option, which defaults to `true`; with it off, Cilium forces default deny on every policy. The placeholder is hidden from the SecurityGroup view on read, so a membership-only group round-trips with no `ingress`/`egress`. `enableDefaultDeny` is what tells it apart from a group whose single ingress rule is genuinely empty, which projects without the override and so keeps Cilium's ingress default deny.
## 6. RBAC
diff --git a/pkg/registry/sdn/securitygroup/cilium.go b/pkg/registry/sdn/securitygroup/cilium.go
index fa0ddb65a..906afcdcd 100644
--- a/pkg/registry/sdn/securitygroup/cilium.go
+++ b/pkg/registry/sdn/securitygroup/cilium.go
@@ -44,25 +44,24 @@ type CiliumNetworkPolicySpec struct {
// it to the SecurityGroup's own membership label.
EndpointSelector metav1.LabelSelector `json:"endpointSelector,omitempty"`
- // Ingress is the list of allowed inbound traffic rules. It is always
- // serialized (an empty list, not omitted) and the projection guarantees it is
- // non-nil. The backing CiliumNetworkPolicy CRD requires at least one rule
- // section to be present (spec anyOf: ingress|ingressDeny|egress|egressDeny),
- // so a rules-less (membership-only) SecurityGroup would otherwise project to a
- // selector-only spec the cilium.io/v2 API rejects synchronously. An empty
- // ingress list is the CRD's documented no-op ("if omitted or empty, this rule
- // does not apply at ingress"), and enableDefaultDeny defaults to false for a
- // direction with no rules, so the always-present list keeps the policy
- // schema-valid while leaving the member pods' connectivity untouched. The
- // non-nil guarantee matters because the CRD has no nullable fields: a nil
- // slice without omitempty would marshal to null and be rejected.
- Ingress []CiliumIngressRule `json:"ingress"`
+ // Ingress is the list of allowed inbound traffic rules.
+ Ingress []CiliumIngressRule `json:"ingress,omitempty"`
- // Egress is the list of allowed outbound traffic rules. Unlike Ingress it
- // keeps omitempty: ingress is the guaranteed-present carrier that satisfies the
- // CRD anyOf in every case, so egress is emitted only when there are egress
- // rules.
+ // Egress is the list of allowed outbound traffic rules.
Egress []CiliumEgressRule `json:"egress,omitempty"`
+
+ // EnableDefaultDeny overrides Cilium's per-direction default-deny posture.
+ // The projection sets it only for a membership-only SecurityGroup; see
+ // membershipOnlySpec.
+ EnableDefaultDeny *CiliumDefaultDeny `json:"enableDefaultDeny,omitempty"`
+}
+
+// CiliumDefaultDeny mirrors the cilium.io/v2 enableDefaultDeny block. A nil
+// direction leaves Cilium's default in place: deny for a direction that has
+// rules, allow otherwise.
+type CiliumDefaultDeny struct {
+ Ingress *bool `json:"ingress,omitempty"`
+ Egress *bool `json:"egress,omitempty"`
}
// CiliumIngressRule mirrors a single cilium.io/v2 ingress rule. fromApp/fromSG
@@ -170,6 +169,26 @@ func (in *CiliumNetworkPolicySpec) DeepCopyInto(out *CiliumNetworkPolicySpec) {
in.Egress[i].DeepCopyInto(&out.Egress[i])
}
}
+ if in.EnableDefaultDeny != nil {
+ out.EnableDefaultDeny = in.EnableDefaultDeny.DeepCopy()
+ }
+}
+
+// DeepCopy returns a deep copy of the receiver.
+func (in *CiliumDefaultDeny) DeepCopy() *CiliumDefaultDeny {
+ if in == nil {
+ return nil
+ }
+ out := new(CiliumDefaultDeny)
+ if in.Ingress != nil {
+ v := *in.Ingress
+ out.Ingress = &v
+ }
+ if in.Egress != nil {
+ v := *in.Egress
+ out.Egress = &v
+ }
+ return out
}
// DeepCopy returns a deep copy of the receiver.
diff --git a/pkg/registry/sdn/securitygroup/rest.go b/pkg/registry/sdn/securitygroup/rest.go
index a06439a75..d4a8d223d 100644
--- a/pkg/registry/sdn/securitygroup/rest.go
+++ b/pkg/registry/sdn/securitygroup/rest.go
@@ -143,11 +143,10 @@ func sgFromSelector(sel metav1.LabelSelector) (string, bool) {
// projectIngress turns the SecurityGroup ingress rules into Cilium ingress
// rules: fromApp peers become lineage-label endpointSelectors, fromSG peers
// become membership-label endpointSelectors, and fromCIDR/toPorts carry over.
-// It always returns a non-nil slice (empty when there are no rules): the backing
-// CiliumNetworkPolicy's ingress section must be present on the wire to satisfy
-// the CRD anyOf, and a non-nil empty slice serializes to an empty list rather
-// than null (which the CRD, having no nullable fields, would reject).
func projectIngress(rules []sdnv1alpha1.IngressRule) []CiliumIngressRule {
+ if rules == nil {
+ return nil
+ }
out := make([]CiliumIngressRule, len(rules))
for i := range rules {
var eps []metav1.LabelSelector
@@ -195,9 +194,8 @@ func projectEgress(rules []sdnv1alpha1.EgressRule) []CiliumEgressRule {
// the lineage-label nor the membership-label shape are ignored, so a
// hand-edited backing policy degrades gracefully rather than erroring.
func reconstructIngress(rules []CiliumIngressRule) []sdnv1alpha1.IngressRule {
- // Collapse an empty list to nil so the always-present empty ingress section the
- // projection emits for a rules-less group (to satisfy the CRD anyOf) is
- // invisible in the SecurityGroup view, keeping the rules-less round-trip exact.
+ // Collapse an empty list to nil: policies written before membershipOnlySpec
+ // carried an always-present empty ingress section.
if len(rules) == 0 {
return nil
}
@@ -224,8 +222,8 @@ func reconstructIngress(rules []CiliumIngressRule) []sdnv1alpha1.IngressRule {
// reconstructEgress is the egress counterpart of reconstructIngress.
func reconstructEgress(rules []CiliumEgressRule) []sdnv1alpha1.EgressRule {
- // Collapse an empty list to nil for the same reason as reconstructIngress, so a
- // SecurityGroup that carries no egress rules round-trips with egress unset.
+ // Collapse an empty list to nil, so a SecurityGroup that carries no egress
+ // rules round-trips with egress unset.
if len(rules) == 0 {
return nil
}
@@ -409,8 +407,10 @@ func policyToSecurityGroup(np *CiliumNetworkPolicy) *sdnv1alpha1.SecurityGroup {
// Attachments live in a storage-owned annotation, not the
// endpointSelector (which is the SecurityGroup's own membership label).
Attachments: decodeAttachments(np.Annotations[attachmentsAnnotation]),
- Ingress: reconstructIngress(spec.Ingress),
- Egress: reconstructEgress(spec.Egress),
+ }
+ if !isMembershipOnlySpec(spec) {
+ sg.Spec.Ingress = reconstructIngress(spec.Ingress)
+ sg.Spec.Egress = reconstructEgress(spec.Egress)
}
}
return sg
@@ -485,6 +485,10 @@ func securityGroupToPolicy(sg *sdnv1alpha1.SecurityGroup, cur *CiliumNetworkPoli
// label onto the attached applications' pods. Rules project app/SG peers into
// endpoint selectors.
spec := sg.Spec.DeepCopy()
+ if len(spec.Ingress) == 0 && len(spec.Egress) == 0 {
+ out.Spec = membershipOnlySpec(sg.Name)
+ return &out
+ }
out.Spec = &CiliumNetworkPolicySpec{
EndpointSelector: buildEndpointSelector(sg.Name),
Ingress: projectIngress(spec.Ingress),
@@ -497,6 +501,41 @@ func securityGroupToPolicy(sg *sdnv1alpha1.SecurityGroup, cur *CiliumNetworkPoli
return &out
}
+// membershipOnlySpec is the backing spec of a SecurityGroup with no rules. The
+// policy must still exist, since it carries the attachments the
+// securitygroup-controller reconciles, but Cilium rejects a rule with no
+// ingress, ingressDeny, egress or egressDeny entry (api.Rule.Sanitize counts
+// entries, so an empty list is as good as absent) and marks the policy
+// Valid=False. One empty ingress rule satisfies that check and allows nothing,
+// and disabling default deny in both directions keeps that rule from putting
+// the member pods into default-deny, so the policy changes nothing in the
+// datapath. That relies on enable-non-default-deny-policies, which defaults to
+// true; with it off Cilium forces default deny on every policy.
+func membershipOnlySpec(name string) *CiliumNetworkPolicySpec {
+ off := false
+ return &CiliumNetworkPolicySpec{
+ EndpointSelector: buildEndpointSelector(name),
+ Ingress: []CiliumIngressRule{{}},
+ EnableDefaultDeny: &CiliumDefaultDeny{Ingress: &off, Egress: &off},
+ }
+}
+
+// isMembershipOnlySpec reports whether spec is what membershipOnlySpec writes,
+// so the reverse projection shows no rules for it. enableDefaultDeny tells it
+// apart from a SecurityGroup whose single ingress rule is genuinely empty,
+// which the projection writes without it, keeping ingress default deny.
+func isMembershipOnlySpec(spec *CiliumNetworkPolicySpec) bool {
+ d := spec.EnableDefaultDeny
+ if d == nil || d.Ingress == nil || *d.Ingress || d.Egress == nil || *d.Egress {
+ return false
+ }
+ if len(spec.Egress) != 0 || len(spec.Ingress) != 1 {
+ return false
+ }
+ in := spec.Ingress[0]
+ return len(in.FromEndpoints) == 0 && len(in.FromCIDR) == 0 && len(in.ToPorts) == 0
+}
+
// normalizePortProtocols upper-cases every port protocol in place so the value
// written to the backing CiliumNetworkPolicy matches its case-sensitive enum.
func normalizePortProtocols(spec *CiliumNetworkPolicySpec) {
diff --git a/pkg/registry/sdn/securitygroup/rest_test.go b/pkg/registry/sdn/securitygroup/rest_test.go
index 5473dfd56..f3ad68fc3 100644
--- a/pkg/registry/sdn/securitygroup/rest_test.go
+++ b/pkg/registry/sdn/securitygroup/rest_test.go
@@ -7,6 +7,7 @@ import (
"context"
"encoding/json"
"errors"
+ "fmt"
"reflect"
"sort"
"strings"
@@ -1195,16 +1196,53 @@ func TestCreateAcceptsEmptyAttachments(t *testing.T) {
}
}
+// specWire returns the backing policy's spec as Cilium decodes it: the CRD and
+// the agent read the wire form, not the Go struct.
+func specWire(t *testing.T, np *CiliumNetworkPolicy) map[string]any {
+ t.Helper()
+ raw, err := json.Marshal(np)
+ if err != nil {
+ t.Fatalf("marshal backing policy: %v", err)
+ }
+ var decoded struct {
+ Spec map[string]any `json:"spec"`
+ }
+ if err := json.Unmarshal(raw, &decoded); err != nil {
+ t.Fatalf("unmarshal backing policy: %v", err)
+ }
+ return decoded.Spec
+}
+
+// ciliumAcceptsRuleSections mirrors the first check of Cilium's
+// api.Rule.Sanitize, which marks a policy Valid=False when none of its rule
+// sections has an entry. The CRD anyOf only needs one of the keys to be
+// present, so it admits an empty list that the agent then rejects.
+func ciliumAcceptsRuleSections(spec map[string]any) bool {
+ for _, k := range []string{"ingress", "ingressDeny", "egress", "egressDeny"} {
+ if l, _ := spec[k].([]any); len(l) > 0 {
+ return true
+ }
+ }
+ return false
+}
+
+func getBackingPolicy(t *testing.T, r *REST, name string) *CiliumNetworkPolicy {
+ t.Helper()
+ np := &CiliumNetworkPolicy{}
+ if err := r.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: name}, np); err != nil {
+ t.Fatalf("backing policy not found: %v", err)
+ }
+ return np
+}
+
func TestRulesLessSecurityGroupProjectsValidPolicy(t *testing.T) {
r := newTestREST(t)
// A membership-only SecurityGroup attaches to an application to stamp the
- // membership label but carries no ingress or egress rules. The backing
- // CiliumNetworkPolicy CRD still requires at least one rule section to be
- // present (spec anyOf: ingress|ingressDeny|egress|egressDeny), so a
- // selector-only spec is rejected synchronously by the cilium.io/v2 API. The
- // projection must emit an explicit empty ingress list — the CRD's documented
- // no-op ("if omitted or empty, this rule does not apply at ingress") — so the
- // policy is schema-valid yet leaves the member pods' connectivity untouched.
+ // membership label but carries no ingress or egress rules. Its backing
+ // policy must be one Cilium accepts and that changes nothing in the
+ // datapath: a rule section with an entry that allows nothing, and default
+ // deny disabled in both directions so that entry does not isolate the
+ // member pods.
createSG(t, r, &sdnv1alpha1.SecurityGroup{
ObjectMeta: metav1.ObjectMeta{Name: "sg-membership", Namespace: testNamespace},
Spec: sdnv1alpha1.SecurityGroupSpec{
@@ -1212,50 +1250,22 @@ func TestRulesLessSecurityGroupProjectsValidPolicy(t *testing.T) {
},
})
- np := &CiliumNetworkPolicy{}
- if err := r.c.Get(context.Background(), types.NamespacedName{Namespace: testNamespace, Name: "sg-membership"}, np); err != nil {
- t.Fatalf("backing policy not found: %v", err)
+ spec := specWire(t, getBackingPolicy(t, r, "sg-membership"))
+ if !ciliumAcceptsRuleSections(spec) {
+ t.Fatalf("rules-less policy has no rule entry, Cilium marks it Valid=False: %v", spec)
+ }
+ if got, want := fmt.Sprint(spec["ingress"]), "[map[]]"; got != want {
+ t.Fatalf("rules-less policy ingress = %s, want %s (one rule that allows nothing)", got, want)
+ }
+ if _, ok := spec["egress"]; ok {
+ t.Fatalf("rules-less policy must not carry an egress key: %v", spec)
+ }
+ if got, want := fmt.Sprint(spec["enableDefaultDeny"]), "map[egress:false ingress:false]"; got != want {
+ t.Fatalf("rules-less policy enableDefaultDeny = %s, want %s", got, want)
}
- // The CRD validates the wire form, not the Go struct, so assert on the
- // marshaled bytes: spec.ingress must be present (an empty list satisfies the
- // anyOf), and spec.egress must be absent (ingress is the guaranteed-present
- // carrier; egress is emitted only when there are egress rules).
- raw, err := json.Marshal(np)
- if err != nil {
- t.Fatalf("marshal backing policy: %v", err)
- }
- var decoded struct {
- Spec map[string]json.RawMessage `json:"spec"`
- }
- if err := json.Unmarshal(raw, &decoded); err != nil {
- t.Fatalf("unmarshal backing policy: %v", err)
- }
- ing, ok := decoded.Spec["ingress"]
- if !ok {
- t.Fatalf("backing policy spec has no ingress key; the CRD anyOf would reject it: %s", raw)
- }
- if string(ing) != "[]" {
- t.Fatalf("rules-less policy ingress = %s, want []", ing)
- }
- if _, ok := decoded.Spec["egress"]; ok {
- t.Fatalf("rules-less policy must not carry an egress key: %s", raw)
- }
-
- // The same key must survive the unstructured conversion path (which honors
- // omitempty independently of json.Marshal), so the empty list cannot be
- // silently dropped there either.
- u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(np)
- if err != nil {
- t.Fatalf("to unstructured: %v", err)
- }
- spec, _ := u["spec"].(map[string]any)
- if _, ok := spec["ingress"]; !ok {
- t.Fatalf("unstructured backing policy spec has no ingress key: %v", spec)
- }
-
- // On read, the synthetic empty ingress list collapses back to no rules, so a
- // rules-less SecurityGroup round-trips with neither ingress nor egress.
+ // On read, the placeholder rule is hidden, so a rules-less SecurityGroup
+ // round-trips with neither ingress nor egress.
out, err := r.Get(ctxNS(), "sg-membership", &metav1.GetOptions{})
if err != nil {
t.Fatalf("Get returned error: %v", err)
@@ -1264,6 +1274,43 @@ func TestRulesLessSecurityGroupProjectsValidPolicy(t *testing.T) {
if got.Spec.Ingress != nil || got.Spec.Egress != nil {
t.Fatalf("rules-less round-trip should carry no ingress/egress, got: %+v", got.Spec)
}
+
+ // Adding a rule must drop the default-deny override, or the group's
+ // rules would allow traffic without isolating the member pods.
+ got.Spec.Egress = []sdnv1alpha1.EgressRule{{ToCIDR: []string{"10.0.0.0/8"}}}
+ if _, _, err := r.Update(ctxNS(), "sg-membership", rest.DefaultUpdatedObjectInfo(got), nil, nil, false, &metav1.UpdateOptions{}); err != nil {
+ t.Fatalf("Update returned error: %v", err)
+ }
+ spec = specWire(t, getBackingPolicy(t, r, "sg-membership"))
+ if _, ok := spec["enableDefaultDeny"]; ok {
+ t.Fatalf("policy with rules must not override default deny: %v", spec)
+ }
+ if _, ok := spec["ingress"]; ok {
+ t.Fatalf("egress-only policy must not carry an ingress key: %v", spec)
+ }
+}
+
+func TestEmptyIngressRuleIsNotMembershipOnly(t *testing.T) {
+ r := newTestREST(t)
+ // An explicit empty ingress rule projects to the same rule as the
+ // membership-only placeholder, but it is tenant-authored: it keeps Cilium's
+ // ingress default deny and must stay visible on read.
+ createSG(t, r, &sdnv1alpha1.SecurityGroup{
+ ObjectMeta: metav1.ObjectMeta{Name: "sg-deny-in", Namespace: testNamespace},
+ Spec: sdnv1alpha1.SecurityGroupSpec{Ingress: []sdnv1alpha1.IngressRule{{}}},
+ })
+
+ spec := specWire(t, getBackingPolicy(t, r, "sg-deny-in"))
+ if _, ok := spec["enableDefaultDeny"]; ok {
+ t.Fatalf("an empty ingress rule must keep default deny: %v", spec)
+ }
+ out, err := r.Get(ctxNS(), "sg-deny-in", &metav1.GetOptions{})
+ if err != nil {
+ t.Fatalf("Get returned error: %v", err)
+ }
+ if got := out.(*sdnv1alpha1.SecurityGroup).Spec.Ingress; len(got) != 1 {
+ t.Fatalf("empty ingress rule should round-trip as one rule, got: %+v", got)
+ }
}
func TestCreateRejectsInvalidAttachmentLabelValue(t *testing.T) {
From f21abe0d35ab533aceebbea95a6b76a88b47cefb Mon Sep 17 00:00:00 2001
From: Mattia Eleuteri
Date: Tue, 27 Oct 2026 09:12:25 +0100
Subject: [PATCH 015/346] test(mariadb-operator): run the package suite from
make test
The helm-unittest sweep reaches a package only through its make test
target, so the suite added with the cluster-name fix would otherwise
never run in CI.
Assisted-by: LLM
Signed-off-by: Mattia Eleuteri
---
packages/system/mariadb-operator/Makefile | 3 +++
1 file changed, 3 insertions(+)
diff --git a/packages/system/mariadb-operator/Makefile b/packages/system/mariadb-operator/Makefile
index 905653cac..e80038778 100644
--- a/packages/system/mariadb-operator/Makefile
+++ b/packages/system/mariadb-operator/Makefile
@@ -3,6 +3,9 @@ export NAMESPACE=cozy-$(NAME)
include ../../../hack/package.mk
+test:
+ helm unittest .
+
update:
rm -rf charts
helm repo add mariadb-operator https://mariadb-operator.github.io/mariadb-operator
From 48779cc03b23b7debb6dc6e4f39958fe9c165e0a Mon Sep 17 00:00:00 2001
From: Mattia Eleuteri
Date: Thu, 29 Oct 2026 10:00:55 +0100
Subject: [PATCH 016/346] fix(redis): run one Sentinel when Redis runs a single
pod
The chart pinned three Sentinels whatever the replica count, so a
single-pod instance paid for three pods that have nothing to fail over
to. One Sentinel is enough for the operator to locate the master: its
quorum is replicas/2+1, which is 1.
The operator gives the Sentinel Deployment a PodDisruptionBudget with
minAvailable 1 whenever Redis has two replicas or fewer. On a single
Sentinel that budget refuses every eviction and would block node
drains, so it is disabled in that case. From two Redis pods up nothing
changes: three Sentinels, budget kept.
Assisted-by: LLM
Signed-off-by: Mattia Eleuteri
---
api/apps/v1alpha1/redis/types.go | 4 +-
packages/apps/redis/README.md | 16 +--
.../apps/redis/templates/redisfailover.yaml | 17 ++-
.../redis/tests/sentinel_replicas_test.yaml | 103 ++++++++++++++++++
packages/apps/redis/values.schema.json | 4 +-
packages/apps/redis/values.yaml | 4 +-
packages/system/redis-rd/cozyrds/redis.yaml | 2 +-
7 files changed, 132 insertions(+), 18 deletions(-)
create mode 100644 packages/apps/redis/tests/sentinel_replicas_test.yaml
diff --git a/api/apps/v1alpha1/redis/types.go b/api/apps/v1alpha1/redis/types.go
index 509c1e6ce..c2363749c 100644
--- a/api/apps/v1alpha1/redis/types.go
+++ b/api/apps/v1alpha1/redis/types.go
@@ -17,7 +17,7 @@ type Config struct {
}
type ConfigSpec struct {
- // Number of Redis replicas.
+ // Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.
// +kubebuilder:default:=2
Replicas int `json:"replicas"`
// Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.
@@ -51,7 +51,7 @@ type ConfigSpec struct {
// Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.
// +kubebuilder:validation:Minimum=0
MinReplicasMaxLag *int `json:"minReplicasMaxLag,omitempty"`
- // Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the three Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.
+ // Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.
// +kubebuilder:default:={}
Sentinel Sentinel `json:"sentinel,omitempty"`
}
diff --git a/packages/apps/redis/README.md b/packages/apps/redis/README.md
index 0922bf821..28152d9f9 100644
--- a/packages/apps/redis/README.md
+++ b/packages/apps/redis/README.md
@@ -17,7 +17,7 @@ Service utilizes the freshworks-oss Redis Operator (a maintained fork of the arc
| Name | Description | Type | Value |
| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------- | --------- |
-| `replicas` | Number of Redis replicas. | `int` | `2` |
+| `replicas` | Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to. | `int` | `2` |
| `resources` | Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied. | `object` | `{}` |
| `resources.cpu` | CPU available to each replica. | `quantity` | `""` |
| `resources.memory` | Memory (RAM) available to each replica. | `quantity` | `""` |
@@ -48,13 +48,13 @@ Service utilizes the freshworks-oss Redis Operator (a maintained fork of the arc
### Sentinel parameters
-| Name | Description | Type | Value |
-| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----- |
-| `sentinel` | Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the three Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does. | `object` | `{}` |
-| `sentinel.resources` | Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`. | `object` | `{}` |
-| `sentinel.resources.cpu` | CPU available to each replica. | `quantity` | `""` |
-| `sentinel.resources.memory` | Memory (RAM) available to each replica. | `quantity` | `""` |
-| `sentinel.resourcesPreset` | Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`. | `string` | `{}` |
+| Name | Description | Type | Value |
+| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----- |
+| `sentinel` | Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does. | `object` | `{}` |
+| `sentinel.resources` | Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`. | `object` | `{}` |
+| `sentinel.resources.cpu` | CPU available to each replica. | `quantity` | `""` |
+| `sentinel.resources.memory` | Memory (RAM) available to each replica. | `quantity` | `""` |
+| `sentinel.resourcesPreset` | Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`. | `string` | `{}` |
## Parameter examples and reference
diff --git a/packages/apps/redis/templates/redisfailover.yaml b/packages/apps/redis/templates/redisfailover.yaml
index 27c7e9047..2dd1fecaf 100644
--- a/packages/apps/redis/templates/redisfailover.yaml
+++ b/packages/apps/redis/templates/redisfailover.yaml
@@ -9,6 +9,14 @@
top-level resources falls back to resourcesPreset.
*/}}
{{- $sentinel := .Values.sentinel | default dict -}}
+{{- /*
+ A lone Redis pod has no replica to promote, so three Sentinels would only
+ watch it. One Sentinel still lets the operator find the master (its quorum is
+ replicas/2+1, so 1). The operator gives the Sentinel Deployment a PDB with
+ minAvailable 1 whenever Redis has two replicas or fewer, which on a single
+ Sentinel would block every node drain, so that PDB is dropped with it.
+*/}}
+{{- $sentinelReplicas := ternary 1 3 (le (int .Values.replicas) 1) -}}
{{- $sentinelPreset := $sentinel.resourcesPreset | default .Values.resourcesPreset -}}
{{- $sentinelResources := .Values.resources -}}
{{- if or $sentinel.resources $sentinel.resourcesPreset -}}
@@ -65,7 +73,10 @@ spec:
# pods they keep no RDB, only a sentinel.conf the init container rewrites
# on every start, so a chart rollback to an older engine is safe here.
image: {{ $redisImage | quote }}
- replicas: 3
+ replicas: {{ $sentinelReplicas }}
+ {{- if eq $sentinelReplicas 1 }}
+ disablePodDisruptionBudget: true
+ {{- end }}
resources: {{- include "cozy-lib.resources.defaultingSanitize" (list $sentinelPreset $sentinelResources $) | nindent 6 }}
redis:
image: {{ $redisImage | quote }}
@@ -152,8 +163,8 @@ metadata:
labels:
workloads.cozystack.io/resource-preset: {{ $sentinelPreset | quote }}
spec:
- minReplicas: 2
- replicas: 3
+ minReplicas: {{ ternary 1 2 (eq $sentinelReplicas 1) }}
+ replicas: {{ $sentinelReplicas }}
kind: redis
type: sentinel
selector:
diff --git a/packages/apps/redis/tests/sentinel_replicas_test.yaml b/packages/apps/redis/tests/sentinel_replicas_test.yaml
new file mode 100644
index 000000000..012f58247
--- /dev/null
+++ b/packages/apps/redis/tests/sentinel_replicas_test.yaml
@@ -0,0 +1,103 @@
+suite: Sentinel count follows the Redis replicas
+
+release:
+ name: my-redis
+ namespace: tenant-test
+
+set:
+ _cluster:
+ cluster-domain: cluster.local
+ _namespace:
+ host: example.com
+
+templates:
+ - templates/redisfailover.yaml
+
+tests:
+ - it: "runs a single Sentinel for a single Redis pod"
+ set:
+ replicas: 1
+ documentSelector:
+ path: kind
+ value: RedisFailover
+ asserts:
+ - equal:
+ path: spec.sentinel.replicas
+ value: 1
+ - equal:
+ path: spec.redis.replicas
+ value: 1
+
+ # The operator's Sentinel PDB is minAvailable 1 at two Redis replicas or
+ # fewer; on one Sentinel it would refuse every eviction and block node drains.
+ - it: "drops the Sentinel PDB when only one Sentinel runs"
+ set:
+ replicas: 1
+ documentSelector:
+ path: kind
+ value: RedisFailover
+ asserts:
+ - equal:
+ path: spec.sentinel.disablePodDisruptionBudget
+ value: true
+
+ - it: "keeps three Sentinels and their PDB from two Redis pods up"
+ set:
+ replicas: 2
+ documentSelector:
+ path: kind
+ value: RedisFailover
+ asserts:
+ - equal:
+ path: spec.sentinel.replicas
+ value: 3
+ - notExists:
+ path: spec.sentinel.disablePodDisruptionBudget
+
+ - it: "keeps three Sentinels for three Redis pods"
+ set:
+ replicas: 3
+ documentSelector:
+ path: kind
+ value: RedisFailover
+ asserts:
+ - equal:
+ path: spec.sentinel.replicas
+ value: 3
+
+ - it: "keeps three Sentinels at the chart default"
+ documentSelector:
+ path: kind
+ value: RedisFailover
+ asserts:
+ - equal:
+ path: spec.sentinel.replicas
+ value: 3
+
+ - it: "reports one expected Sentinel pod to the workload monitor"
+ set:
+ replicas: 1
+ documentSelector:
+ path: metadata.name
+ value: my-redis-sentinel
+ asserts:
+ - equal:
+ path: spec.replicas
+ value: 1
+ - equal:
+ path: spec.minReplicas
+ value: 1
+
+ - it: "reports three expected Sentinel pods to the workload monitor"
+ set:
+ replicas: 2
+ documentSelector:
+ path: metadata.name
+ value: my-redis-sentinel
+ asserts:
+ - equal:
+ path: spec.replicas
+ value: 3
+ - equal:
+ path: spec.minReplicas
+ value: 2
diff --git a/packages/apps/redis/values.schema.json b/packages/apps/redis/values.schema.json
index 4bf3e0964..1a301615f 100644
--- a/packages/apps/redis/values.schema.json
+++ b/packages/apps/redis/values.schema.json
@@ -3,7 +3,7 @@
"type": "object",
"properties": {
"replicas": {
- "description": "Number of Redis replicas.",
+ "description": "Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.",
"type": "integer",
"default": 2
},
@@ -172,7 +172,7 @@
"minimum": 0
},
"sentinel": {
- "description": "Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the three Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.",
+ "description": "Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.",
"type": "object",
"default": {},
"properties": {
diff --git a/packages/apps/redis/values.yaml b/packages/apps/redis/values.yaml
index 2a580c8e6..7917ca34d 100644
--- a/packages/apps/redis/values.yaml
+++ b/packages/apps/redis/values.yaml
@@ -55,7 +55,7 @@
## @value xlarge
## @value 2xlarge
-## @param {int} replicas - Number of Redis replicas.
+## @param {int} replicas - Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.
replicas: 2
## @param {Resources} [resources] - Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.
@@ -127,5 +127,5 @@ authEnabled: true
## @field {Resources} [resources] - Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.
## @field {ResourcesPreset} [resourcesPreset] - Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.
-## @param {Sentinel} [sentinel] - Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the three Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.
+## @param {Sentinel} [sentinel] - Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.
sentinel: {}
diff --git a/packages/system/redis-rd/cozyrds/redis.yaml b/packages/system/redis-rd/cozyrds/redis.yaml
index a1d619c34..cfcaeb01d 100644
--- a/packages/system/redis-rd/cozyrds/redis.yaml
+++ b/packages/system/redis-rd/cozyrds/redis.yaml
@@ -8,7 +8,7 @@ spec:
plural: redises
singular: redis
openAPISchema: |-
- {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the three Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}
+ {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}
release:
prefix: redis-
labels:
From 7c1bd33b08a6cb690cf574de7fe392f15cae771d Mon Sep 17 00:00:00 2001
From: Mattia Eleuteri
Date: Mon, 2 Nov 2026 11:23:46 +0100
Subject: [PATCH 017/346] fix(redis): delete the Sentinel PDB left behind on
scale-down
Setting sentinel.disablePodDisruptionBudget only stops the operator from
ensuring the PDB; it never deletes the one it created while three
Sentinels ran. An existing instance scaled to one Redis pod therefore
kept minAvailable 1 on its single Sentinel, which refuses every
eviction. A post-upgrade hook deletes rfs- in that case.
Assisted-by: LLM
Signed-off-by: Mattia Eleuteri
---
packages/apps/redis/README.md | 2 +-
packages/apps/redis/templates/_resources.tpl | 4 +
.../redis/templates/hooks/sentinel-pdb.yaml | 119 ++++++++++++++++++
.../apps/redis/templates/redisfailover.yaml | 6 +-
.../redis/tests/sentinel_pdb_hook_test.yaml | 78 ++++++++++++
5 files changed, 206 insertions(+), 3 deletions(-)
create mode 100644 packages/apps/redis/templates/hooks/sentinel-pdb.yaml
create mode 100644 packages/apps/redis/tests/sentinel_pdb_hook_test.yaml
diff --git a/packages/apps/redis/README.md b/packages/apps/redis/README.md
index 28152d9f9..478f75986 100644
--- a/packages/apps/redis/README.md
+++ b/packages/apps/redis/README.md
@@ -79,7 +79,7 @@ See [`docs/operations/resource-presets.md`](../../../docs/operations/resource-pr
### sentinel
-`sentinel.resources` and `sentinel.resourcesPreset` size the three Sentinel pods separately from the Redis pods. A Sentinel holds only its configuration and the failover state, so it needs a small fraction of what a Redis replica holding the dataset does. When the `sentinel` block sets neither, the Sentinel pods are sized like the Redis pods, which is how every instance was sized before these parameters existed.
+`sentinel.resources` and `sentinel.resourcesPreset` size the Sentinel pods separately from the Redis pods. A Sentinel holds only its configuration and the failover state, so it needs a small fraction of what a Redis replica holding the dataset does. When the `sentinel` block sets neither, the Sentinel pods are sized like the Redis pods, which is how every instance was sized before these parameters existed.
```yaml
resourcesPreset: u1.small
diff --git a/packages/apps/redis/templates/_resources.tpl b/packages/apps/redis/templates/_resources.tpl
index 6539c99a1..98119dbb2 100644
--- a/packages/apps/redis/templates/_resources.tpl
+++ b/packages/apps/redis/templates/_resources.tpl
@@ -47,3 +47,7 @@ These presets are for basic testing and not meant to be used in production
{{- printf "ERROR: Preset key '%s' invalid. Allowed values are %s" .type (join "," (keys $presets)) | fail -}}
{{- end -}}
{{- end -}}
+
+{{- define "redis.sentinelReplicas" -}}
+{{- ternary 1 3 (le (int .Values.replicas) 1) -}}
+{{- end -}}
diff --git a/packages/apps/redis/templates/hooks/sentinel-pdb.yaml b/packages/apps/redis/templates/hooks/sentinel-pdb.yaml
new file mode 100644
index 000000000..e29613c8a
--- /dev/null
+++ b/packages/apps/redis/templates/hooks/sentinel-pdb.yaml
@@ -0,0 +1,119 @@
+{{- /*
+ An instance scaled down to one Redis pod keeps the Sentinel PDB the operator
+ created while it ran three Sentinels: setting disablePodDisruptionBudget only
+ stops the operator from ensuring it, never deletes it, and minAvailable 1 on
+ a single Sentinel refuses every eviction. A fresh install with one replica
+ never gets that PDB, so this runs on upgrade only.
+*/}}
+{{- if and .Release.IsUpgrade (eq (include "redis.sentinelReplicas" . | int) 1) }}
+{{- $name := printf "%s-sentinel-pdb" .Release.Name }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ $name }}
+ labels:
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ annotations:
+ helm.sh/hook: post-upgrade
+ helm.sh/hook-weight: "0"
+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: {{ $name }}
+ labels:
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ annotations:
+ helm.sh/hook: post-upgrade
+ helm.sh/hook-weight: "0"
+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+rules:
+ - apiGroups: ["policy"]
+ resources: ["poddisruptionbudgets"]
+ resourceNames: ["rfs-{{ .Release.Name }}"]
+ verbs: ["get", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ $name }}
+ labels:
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ annotations:
+ helm.sh/hook: post-upgrade
+ helm.sh/hook-weight: "0"
+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: {{ $name }}
+subjects:
+ - kind: ServiceAccount
+ name: {{ $name }}
+ namespace: {{ .Release.Namespace }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: {{ $name }}
+ labels:
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ annotations:
+ helm.sh/hook: post-upgrade
+ helm.sh/hook-weight: "5"
+ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+spec:
+ ttlSecondsAfterFinished: 3600
+ # Bounds the kubectl call too: a --request-timeout would make kubectl skip
+ # its in-cluster credentials (hack/check-hook-request-timeout.bats).
+ activeDeadlineSeconds: 120
+ backoffLimit: 2
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ policy.cozystack.io/allow-to-apiserver: "true"
+ spec:
+ serviceAccountName: {{ $name }}
+ restartPolicy: Never
+ terminationGracePeriodSeconds: 10
+ securityContext:
+ runAsNonRoot: true
+ runAsUser: 65534
+ runAsGroup: 65534
+ seccompProfile:
+ type: RuntimeDefault
+ containers:
+ - name: delete-pdb
+ image: '{{ include "cozy-lib.image" (list "clastix/kubectl:v1.32@sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c" $) }}'
+ securityContext:
+ allowPrivilegeEscalation: false
+ readOnlyRootFilesystem: true
+ capabilities:
+ drop: ["ALL"]
+ resources:
+ requests:
+ cpu: 10m
+ memory: 64Mi
+ limits:
+ cpu: 200m
+ memory: 128Mi
+ env:
+ - name: HOME
+ value: /tmp
+ command:
+ - kubectl
+ - --namespace={{ .Release.Namespace }}
+ - delete
+ - poddisruptionbudget
+ - rfs-{{ .Release.Name }}
+ - --ignore-not-found=true
+ volumeMounts:
+ - name: tmp
+ mountPath: /tmp
+ volumes:
+ - name: tmp
+ emptyDir: {}
+{{- end }}
diff --git a/packages/apps/redis/templates/redisfailover.yaml b/packages/apps/redis/templates/redisfailover.yaml
index 2dd1fecaf..05a96e3cc 100644
--- a/packages/apps/redis/templates/redisfailover.yaml
+++ b/packages/apps/redis/templates/redisfailover.yaml
@@ -14,9 +14,11 @@
watch it. One Sentinel still lets the operator find the master (its quorum is
replicas/2+1, so 1). The operator gives the Sentinel Deployment a PDB with
minAvailable 1 whenever Redis has two replicas or fewer, which on a single
- Sentinel would block every node drain, so that PDB is dropped with it.
+ Sentinel would block every node drain, so that PDB is dropped with it; the
+ operator does not delete a PDB it stops managing, hence
+ hooks/sentinel-pdb.yaml.
*/}}
-{{- $sentinelReplicas := ternary 1 3 (le (int .Values.replicas) 1) -}}
+{{- $sentinelReplicas := include "redis.sentinelReplicas" . | int -}}
{{- $sentinelPreset := $sentinel.resourcesPreset | default .Values.resourcesPreset -}}
{{- $sentinelResources := .Values.resources -}}
{{- if or $sentinel.resources $sentinel.resourcesPreset -}}
diff --git a/packages/apps/redis/tests/sentinel_pdb_hook_test.yaml b/packages/apps/redis/tests/sentinel_pdb_hook_test.yaml
new file mode 100644
index 000000000..71ef30e23
--- /dev/null
+++ b/packages/apps/redis/tests/sentinel_pdb_hook_test.yaml
@@ -0,0 +1,78 @@
+suite: stale Sentinel PDB removed on scale-down to one Redis pod
+
+# The operator only stops ensuring the Sentinel PDB once
+# disablePodDisruptionBudget is set; it never deletes the one it created while
+# three Sentinels ran. Left in place, its minAvailable 1 refuses every eviction
+# of the single Sentinel and blocks node drains.
+
+templates:
+ - templates/hooks/sentinel-pdb.yaml
+
+release:
+ name: my-redis
+ namespace: tenant-test
+
+set:
+ _cluster:
+ cluster-domain: cluster.local
+ _namespace:
+ host: example.com
+
+tests:
+ - it: deletes the operator's Sentinel PDB on an upgrade to one replica
+ release:
+ upgrade: true
+ set:
+ replicas: 1
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.annotations["helm.sh/hook"]
+ value: post-upgrade
+ - equal:
+ path: spec.template.spec.containers[0].command
+ value:
+ - kubectl
+ - --namespace=tenant-test
+ - delete
+ - poddisruptionbudget
+ - rfs-my-redis
+ - --ignore-not-found=true
+ - notMatchRegex:
+ path: spec.template.spec.containers[0].command[4]
+ pattern: request-timeout
+
+ - it: lets the hook delete only that PDB
+ release:
+ upgrade: true
+ set:
+ replicas: 1
+ documentSelector:
+ path: kind
+ value: Role
+ asserts:
+ - equal:
+ path: rules
+ value:
+ - apiGroups: ["policy"]
+ resources: ["poddisruptionbudgets"]
+ resourceNames: ["rfs-my-redis"]
+ verbs: ["get", "delete"]
+
+ - it: renders nothing on install, where no such PDB exists
+ set:
+ replicas: 1
+ asserts:
+ - hasDocuments:
+ count: 0
+
+ - it: renders nothing while three Sentinels run
+ release:
+ upgrade: true
+ set:
+ replicas: 2
+ asserts:
+ - hasDocuments:
+ count: 0
From 2541aec62550e9b5cee12ec6e15cc5b58bc5adc9 Mon Sep 17 00:00:00 2001
From: Mattia Eleuteri
Date: Mon, 2 Nov 2026 11:32:58 +0100
Subject: [PATCH 018/346] fix(redis): keep the Sentinel PDB hook Job name
within 63 characters
The API server copies a Job's name into its pods' job-name label, which
takes 63 characters at most, and a release name may take 53, so
-sentinel-pdb could be refused and leave the stale PDB in
place. The Job name is truncated; the ServiceAccount, Role and
RoleBinding keep the full name.
Assisted-by: LLM
Signed-off-by: Mattia Eleuteri
---
.../redis/templates/hooks/sentinel-pdb.yaml | 5 ++-
.../redis/tests/sentinel_pdb_hook_test.yaml | 36 +++++++++++++++++++
2 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/packages/apps/redis/templates/hooks/sentinel-pdb.yaml b/packages/apps/redis/templates/hooks/sentinel-pdb.yaml
index e29613c8a..1649caa79 100644
--- a/packages/apps/redis/templates/hooks/sentinel-pdb.yaml
+++ b/packages/apps/redis/templates/hooks/sentinel-pdb.yaml
@@ -57,7 +57,10 @@ subjects:
apiVersion: batch/v1
kind: Job
metadata:
- name: {{ $name }}
+ {{- /* The API server copies a Job's name into its pods' job-name label,
+ which takes 63 characters at most, while a release name may take 53.
+ The objects above have no such limit and keep the full name. */}}
+ name: {{ $name | trunc 63 | trimSuffix "-" }}
labels:
app.kubernetes.io/instance: {{ .Release.Name }}
annotations:
diff --git a/packages/apps/redis/tests/sentinel_pdb_hook_test.yaml b/packages/apps/redis/tests/sentinel_pdb_hook_test.yaml
index 71ef30e23..bc6aea706 100644
--- a/packages/apps/redis/tests/sentinel_pdb_hook_test.yaml
+++ b/packages/apps/redis/tests/sentinel_pdb_hook_test.yaml
@@ -76,3 +76,39 @@ tests:
asserts:
- hasDocuments:
count: 0
+
+ # The pods' job-name label takes 63 characters at most; a 53-character
+ # release name plus -sentinel-pdb would pass it and Kubernetes would refuse
+ # the Job, leaving the stale PDB in place.
+ - it: keeps the Job name within 63 characters for the longest release name
+ release:
+ name: redis-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ upgrade: true
+ set:
+ replicas: 1
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - matchRegex:
+ path: metadata.name
+ pattern: ^[a-z0-9-]{1,63}$
+ - matchRegex:
+ path: metadata.name
+ pattern: '[a-z0-9]$'
+ - equal:
+ path: spec.template.spec.containers[0].command[4]
+ value: rfs-redis-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+
+ - it: keeps the full Job name for a short release name
+ release:
+ upgrade: true
+ set:
+ replicas: 1
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: my-redis-sentinel-pdb
From 54695fc83b3328a5401710b9b9d5debbbd05809c Mon Sep 17 00:00:00 2001
From: Aleksei Sviridkin
Date: Tue, 3 Nov 2026 03:47:58 +0300
Subject: [PATCH 019/346] fix(api): read pool usage under the quota key the
controller counts
Hierarchical quota admission subtracts the parent pool's current usage
from what a child tenant may still claim, and reads that usage from
ResourceQuota.status.used. It looked every non-raw key up under
limits., but the ResourceQuota pod evaluator reports usage under
limits.* only for cpu, memory and ephemeral-storage. Extended resources
and hugepages are counted under requests.* alone, so their usage always
read as zero and a child could be granted budget the parent pool had
already consumed.
Read usage from requests. for every resource the evaluator counts
only on the request side. flatten applies no allocation ratio there, so
that key holds the declared amount just as limits. does for cpu,
memory and ephemeral-storage.
Fixes: e3f09bab1bcc ("fix(apiserver): subtract parent pool usage in hierarchical tenant quota admission")
Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin
---
pkg/registry/apps/application/quota.go | 44 ++++++++++++++-------
pkg/registry/apps/application/quota_test.go | 29 ++++++++++++--
2 files changed, 54 insertions(+), 19 deletions(-)
diff --git a/pkg/registry/apps/application/quota.go b/pkg/registry/apps/application/quota.go
index eb6f20622..5376139c0 100644
--- a/pkg/registry/apps/application/quota.go
+++ b/pkg/registry/apps/application/quota.go
@@ -233,7 +233,7 @@ func (r *REST) validateTenantResourceQuotas(ctx context.Context, app *appsv1alph
}
parentLimit := parentQuota[res]
allocated := siblings[res] // zero value when no sibling bounds this resource
- used := poolUsed[renderedLimitKey(res)]
+ used := poolUsed[usageKey(res)]
remaining := parentLimit.DeepCopy()
remaining.Sub(allocated)
remaining.Sub(used)
@@ -285,25 +285,39 @@ var rawQuotaKeys = map[string]struct{}{
"resourcequotas": {},
}
-// renderedLimitKey maps a shorthand resourceQuotas key (the vocabulary admission
-// works in) to the rendered ResourceQuota key that carries its limit-side
-// budget, mirroring cozy-lib.resources.flatten. The mapping is allocation-ratio
-// independent: ratios only scale the requests.* side that flatten derives from
-// the same shorthand value, so the limit-side key always holds the operator's
-// declared amount. This lets admission read a tenant's current usage from its
-// rendered ResourceQuota.status.used without replicating the chart's ratio math.
+// limitCountedKeys are the shorthand keys whose usage the ResourceQuota pod
+// evaluator reports under limits.*. For every other resource it counts only
+// requests.*, so limits. or limits.hugepages-* never carries
+// usage even though the chart renders it.
+var limitCountedKeys = map[string]struct{}{
+ "cpu": {},
+ "memory": {},
+ "ephemeral-storage": {},
+}
+
+// usageKey maps a shorthand resourceQuotas key (the vocabulary admission works
+// in) to the rendered ResourceQuota key whose status.used carries its usage.
+// cozy-lib.resources.flatten renders a shorthand key both as limits. and as
+// requests.; usage is read from whichever of the two the quota controller
+// counts. For cpu, memory and ephemeral-storage that is the limit side, which
+// holds the operator's declared amount whatever allocation ratio scales the
+// request side. Every other resource is counted only on the request side, where
+// flatten applies no ratio, so requests. holds the declared amount too.
+// This lets admission read a tenant's current usage from its rendered
+// ResourceQuota.status.used without replicating the chart's ratio math.
//
-// cpu, memory, ephemeral-storage, custom resources -> "limits."
+// cpu, memory, ephemeral-storage -> "limits."
// storage -> "requests.storage" (no hard limits.storage)
// pods, services, services.loadbalancers, ... -> "" (verbatim)
-func renderedLimitKey(raw string) string {
- if raw == "storage" {
- return "requests.storage"
- }
+// extended resources, hugepages-, ... -> "requests."
+func usageKey(raw string) string {
if _, ok := rawQuotaKeys[raw]; ok {
return raw
}
- return "limits." + raw
+ if _, ok := limitCountedKeys[raw]; ok {
+ return "limits." + raw
+ }
+ return "requests." + raw
}
// parentNamespaceOf returns the namespace owned by the parent of the tenant that
@@ -341,7 +355,7 @@ func (r *REST) poolRootOf(ns string, bounded map[string]bool) string {
// created/updated). Usage is read from each member's ResourceQuota.status.used,
// taking a per-resource max within a namespace to avoid double-counting the
// chart-rendered and controller-allocated quotas, and is keyed in the rendered
-// quota key space (see renderedLimitKey). Boundedness is decided by declared
+// quota key space (see usageKey). Boundedness is decided by declared
// resourceQuotas so the result does not depend on whether Flux has rendered a
// freshly-created tenant's quota yet.
func (r *REST) parentPoolUsage(ctx context.Context, parentNS, excludeNS string) (map[string]resource.Quantity, error) {
diff --git a/pkg/registry/apps/application/quota_test.go b/pkg/registry/apps/application/quota_test.go
index f6e5d3054..3ea2b7a2b 100644
--- a/pkg/registry/apps/application/quota_test.go
+++ b/pkg/registry/apps/application/quota_test.go
@@ -136,19 +136,20 @@ func usageQuota(t *testing.T, namespace, name string, used map[string]string) *c
}
}
-func TestRenderedLimitKey(t *testing.T) {
+func TestUsageKey(t *testing.T) {
cases := map[string]string{
"cpu": "limits.cpu",
"memory": "limits.memory",
"ephemeral-storage": "limits.ephemeral-storage",
- "devices.com/nvidia": "limits.devices.com/nvidia",
+ "devices.com/nvidia": "requests.devices.com/nvidia",
+ "hugepages-2Mi": "requests.hugepages-2Mi",
"storage": "requests.storage",
"pods": "pods",
"services.loadbalancers": "services.loadbalancers",
}
for raw, want := range cases {
- if got := renderedLimitKey(raw); got != want {
- t.Errorf("renderedLimitKey(%q) = %q, want %q", raw, got, want)
+ if got := usageKey(raw); got != want {
+ t.Errorf("usageKey(%q) = %q, want %q", raw, got, want)
}
}
}
@@ -339,6 +340,26 @@ func TestValidateTenantResourceQuotas_PoolUsage(t *testing.T) {
}
})
+ t.Run("parent usage of an extended resource shrinks remaining", func(t *testing.T) {
+ parent := tenantHelmRelease(t, "foo", "tenant-root", map[string]string{"example.com/device": "2"})
+ used := usageQuota(t, "tenant-foo", "tenant-quota", map[string]string{"requests.example.com/device": "2"})
+ r := newTenantREST(t, parent, used)
+
+ if errs := r.validateTenantResourceQuotas(context.Background(), childApplication(t, "bar", "tenant-foo", map[string]string{"example.com/device": "1"})); len(errs) == 0 {
+ t.Fatalf("child example.com/device=1 should be denied when the parent pool already uses 2 of 2")
+ }
+ })
+
+ t.Run("parent usage of an extended resource leaves the rest grantable", func(t *testing.T) {
+ parent := tenantHelmRelease(t, "foo", "tenant-root", map[string]string{"example.com/device": "2"})
+ used := usageQuota(t, "tenant-foo", "tenant-quota", map[string]string{"requests.example.com/device": "1"})
+ r := newTenantREST(t, parent, used)
+
+ if errs := r.validateTenantResourceQuotas(context.Background(), childApplication(t, "bar", "tenant-foo", map[string]string{"example.com/device": "1"})); len(errs) > 0 {
+ t.Fatalf("child example.com/device=1 should be allowed when the parent pool uses 1 of 2, got: %v", errs)
+ }
+ })
+
// No rendered usage yet (freshly-created parent, Flux has not rendered the
// quota): usage degrades to zero and behaviour matches the pre-usage gate.
t.Run("absent usage degrades to budget-only check", func(t *testing.T) {
From 7de70ac2fc64a03698f131c883474da73945620a Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 04:11:15 +0700
Subject: [PATCH 020/346] feat(monitoring-agents): export PVC labels from
kube-state-metrics
The tenant metrics API scopes a query to one application by looking up
its pods and PVCs through their apps.cozystack.io/application.* lineage
labels. kube-state-metrics emits kube_persistentvolumeclaim_labels only
for resources on --metric-labels-allowlist, so without PVCs there the
PVC half of that lookup always comes back empty and volume series of
an application fall out of its scope.
All labels are allowed, as for pods and deployments: kube-state-metrics
emits one series per PVC whatever keys are listed, and PVCs are far
fewer than pods, which already export every label.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../tests/kube_state_metrics_labels_test.yaml | 15 +++++++++++++++
packages/system/monitoring-agents/values.yaml | 14 ++++++++++----
2 files changed, 25 insertions(+), 4 deletions(-)
create mode 100644 packages/system/monitoring-agents/tests/kube_state_metrics_labels_test.yaml
diff --git a/packages/system/monitoring-agents/tests/kube_state_metrics_labels_test.yaml b/packages/system/monitoring-agents/tests/kube_state_metrics_labels_test.yaml
new file mode 100644
index 000000000..578371a1e
--- /dev/null
+++ b/packages/system/monitoring-agents/tests/kube_state_metrics_labels_test.yaml
@@ -0,0 +1,15 @@
+suite: kube-state-metrics exports the labels of pods, deployments and PVCs
+# The tenant metrics API scopes a query to one application by looking up the
+# application's pods and PVCs in kube_pod_labels and
+# kube_persistentvolumeclaim_labels by their apps.cozystack.io/application.*
+# lineage labels. kube-state-metrics emits no kube_*_labels series at all for
+# a resource missing from this allowlist, so dropping persistentvolumeclaims
+# silently empties the PVC branch of that lookup.
+templates:
+ - charts/kube-state-metrics/templates/deployment.yaml
+tests:
+ - it: allowlists all labels of pods, deployments and persistentvolumeclaims
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].args
+ content: --metric-labels-allowlist=pods=[*],deployments=[*],persistentvolumeclaims=[*]
diff --git a/packages/system/monitoring-agents/values.yaml b/packages/system/monitoring-agents/values.yaml
index a9c22cb14..84c39ce44 100644
--- a/packages/system/monitoring-agents/values.yaml
+++ b/packages/system/monitoring-agents/values.yaml
@@ -6,9 +6,10 @@ kube-state-metrics:
# 322 pods and roughly 25.6k series, which is the smallest cluster we run. The
# request sits above the 176Mi mean rather than at the peak, and the limit
# carries wide headroom on purpose: this component's memory scales with the
- # number of objects in the cluster, and --metric-labels-allowlist below widens
- # every pod and deployment series further, so a cluster larger than the one
- # measured will land well above the peak seen here.
+ # number of objects in the cluster, and --metric-labels-allowlist below copies
+ # every label of each pod, deployment and PVC onto its kube_*_labels series,
+ # so a cluster larger than the one measured will land well above the peak
+ # seen here.
# 2Gi is 6.7x that peak, so scaling linearly with object count it stays a
# ceiling rather than a bound up to roughly 2100 pods and 170k series.
#
@@ -27,7 +28,12 @@ kube-state-metrics:
limits:
memory: 2Gi
extraArgs:
- - --metric-labels-allowlist=pods=[*],deployments=[*]
+ # PVCs are listed so the tenant metrics API can find an application's PVCs
+ # in kube_persistentvolumeclaim_labels by their apps.cozystack.io/application.*
+ # lineage labels. kube-state-metrics emits that metric only for resources on
+ # this list, one series per PVC; naming only the lineage keys would emit the
+ # same number of series.
+ - --metric-labels-allowlist=pods=[*],deployments=[*],persistentvolumeclaims=[*]
rbac:
extraRules:
- apiGroups:
From f98ee76d2db3ab3acea796ce981374461e82caaa Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 04:16:56 +0700
Subject: [PATCH 021/346] feat(monitoring): stamp application identity on
cluster-wide scrapes
Scoping a metrics query to one application needs the application's
identity on the series themselves, and it is only a boundary if the
scraped pod cannot choose it. A default scrape class on the cluster-wide
vmagent copies the pod's lineage labels into
apps_cozystack_io_application_{group,kind,name} at scrape time.
The target always carries all three labels, with a placeholder when the
pod has no lineage, so an exporter's own copy always collides, is
renamed to exported_* under honorLabels false, and is dropped. Without
the placeholder a pod lacking lineage could publish any identity
unchallenged. The placeholder is removed again before storage, so series
of pods without lineage look as they did before.
Scrapes with honorLabels true keep the exporter's value; none of them
scrapes a tenant-controlled exporter.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../monitoring-agents/templates/vmagent.yaml | 25 +++++++
.../vmagent_application_identity_test.yaml | 67 +++++++++++++++++++
2 files changed, 92 insertions(+)
create mode 100644 packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml
diff --git a/packages/system/monitoring-agents/templates/vmagent.yaml b/packages/system/monitoring-agents/templates/vmagent.yaml
index 00ab63c10..f0b66bd59 100644
--- a/packages/system/monitoring-agents/templates/vmagent.yaml
+++ b/packages/system/monitoring-agents/templates/vmagent.yaml
@@ -20,6 +20,31 @@ spec:
scrapeInterval: 30s
selectAllByDefault: true
+ # The target always carries all three identity labels, "-" when the pod
+ # has no lineage ("-" is not a valid Kubernetes label value, so it never
+ # collides with a real one). That makes every exporter-supplied copy
+ # collide and be renamed to exported_* (honorLabels false), where it is
+ # dropped; the placeholder itself is removed before storage.
+ scrapeClasses:
+ - name: application-identity
+ default: true
+ relabelConfigs:
+ {{- range list "group" "kind" "name" }}
+ - targetLabel: apps_cozystack_io_application_{{ . }}
+ replacement: "-"
+ - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_{{ . }}]
+ regex: (.+)
+ targetLabel: apps_cozystack_io_application_{{ . }}
+ {{- end }}
+ metricRelabelConfigs:
+ - action: labeldrop
+ regex: (exported_)+apps_cozystack_io_application_(group|kind|name)
+ {{- range list "group" "kind" "name" }}
+ - sourceLabels: [apps_cozystack_io_application_{{ . }}]
+ regex: "-"
+ targetLabel: apps_cozystack_io_application_{{ . }}
+ replacement: ""
+ {{- end }}
additionalScrapeConfigs:
name: additional-scrape-configs
key: prometheus-additional.yaml
diff --git a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml
new file mode 100644
index 000000000..601f1c926
--- /dev/null
+++ b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml
@@ -0,0 +1,67 @@
+suite: cluster-wide vmagent stamps application identity on exporter series
+templates:
+ - templates/vmagent.yaml
+release:
+ name: monitoring-agents
+ namespace: cozy-monitoring
+
+# The tenant-resource-metrics proxy scopes a query to one application by
+# apps_cozystack_io_application_{group,kind,name}. The labels are only a
+# boundary if the scraped pod cannot choose them: the target always carries
+# all three (a placeholder when the pod has no lineage), so with
+# honorLabels false an exporter's own copy is renamed to exported_* and
+# dropped. The same rules live in the tenant vmagent of the monitoring
+# package and are pinned there by an identical suite.
+tests:
+ - it: every scrape gets the application identity class unless it names another
+ asserts:
+ - equal:
+ path: spec.scrapeClasses[0].name
+ value: application-identity
+ - equal:
+ path: spec.scrapeClasses[0].default
+ value: true
+ - lengthEqual:
+ path: spec.scrapeClasses
+ count: 1
+
+ - it: the identity comes from the scraped pod's lineage labels, with a placeholder when the pod has none
+ asserts:
+ - equal:
+ path: spec.scrapeClasses[0].relabelConfigs
+ value:
+ - targetLabel: apps_cozystack_io_application_group
+ replacement: "-"
+ - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_group]
+ regex: (.+)
+ targetLabel: apps_cozystack_io_application_group
+ - targetLabel: apps_cozystack_io_application_kind
+ replacement: "-"
+ - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_kind]
+ regex: (.+)
+ targetLabel: apps_cozystack_io_application_kind
+ - targetLabel: apps_cozystack_io_application_name
+ replacement: "-"
+ - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_name]
+ regex: (.+)
+ targetLabel: apps_cozystack_io_application_name
+
+ - it: an exporter-supplied identity is dropped and the placeholder never reaches storage
+ asserts:
+ - equal:
+ path: spec.scrapeClasses[0].metricRelabelConfigs
+ value:
+ - action: labeldrop
+ regex: (exported_)+apps_cozystack_io_application_(group|kind|name)
+ - sourceLabels: [apps_cozystack_io_application_group]
+ regex: "-"
+ targetLabel: apps_cozystack_io_application_group
+ replacement: ""
+ - sourceLabels: [apps_cozystack_io_application_kind]
+ regex: "-"
+ targetLabel: apps_cozystack_io_application_kind
+ replacement: ""
+ - sourceLabels: [apps_cozystack_io_application_name]
+ regex: "-"
+ targetLabel: apps_cozystack_io_application_name
+ replacement: ""
From 14f815a5cc543ba35eeec1d407cc07a3b5b7fea1 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 04:16:56 +0700
Subject: [PATCH 022/346] feat(monitoring): stamp application identity on
tenant vmagent scrapes
Application pods in a tenant's namespaces are also scraped by the
tenant's own vmagent, and its series need the same identity labels and
the same protection against an exporter choosing them, or the two stores
would disagree about which application a series belongs to. The class
is identical to the cluster-wide one in monitoring-agents.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../monitoring/templates/vm/vmagent.yaml | 25 ++++++++
.../vmagent_application_identity_test.yaml | 62 +++++++++++++++++++
2 files changed, 87 insertions(+)
create mode 100644 packages/system/monitoring/tests/vmagent_application_identity_test.yaml
diff --git a/packages/system/monitoring/templates/vm/vmagent.yaml b/packages/system/monitoring/templates/vm/vmagent.yaml
index 98bff1e14..f65cd287e 100644
--- a/packages/system/monitoring/templates/vm/vmagent.yaml
+++ b/packages/system/monitoring/templates/vm/vmagent.yaml
@@ -24,6 +24,31 @@ spec:
scrapeInterval: 30s
selectAllByDefault: false
+ # The target always carries all three identity labels, "-" when the pod
+ # has no lineage ("-" is not a valid Kubernetes label value, so it never
+ # collides with a real one). That makes every exporter-supplied copy
+ # collide and be renamed to exported_* (honorLabels false), where it is
+ # dropped; the placeholder itself is removed before storage.
+ scrapeClasses:
+ - name: application-identity
+ default: true
+ relabelConfigs:
+ {{- range list "group" "kind" "name" }}
+ - targetLabel: apps_cozystack_io_application_{{ . }}
+ replacement: "-"
+ - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_{{ . }}]
+ regex: (.+)
+ targetLabel: apps_cozystack_io_application_{{ . }}
+ {{- end }}
+ metricRelabelConfigs:
+ - action: labeldrop
+ regex: (exported_)+apps_cozystack_io_application_(group|kind|name)
+ {{- range list "group" "kind" "name" }}
+ - sourceLabels: [apps_cozystack_io_application_{{ . }}]
+ regex: "-"
+ targetLabel: apps_cozystack_io_application_{{ . }}
+ replacement: ""
+ {{- end }}
{{- with .Values.vmagent.inlineScrapeConfig }}
inlineScrapeConfig: |
{{- . | nindent 4 }}
diff --git a/packages/system/monitoring/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml
new file mode 100644
index 000000000..f36105f83
--- /dev/null
+++ b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml
@@ -0,0 +1,62 @@
+suite: tenant vmagent stamps application identity on exporter series
+templates:
+ - templates/vm/vmagent.yaml
+release:
+ name: monitoring
+ namespace: tenant-foo
+
+# Same rules as the cluster-wide vmagent in monitoring-agents, pinned there
+# by an identical suite; see it for why the placeholder and the drop exist.
+tests:
+ - it: every scrape gets the application identity class unless it names another
+ asserts:
+ - equal:
+ path: spec.scrapeClasses[0].name
+ value: application-identity
+ - equal:
+ path: spec.scrapeClasses[0].default
+ value: true
+ - lengthEqual:
+ path: spec.scrapeClasses
+ count: 1
+
+ - it: the identity comes from the scraped pod's lineage labels, with a placeholder when the pod has none
+ asserts:
+ - equal:
+ path: spec.scrapeClasses[0].relabelConfigs
+ value:
+ - targetLabel: apps_cozystack_io_application_group
+ replacement: "-"
+ - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_group]
+ regex: (.+)
+ targetLabel: apps_cozystack_io_application_group
+ - targetLabel: apps_cozystack_io_application_kind
+ replacement: "-"
+ - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_kind]
+ regex: (.+)
+ targetLabel: apps_cozystack_io_application_kind
+ - targetLabel: apps_cozystack_io_application_name
+ replacement: "-"
+ - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_name]
+ regex: (.+)
+ targetLabel: apps_cozystack_io_application_name
+
+ - it: an exporter-supplied identity is dropped and the placeholder never reaches storage
+ asserts:
+ - equal:
+ path: spec.scrapeClasses[0].metricRelabelConfigs
+ value:
+ - action: labeldrop
+ regex: (exported_)+apps_cozystack_io_application_(group|kind|name)
+ - sourceLabels: [apps_cozystack_io_application_group]
+ regex: "-"
+ targetLabel: apps_cozystack_io_application_group
+ replacement: ""
+ - sourceLabels: [apps_cozystack_io_application_kind]
+ regex: "-"
+ targetLabel: apps_cozystack_io_application_kind
+ replacement: ""
+ - sourceLabels: [apps_cozystack_io_application_name]
+ regex: "-"
+ targetLabel: apps_cozystack_io_application_name
+ replacement: ""
From e5f17148368f6187fd36c7fbd0348ed033d98630 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 04:12:02 +0700
Subject: [PATCH 023/346] feat(monitoring): keep application identity on
container log lines
A reader that scopes logs to one application needs the owning
application on every line, and the lineage webhook already puts it on
the pod as apps.cozystack.io/application.{group,kind,name}. Fluent Bit
fetched those labels and then dropped them with the rest of the
kubernetes map, so the store could only be scoped by namespace.
Keep the three labels as apps_cozystack_io_application_{group,kind,name},
the names the metrics side uses, and make them stream fields on every
container-log output so the scope is an indexed stream filter. A pod
without the labels produces no such fields, and VictoriaLogs builds the
stream from the fields present, so existing streams keep their
identity; a labelled pod starts a new stream once.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../fluent-bit_application_identity_test.yaml | 67 +++++++++++++++++++
packages/system/monitoring-agents/values.yaml | 20 +++++-
2 files changed, 85 insertions(+), 2 deletions(-)
create mode 100644 packages/system/monitoring-agents/tests/fluent-bit_application_identity_test.yaml
diff --git a/packages/system/monitoring-agents/tests/fluent-bit_application_identity_test.yaml b/packages/system/monitoring-agents/tests/fluent-bit_application_identity_test.yaml
new file mode 100644
index 000000000..d1815eb1b
--- /dev/null
+++ b/packages/system/monitoring-agents/tests/fluent-bit_application_identity_test.yaml
@@ -0,0 +1,67 @@
+suite: fluent-bit keeps the application identity of a container log line
+
+# The lineage webhook labels a pod with apps.cozystack.io/application.{group,
+# kind,name}. A container log line keeps them as top-level
+# apps_cozystack_io_application_{group,kind,name}, the names the metrics side
+# uses, and they are stream fields so a reader can scope to one application
+# with extra_stream_filters. The kubernetes map, labels included, is otherwise
+# dropped by Remove_wildcard kubernetes_, so the three keys have to be lifted
+# and renamed out of the kubernetes_ prefix before that filter runs.
+templates:
+ - charts/fluent-bit/templates/configmap.yaml
+release:
+ name: monitoring-agents
+ namespace: cozy-monitoring
+set:
+ global:
+ target: tenant-root
+kubernetesProvider:
+ scheme:
+ "v1/Namespace":
+ gvr:
+ version: v1
+ resource: namespaces
+ namespaced: false
+ objects:
+ - kind: Namespace
+ apiVersion: v1
+ metadata:
+ name: tenant-a
+ labels:
+ namespace.cozystack.io/monitoring: tenant-a
+tests:
+ - it: lifts the pod labels after the kubernetes map is lifted
+ asserts:
+ - matchRegex:
+ path: data["fluent-bit.conf"]
+ pattern: '(?s)Operation lift\n\s+Nested_under kubernetes\n.*Name nest\n\s+Match kube\.\*\n\s+Operation lift\n\s+Nested_under kubernetes_labels\n\s+Add_prefix kubernetes_labels_\n'
+
+ - it: renames the three lineage labels out of the kubernetes_ prefix before Remove_wildcard
+ asserts:
+ - matchRegex:
+ path: data["fluent-bit.conf"]
+ pattern: '(?s)Nested_under kubernetes_labels\n.*Rename kubernetes_labels_apps\.cozystack\.io/application\.group apps_cozystack_io_application_group\n\s+Rename kubernetes_labels_apps\.cozystack\.io/application\.kind apps_cozystack_io_application_kind\n\s+Rename kubernetes_labels_apps\.cozystack\.io/application\.name apps_cozystack_io_application_name\n.*Remove_wildcard kubernetes_\n'
+
+ - it: makes the lineage keys stream fields on the default container-log output
+ asserts:
+ - matchRegex:
+ path: data["fluent-bit.conf"]
+ pattern: 'Match kube\.\*\n\s+Host vlinsert-generic\.tenant-root\.svc\n\s+port 9481\n\s+compress gzip\n\s+uri /insert/jsonline\?_stream_fields=log_source,stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name,apps_cozystack_io_application_group,apps_cozystack_io_application_kind,apps_cozystack_io_application_name&'
+
+ - it: makes the lineage keys stream fields on a routed tenant's container-log output
+ asserts:
+ - matchRegex:
+ path: data["fluent-bit.conf"]
+ pattern: 'Match tenant\.tenant-a\.\*\n\s+Host vlinsert-generic\.tenant-a\.svc\n\s+port 9481\n\s+compress gzip\n\s+uri /insert/jsonline\?_stream_fields=log_source,stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name,apps_cozystack_io_application_group,apps_cozystack_io_application_kind,apps_cozystack_io_application_name&'
+
+ - it: leaves the events and audit outputs without lineage stream fields
+ asserts:
+ - matchRegex:
+ path: data["fluent-bit.conf"]
+ pattern: 'uri /insert/jsonline\?_stream_fields=log_source,reason,metadata_namespace,metadata_name&'
+ - matchRegex:
+ path: data["fluent-bit.conf"]
+ pattern: 'uri /insert/jsonline\?_stream_fields=log_source,stage,user_username,verb,requestUri&'
+ - notMatchRegex:
+ path: data["fluent-bit.conf"]
+ pattern: 'Match (events\.\*|audit\.\*|\*)\n[^\[]*Nested_under kubernetes_labels'
diff --git a/packages/system/monitoring-agents/values.yaml b/packages/system/monitoring-agents/values.yaml
index a9c22cb14..186828f70 100644
--- a/packages/system/monitoring-agents/values.yaml
+++ b/packages/system/monitoring-agents/values.yaml
@@ -405,7 +405,7 @@ fluent-bit:
Host vlinsert-generic.{{ .Values.global.target }}.svc
port 9481
compress gzip
- uri /insert/jsonline?_stream_fields=log_source,stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name&_msg_field=log&_time_field=date
+ uri /insert/jsonline?_stream_fields=log_source,stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name,apps_cozystack_io_application_group,apps_cozystack_io_application_kind,apps_cozystack_io_application_name&_msg_field=log&_time_field=date
format json_lines
json_date_format iso8601
header AccountID 0
@@ -439,7 +439,7 @@ fluent-bit:
Host vlinsert-generic.{{ $target }}.svc
port 9481
compress gzip
- uri /insert/jsonline?_stream_fields=log_source,stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name&_msg_field=log&_time_field=date
+ uri /insert/jsonline?_stream_fields=log_source,stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name,apps_cozystack_io_application_group,apps_cozystack_io_application_kind,apps_cozystack_io_application_name&_msg_field=log&_time_field=date
format json_lines
json_date_format iso8601
header AccountID 0
@@ -461,6 +461,19 @@ fluent-bit:
Operation lift
Nested_under kubernetes
Add_prefix kubernetes_
+ {{- /*
+ The pod's lineage labels name the application it belongs to. lift
+ cannot pick keys, so every label comes up; all but these three keep
+ the kubernetes_ prefix and fall to Remove_wildcard below. An absent
+ label stays absent, and VictoriaLogs builds _stream from the stream
+ fields that are present, so an unlabelled pod keeps its stream.
+ */}}
+ [FILTER]
+ Name nest
+ Match kube.*
+ Operation lift
+ Nested_under kubernetes_labels
+ Add_prefix kubernetes_labels_
[FILTER]
Name modify
Match kube.*
@@ -501,6 +514,9 @@ fluent-bit:
Rename kubernetes_pod_name keep_kubernetes_pod_name
Rename kubernetes_namespace_name keep_kubernetes_namespace_name
Rename kubernetes_container_name keep_kubernetes_container_name
+ Rename kubernetes_labels_apps.cozystack.io/application.group apps_cozystack_io_application_group
+ Rename kubernetes_labels_apps.cozystack.io/application.kind apps_cozystack_io_application_kind
+ Rename kubernetes_labels_apps.cozystack.io/application.name apps_cozystack_io_application_name
[FILTER]
Name modify
Match kube.*
From 95dc5911981adf1ad37aca6c949f33c90708d2a2 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 04:12:02 +0700
Subject: [PATCH 024/346] test(kubernetes): keep application identity out of
guest log streams
Guest pods are tenant-controlled, so their lineage labels are not a
trustworthy application identity, and the platform must not promote
them to the stream fields the parent scopes application logs by. The
guest release replaces the monitoring-agents filters and outputs
wholesale, which is what keeps the parent's label lift out of guests;
pin that so a later change to the guest overrides cannot bring it in.
This only covers what the platform ships. A guest admin can still edit
the guest's fluent-bit or write to the parent's vlinsert directly; that
needs a write-path control, which the logs proposal leaves open.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
...ring_agents_application_identity_test.yaml | 38 +++++++++++++++++++
1 file changed, 38 insertions(+)
create mode 100644 packages/apps/kubernetes/tests/monitoring_agents_application_identity_test.yaml
diff --git a/packages/apps/kubernetes/tests/monitoring_agents_application_identity_test.yaml b/packages/apps/kubernetes/tests/monitoring_agents_application_identity_test.yaml
new file mode 100644
index 000000000..9fbc1c45b
--- /dev/null
+++ b/packages/apps/kubernetes/tests/monitoring_agents_application_identity_test.yaml
@@ -0,0 +1,38 @@
+suite: guest fluent-bit never ships application identity stream fields
+
+templates:
+ - templates/helmreleases/monitoring-agents.yaml
+
+values:
+ - values/common.yaml
+
+release:
+ name: kubernetes-test
+ namespace: tenant-test
+
+# Pods in a guest cluster are tenant-controlled, so their
+# apps.cozystack.io/application.* labels are not a trustworthy application
+# identity, and the platform must not promote them to the stream fields the
+# parent scopes application logs by. The guest release replaces the
+# monitoring-agents filters and outputs, so it is the guest overrides that keep
+# the parent's label lift out. This does not stop a guest admin who edits the
+# guest's own fluent-bit or writes to the parent's vlinsert directly; that is
+# the write-path control the logs proposal leaves open.
+
+tests:
+ - it: does not lift pod labels or declare lineage stream fields in the guest
+ set:
+ addons.monitoringAgents.enabled: true
+ _namespace.monitoring: tenant-test
+ asserts:
+ - isKind:
+ of: HelmRelease
+ - notMatchRegex:
+ path: spec.values.fluent-bit.config.filters
+ pattern: 'kubernetes_labels|apps_cozystack_io_application_'
+ - notMatchRegex:
+ path: spec.values.fluent-bit.config.outputs
+ pattern: 'apps_cozystack_io_application_'
+ - matchRegex:
+ path: spec.values.fluent-bit.config.outputs
+ pattern: '_stream_fields=stream,kubernetes_pod_name,kubernetes_container_name,kubernetes_namespace_name&'
From 0e5557d0baef802125b88d438d741055da5ee408 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 04:26:04 +0700
Subject: [PATCH 025/346] fix(ingress): keep application identity off
ingress-nginx series
The vmagent scrape class stamps the scraped pod's application identity
on its series. ingress-nginx series describe the Ingress objects the
controller serves, in whatever namespace they live, so the controller's
own identity would attribute another tenant's traffic to the Ingress
application. And because these scrapes honor exporter labels, an
exporter-supplied identity would win over the class. Dropping the labels
leaves these series to the pod and namespace scopes.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../extra/ingress/templates/nginx-scrape.yaml | 10 +++++++++
.../tests/application_identity_test.yaml | 21 +++++++++++++++++++
2 files changed, 31 insertions(+)
create mode 100644 packages/extra/ingress/tests/application_identity_test.yaml
diff --git a/packages/extra/ingress/templates/nginx-scrape.yaml b/packages/extra/ingress/templates/nginx-scrape.yaml
index d04c5b651..780f5e3a9 100644
--- a/packages/extra/ingress/templates/nginx-scrape.yaml
+++ b/packages/extra/ingress/templates/nginx-scrape.yaml
@@ -9,6 +9,10 @@ spec:
matchNames:
- {{ .Release.Namespace }}
podMetricsEndpoints:
+ # The series describe the Ingress objects served, in any namespace, not
+ # this controller, so the application identity the vmagent scrape class
+ # stamps from the controller pod would misattribute them; and with
+ # honorLabels an exporter-supplied identity would win over the class.
- port: metrics
honorLabels: true
interval: 30s
@@ -19,6 +23,9 @@ spec:
targetLabel: node
- targetLabel: tier
replacement: cluster
+ metricRelabelConfigs:
+ - action: labeldrop
+ regex: apps_cozystack_io_application_(group|kind|name)
selector:
matchLabels:
app.kubernetes.io/component: controller
@@ -46,6 +53,9 @@ spec:
targetLabel: controller
- targetLabel: tier
replacement: cluster
+ metricRelabelConfigs:
+ - action: labeldrop
+ regex: apps_cozystack_io_application_(group|kind|name)
selector:
matchLabels:
app.kubernetes.io/component: controller
diff --git a/packages/extra/ingress/tests/application_identity_test.yaml b/packages/extra/ingress/tests/application_identity_test.yaml
new file mode 100644
index 000000000..e1be5ca4f
--- /dev/null
+++ b/packages/extra/ingress/tests/application_identity_test.yaml
@@ -0,0 +1,21 @@
+suite: ingress-nginx series do not carry an application identity
+templates:
+ - templates/nginx-scrape.yaml
+release:
+ name: ingress
+ namespace: tenant-root
+
+# The vmagent scrape class stamps the scraped pod's application identity on
+# every series. ingress-nginx series describe the Ingress objects it serves,
+# in whatever namespace they live (honorLabels keeps their namespace), so
+# the controller's identity would misattribute them, and with honorLabels
+# true an exporter-supplied identity would win over the class. Both
+# endpoints drop the identity instead.
+tests:
+ - it: drops the identity on every endpoint
+ asserts:
+ - equal:
+ path: spec.podMetricsEndpoints[0].metricRelabelConfigs
+ value:
+ - action: labeldrop
+ regex: apps_cozystack_io_application_(group|kind|name)
From dca95eb753fa31271eb7b3cbbe3a0312da138846 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 04:26:04 +0700
Subject: [PATCH 026/346] docs(monitoring): state when the identity scrape
class can be overridden
The exported_* rename only protects scrapes that do not honor exporter
labels, and a scrape object's own relabel rules run after the class, so
both can undo it. Saying so next to the rules keeps the next scrape
author from assuming the protection is unconditional.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
packages/system/monitoring-agents/templates/vmagent.yaml | 8 +++++---
packages/system/monitoring/templates/vm/vmagent.yaml | 8 +++++---
2 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/packages/system/monitoring-agents/templates/vmagent.yaml b/packages/system/monitoring-agents/templates/vmagent.yaml
index f0b66bd59..29349f17e 100644
--- a/packages/system/monitoring-agents/templates/vmagent.yaml
+++ b/packages/system/monitoring-agents/templates/vmagent.yaml
@@ -22,9 +22,11 @@ spec:
selectAllByDefault: true
# The target always carries all three identity labels, "-" when the pod
# has no lineage ("-" is not a valid Kubernetes label value, so it never
- # collides with a real one). That makes every exporter-supplied copy
- # collide and be renamed to exported_* (honorLabels false), where it is
- # dropped; the placeholder itself is removed before storage.
+ # collides with a real one). Under honorLabels false every exporter-supplied
+ # copy then collides, is renamed to exported_* and dropped. Under
+ # honorLabels true the exporter's copy wins, so such a scrape must drop
+ # these labels itself. A scrape object's own relabel rules run after these
+ # and must leave the labels alone. The placeholder is removed before storage.
scrapeClasses:
- name: application-identity
default: true
diff --git a/packages/system/monitoring/templates/vm/vmagent.yaml b/packages/system/monitoring/templates/vm/vmagent.yaml
index f65cd287e..fa4c4fc93 100644
--- a/packages/system/monitoring/templates/vm/vmagent.yaml
+++ b/packages/system/monitoring/templates/vm/vmagent.yaml
@@ -26,9 +26,11 @@ spec:
selectAllByDefault: false
# The target always carries all three identity labels, "-" when the pod
# has no lineage ("-" is not a valid Kubernetes label value, so it never
- # collides with a real one). That makes every exporter-supplied copy
- # collide and be renamed to exported_* (honorLabels false), where it is
- # dropped; the placeholder itself is removed before storage.
+ # collides with a real one). Under honorLabels false every exporter-supplied
+ # copy then collides, is renamed to exported_* and dropped. Under
+ # honorLabels true the exporter's copy wins, so such a scrape must drop
+ # these labels itself. A scrape object's own relabel rules run after these
+ # and must leave the labels alone. The placeholder is removed before storage.
scrapeClasses:
- name: application-identity
default: true
From 5a6086c7a1756499f10fd869e20e7bd3c14a5e17 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 04:30:32 +0700
Subject: [PATCH 027/346] test(monitoring): pin the endpoint count the ingress
identity test relies on
The ingress assertion reads only the first endpoint of each scrape, so a
second endpoint added without the label drop would go unnoticed. Pinning
the count makes such an addition fail the suite.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
packages/extra/ingress/tests/application_identity_test.yaml | 3 +++
.../tests/vmagent_application_identity_test.yaml | 2 +-
.../monitoring/tests/vmagent_application_identity_test.yaml | 3 ++-
3 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/packages/extra/ingress/tests/application_identity_test.yaml b/packages/extra/ingress/tests/application_identity_test.yaml
index e1be5ca4f..0e53a2756 100644
--- a/packages/extra/ingress/tests/application_identity_test.yaml
+++ b/packages/extra/ingress/tests/application_identity_test.yaml
@@ -14,6 +14,9 @@ release:
tests:
- it: drops the identity on every endpoint
asserts:
+ - lengthEqual:
+ path: spec.podMetricsEndpoints
+ count: 1
- equal:
path: spec.podMetricsEndpoints[0].metricRelabelConfigs
value:
diff --git a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml
index 601f1c926..b59f9e3ee 100644
--- a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml
+++ b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml
@@ -11,7 +11,7 @@ release:
# all three (a placeholder when the pod has no lineage), so with
# honorLabels false an exporter's own copy is renamed to exported_* and
# dropped. The same rules live in the tenant vmagent of the monitoring
-# package and are pinned there by an identical suite.
+# package and are pinned there by the same assertions.
tests:
- it: every scrape gets the application identity class unless it names another
asserts:
diff --git a/packages/system/monitoring/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml
index f36105f83..da14330de 100644
--- a/packages/system/monitoring/tests/vmagent_application_identity_test.yaml
+++ b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml
@@ -6,7 +6,8 @@ release:
namespace: tenant-foo
# Same rules as the cluster-wide vmagent in monitoring-agents, pinned there
-# by an identical suite; see it for why the placeholder and the drop exist.
+# by the same assertions; see that suite for why the placeholder and the
+# drop exist.
tests:
- it: every scrape gets the application identity class unless it names another
asserts:
From e96ceedd7ebd6a8e138e8b363ef36a2ffbf7e20f Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:25:58 +0700
Subject: [PATCH 028/346] feat(dashboard): add a console client for the
observability proxy
The observability proxy serves metrics and logs under Kubernetes-shaped
paths but answers in native Prometheus JSON and VictoriaLogs NDJSON, so
the typed K8sClient methods cannot read it. A raw fetch on the client
keeps the same credentials and 401 handling the rest of the console
uses, and the proxy module maps each documented status to its own
error kind so a cluster without the proxy (404) can be told apart from
a denied request or a store that is down.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../__tests__/k8s-client/client.raw.test.ts | 47 ++++
.../src/lib/observability/proxy.test.ts | 179 +++++++++++++++
.../console/src/lib/observability/proxy.ts | 203 ++++++++++++++++++
.../console/packages/k8s-client/src/client.ts | 18 ++
4 files changed, 447 insertions(+)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
diff --git a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
new file mode 100644
index 000000000..ab6c056f5
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
@@ -0,0 +1,47 @@
+import { describe, it, expect, vi, afterEach } from "vitest"
+import { K8sClient } from "@cozystack/k8s-client"
+
+afterEach(() => {
+ vi.unstubAllGlobals()
+ vi.restoreAllMocks()
+})
+
+describe("K8sClient.raw", () => {
+ it("returns the response untouched, including a non-2xx one", async () => {
+ const res = new Response("not found", { status: 404 })
+ vi.stubGlobal("fetch", vi.fn(async () => res))
+ const client = new K8sClient()
+
+ await expect(client.raw("/x")).resolves.toBe(res)
+ })
+
+ it("prefixes the base URL and attaches the bearer token", async () => {
+ const fetchMock = vi.fn(async () => new Response("{}"))
+ vi.stubGlobal("fetch", fetchMock)
+ const client = new K8sClient({ baseUrl: "/base", getToken: async () => "t0k" })
+
+ await client.raw("/apis/x", { headers: { Accept: "application/json" } })
+
+ const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
+ expect(url).toBe("/base/apis/x")
+ expect(init.headers).toEqual({ Accept: "application/json", Authorization: "Bearer t0k" })
+ })
+
+ it("routes a 401 through the client's unauthorized handler", async () => {
+ const onUnauthorized = vi.fn()
+ vi.stubGlobal("fetch", vi.fn(async () => new Response("", { status: 401 })))
+ const client = new K8sClient({ onUnauthorized })
+
+ const res = await client.raw("/x")
+
+ expect(res.status).toBe(401)
+ expect(onUnauthorized).toHaveBeenCalledTimes(1)
+ })
+
+ it("does not swallow a network failure", async () => {
+ vi.stubGlobal("fetch", vi.fn(async () => { throw new TypeError("offline") }))
+ const client = new K8sClient()
+
+ await expect(client.raw("/x")).rejects.toThrow("offline")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
new file mode 100644
index 000000000..f06835ff7
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
@@ -0,0 +1,179 @@
+import { describe, it, expect, vi } from "vitest"
+import { K8sClient } from "@cozystack/k8s-client"
+import {
+ ProxyError,
+ errorKindForStatus,
+ fieldValues,
+ logsRoot,
+ metricsRoot,
+ queryLogs,
+ queryRange,
+ tailLogs,
+} from "./proxy.ts"
+
+const app = { namespace: "tenant-a", plural: "postgreses", name: "db" }
+const ns = { namespace: "tenant-a" }
+
+function clientReturning(...responses: Response[]) {
+ const client = new K8sClient({ onUnauthorized: () => {} })
+ const raw = vi.spyOn(client, "raw")
+ for (const r of responses) raw.mockResolvedValueOnce(r)
+ return { client, raw }
+}
+
+function calledUrl(raw: ReturnType["raw"], call = 0): URL {
+ return new URL(raw.mock.calls[call][0], "http://x")
+}
+
+describe("roots", () => {
+ it("builds the application roots under apps.cozystack.io", () => {
+ expect(metricsRoot(app)).toBe("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/")
+ expect(logsRoot(app)).toBe("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/")
+ })
+
+ it("builds the namespace roots under cozystack.io", () => {
+ expect(metricsRoot(ns)).toBe("/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/")
+ expect(logsRoot(ns)).toBe("/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/")
+ })
+})
+
+describe("errorKindForStatus", () => {
+ it.each([
+ [400, "bad-request"],
+ [401, "unauthorized"],
+ [403, "forbidden"],
+ [404, "not-available"],
+ [422, "limit"],
+ [429, "rate-limited"],
+ [502, "store-unreachable"],
+ [503, "unavailable"],
+ [504, "timeout"],
+ [500, "unknown"],
+ ])("maps %i to %s", (status, kind) => {
+ expect(errorKindForStatus(status)).toBe(kind)
+ })
+})
+
+describe("queryRange", () => {
+ it("GETs query_range under the root and returns the matrix", async () => {
+ const body = { status: "success", data: { resultType: "matrix", result: [{ metric: { pod: "p" }, values: [[1, "2"]] }] } }
+ const { client, raw } = clientReturning(Response.json(body))
+
+ const result = await queryRange(client, metricsRoot(app), { query: "up", start: 10, end: 20, step: 5 })
+
+ const url = calledUrl(raw)
+ expect(url.pathname).toBe(`${metricsRoot(app)}api/v1/query_range`)
+ expect(Object.fromEntries(url.searchParams)).toEqual({ query: "up", start: "10", end: "20", step: "5" })
+ expect(result).toEqual(body.data.result)
+ })
+
+ it("throws a ProxyError carrying the Prometheus error text", async () => {
+ const { client } = clientReturning(
+ Response.json({ status: "error", error: "range too long" }, { status: 422 }),
+ )
+
+ const err = await queryRange(client, metricsRoot(app), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
+
+ expect(err).toBeInstanceOf(ProxyError)
+ expect(err).toMatchObject({ kind: "limit", status: 422, message: "range too long" })
+ })
+
+ it("reports a fetch failure as a network error", async () => {
+ const client = new K8sClient()
+ vi.spyOn(client, "raw").mockRejectedValue(new TypeError("offline"))
+
+ const err = await queryRange(client, metricsRoot(app), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
+
+ expect(err).toMatchObject({ kind: "network" })
+ })
+})
+
+describe("queryLogs", () => {
+ it("parses NDJSON lines and sorts them by _time ascending", async () => {
+ const ndjson = [
+ JSON.stringify({ _time: "2026-11-04T10:00:02Z", _msg: "b" }),
+ "",
+ JSON.stringify({ _time: "2026-11-04T10:00:01Z", _msg: "a" }),
+ ].join("\n")
+ const { client, raw } = clientReturning(new Response(ndjson))
+
+ const entries = await queryLogs(client, logsRoot(ns), { query: "*", start: "s", end: "e", limit: 100 })
+
+ expect(calledUrl(raw).pathname).toBe(`${logsRoot(ns)}select/logsql/query`)
+ expect(Object.fromEntries(calledUrl(raw).searchParams)).toEqual({ query: "*", start: "s", end: "e", limit: "100" })
+ expect(entries.map((e) => e._msg)).toEqual(["a", "b"])
+ })
+
+ it("keeps a plain-text error body as the message", async () => {
+ const { client } = clientReturning(new Response("cannot parse query", { status: 400 }))
+
+ const err = await queryLogs(client, logsRoot(ns), { query: "(", start: "s", end: "e", limit: 1 }).catch((e) => e)
+
+ expect(err).toMatchObject({ kind: "bad-request", message: "cannot parse query" })
+ })
+})
+
+describe("fieldValues", () => {
+ it("uses stream_field_values for stream fields and returns the values", async () => {
+ const { client, raw } = clientReturning(
+ Response.json({ values: [{ value: "pod-b", hits: 1 }, { value: "pod-a", hits: 9 }] }),
+ )
+
+ const values = await fieldValues(client, logsRoot(app), {
+ field: "kubernetes_pod_name",
+ stream: true,
+ query: "*",
+ start: "s",
+ end: "e",
+ })
+
+ expect(calledUrl(raw).pathname).toBe(`${logsRoot(app)}select/logsql/stream_field_values`)
+ expect(calledUrl(raw).searchParams.get("field")).toBe("kubernetes_pod_name")
+ expect(values).toEqual(["pod-a", "pod-b"])
+ })
+
+ it("uses field_values for other fields", async () => {
+ const { client, raw } = clientReturning(Response.json({ values: [] }))
+
+ await fieldValues(client, logsRoot(app), { field: "level", stream: false, query: "*", start: "s", end: "e" })
+
+ expect(calledUrl(raw).pathname).toBe(`${logsRoot(app)}select/logsql/field_values`)
+ })
+})
+
+describe("tailLogs", () => {
+ it("delivers every line as it arrives, including one split across chunks", async () => {
+ const enc = new TextEncoder()
+ const chunks = ['{"_time":"t1","_msg":"a"}\n{"_ti', 'me":"t2","_msg":"b"}\n']
+ const body = new ReadableStream({
+ start(c) {
+ for (const ch of chunks) c.enqueue(enc.encode(ch))
+ c.close()
+ },
+ })
+ const { client, raw } = clientReturning(new Response(body))
+ const seen: string[] = []
+
+ await tailLogs(client, logsRoot(ns), { query: "*", since: "t0" }, (e) => seen.push(e._msg))
+
+ expect(calledUrl(raw).pathname).toBe(`${logsRoot(ns)}select/logsql/tail`)
+ expect(calledUrl(raw).searchParams.get("since")).toBe("t0")
+ expect(seen).toEqual(["a", "b"])
+ })
+
+ it("omits since on the first connect", async () => {
+ const { client, raw } = clientReturning(new Response(""))
+
+ await tailLogs(client, logsRoot(ns), { query: "*" }, () => {})
+
+ expect(calledUrl(raw).searchParams.has("since")).toBe(false)
+ })
+
+ it("throws a ProxyError when the tail is refused", async () => {
+ const { client } = clientReturning(new Response("", { status: 403 }))
+
+ const err = await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}).catch((e) => e)
+
+ expect(err).toMatchObject({ kind: "forbidden" })
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
new file mode 100644
index 000000000..a257825c9
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -0,0 +1,203 @@
+import type { K8sClient } from "@cozystack/k8s-client"
+
+export type ObservabilityScope =
+ | { namespace: string }
+ | { namespace: string; plural: string; name: string }
+
+function appBase(scope: ObservabilityScope): string | null {
+ if (!("plural" in scope)) return null
+ return `/apis/apps.cozystack.io/v1alpha1/namespaces/${scope.namespace}/${scope.plural}/${scope.name}`
+}
+
+export function metricsRoot(scope: ObservabilityScope): string {
+ const app = appBase(scope)
+ return app ? `${app}/metrics/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/metrics/`
+}
+
+export function logsRoot(scope: ObservabilityScope): string {
+ const app = appBase(scope)
+ return app ? `${app}/log/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/logs/`
+}
+
+export type ProxyErrorKind =
+ | "bad-request"
+ | "unauthorized"
+ | "forbidden"
+ | "not-available"
+ | "limit"
+ | "rate-limited"
+ | "store-unreachable"
+ | "unavailable"
+ | "timeout"
+ | "network"
+ | "unknown"
+
+const KIND_BY_STATUS: Record = {
+ 400: "bad-request",
+ 401: "unauthorized",
+ 403: "forbidden",
+ 404: "not-available",
+ 422: "limit",
+ 429: "rate-limited",
+ 502: "store-unreachable",
+ 503: "unavailable",
+ 504: "timeout",
+}
+
+export function errorKindForStatus(status: number): ProxyErrorKind {
+ return KIND_BY_STATUS[status] ?? "unknown"
+}
+
+export class ProxyError extends Error {
+ kind: ProxyErrorKind
+ status: number
+
+ constructor(kind: ProxyErrorKind, status: number, message: string) {
+ super(message)
+ this.name = "ProxyError"
+ this.kind = kind
+ this.status = status
+ }
+}
+
+// Prometheus answers {"error": "..."}, VictoriaLogs and nginx answer plain text.
+function errorMessage(text: string, status: number): string {
+ try {
+ const parsed: unknown = JSON.parse(text)
+ if (parsed && typeof parsed === "object" && "error" in parsed && typeof parsed.error === "string") {
+ return parsed.error
+ }
+ } catch {
+ // not JSON
+ }
+ const trimmed = text.trim()
+ return trimmed && trimmed.length < 300 && !trimmed.startsWith("<") ? trimmed : `Server returned ${status}`
+}
+
+async function proxyGet(
+ client: K8sClient,
+ path: string,
+ params: Record,
+ signal?: AbortSignal,
+): Promise {
+ const search = new URLSearchParams()
+ for (const [k, v] of Object.entries(params)) if (v !== undefined) search.set(k, String(v))
+ let res: Response
+ try {
+ res = await client.raw(`${path}?${search}`, { signal })
+ } catch (err) {
+ if (signal?.aborted) throw err
+ throw new ProxyError("network", 0, "Unable to connect to the server")
+ }
+ if (!res.ok) {
+ const text = await res.text().catch(() => "")
+ throw new ProxyError(errorKindForStatus(res.status), res.status, errorMessage(text, res.status))
+ }
+ return res
+}
+
+export interface PromSeries {
+ metric: Record
+ values: [number, string][]
+}
+
+export interface RangeParams {
+ query: string
+ start: number
+ end: number
+ step: number
+}
+
+export async function queryRange(
+ client: K8sClient,
+ root: string,
+ params: RangeParams,
+ signal?: AbortSignal,
+): Promise {
+ const res = await proxyGet(client, `${root}api/v1/query_range`, { ...params }, signal)
+ const body = (await res.json()) as { data?: { result?: PromSeries[] } }
+ return body.data?.result ?? []
+}
+
+export interface LogEntry {
+ _time: string
+ _msg: string
+ _stream_id?: string
+ [field: string]: string | undefined
+}
+
+function parseLine(line: string): LogEntry | null {
+ if (!line.trim()) return null
+ try {
+ const value: unknown = JSON.parse(line)
+ if (value && typeof value === "object" && "_time" in value) return value as LogEntry
+ } catch {
+ // a torn or non-JSON line carries nothing to show
+ }
+ return null
+}
+
+export interface LogsQueryParams {
+ query: string
+ start: string
+ end: string
+ limit: number
+}
+
+export async function queryLogs(
+ client: K8sClient,
+ root: string,
+ params: LogsQueryParams,
+ signal?: AbortSignal,
+): Promise {
+ const res = await proxyGet(client, `${root}select/logsql/query`, { ...params }, signal)
+ const entries = (await res.text()).split("\n").map(parseLine).filter((e) => e !== null)
+ return entries.sort((a, b) => (a._time < b._time ? -1 : a._time > b._time ? 1 : 0))
+}
+
+export interface FieldValuesParams {
+ field: string
+ stream: boolean
+ query: string
+ start: string
+ end: string
+}
+
+export async function fieldValues(
+ client: K8sClient,
+ root: string,
+ { stream, ...params }: FieldValuesParams,
+ signal?: AbortSignal,
+): Promise {
+ const endpoint = stream ? "stream_field_values" : "field_values"
+ const res = await proxyGet(client, `${root}select/logsql/${endpoint}`, { ...params }, signal)
+ const body = (await res.json()) as { values?: { value: string }[] }
+ return (body.values ?? []).map((v) => v.value).sort()
+}
+
+export async function tailLogs(
+ client: K8sClient,
+ root: string,
+ params: { query: string; since?: string },
+ onEntry: (entry: LogEntry) => void,
+ signal?: AbortSignal,
+): Promise {
+ const res = await proxyGet(client, `${root}select/logsql/tail`, params, signal)
+ if (!res.body) return
+ const reader = res.body.getReader()
+ const decoder = new TextDecoder()
+ let buffer = ""
+ for (;;) {
+ const { done, value } = await reader.read()
+ if (done) break
+ buffer += decoder.decode(value, { stream: true })
+ const lines = buffer.split("\n")
+ buffer = lines.pop() ?? ""
+ for (const line of lines) {
+ const entry = parseLine(line)
+ if (entry) onEntry(entry)
+ }
+ }
+ const last = parseLine(buffer)
+ if (last) onEntry(last)
+}
diff --git a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
index 12f50e0fd..04c528760 100644
--- a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
+++ b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
@@ -241,6 +241,24 @@ export class K8sClient {
})
}
+ /**
+ * Fetch a same-origin path with the client's credentials and 401 handling,
+ * leaving status and body to the caller. For endpoints that do not speak
+ * Kubernetes JSON, such as the observability proxy's streamed NDJSON.
+ */
+ async raw(path: string, init?: RequestInit): Promise {
+ const headers: Record = {
+ ...(init?.headers as Record),
+ }
+ if (this.getToken) {
+ headers["Authorization"] = `Bearer ${await this.getToken()}`
+ }
+ const res = await fetch(`${this.baseUrl}${path}`, { ...init, headers })
+ if (res.status === 401) this.handleUnauthorized()
+ else if (res.ok) this.unauthorizedHandled = false
+ return res
+ }
+
getApiGroups(): Promise {
return this.request("/apis")
}
From 063e56ede57a481b2db223a37641e601a2d51a05 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:30:03 +0700
Subject: [PATCH 029/346] feat(dashboard): show resource consumption on
application pages
Tenants had no way to see what their applications consume. A Metrics
tab on every application detail page (except VM disks, which run no
containers) charts CPU, memory and network for a selectable range
through the observability proxy's application root, which injects the
scope server-side, so the queries stay kind-agnostic and no per-kind
UI is needed.
Charts use uPlot: about 25 KB gzipped with no dependencies, built for
time series, and loaded only with the Metrics tab so other pages do
not pay for it. A cluster without the proxy answers 404 and the tab
says metrics are not available there instead of showing an error.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../images/console/apps/console/package.json | 3 +-
.../observability/MetricsView.test.tsx | 88 +++++++++++++
.../components/observability/MetricsView.tsx | 72 +++++++++++
.../observability/ProxyNotice.test.tsx | 43 +++++++
.../components/observability/ProxyNotice.tsx | 49 +++++++
.../components/observability/RangePicker.tsx | 30 +++++
.../observability/TimeSeriesChart.test.tsx | 46 +++++++
.../observability/TimeSeriesChart.tsx | 51 ++++++++
.../src/lib/observability/metrics.test.ts | 107 ++++++++++++++++
.../console/src/lib/observability/metrics.ts | 121 ++++++++++++++++++
.../src/lib/observability/proxy.test.ts | 10 ++
.../console/src/lib/observability/proxy.ts | 8 ++
.../detail/ApplicationDetailPage.test.tsx | 28 ++++
.../routes/detail/ApplicationDetailPage.tsx | 12 ++
.../console/src/routes/detail/MetricsTab.tsx | 15 +++
.../dashboard/images/console/pnpm-lock.yaml | 8 ++
16 files changed, 690 insertions(+), 1 deletion(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/package.json b/packages/system/dashboard/images/console/apps/console/package.json
index fcddb7023..254195f28 100644
--- a/packages/system/dashboard/images/console/apps/console/package.json
+++ b/packages/system/dashboard/images/console/apps/console/package.json
@@ -31,7 +31,8 @@
"react": "^19.2.4",
"react-dom": "^19.2.4",
"react-router": "^7.6.1",
- "simple-icons": "^16.22.0"
+ "simple-icons": "^16.22.0",
+ "uplot": "^1.6.32"
},
"devDependencies": {
"@tailwindcss/vite": "^4.2.2",
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
new file mode 100644
index 000000000..f09e15201
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
@@ -0,0 +1,88 @@
+import { describe, it, expect, vi } from "vitest"
+import { screen, waitFor } from "@testing-library/react"
+import userEvent from "@testing-library/user-event"
+import { K8sClient } from "@cozystack/k8s-client"
+import { renderWithK8sProvider } from "../../test-utils/render.tsx"
+import type { ChartData } from "../../lib/observability/metrics.ts"
+
+vi.mock("./TimeSeriesChart.tsx", () => ({
+ TimeSeriesChart: ({ data }: { data: ChartData }) => (
+ {data.series.map((s) => s.label).join(",")}
+ ),
+}))
+
+const { MetricsView } = await import("./MetricsView.tsx")
+
+const scope = { namespace: "tenant-a", plural: "postgreses", name: "db" }
+
+function matrix(pod: string) {
+ return Response.json({
+ status: "success",
+ data: { resultType: "matrix", result: [{ metric: { pod }, values: [[1, "1"]] }] },
+ })
+}
+
+function clientWith(handler: (url: URL) => Response) {
+ const client = new K8sClient()
+ const raw = vi.spyOn(client, "raw").mockImplementation(async (path) => handler(new URL(path, "http://x")))
+ return { client, raw }
+}
+
+describe("MetricsView", () => {
+ it("draws CPU, memory and network from the application root", async () => {
+ const { client, raw } = clientWith(() => matrix("db-0"))
+
+ renderWithK8sProvider( , { client })
+
+ await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3))
+ expect(screen.getByText("CPU (cores)")).toBeInTheDocument()
+ expect(screen.getByText("Memory (working set)")).toBeInTheDocument()
+ expect(screen.getByText("Network")).toBeInTheDocument()
+ for (const [path] of raw.mock.calls) {
+ expect(path.startsWith("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query_range?")).toBe(true)
+ }
+ })
+
+ it("requeries with a coarser step when a longer range is picked", async () => {
+ const { client, raw } = clientWith(() => matrix("db-0"))
+ renderWithK8sProvider( , { client })
+ await waitFor(() => expect(raw).toHaveBeenCalled())
+
+ await userEvent.click(screen.getByRole("button", { name: "7d" }))
+
+ await waitFor(() =>
+ expect(raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("step") === "2520")).toBe(true),
+ )
+ })
+
+ it("shows one quiet notice instead of three errors when the proxy is absent", async () => {
+ const { client } = clientWith(() => new Response("404 page not found", { status: 404 }))
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("Metrics are not available on this cluster.")).toBeInTheDocument()
+ expect(screen.getAllByRole("status")).toHaveLength(1)
+ expect(screen.queryByText("CPU (cores)")).toBeNull()
+ })
+
+ it("keeps the other panels when one query hits a limit", async () => {
+ const { client } = clientWith((url) =>
+ url.searchParams.get("query")?.includes("memory")
+ ? Response.json({ status: "error", error: "too many series" }, { status: 422 })
+ : matrix("db-0"),
+ )
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("too many series. Choose a shorter range.")).toBeInTheDocument()
+ await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(2))
+ })
+
+ it("says so when a panel has no data", async () => {
+ const { client } = clientWith(() => Response.json({ status: "success", data: { resultType: "matrix", result: [] } }))
+
+ renderWithK8sProvider( , { client })
+
+ await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(3))
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
new file mode 100644
index 000000000..35c8e8417
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
@@ -0,0 +1,72 @@
+import { useState } from "react"
+import { useQueries } from "@tanstack/react-query"
+import { useK8sClient } from "@cozystack/k8s-client"
+import { Section, Spinner } from "@cozystack/ui"
+import {
+ METRIC_PANELS,
+ RANGES,
+ rangeWindow,
+ toChartData,
+ type RangeKey,
+} from "../../lib/observability/metrics.ts"
+import { isScopeWide, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts"
+import { ProxyNotice } from "./ProxyNotice.tsx"
+import { TimeSeriesChart } from "./TimeSeriesChart.tsx"
+import { RangePicker } from "./RangePicker.tsx"
+
+const REFRESH_MS = 60_000
+
+export function MetricsView({ scope }: { scope: ObservabilityScope }) {
+ const client = useK8sClient()
+ const root = metricsRoot(scope)
+ const [range, setRange] = useState("1h")
+
+ const results = useQueries({
+ queries: METRIC_PANELS.map((panel) => ({
+ queryKey: ["observability", "metrics", root, panel.id, range],
+ queryFn: async ({ signal }: { signal: AbortSignal }) => {
+ const window = rangeWindow(range, Math.floor(Date.now() / 1000))
+ const series = await Promise.all(
+ panel.queries(window).map(async (q) => ({
+ label: q.label,
+ series: await queryRange(client, root, { query: q.expr, ...window }, signal),
+ })),
+ )
+ return toChartData(series)
+ },
+ retry: false,
+ refetchInterval: REFRESH_MS,
+ })),
+ })
+
+ const wide = results.find((r) => isScopeWide(r.error))?.error
+
+ return (
+
+
+
Resource consumption
+
+
+ {wide ? (
+
+ ) : (
+ METRIC_PANELS.map((panel, i) => {
+ const { data, error, isLoading } = results[i]
+ return (
+
+ {error ? (
+
+ ) : isLoading || !data ? (
+
+ ) : data.series.length === 0 ? (
+ No data for this range.
+ ) : (
+
+ )}
+
+ )
+ })
+ )}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
new file mode 100644
index 000000000..fd001bcae
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
@@ -0,0 +1,43 @@
+import { describe, it, expect } from "vitest"
+import { render, screen } from "@testing-library/react"
+import { ProxyError } from "../../lib/observability/proxy.ts"
+import { ProxyNotice } from "./ProxyNotice.tsx"
+
+describe("ProxyNotice", () => {
+ it("renders a 404 as not available on this cluster, not as an error", () => {
+ render( )
+
+ expect(screen.getByRole("status")).toHaveTextContent("Metrics are not available on this cluster.")
+ expect(screen.queryByRole("alert")).toBeNull()
+ })
+
+ it("renders a 502 as a quiet notice about the store", () => {
+ render( )
+
+ expect(screen.getByRole("status")).toHaveTextContent("The logs store is unreachable right now.")
+ })
+
+ it.each([
+ ["forbidden", 403, "You do not have permission to read metrics here."],
+ ["limit", 422, "too many points. Choose a shorter range."],
+ ["rate-limited", 429, "Too many requests. Try again in a moment."],
+ ["timeout", 504, "The store did not answer in time. Choose a shorter range or a narrower filter."],
+ ["unavailable", 503, "The service is temporarily unavailable. Try again in a moment."],
+ ] as const)("explains %s", (kind, status, text) => {
+ render( )
+
+ expect(screen.getByRole("alert")).toHaveTextContent(text)
+ })
+
+ it("shows the server message for a rejected query", () => {
+ render( )
+
+ expect(screen.getByRole("alert")).toHaveTextContent("cannot parse query")
+ })
+
+ it("falls back to the message of an unexpected error", () => {
+ render( )
+
+ expect(screen.getByRole("alert")).toHaveTextContent("boom")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
new file mode 100644
index 000000000..a60b831b0
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
@@ -0,0 +1,49 @@
+import { ProxyError } from "../../lib/observability/proxy.ts"
+
+interface ProxyNoticeProps {
+ what: "Metrics" | "Logs"
+ error: unknown
+}
+
+function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: boolean; text: string } {
+ const lower = what.toLowerCase()
+ if (!(error instanceof ProxyError)) {
+ return { quiet: false, text: error instanceof Error ? error.message : String(error) }
+ }
+ switch (error.kind) {
+ case "not-available":
+ return { quiet: true, text: `${what} are not available on this cluster.` }
+ case "store-unreachable":
+ return { quiet: true, text: `The ${lower} store is unreachable right now.` }
+ case "unauthorized":
+ return { quiet: false, text: "Your session has expired. Sign in again." }
+ case "forbidden":
+ return { quiet: false, text: `You do not have permission to read ${lower} here.` }
+ case "limit":
+ return { quiet: false, text: `${error.message}. Choose a shorter range.` }
+ case "rate-limited":
+ return { quiet: false, text: "Too many requests. Try again in a moment." }
+ case "timeout":
+ return { quiet: false, text: "The store did not answer in time. Choose a shorter range or a narrower filter." }
+ case "unavailable":
+ return { quiet: false, text: "The service is temporarily unavailable. Try again in a moment." }
+ default:
+ return { quiet: false, text: error.message }
+ }
+}
+
+export function ProxyNotice({ what, error }: ProxyNoticeProps) {
+ const { quiet, text } = describe(what, error)
+ if (quiet) {
+ return (
+
+ {text}
+
+ )
+ }
+ return (
+
+ {text}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
new file mode 100644
index 000000000..22030c706
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
@@ -0,0 +1,30 @@
+import { cn } from "@cozystack/ui"
+
+export function RangePicker({
+ options,
+ value,
+ onChange,
+}: {
+ options: readonly K[]
+ value: K
+ onChange: (value: K) => void
+}) {
+ return (
+
+ {options.map((key) => (
+ onChange(key)}
+ className={cn(
+ "rounded px-2.5 py-1 text-xs",
+ value === key ? "bg-slate-900 text-white" : "text-slate-600 hover:bg-slate-100",
+ )}
+ >
+ {key}
+
+ ))}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
new file mode 100644
index 000000000..57cf6fb99
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
@@ -0,0 +1,46 @@
+import { describe, it, expect, vi, beforeEach } from "vitest"
+import { render } from "@testing-library/react"
+
+const created: { opts: { series: { label?: string }[] }; data: unknown[]; destroy: ReturnType }[] = []
+
+vi.mock("uplot", () => ({
+ default: class {
+ destroy = vi.fn()
+ constructor(opts: { series: { label?: string }[] }, data: unknown[]) {
+ created.push({ opts, data, destroy: this.destroy })
+ }
+ setSize() {}
+ },
+}))
+
+const { TimeSeriesChart } = await import("./TimeSeriesChart.tsx")
+
+beforeEach(() => {
+ created.length = 0
+})
+
+describe("TimeSeriesChart", () => {
+ const data = {
+ timestamps: [1, 2],
+ series: [
+ { label: "db-0", values: [1, 2] },
+ { label: "db-1", values: [null, 3] },
+ ],
+ }
+
+ it("hands uPlot the timestamps first and one labelled series per input", () => {
+ render( )
+
+ expect(created).toHaveLength(1)
+ expect(created[0].data).toEqual([[1, 2], [1, 2], [null, 3]])
+ expect(created[0].opts.series.slice(1).map((s) => s.label)).toEqual(["db-0", "db-1"])
+ })
+
+ it("destroys the chart on unmount", () => {
+ const { unmount } = render( )
+
+ unmount()
+
+ expect(created[0].destroy).toHaveBeenCalled()
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
new file mode 100644
index 000000000..6b596e17f
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
@@ -0,0 +1,51 @@
+import { useEffect, useRef } from "react"
+import uPlot from "uplot"
+import "uplot/dist/uPlot.min.css"
+import type { ChartData } from "../../lib/observability/metrics.ts"
+
+const COLORS = ["#2563eb", "#16a34a", "#d97706", "#9333ea", "#dc2626", "#0891b2", "#4b5563", "#db2777"]
+const HEIGHT = 200
+
+interface TimeSeriesChartProps {
+ data: ChartData
+ format: (value: number) => string
+}
+
+export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) {
+ const ref = useRef(null)
+
+ useEffect(() => {
+ const el = ref.current
+ if (!el) return
+ const fmt = (v: number | null) => (v === null ? "–" : format(v))
+ const chart = new uPlot(
+ {
+ width: el.clientWidth || 600,
+ height: HEIGHT,
+ scales: { x: { time: true } },
+ legend: { live: true },
+ series: [
+ {},
+ ...data.series.map((s, i) => ({
+ label: s.label,
+ stroke: COLORS[i % COLORS.length],
+ width: 1.5,
+ value: (_: uPlot, v: number | null) => fmt(v),
+ })),
+ ],
+ axes: [{}, { size: 70, values: (_: uPlot, splits: number[]) => splits.map(format) }],
+ },
+ [data.timestamps, ...data.series.map((s) => s.values)],
+ el,
+ )
+ if (typeof ResizeObserver === "undefined") return () => chart.destroy()
+ const observer = new ResizeObserver(() => chart.setSize({ width: el.clientWidth, height: HEIGHT }))
+ observer.observe(el)
+ return () => {
+ observer.disconnect()
+ chart.destroy()
+ }
+ }, [data, format])
+
+ return
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts
new file mode 100644
index 000000000..0803940bf
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts
@@ -0,0 +1,107 @@
+import { describe, it, expect } from "vitest"
+import {
+ METRIC_PANELS,
+ formatBytes,
+ formatCores,
+ rangeWindow,
+ toChartData,
+} from "./metrics.ts"
+
+describe("rangeWindow", () => {
+ it("ends now and spans the range in about 240 points", () => {
+ const w = rangeWindow("6h", 1_000_000)
+
+ expect(w.end).toBe(1_000_000)
+ expect(w.start).toBe(1_000_000 - 6 * 3600)
+ expect(w.step).toBe(90)
+ })
+
+ it("never steps finer than 15s", () => {
+ expect(rangeWindow("1h", 0).step).toBe(15)
+ })
+})
+
+describe("METRIC_PANELS", () => {
+ const window = { start: 0, end: 3600, step: 15 }
+
+ it("rates CPU over at least five minutes so sparse scrapes still draw", () => {
+ const [query] = METRIC_PANELS.find((p) => p.id === "cpu")!.queries(window)
+
+ expect(query.expr).toBe(
+ 'sum by (pod) (rate(container_cpu_usage_seconds_total{container!="",container!="POD"}[300s]))',
+ )
+ })
+
+ it("widens the rate window with the step on long ranges", () => {
+ const [query] = METRIC_PANELS.find((p) => p.id === "cpu")!.queries({ start: 0, end: 1, step: 2520 })
+
+ expect(query.expr).toContain("[5040s]")
+ })
+
+ it("reads memory as working set per pod", () => {
+ const [query] = METRIC_PANELS.find((p) => p.id === "memory")!.queries(window)
+
+ expect(query.expr).toBe('sum by (pod) (container_memory_working_set_bytes{container!="",container!="POD"})')
+ })
+
+ it("draws network receive and transmit as two named series", () => {
+ const queries = METRIC_PANELS.find((p) => p.id === "network")!.queries(window)
+
+ expect(queries.map((q) => q.label)).toEqual(["receive", "transmit"])
+ expect(queries[0].expr).toBe("sum(rate(container_network_receive_bytes_total[300s]))")
+ expect(queries[1].expr).toBe("sum(rate(container_network_transmit_bytes_total[300s]))")
+ })
+
+ it("names no application kind in any query", () => {
+ for (const panel of METRIC_PANELS) {
+ for (const q of panel.queries(window)) expect(q.expr).not.toMatch(/apps_cozystack_io|namespace=/)
+ }
+ })
+})
+
+describe("toChartData", () => {
+ it("aligns series on the union of timestamps and leaves gaps as null", () => {
+ const data = toChartData([
+ { label: "a", series: [{ metric: {}, values: [[10, "1"], [20, "2"]] }] },
+ { label: "b", series: [{ metric: {}, values: [[20, "5"], [30, "6"]] }] },
+ ])
+
+ expect(data.timestamps).toEqual([10, 20, 30])
+ expect(data.series).toEqual([
+ { label: "a", values: [1, 2, null] },
+ { label: "b", values: [null, 5, 6] },
+ ])
+ })
+
+ it("labels each series by its pod, falling back to the query label", () => {
+ const data = toChartData([
+ {
+ label: "cpu",
+ series: [
+ { metric: { pod: "db-1" }, values: [[1, "0.5"]] },
+ { metric: {}, values: [[1, "0.1"]] },
+ ],
+ },
+ ])
+
+ expect(data.series.map((s) => s.label)).toEqual(["db-1", "cpu"])
+ })
+
+ it("drops NaN samples instead of drawing them as zero", () => {
+ const data = toChartData([{ label: "x", series: [{ metric: {}, values: [[1, "NaN"]] }] }])
+
+ expect(data.series[0].values).toEqual([null])
+ })
+})
+
+describe("formatters", () => {
+ it("formats CPU in millicores below one core", () => {
+ expect(formatCores(0.25)).toBe("250m")
+ expect(formatCores(2)).toBe("2")
+ })
+
+ it("formats bytes with binary units", () => {
+ expect(formatBytes(512)).toBe("512 B")
+ expect(formatBytes(3 * 1024 * 1024)).toBe("3.0 MiB")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
new file mode 100644
index 000000000..b23a51e27
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
@@ -0,0 +1,121 @@
+import type { PromSeries } from "./proxy.ts"
+
+export const RANGES = {
+ "1h": 3600,
+ "6h": 6 * 3600,
+ "24h": 24 * 3600,
+ "7d": 7 * 24 * 3600,
+} as const
+
+export type RangeKey = keyof typeof RANGES
+
+export interface RangeWindow {
+ start: number
+ end: number
+ step: number
+}
+
+const POINTS = 240
+const MIN_STEP = 15
+
+export function rangeWindow(range: RangeKey, nowSeconds: number): RangeWindow {
+ const span = RANGES[range]
+ return { start: nowSeconds - span, end: nowSeconds, step: Math.max(MIN_STEP, Math.round(span / POINTS)) }
+}
+
+// A rate needs several scrapes inside its window, and on long ranges the window
+// must also cover the step or samples between points are skipped.
+function rateWindow(step: number): string {
+ return `${Math.max(300, step * 2)}s`
+}
+
+const CONTAINERS = 'container!="",container!="POD"'
+
+export interface MetricQuery {
+ label: string
+ expr: string
+}
+
+export interface MetricPanel {
+ id: "cpu" | "memory" | "network"
+ title: string
+ format: (value: number) => string
+ queries: (window: RangeWindow) => MetricQuery[]
+}
+
+export function formatCores(value: number): string {
+ if (value === 0) return "0"
+ if (Math.abs(value) < 1) return `${Math.round(value * 1000)}m`
+ return `${Number(value.toFixed(2))}`
+}
+
+const BYTE_UNITS = ["B", "KiB", "MiB", "GiB", "TiB"]
+
+export function formatBytes(value: number): string {
+ let v = value
+ let unit = 0
+ while (Math.abs(v) >= 1024 && unit < BYTE_UNITS.length - 1) {
+ v /= 1024
+ unit++
+ }
+ return unit === 0 ? `${Math.round(v)} B` : `${v.toFixed(1)} ${BYTE_UNITS[unit]}`
+}
+
+// The proxy injects the namespace or application scope, so these stay kind-agnostic.
+export const METRIC_PANELS: MetricPanel[] = [
+ {
+ id: "cpu",
+ title: "CPU (cores)",
+ format: formatCores,
+ queries: ({ step }) => [
+ {
+ label: "cpu",
+ expr: `sum by (pod) (rate(container_cpu_usage_seconds_total{${CONTAINERS}}[${rateWindow(step)}]))`,
+ },
+ ],
+ },
+ {
+ id: "memory",
+ title: "Memory (working set)",
+ format: formatBytes,
+ queries: () => [
+ { label: "memory", expr: `sum by (pod) (container_memory_working_set_bytes{${CONTAINERS}})` },
+ ],
+ },
+ {
+ id: "network",
+ title: "Network",
+ format: (v) => `${formatBytes(v)}/s`,
+ queries: ({ step }) => [
+ { label: "receive", expr: `sum(rate(container_network_receive_bytes_total[${rateWindow(step)}]))` },
+ { label: "transmit", expr: `sum(rate(container_network_transmit_bytes_total[${rateWindow(step)}]))` },
+ ],
+ },
+]
+
+export interface ChartSeries {
+ label: string
+ values: (number | null)[]
+}
+
+export interface ChartData {
+ timestamps: number[]
+ series: ChartSeries[]
+}
+
+export function toChartData(results: { label: string; series: PromSeries[] }[]): ChartData {
+ const flat = results.flatMap(({ label, series }) =>
+ series.map((s) => ({ label: s.metric.pod ?? label, points: new Map(s.values.map(([t, v]) => [t, Number(v)])) })),
+ )
+ const timestamps = [...new Set(flat.flatMap((s) => [...s.points.keys()]))].sort((a, b) => a - b)
+ return {
+ timestamps,
+ series: flat.map(({ label, points }) => ({
+ label,
+ values: timestamps.map((t) => {
+ const v = points.get(t)
+ return v === undefined || Number.isNaN(v) ? null : v
+ }),
+ })),
+ }
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
index f06835ff7..f8cf44656 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
@@ -4,6 +4,7 @@ import {
ProxyError,
errorKindForStatus,
fieldValues,
+ isScopeWide,
logsRoot,
metricsRoot,
queryLogs,
@@ -177,3 +178,12 @@ describe("tailLogs", () => {
expect(err).toMatchObject({ kind: "forbidden" })
})
})
+
+describe("isScopeWide", () => {
+ it("is true for answers that do not depend on the query", () => {
+ expect(isScopeWide(new ProxyError("not-available", 404, ""))).toBe(true)
+ expect(isScopeWide(new ProxyError("forbidden", 403, ""))).toBe(true)
+ expect(isScopeWide(new ProxyError("limit", 422, ""))).toBe(false)
+ expect(isScopeWide(new Error("x"))).toBe(false)
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
index a257825c9..40b451b0b 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -60,6 +60,14 @@ export class ProxyError extends Error {
}
}
+// These answers hold for every query in the scope, so one notice replaces the whole view.
+export function isScopeWide(error: unknown): boolean {
+ return (
+ error instanceof ProxyError &&
+ (error.kind === "not-available" || error.kind === "forbidden" || error.kind === "unauthorized")
+ )
+}
+
// Prometheus answers {"error": "..."}, VictoriaLogs and nginx answer plain text.
function errorMessage(text: string, status: number): string {
try {
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx
index a015a98bf..40b923f62 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx
@@ -59,6 +59,7 @@ vi.mock("./EventsTab.tsx", () => ({ EventsTab: () => null }))
vi.mock("./VncTab.tsx", () => ({ VncTab: () => null }))
vi.mock("./SerialTab.tsx", () => ({ SerialTab: () => null }))
vi.mock("./VMPowerControls.tsx", () => ({ VMPowerControls: () => null }))
+vi.mock("./MetricsTab.tsx", () => ({ MetricsTab: () => null }))
const { ApplicationDetailPage } = await import("./ApplicationDetailPage.tsx")
@@ -111,11 +112,38 @@ describe("ApplicationDetailPage tabs for a virtual machine", () => {
"VNC",
"Workloads",
"Services",
+ "Metrics",
"Events",
])
})
})
+describe("ApplicationDetailPage observability tabs", () => {
+ function showKind(kind: string, plural: string) {
+ h.appKind = kind
+ h.appPlural = plural
+ h.get = {
+ data: { kind, metadata: { name: "demo", namespace: "tenant-test" } },
+ isLoading: false,
+ error: undefined,
+ }
+ h.configMaps = []
+ h.configMapsError = undefined
+ h.tabLabels = []
+ renderPage()
+ }
+
+ it("offers Metrics on any application kind, ahead of Events", () => {
+ showKind("Postgres", "postgreses")
+ expect(h.tabLabels.slice(-3)).toEqual(["Secrets", "Metrics", "Events"])
+ })
+
+ it("leaves Metrics off a VM disk, which runs no containers", () => {
+ showKind("VMDisk", "vmdisks")
+ expect(h.tabLabels).not.toContain("Metrics")
+ })
+})
+
describe("ApplicationDetailPage configuration", () => {
it("offers ConfigMaps declared by the application's resource map", () => {
h.appKind = "FoundationDB"
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
index 55fa684b8..4520244d5 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
@@ -37,6 +37,8 @@ import { VncTab } from "./VncTab.tsx"
// fetched only when a VM's console is opened. Every dashboard page reaches this
// file, and most of them never touch a VM.
const SerialTab = lazy(() => import("./SerialTab.tsx").then((m) => ({ default: m.SerialTab })))
+// Same for the charting library behind the Metrics tab.
+const MetricsTab = lazy(() => import("./MetricsTab.tsx").then((m) => ({ default: m.MetricsTab })))
import { VMPowerControls } from "./VMPowerControls.tsx"
import { useResourceBasePath } from "../../lib/portal.ts"
import { useResourcePresence } from "./use-resource-presence.ts"
@@ -143,6 +145,7 @@ export function ApplicationDetailPage() {
{ to: `${base}/vnc`, label: "VNC", end: false },
{ to: `${base}/workloads`, label: "Workloads", end: false },
{ to: `${base}/services`, label: "Services", end: false },
+ { to: `${base}/metrics`, label: "Metrics", end: false },
{ to: `${base}/events`, label: "Events", end: false },
)
} else {
@@ -163,6 +166,7 @@ export function ApplicationDetailPage() {
}
tabs.push(
{ to: `${base}/secrets`, label: "Secrets", end: false },
+ { to: `${base}/metrics`, label: "Metrics", end: false },
{ to: `${base}/events`, label: "Events", end: false },
)
}
@@ -262,6 +266,14 @@ export function ApplicationDetailPage() {
}
/>
} />
+ }>
+
+
+ }
+ />
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx
new file mode 100644
index 000000000..a928e633d
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx
@@ -0,0 +1,15 @@
+import type { ApplicationDefinition, ApplicationInstance } from "@cozystack/types"
+import { useTenantContext } from "../../lib/tenant-context.tsx"
+import { MetricsView } from "../../components/observability/MetricsView.tsx"
+
+interface MetricsTabProps {
+ ad: ApplicationDefinition
+ instance: ApplicationInstance
+}
+
+export function MetricsTab({ ad, instance }: MetricsTabProps) {
+ const { tenantNamespace } = useTenantContext()
+ const plural = ad.spec?.application.plural
+ if (!tenantNamespace || !plural) return null
+ return
+}
diff --git a/packages/system/dashboard/images/console/pnpm-lock.yaml b/packages/system/dashboard/images/console/pnpm-lock.yaml
index 9d90b0c5c..4661c444a 100644
--- a/packages/system/dashboard/images/console/pnpm-lock.yaml
+++ b/packages/system/dashboard/images/console/pnpm-lock.yaml
@@ -92,6 +92,9 @@ importers:
simple-icons:
specifier: ^16.22.0
version: 16.22.0
+ uplot:
+ specifier: ^1.6.32
+ version: 1.6.32
devDependencies:
'@tailwindcss/vite':
specifier: ^4.2.2
@@ -1773,6 +1776,9 @@ packages:
peerDependencies:
browserslist: '>= 4.21.0'
+ uplot@1.6.32:
+ resolution: {integrity: sha512-KIMVnG68zvu5XXUbC4LQEPnhwOxBuLyW1AHtpm6IKTXImkbLgkMy+jabjLgSLMasNuGGzQm/ep3tOkyTxpiQIw==}
+
uri-js@4.4.1:
resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
@@ -3405,6 +3411,8 @@ snapshots:
escalade: 3.2.0
picocolors: 1.1.1
+ uplot@1.6.32: {}
+
uri-js@4.4.1:
dependencies:
punycode: 2.3.1
From b5a0ab8de6db7e68a6f4396f01becd8cb98b1dff Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:34:01 +0700
Subject: [PATCH 030/346] feat(dashboard): add a log viewer for the
observability proxy
Tenants could not read their applications' logs without Grafana. The
viewer reads VictoriaLogs' native responses through a proxy root: a
time-ranged LogsQL query, pod and container pickers fed by the stream
field values of the same scope, and a follow mode over the streamed
tail.
The picked pod and container wrap the filter part of the query rather
than being appended to it, because AND binds tighter than OR in LogsQL
and the user's pipes must still run last. A dropped tail resumes from
the last line's _time; the proxy may replay lines around that point,
so the client drops replayed lines it already holds by stream and
time, which VictoriaLogs does not guarantee unique and is therefore
applied only at the resume boundary. Follow keeps the newest 1000
lines so a chatty stream cannot grow the page without bound.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/LogsView.test.tsx | 111 +++++++++++++
.../src/components/observability/LogsView.tsx | 152 ++++++++++++++++++
.../src/lib/observability/logs.test.ts | 65 ++++++++
.../console/src/lib/observability/logs.ts | 73 +++++++++
.../console/src/lib/observability/proxy.ts | 3 +-
.../lib/observability/use-log-tail.test.tsx | 65 ++++++++
.../src/lib/observability/use-log-tail.ts | 95 +++++++++++
7 files changed, 563 insertions(+), 1 deletion(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
new file mode 100644
index 000000000..693adf70a
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -0,0 +1,111 @@
+import { describe, it, expect, vi } from "vitest"
+import { screen, waitFor } from "@testing-library/react"
+import userEvent from "@testing-library/user-event"
+import { K8sClient } from "@cozystack/k8s-client"
+import { renderWithK8sProvider } from "../../test-utils/render.tsx"
+import { LogsView } from "./LogsView.tsx"
+
+const app = { namespace: "tenant-a", plural: "postgreses", name: "db" }
+const ROOT = "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/"
+
+const line = (time: string, msg: string, pod = "db-0") =>
+ JSON.stringify({ _time: time, _msg: msg, _stream_id: "s", kubernetes_pod_name: pod, kubernetes_container_name: "postgres" })
+
+function clientWith(handler: (endpoint: string, params: URLSearchParams) => Response | Promise) {
+ const client = new K8sClient()
+ const raw = vi.spyOn(client, "raw").mockImplementation(async (path) => {
+ const url = new URL(path, "http://x")
+ return handler(url.pathname, url.searchParams)
+ })
+ return { client, raw }
+}
+
+const values = (...v: string[]) => Response.json({ values: v.map((value) => ({ value, hits: 1 })) })
+
+function defaultHandler(endpoint: string, params?: URLSearchParams): Response | Promise {
+ if (endpoint.endsWith("stream_field_values")) {
+ return params?.get("field") === "kubernetes_pod_name" ? values("db-0", "db-1") : values("postgres")
+ }
+ if (endpoint.endsWith("query")) return new Response(`${line("2026-11-04T10:00:01Z", "ready to accept connections")}\n`)
+ return new Promise(() => {})
+}
+
+describe("LogsView", () => {
+ it("shows recent lines from the application root", async () => {
+ const { client, raw } = clientWith(defaultHandler)
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("ready to accept connections")).toBeInTheDocument()
+ const call = raw.mock.calls.find(([p]) => p.includes("select/logsql/query?"))!
+ expect(call[0].startsWith(`${ROOT}select/logsql/query?`)).toBe(true)
+ const params = new URL(call[0], "http://x").searchParams
+ expect(params.get("query")).toBe("*")
+ expect(params.get("limit")).toBe("500")
+ })
+
+ it("narrows to the picked pod with a stream filter", async () => {
+ const { client, raw } = clientWith(defaultHandler)
+ renderWithK8sProvider( , { client })
+ await screen.findByRole("option", { name: "db-1" })
+
+ await userEvent.selectOptions(screen.getByLabelText("Pod"), "db-1")
+
+ await waitFor(() =>
+ expect(
+ raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("query") === '{kubernetes_pod_name="db-1"} (*)'),
+ ).toBe(true),
+ )
+ })
+
+ it("applies the LogsQL filter on submit", async () => {
+ const { client, raw } = clientWith(defaultHandler)
+ renderWithK8sProvider( , { client })
+
+ await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}")
+
+ await waitFor(() =>
+ expect(raw.mock.calls.some(([p]) => p.includes("select/logsql/query?") && p.includes("query=error"))).toBe(true),
+ )
+ })
+
+ it("follows the live tail when Follow is pressed", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("tail") ? new Response(`${line("2026-11-04T10:00:05Z", "checkpoint complete")}\n`) : defaultHandler(endpoint, params),
+ )
+ renderWithK8sProvider( , { client })
+
+ await userEvent.click(screen.getByRole("button", { name: "Follow" }))
+
+ expect(await screen.findByText("checkpoint complete")).toBeInTheDocument()
+ expect(screen.getByRole("button", { name: "Follow" })).toHaveAttribute("aria-pressed", "true")
+ })
+
+ it("renders a cluster without the proxy as not available", async () => {
+ const { client } = clientWith(() => new Response("404 page not found", { status: 404 }))
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("Logs are not available on this cluster.")).toBeInTheDocument()
+ expect(screen.queryByLabelText("LogsQL filter")).toBeNull()
+ })
+
+ it("keeps the controls and explains a rejected query", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query") ? new Response("cannot parse query", { status: 400 }) : defaultHandler(endpoint, params),
+ )
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("cannot parse query")).toBeInTheDocument()
+ expect(screen.getByLabelText("LogsQL filter")).toBeInTheDocument()
+ })
+
+ it("names the source pod and container of each line", async () => {
+ const { client } = clientWith(defaultHandler)
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("db-0/postgres")).toBeInTheDocument()
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
new file mode 100644
index 000000000..03235cda3
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -0,0 +1,152 @@
+import { useEffect, useRef, useState } from "react"
+import { useQuery } from "@tanstack/react-query"
+import { useK8sClient } from "@cozystack/k8s-client"
+import { Button, Spinner } from "@cozystack/ui"
+import {
+ CONTAINER_FIELD,
+ LOG_RANGES,
+ POD_FIELD,
+ buildLogsQuery,
+ type LogRangeKey,
+} from "../../lib/observability/logs.ts"
+import {
+ fieldValues,
+ isScopeWide,
+ logsRoot,
+ queryLogs,
+ type LogEntry,
+ type ObservabilityScope,
+} from "../../lib/observability/proxy.ts"
+import { useLogTail } from "../../lib/observability/use-log-tail.ts"
+import { ProxyNotice } from "./ProxyNotice.tsx"
+import { RangePicker } from "./RangePicker.tsx"
+
+const HISTORY_LIMIT = 500
+const FOLLOW_CAP = 1000
+
+function windowFor(range: LogRangeKey) {
+ const end = Date.now()
+ return { start: new Date(end - LOG_RANGES[range] * 1000).toISOString(), end: new Date(end).toISOString() }
+}
+
+function Picker({ label, value, options, onChange }: { label: string; value: string; options: string[]; onChange: (v: string) => void }) {
+ return (
+
+ {label}
+ onChange(e.target.value)}
+ className="rounded-md border border-slate-200 bg-white px-2 py-1 text-xs text-slate-900"
+ >
+ All
+ {options.map((o) => (
+
+ {o}
+
+ ))}
+
+
+ )
+}
+
+function LogLines({ entries, follow }: { entries: LogEntry[]; follow: boolean }) {
+ const ref = useRef(null)
+ useEffect(() => {
+ if (follow && ref.current) ref.current.scrollTop = ref.current.scrollHeight
+ }, [entries, follow])
+ if (entries.length === 0) return No log lines for this range and filter.
+ return (
+
+ {entries.map((e, i) => (
+
+ {new Date(e._time).toLocaleTimeString()} {" "}
+ {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`} {" "}
+ {e._msg}
+
+ ))}
+
+ )
+}
+
+export function LogsView({ scope }: { scope: ObservabilityScope }) {
+ const client = useK8sClient()
+ const root = logsRoot(scope)
+ const [range, setRange] = useState("1h")
+ const [draft, setDraft] = useState("")
+ const [filter, setFilter] = useState("")
+ const [pod, setPod] = useState("")
+ const [container, setContainer] = useState("")
+ const [follow, setFollow] = useState(false)
+ const query = buildLogsQuery({ filter, pod, container })
+
+ const pickerQuery = (field: string, narrowTo?: string) => ({
+ queryKey: ["observability", "logs-fields", root, field, range, narrowTo ?? ""],
+ queryFn: ({ signal }: { signal: AbortSignal }) =>
+ fieldValues(client, root, {
+ field,
+ stream: true,
+ query: buildLogsQuery({ filter: "", pod: narrowTo }),
+ ...windowFor(range),
+ }, signal),
+ retry: false,
+ })
+ const pods = useQuery(pickerQuery(POD_FIELD))
+ const containers = useQuery(pickerQuery(CONTAINER_FIELD, pod || undefined))
+
+ const history = useQuery({
+ queryKey: ["observability", "logs", root, query, range],
+ queryFn: ({ signal }) => queryLogs(client, root, { query, limit: HISTORY_LIMIT, ...windowFor(range) }, signal),
+ enabled: !follow,
+ retry: false,
+ })
+ const tail = useLogTail(root, query, follow, { cap: FOLLOW_CAP })
+
+ const wide = [pods.error, history.error, tail.error].find(isScopeWide)
+ if (wide) {
+ return (
+
+ )
+ }
+
+ const error = follow ? tail.error : history.error
+ const entries = follow ? tail.entries : (history.data ?? [])
+
+ return (
+
+
+
+
{ setPod(v); setContainer("") }} />
+
+ {!follow && }
+ setFollow(!follow)}>
+ Follow
+
+
+ {follow && (
+
+ {tail.state === "live" ? "Live" : tail.state === "reconnecting" ? "Reconnecting…" : "Connecting…"}
+ {` · newest ${FOLLOW_CAP} lines kept`}
+
+ )}
+ {error ?
: null}
+ {!follow && history.isLoading ?
:
}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts
new file mode 100644
index 000000000..25d26a23a
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts
@@ -0,0 +1,65 @@
+import { describe, it, expect } from "vitest"
+import { buildLogsQuery, compareTime, mergeTail } from "./logs.ts"
+
+describe("buildLogsQuery", () => {
+ it("passes the filter through when nothing is picked", () => {
+ expect(buildLogsQuery({ filter: "error or warn" })).toBe("error or warn")
+ expect(buildLogsQuery({ filter: " " })).toBe("*")
+ })
+
+ it("ANDs the picked pod and container as a stream filter around the whole filter", () => {
+ expect(buildLogsQuery({ filter: "error or warn", pod: "db-0", container: "postgres" })).toBe(
+ '{kubernetes_pod_name="db-0",kubernetes_container_name="postgres"} (error or warn)',
+ )
+ })
+
+ it("keeps the user's pipes after the scoped filter", () => {
+ expect(buildLogsQuery({ filter: 'a or "x|y" | fields _msg', pod: "db-0" })).toBe(
+ '{kubernetes_pod_name="db-0"} (a or "x|y") | fields _msg',
+ )
+ })
+
+ it("escapes quotes in picked values", () => {
+ expect(buildLogsQuery({ filter: "", pod: 'a"b' })).toBe('{kubernetes_pod_name="a\\"b"} (*)')
+ })
+})
+
+describe("compareTime", () => {
+ it("orders RFC3339 times with differing fraction lengths", () => {
+ expect(compareTime("2026-11-04T10:00:01Z", "2026-11-04T10:00:01.1Z")).toBeLessThan(0)
+ expect(compareTime("2026-11-04T10:00:01.5Z", "2026-11-04T10:00:01.123Z")).toBeGreaterThan(0)
+ expect(compareTime("2026-11-04T10:00:01.10Z", "2026-11-04T10:00:01.1Z")).toBe(0)
+ })
+})
+
+const e = (time: string, msg: string, stream = "s1") => ({ _time: time, _msg: msg, _stream_id: stream })
+
+describe("mergeTail", () => {
+ it("appends and keeps only the newest lines over the cap", () => {
+ const out = mergeTail([e("t1", "a"), e("t2", "b")], [e("t3", "c")], 2)
+
+ expect(out.map((x) => x._msg)).toEqual(["b", "c"])
+ })
+
+ it("drops replayed lines at or before the resume point", () => {
+ const buffer = [e("2026-01-01T00:00:01Z", "a"), e("2026-01-01T00:00:02Z", "b")]
+
+ const out = mergeTail(buffer, [e("2026-01-01T00:00:02Z", "b"), e("2026-01-01T00:00:03Z", "c")], 10, "2026-01-01T00:00:02Z")
+
+ expect(out.map((x) => x._msg)).toEqual(["a", "b", "c"])
+ })
+
+ it("keeps a line at the resume point from another stream", () => {
+ const buffer = [e("2026-01-01T00:00:02Z", "b", "s1")]
+
+ const out = mergeTail(buffer, [e("2026-01-01T00:00:02Z", "other", "s2")], 10, "2026-01-01T00:00:02Z")
+
+ expect(out).toHaveLength(2)
+ })
+
+ it("does not deduplicate outside a resume", () => {
+ const out = mergeTail([e("t1", "a")], [e("t1", "a")], 10)
+
+ expect(out).toHaveLength(2)
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts
new file mode 100644
index 000000000..a6255e541
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts
@@ -0,0 +1,73 @@
+import type { LogEntry } from "./proxy.ts"
+
+export const LOG_RANGES = {
+ "15m": 15 * 60,
+ "1h": 3600,
+ "6h": 6 * 3600,
+ "24h": 24 * 3600,
+} as const
+
+export type LogRangeKey = keyof typeof LOG_RANGES
+
+export const POD_FIELD = "kubernetes_pod_name"
+export const CONTAINER_FIELD = "kubernetes_container_name"
+
+function quote(value: string): string {
+ return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`
+}
+
+function splitAtFirstPipe(query: string): [string, string] {
+ let quoteChar = ""
+ for (let i = 0; i < query.length; i++) {
+ const c = query[i]
+ if (quoteChar) {
+ if (c === "\\") i++
+ else if (c === quoteChar) quoteChar = ""
+ } else if (c === '"' || c === "'" || c === "`") {
+ quoteChar = c
+ } else if (c === "|") {
+ return [query.slice(0, i), query.slice(i + 1)]
+ }
+ }
+ return [query, ""]
+}
+
+// The picker narrows the filter part only: AND binds tighter than OR in
+// LogsQL, and the user's pipes must still run after the narrowed filter.
+export function buildLogsQuery({ filter, pod, container }: { filter: string; pod?: string; container?: string }): string {
+ const trimmed = filter.trim() || "*"
+ const matchers = [
+ pod ? `${POD_FIELD}=${quote(pod)}` : "",
+ container ? `${CONTAINER_FIELD}=${quote(container)}` : "",
+ ].filter(Boolean)
+ if (matchers.length === 0) return trimmed
+ const [head, pipes] = splitAtFirstPipe(trimmed)
+ const scoped = `{${matchers.join(",")}} (${head.trim() || "*"})`
+ return pipes.trim() ? `${scoped} | ${pipes.trim()}` : scoped
+}
+
+// VictoriaLogs trims trailing zeros from the fraction, so the strings do not sort lexically.
+function padFraction(time: string): string {
+ const m = /^([^.Z]+)(?:\.(\d+))?Z$/.exec(time)
+ return m ? `${m[1]}.${(m[2] ?? "").padEnd(9, "0")}Z` : time
+}
+
+export function compareTime(a: string, b: string): number {
+ const x = padFraction(a)
+ const y = padFraction(b)
+ return x < y ? -1 : x > y ? 1 : 0
+}
+
+const entryKey = (e: LogEntry) => `${e._stream_id ?? ""}\u0000${e._time}`
+
+// (_stream_id, _time) is not unique in VictoriaLogs, so dedup applies only to
+// lines a resumed tail replays at or before the resume point.
+export function mergeTail(buffer: LogEntry[], incoming: LogEntry[], cap: number, resumedFrom?: string): LogEntry[] {
+ let fresh = incoming
+ if (resumedFrom) {
+ const seen = new Set(buffer.map(entryKey))
+ fresh = incoming.filter((e) => compareTime(e._time, resumedFrom) > 0 || !seen.has(entryKey(e)))
+ }
+ const merged = buffer.concat(fresh)
+ return merged.length > cap ? merged.slice(merged.length - cap) : merged
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
index 40b451b0b..345d49815 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -1,4 +1,5 @@
import type { K8sClient } from "@cozystack/k8s-client"
+import { compareTime } from "./logs.ts"
export type ObservabilityScope =
| { namespace: string }
@@ -160,7 +161,7 @@ export async function queryLogs(
): Promise {
const res = await proxyGet(client, `${root}select/logsql/query`, { ...params }, signal)
const entries = (await res.text()).split("\n").map(parseLine).filter((e) => e !== null)
- return entries.sort((a, b) => (a._time < b._time ? -1 : a._time > b._time ? 1 : 0))
+ return entries.sort((a, b) => compareTime(a._time, b._time))
}
export interface FieldValuesParams {
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
new file mode 100644
index 000000000..5c420314a
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
@@ -0,0 +1,65 @@
+import { describe, it, expect, vi } from "vitest"
+import { renderHook, waitFor } from "@testing-library/react"
+import type { ReactNode } from "react"
+import { K8sClient, K8sProvider } from "@cozystack/k8s-client"
+import { useLogTail } from "./use-log-tail.ts"
+
+const line = (time: string, msg: string) => JSON.stringify({ _time: time, _msg: msg, _stream_id: "s" })
+
+function ndjson(...lines: string[]): Response {
+ return new Response(lines.map((l) => `${l}\n`).join(""))
+}
+
+function setup(...responses: Response[]) {
+ const client = new K8sClient()
+ const raw = vi.spyOn(client, "raw")
+ for (const r of responses) raw.mockResolvedValueOnce(r)
+ raw.mockImplementation(() => new Promise(() => {}))
+ const wrapper = ({ children }: { children: ReactNode }) => {children}
+ return { raw, wrapper }
+}
+
+const T1 = "2026-11-04T10:00:01Z"
+const T2 = "2026-11-04T10:00:02Z"
+const T3 = "2026-11-04T10:00:03Z"
+
+describe("useLogTail", () => {
+ it("streams lines and resumes after a disconnect from the last _time without duplicates", async () => {
+ const { raw, wrapper } = setup(ndjson(line(T1, "a"), line(T2, "b")), ndjson(line(T2, "b"), line(T3, "c")))
+
+ const { result } = renderHook(() => useLogTail("/root/", "*", true, { cap: 100, retryDelayMs: 1 }), { wrapper })
+
+ await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a", "b", "c"]))
+ expect(new URL(raw.mock.calls[0][0], "http://x").searchParams.has("since")).toBe(false)
+ expect(new URL(raw.mock.calls[1][0], "http://x").searchParams.get("since")).toBe(T2)
+ })
+
+ it("stops and reports an error the proxy will repeat on every retry", async () => {
+ const { raw, wrapper } = setup(new Response("", { status: 403 }))
+
+ const { result } = renderHook(() => useLogTail("/root/", "*", true, { cap: 100, retryDelayMs: 1 }), { wrapper })
+
+ await waitFor(() => expect(result.current.state).toBe("stopped"))
+ expect(result.current.error).toMatchObject({ kind: "forbidden" })
+ expect(raw).toHaveBeenCalledTimes(1)
+ })
+
+ it("does not connect while disabled", () => {
+ const { raw, wrapper } = setup()
+
+ renderHook(() => useLogTail("/root/", "*", false), { wrapper })
+
+ expect(raw).not.toHaveBeenCalled()
+ })
+
+ it("aborts the stream on unmount", async () => {
+ const { raw, wrapper } = setup()
+ const { unmount } = renderHook(() => useLogTail("/root/", "*", true), { wrapper })
+ await waitFor(() => expect(raw).toHaveBeenCalled())
+
+ unmount()
+
+ const init = raw.mock.calls[0][1] as RequestInit
+ expect(init.signal?.aborted).toBe(true)
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
new file mode 100644
index 000000000..cb9222f48
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
@@ -0,0 +1,95 @@
+import { useEffect, useState } from "react"
+import { useK8sClient } from "@cozystack/k8s-client"
+import { mergeTail } from "./logs.ts"
+import { ProxyError, tailLogs, type LogEntry } from "./proxy.ts"
+
+export type TailState = "connecting" | "live" | "reconnecting" | "stopped"
+
+interface TailSnapshot {
+ key: string
+ entries: LogEntry[]
+ state: TailState
+ error: unknown
+}
+
+const FLUSH_MS = 100
+const MAX_RETRY_MS = 30_000
+const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forbidden", "not-available", "limit"])
+
+const sleep = (ms: number, signal: AbortSignal) =>
+ new Promise((resolve) => {
+ const t = setTimeout(resolve, ms)
+ signal.addEventListener("abort", () => {
+ clearTimeout(t)
+ resolve()
+ })
+ })
+
+export function useLogTail(
+ root: string,
+ query: string,
+ enabled: boolean,
+ { cap = 1000, retryDelayMs = 2000 }: { cap?: number; retryDelayMs?: number } = {},
+): Omit {
+ const client = useK8sClient()
+ const key = `${root}\n${query}\n${enabled}`
+ const [snap, setSnap] = useState({ key, entries: [], state: "connecting", error: null })
+
+ useEffect(() => {
+ if (!enabled) return
+ const controller = new AbortController()
+ const { signal } = controller
+ let pending: LogEntry[] = []
+ let timer: ReturnType | undefined
+ let resumedFrom: string | undefined
+ let lastTime: string | undefined
+
+ const update = (patch: Partial, batch: LogEntry[] = [], from = resumedFrom) =>
+ setSnap((prev) => {
+ const base = prev.key === key ? prev : { key, entries: [], state: "connecting" as const, error: null }
+ return { ...base, ...patch, entries: batch.length ? mergeTail(base.entries, batch, cap, from) : base.entries }
+ })
+ const flush = () => {
+ clearTimeout(timer)
+ timer = undefined
+ if (pending.length === 0 || signal.aborted) return
+ const batch = pending
+ pending = []
+ update({ state: "live", error: null }, batch)
+ }
+
+ const run = async () => {
+ for (let attempt = 0; !signal.aborted; attempt++) {
+ resumedFrom = lastTime
+ try {
+ await tailLogs(client, root, { query, since: lastTime }, (entry) => {
+ attempt = 0
+ lastTime = entry._time
+ pending.push(entry)
+ timer ??= setTimeout(flush, FLUSH_MS)
+ }, signal)
+ flush()
+ } catch (err) {
+ flush()
+ if (signal.aborted) return
+ if (err instanceof ProxyError && FATAL.has(err.kind)) {
+ update({ state: "stopped", error: err })
+ return
+ }
+ update({ error: err })
+ }
+ if (signal.aborted) return
+ update({ state: "reconnecting" })
+ await sleep(Math.min(retryDelayMs * 2 ** attempt, MAX_RETRY_MS), signal)
+ }
+ }
+ void run()
+ return () => {
+ controller.abort()
+ clearTimeout(timer)
+ }
+ }, [client, root, query, enabled, key, cap, retryDelayMs])
+
+ const current = snap.key === key ? snap : { key, entries: [], state: "connecting" as const, error: null }
+ return { entries: current.entries, state: current.state, error: current.error }
+}
From f9ed2afad749646bb5c1de0a56b4e8d17e773bc1 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:34:01 +0700
Subject: [PATCH 031/346] feat(dashboard): show logs on application pages and
per tenant
A Logs tab on application detail pages reads the application root of
the observability proxy, and a Logs page under a new Observability
sidebar group reads the tenant namespace root, so lines from before
the application identity labels existed stay reachable there.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../apps/console/src/routes/ConsolePage.tsx | 2 ++
.../apps/console/src/routes/LogsPage.test.tsx | 20 +++++++++++++++++++
.../apps/console/src/routes/LogsPage.tsx | 15 ++++++++++++++
.../detail/ApplicationDetailPage.test.tsx | 9 ++++++---
.../routes/detail/ApplicationDetailPage.tsx | 4 ++++
.../console/src/routes/detail/LogsTab.tsx | 15 ++++++++++++++
.../src/routes/sidebar-sections.test.tsx | 10 ++++++++++
.../console/src/routes/sidebar-sections.tsx | 8 +++++++-
8 files changed, 79 insertions(+), 4 deletions(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx
index 37caae92f..5e204e84f 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx
@@ -15,6 +15,7 @@ import { MigrationSourceEditPage } from "./MigrationSourceEditPage.tsx"
import { MigrationTaskCreatePage } from "./MigrationTaskCreatePage.tsx"
import { MigrationTaskDetailPage } from "./MigrationTaskDetailPage.tsx"
import { ApplicationOrderPage } from "./ApplicationOrderPage.tsx"
+import { LogsPage } from "./LogsPage.tsx"
export function ConsolePage() {
return (
@@ -96,6 +97,7 @@ export function ConsolePage() {
path="migration/vmimporttasks/:name"
element={ }
/>
+ } />
} />
} />
} />
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx
new file mode 100644
index 000000000..1f4263cf1
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx
@@ -0,0 +1,20 @@
+import { describe, it, expect, vi } from "vitest"
+import { render, screen } from "@testing-library/react"
+
+vi.mock("../lib/tenant-context.tsx", () => ({
+ useTenantContext: () => ({ tenantNamespace: "tenant-a" }),
+}))
+vi.mock("../components/observability/LogsView.tsx", () => ({
+ LogsView: ({ scope }: { scope: object }) => {JSON.stringify(scope)} ,
+}))
+
+const { LogsPage } = await import("./LogsPage.tsx")
+
+describe("LogsPage", () => {
+ it("reads logs from the selected tenant's namespace root", () => {
+ render( )
+
+ expect(screen.getByRole("heading", { name: "Logs" })).toBeInTheDocument()
+ expect(JSON.parse(screen.getByTestId("scope").textContent ?? "")).toEqual({ namespace: "tenant-a" })
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx
new file mode 100644
index 000000000..7612dfbb2
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx
@@ -0,0 +1,15 @@
+import { useTenantContext } from "../lib/tenant-context.tsx"
+import { LogsView } from "../components/observability/LogsView.tsx"
+
+export function LogsPage() {
+ const { tenantNamespace } = useTenantContext()
+ return (
+
+
+
Logs
+
Every application in {tenantNamespace ?? "this tenant"}
+
+ {tenantNamespace &&
}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx
index 40b923f62..a0c3a6b84 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx
@@ -60,6 +60,7 @@ vi.mock("./VncTab.tsx", () => ({ VncTab: () => null }))
vi.mock("./SerialTab.tsx", () => ({ SerialTab: () => null }))
vi.mock("./VMPowerControls.tsx", () => ({ VMPowerControls: () => null }))
vi.mock("./MetricsTab.tsx", () => ({ MetricsTab: () => null }))
+vi.mock("./LogsTab.tsx", () => ({ LogsTab: () => null }))
const { ApplicationDetailPage } = await import("./ApplicationDetailPage.tsx")
@@ -113,6 +114,7 @@ describe("ApplicationDetailPage tabs for a virtual machine", () => {
"Workloads",
"Services",
"Metrics",
+ "Logs",
"Events",
])
})
@@ -133,14 +135,15 @@ describe("ApplicationDetailPage observability tabs", () => {
renderPage()
}
- it("offers Metrics on any application kind, ahead of Events", () => {
+ it("offers Metrics and Logs on any application kind, ahead of Events", () => {
showKind("Postgres", "postgreses")
- expect(h.tabLabels.slice(-3)).toEqual(["Secrets", "Metrics", "Events"])
+ expect(h.tabLabels.slice(-4)).toEqual(["Secrets", "Metrics", "Logs", "Events"])
})
- it("leaves Metrics off a VM disk, which runs no containers", () => {
+ it("leaves Metrics and Logs off a VM disk, which runs no containers", () => {
showKind("VMDisk", "vmdisks")
expect(h.tabLabels).not.toContain("Metrics")
+ expect(h.tabLabels).not.toContain("Logs")
})
})
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
index 4520244d5..ca499aac1 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx
@@ -44,6 +44,7 @@ import { useResourceBasePath } from "../../lib/portal.ts"
import { useResourcePresence } from "./use-resource-presence.ts"
import { useApplicationConfigMaps } from "./use-app-configmaps.ts"
import { ConfigMapsTab } from "./ConfigMapsTab.tsx"
+import { LogsTab } from "./LogsTab.tsx"
export function ApplicationDetailPage() {
const { plural, name } = useParams<{ plural: string; name: string }>()
@@ -146,6 +147,7 @@ export function ApplicationDetailPage() {
{ to: `${base}/workloads`, label: "Workloads", end: false },
{ to: `${base}/services`, label: "Services", end: false },
{ to: `${base}/metrics`, label: "Metrics", end: false },
+ { to: `${base}/logs`, label: "Logs", end: false },
{ to: `${base}/events`, label: "Events", end: false },
)
} else {
@@ -167,6 +169,7 @@ export function ApplicationDetailPage() {
tabs.push(
{ to: `${base}/secrets`, label: "Secrets", end: false },
{ to: `${base}/metrics`, label: "Metrics", end: false },
+ { to: `${base}/logs`, label: "Logs", end: false },
{ to: `${base}/events`, label: "Events", end: false },
)
}
@@ -274,6 +277,7 @@ export function ApplicationDetailPage() {
}
/>
+ } />
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx
new file mode 100644
index 000000000..4ac5f878d
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx
@@ -0,0 +1,15 @@
+import type { ApplicationDefinition, ApplicationInstance } from "@cozystack/types"
+import { useTenantContext } from "../../lib/tenant-context.tsx"
+import { LogsView } from "../../components/observability/LogsView.tsx"
+
+interface LogsTabProps {
+ ad: ApplicationDefinition
+ instance: ApplicationInstance
+}
+
+export function LogsTab({ ad, instance }: LogsTabProps) {
+ const { tenantNamespace } = useTenantContext()
+ const plural = ad.spec?.application.plural
+ if (!tenantNamespace || !plural) return null
+ return
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx
index efb69dc7b..da40081bc 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx
@@ -110,6 +110,16 @@ describe("useConsoleSidebarSections — admin areas moved out", () => {
expect(findItem(result.current, "Imports")?.to).toBe("/console/migration/vmimporttasks")
expect(result.current.some((s) => s.title === "Migration")).toBe(true)
})
+
+ it("offers the tenant's namespace logs in an Observability group", async () => {
+ const client = makeClient({ nodes: true, backupclasses: true })
+ const { result } = renderHook(() => useConsoleSidebarSections(), {
+ wrapper: makeWrapper(client),
+ })
+ await waitFor(() => expect(result.current.length).toBeGreaterThan(0))
+ expect(findItem(result.current, "Logs")?.to).toBe("/console/logs")
+ expect(result.current.find((s) => s.title === "Observability")?.items).toHaveLength(1)
+ })
})
function appDef(name: string, kind: string, category: string, module = false) {
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx
index d5d100813..4cec3a94c 100644
--- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx
@@ -11,6 +11,7 @@ import {
Layers,
Network,
Package,
+ ScrollText,
Plug,
Server,
ToyBrick,
@@ -118,7 +119,12 @@ export function useConsoleSidebarSections(): SidebarSection[] {
],
}
- return [...categorySections, backupsSection, migrationSection]
+ const observabilitySection: SidebarSection = {
+ title: "Observability",
+ items: [{ label: "Logs", to: "/console/logs", icon: ScrollText }],
+ }
+
+ return [...categorySections, backupsSection, migrationSection, observabilitySection]
}, [grouped])
}
From c5b57bffd173ea5e4bb371edb75a13a60f549d96 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:39:27 +0700
Subject: [PATCH 032/346] fix(dashboard): open log history at the newest line
A historical query lists lines oldest first, so the viewer opened at
the top of a 500-line window while the line a user looks for is
usually the latest. The chart's value axis also clipped wide labels
such as "476.8 MiB".
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../console/src/components/observability/LogsView.tsx | 8 ++++----
.../src/components/observability/TimeSeriesChart.tsx | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index 03235cda3..707b037b9 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -50,11 +50,11 @@ function Picker({ label, value, options, onChange }: { label: string; value: str
)
}
-function LogLines({ entries, follow }: { entries: LogEntry[]; follow: boolean }) {
+function LogLines({ entries }: { entries: LogEntry[] }) {
const ref = useRef(null)
useEffect(() => {
- if (follow && ref.current) ref.current.scrollTop = ref.current.scrollHeight
- }, [entries, follow])
+ if (ref.current) ref.current.scrollTop = ref.current.scrollHeight
+ }, [entries])
if (entries.length === 0) return No log lines for this range and filter.
return (
@@ -146,7 +146,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
)}
{error ? : null}
- {!follow && history.isLoading ? : }
+ {!follow && history.isLoading ? : }
)
}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
index 6b596e17f..683df2dff 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
@@ -33,7 +33,7 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) {
value: (_: uPlot, v: number | null) => fmt(v),
})),
],
- axes: [{}, { size: 70, values: (_: uPlot, splits: number[]) => splits.map(format) }],
+ axes: [{}, { size: 84, values: (_: uPlot, splits: number[]) => splits.map(format) }],
},
[data.timestamps, ...data.series.map((s) => s.values)],
el,
From 97dcf4c4f02d78dc2dcd4ec862cdaa77631de4f3 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:45:42 +0700
Subject: [PATCH 033/346] fix(dashboard): treat an apiserver answer on a proxy
path as no proxy
Until the console nginx routes the proxy roots, those paths reach
kube-apiserver, which RBAC-denies the namespace root with 403 for a
tenant, so the Logs page told every tenant they lacked permission. The
proxy itself answers in plain text, so a Kubernetes Status body is a
reliable sign that the path did not reach it.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../components/observability/LogsView.test.tsx | 10 ++++++++++
.../console/src/lib/observability/proxy.test.ts | 17 +++++++++++++++++
.../apps/console/src/lib/observability/proxy.ts | 14 +++++++++++++-
3 files changed, 40 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 693adf70a..fae3f6a4b 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -90,6 +90,16 @@ describe("LogsView", () => {
expect(screen.queryByLabelText("LogsQL filter")).toBeNull()
})
+ it("renders the tenant logs page as not available when kube-apiserver answers instead of the proxy", async () => {
+ const { client } = clientWith(() =>
+ Response.json({ kind: "Status", apiVersion: "v1", status: "Failure", reason: "Forbidden", code: 403 }, { status: 403 }),
+ )
+
+ renderWithK8sProvider( , { client })
+
+ expect(await screen.findByText("Logs are not available on this cluster.")).toBeInTheDocument()
+ })
+
it("keeps the controls and explains a rejected query", async () => {
const { client } = clientWith((endpoint, params) =>
endpoint.endsWith("query") ? new Response("cannot parse query", { status: 400 }) : defaultHandler(endpoint, params),
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
index f8cf44656..69a8c7c8c 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
@@ -79,6 +79,23 @@ describe("queryRange", () => {
expect(err).toMatchObject({ kind: "limit", status: 422, message: "range too long" })
})
+ it("treats a Kubernetes Status 403 as a cluster without the proxy route", async () => {
+ const status = { kind: "Status", apiVersion: "v1", status: "Failure", reason: "Forbidden", code: 403, message: "logs.cozystack.io \"select\" is forbidden" }
+ const { client } = clientReturning(Response.json(status, { status: 403 }))
+
+ const err = await queryRange(client, metricsRoot(ns), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
+
+ expect(err).toMatchObject({ kind: "not-available", status: 403 })
+ })
+
+ it("keeps the proxy's own plain-text 403 as forbidden", async () => {
+ const { client } = clientReturning(new Response("forbidden\n", { status: 403 }))
+
+ const err = await queryRange(client, metricsRoot(ns), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
+
+ expect(err).toMatchObject({ kind: "forbidden" })
+ })
+
it("reports a fetch failure as a network error", async () => {
const client = new K8sClient()
vi.spyOn(client, "raw").mockRejectedValue(new TypeError("offline"))
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
index 345d49815..c9fce3d4f 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -69,6 +69,17 @@ export function isScopeWide(error: unknown): boolean {
)
}
+// The proxy answers in plain text; a Kubernetes Status body means nginx sent
+// the path to kube-apiserver, which RBAC-denies or 404s a route it does not serve.
+function isKubernetesStatus(text: string): boolean {
+ try {
+ const parsed: unknown = JSON.parse(text)
+ return !!parsed && typeof parsed === "object" && "kind" in parsed && parsed.kind === "Status"
+ } catch {
+ return false
+ }
+}
+
// Prometheus answers {"error": "..."}, VictoriaLogs and nginx answer plain text.
function errorMessage(text: string, status: number): string {
try {
@@ -100,7 +111,8 @@ async function proxyGet(
}
if (!res.ok) {
const text = await res.text().catch(() => "")
- throw new ProxyError(errorKindForStatus(res.status), res.status, errorMessage(text, res.status))
+ const kind = isKubernetesStatus(text) ? "not-available" : errorKindForStatus(res.status)
+ throw new ProxyError(kind, res.status, errorMessage(text, res.status))
}
return res
}
From 4b44971d125cce2c2fd06457c68f23cf1645ac87 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:47:13 +0700
Subject: [PATCH 034/346] fix(dashboard): start each log follow run with its
own buffer
Turning Follow off and on again reused the previous run's buffer, so
new lines were appended after a silent gap and, on a quick toggle,
duplicated. A follow run is now identified by a session value, so the
buffer starts empty each time.
A resume now uses the newest _time seen rather than the last line
received, so an out-of-order line cannot move since backwards, and
the backoff sleep no longer leaves an abort listener per retry.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../lib/observability/use-log-tail.test.tsx | 31 ++++++++++++++++---
.../src/lib/observability/use-log-tail.ts | 23 ++++++++------
2 files changed, 41 insertions(+), 13 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
index 5c420314a..46df47970 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
@@ -27,7 +27,7 @@ describe("useLogTail", () => {
it("streams lines and resumes after a disconnect from the last _time without duplicates", async () => {
const { raw, wrapper } = setup(ndjson(line(T1, "a"), line(T2, "b")), ndjson(line(T2, "b"), line(T3, "c")))
- const { result } = renderHook(() => useLogTail("/root/", "*", true, { cap: 100, retryDelayMs: 1 }), { wrapper })
+ const { result } = renderHook(() => useLogTail("/root/", "*", 1, { cap: 100, retryDelayMs: 1 }), { wrapper })
await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a", "b", "c"]))
expect(new URL(raw.mock.calls[0][0], "http://x").searchParams.has("since")).toBe(false)
@@ -37,24 +37,47 @@ describe("useLogTail", () => {
it("stops and reports an error the proxy will repeat on every retry", async () => {
const { raw, wrapper } = setup(new Response("", { status: 403 }))
- const { result } = renderHook(() => useLogTail("/root/", "*", true, { cap: 100, retryDelayMs: 1 }), { wrapper })
+ const { result } = renderHook(() => useLogTail("/root/", "*", 1, { cap: 100, retryDelayMs: 1 }), { wrapper })
await waitFor(() => expect(result.current.state).toBe("stopped"))
expect(result.current.error).toMatchObject({ kind: "forbidden" })
expect(raw).toHaveBeenCalledTimes(1)
})
+ it("starts a new follow session with an empty buffer", async () => {
+ const { wrapper } = setup(ndjson(line(T1, "first")), ndjson(line(T3, "second")))
+ const { result, rerender } = renderHook(
+ ({ session }: { session: number | null }) => useLogTail("/root/", "*", session, { retryDelayMs: 60_000 }),
+ { wrapper, initialProps: { session: 1 as number | null } },
+ )
+ await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["first"]))
+
+ rerender({ session: null })
+ rerender({ session: 2 })
+
+ await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["second"]))
+ })
+
+ it("resumes from the newest line seen, not the last one received", async () => {
+ const { raw, wrapper } = setup(ndjson(line(T3, "c"), line(T1, "a")), ndjson())
+
+ renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper })
+
+ await waitFor(() => expect(raw.mock.calls.length).toBeGreaterThan(1))
+ expect(new URL(raw.mock.calls[1][0], "http://x").searchParams.get("since")).toBe(T3)
+ })
+
it("does not connect while disabled", () => {
const { raw, wrapper } = setup()
- renderHook(() => useLogTail("/root/", "*", false), { wrapper })
+ renderHook(() => useLogTail("/root/", "*", null), { wrapper })
expect(raw).not.toHaveBeenCalled()
})
it("aborts the stream on unmount", async () => {
const { raw, wrapper } = setup()
- const { unmount } = renderHook(() => useLogTail("/root/", "*", true), { wrapper })
+ const { unmount } = renderHook(() => useLogTail("/root/", "*", 1), { wrapper })
await waitFor(() => expect(raw).toHaveBeenCalled())
unmount()
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
index cb9222f48..ddc835821 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
@@ -1,6 +1,6 @@
import { useEffect, useState } from "react"
import { useK8sClient } from "@cozystack/k8s-client"
-import { mergeTail } from "./logs.ts"
+import { compareTime, mergeTail } from "./logs.ts"
import { ProxyError, tailLogs, type LogEntry } from "./proxy.ts"
export type TailState = "connecting" | "live" | "reconnecting" | "stopped"
@@ -18,25 +18,30 @@ const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forb
const sleep = (ms: number, signal: AbortSignal) =>
new Promise((resolve) => {
- const t = setTimeout(resolve, ms)
- signal.addEventListener("abort", () => {
+ const onAbort = () => {
clearTimeout(t)
resolve()
- })
+ }
+ const t = setTimeout(() => {
+ signal.removeEventListener("abort", onAbort)
+ resolve()
+ }, ms)
+ signal.addEventListener("abort", onAbort, { once: true })
})
+// Each non-null session is one follow run with its own buffer; null stops following.
export function useLogTail(
root: string,
query: string,
- enabled: boolean,
+ session: number | null,
{ cap = 1000, retryDelayMs = 2000 }: { cap?: number; retryDelayMs?: number } = {},
): Omit {
const client = useK8sClient()
- const key = `${root}\n${query}\n${enabled}`
+ const key = `${root}\n${query}\n${session}`
const [snap, setSnap] = useState({ key, entries: [], state: "connecting", error: null })
useEffect(() => {
- if (!enabled) return
+ if (session === null) return
const controller = new AbortController()
const { signal } = controller
let pending: LogEntry[] = []
@@ -64,7 +69,7 @@ export function useLogTail(
try {
await tailLogs(client, root, { query, since: lastTime }, (entry) => {
attempt = 0
- lastTime = entry._time
+ if (!lastTime || compareTime(entry._time, lastTime) > 0) lastTime = entry._time
pending.push(entry)
timer ??= setTimeout(flush, FLUSH_MS)
}, signal)
@@ -88,7 +93,7 @@ export function useLogTail(
controller.abort()
clearTimeout(timer)
}
- }, [client, root, query, enabled, key, cap, retryDelayMs])
+ }, [client, root, query, session, key, cap, retryDelayMs])
const current = snap.key === key ? snap : { key, entries: [], state: "connecting" as const, error: null }
return { entries: current.entries, state: current.state, error: current.error }
From 290490769eacf8906256290d3078bba02e4e9d04 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:47:13 +0700
Subject: [PATCH 035/346] fix(dashboard): let the reader scroll up while logs
stream
The viewer jumped to the newest line on every 100 ms flush, so a line
could not be read on a busy application. It now follows the bottom
only while the reader is already there, and says when a refused tail
has stopped instead of showing it as connecting.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/LogLines.test.tsx | 34 +++++++++++++++++++
.../src/components/observability/LogLines.tsx | 34 +++++++++++++++++++
.../observability/LogsView.test.tsx | 12 +++++++
.../src/components/observability/LogsView.tsx | 33 +++++-------------
4 files changed, 88 insertions(+), 25 deletions(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
new file mode 100644
index 000000000..775c5fcc6
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
@@ -0,0 +1,34 @@
+import { describe, it, expect } from "vitest"
+import { fireEvent, render, screen } from "@testing-library/react"
+import { LogLines } from "./LogLines.tsx"
+
+const entry = (n: number) => ({ _time: `2026-11-04T10:00:0${n}Z`, _msg: `line ${n}`, _stream_id: "s" })
+
+function sized(list: HTMLElement, scrollHeight: number) {
+ Object.defineProperty(list, "scrollHeight", { configurable: true, value: scrollHeight })
+ Object.defineProperty(list, "clientHeight", { configurable: true, value: 100 })
+}
+
+describe("LogLines", () => {
+ it("keeps following the newest line while the reader is at the bottom", () => {
+ const { rerender } = render( )
+ const list = screen.getByRole("list")
+ sized(list, 500)
+
+ rerender( )
+
+ expect(list.scrollTop).toBe(500)
+ })
+
+ it("leaves the position alone once the reader scrolls up", () => {
+ const { rerender } = render( )
+ const list = screen.getByRole("list")
+ sized(list, 500)
+ list.scrollTop = 120
+ fireEvent.scroll(list)
+
+ rerender( )
+
+ expect(list.scrollTop).toBe(120)
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
new file mode 100644
index 000000000..a6f4b5861
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
@@ -0,0 +1,34 @@
+import { useEffect, useRef } from "react"
+import { CONTAINER_FIELD, POD_FIELD } from "../../lib/observability/logs.ts"
+import type { LogEntry } from "../../lib/observability/proxy.ts"
+
+const BOTTOM_SLACK_PX = 24
+
+export function LogLines({ entries }: { entries: LogEntry[] }) {
+ const ref = useRef(null)
+ const pinned = useRef(true)
+
+ useEffect(() => {
+ if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight
+ }, [entries])
+
+ if (entries.length === 0) return No log lines for this range and filter.
+ return (
+ {
+ const el = e.currentTarget
+ pinned.current = el.scrollHeight - el.scrollTop - el.clientHeight <= BOTTOM_SLACK_PX
+ }}
+ className="max-h-[60vh] overflow-auto rounded-md bg-slate-950 p-3 font-mono text-xs leading-5 text-slate-100"
+ >
+ {entries.map((e, i) => (
+
+ {new Date(e._time).toLocaleTimeString()} {" "}
+ {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`} {" "}
+ {e._msg}
+
+ ))}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index fae3f6a4b..526efeb23 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -81,6 +81,18 @@ describe("LogsView", () => {
expect(screen.getByRole("button", { name: "Follow" })).toHaveAttribute("aria-pressed", "true")
})
+ it("says the tail stopped when the proxy refuses the query", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("tail") ? new Response("pipe not allowed in tail", { status: 400 }) : defaultHandler(endpoint, params),
+ )
+ renderWithK8sProvider( , { client })
+
+ await userEvent.click(screen.getByRole("button", { name: "Follow" }))
+
+ expect(await screen.findByText(/Stopped/)).toBeInTheDocument()
+ expect(screen.getByText("pipe not allowed in tail")).toBeInTheDocument()
+ })
+
it("renders a cluster without the proxy as not available", async () => {
const { client } = clientWith(() => new Response("404 page not found", { status: 404 }))
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index 707b037b9..f21f2eb86 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -1,4 +1,4 @@
-import { useEffect, useRef, useState } from "react"
+import { useState } from "react"
import { useQuery } from "@tanstack/react-query"
import { useK8sClient } from "@cozystack/k8s-client"
import { Button, Spinner } from "@cozystack/ui"
@@ -14,15 +14,16 @@ import {
isScopeWide,
logsRoot,
queryLogs,
- type LogEntry,
type ObservabilityScope,
} from "../../lib/observability/proxy.ts"
import { useLogTail } from "../../lib/observability/use-log-tail.ts"
import { ProxyNotice } from "./ProxyNotice.tsx"
import { RangePicker } from "./RangePicker.tsx"
+import { LogLines } from "./LogLines.tsx"
const HISTORY_LIMIT = 500
const FOLLOW_CAP = 1000
+const TAIL_STATE_LABEL = { connecting: "Connecting…", live: "Live", reconnecting: "Reconnecting…", stopped: "Stopped" }
function windowFor(range: LogRangeKey) {
const end = Date.now()
@@ -50,25 +51,6 @@ function Picker({ label, value, options, onChange }: { label: string; value: str
)
}
-function LogLines({ entries }: { entries: LogEntry[] }) {
- const ref = useRef(null)
- useEffect(() => {
- if (ref.current) ref.current.scrollTop = ref.current.scrollHeight
- }, [entries])
- if (entries.length === 0) return No log lines for this range and filter.
- return (
-
- {entries.map((e, i) => (
-
- {new Date(e._time).toLocaleTimeString()} {" "}
- {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`} {" "}
- {e._msg}
-
- ))}
-
- )
-}
-
export function LogsView({ scope }: { scope: ObservabilityScope }) {
const client = useK8sClient()
const root = logsRoot(scope)
@@ -77,7 +59,8 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
const [filter, setFilter] = useState("")
const [pod, setPod] = useState("")
const [container, setContainer] = useState("")
- const [follow, setFollow] = useState(false)
+ const [followRun, setFollowRun] = useState(null)
+ const follow = followRun !== null
const query = buildLogsQuery({ filter, pod, container })
const pickerQuery = (field: string, narrowTo?: string) => ({
@@ -100,7 +83,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
enabled: !follow,
retry: false,
})
- const tail = useLogTail(root, query, follow, { cap: FOLLOW_CAP })
+ const tail = useLogTail(root, query, followRun, { cap: FOLLOW_CAP })
const wide = [pods.error, history.error, tail.error].find(isScopeWide)
if (wide) {
@@ -135,13 +118,13 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
{ setPod(v); setContainer("") }} />
{!follow && }
- setFollow(!follow)}>
+ setFollowRun(follow ? null : Date.now())}>
Follow
{follow && (
- {tail.state === "live" ? "Live" : tail.state === "reconnecting" ? "Reconnecting…" : "Connecting…"}
+ {TAIL_STATE_LABEL[tail.state]}
{` · newest ${FOLLOW_CAP} lines kept`}
)}
From b05323f6bee913256c3e4ae6eedc764689ec508c Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:47:22 +0700
Subject: [PATCH 036/346] docs(dashboard): note the observability proxy client
in the console guide
The next session needs to know why one module bypasses the typed
Kubernetes client and where the charting dependency is loaded.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
packages/system/dashboard/images/console/CLAUDE.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/packages/system/dashboard/images/console/CLAUDE.md b/packages/system/dashboard/images/console/CLAUDE.md
index 2acf80504..96419d118 100644
--- a/packages/system/dashboard/images/console/CLAUDE.md
+++ b/packages/system/dashboard/images/console/CLAUDE.md
@@ -32,6 +32,8 @@ Implications:
- **RJSF** (`@rjsf/core` + `validator-ajv8`) for application config forms.
- **Vitest + jsdom + @testing-library/react** for tests.
- **Monaco** for YAML view, **noVNC** for the VM console tab.
+- **uPlot** for the Metrics tab charts, lazy-loaded with that tab.
+- Metrics and logs come from the observability proxy, not kube-apiserver: `lib/observability/proxy.ts` reads its native Prometheus JSON and VictoriaLogs NDJSON through `K8sClient.raw`, under the paths nginx routes to it ahead of `/apis`.
## Dev loop
From 84ed3aaefd34159549534dd8ff6f7e0e3111c3c9 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:50:33 +0700
Subject: [PATCH 037/346] fix(dashboard): follow the newest line again after
the log list clears
A reader who scrolled up stayed unpinned across a new follow run or a
new filter, so the fresh list sat at its top while lines streamed in
below. An empty list now means the reader is back at the bottom.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../components/observability/LogLines.test.tsx | 16 ++++++++++++++++
.../src/components/observability/LogLines.tsx | 3 ++-
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
index 775c5fcc6..7110fe464 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
@@ -31,4 +31,20 @@ describe("LogLines", () => {
expect(list.scrollTop).toBe(120)
})
+
+ it("follows again once the list is cleared for a new run", () => {
+ const { rerender } = render( )
+ const first = screen.getByRole("list")
+ sized(first, 500)
+ first.scrollTop = 120
+ fireEvent.scroll(first)
+
+ rerender( )
+ rerender( )
+ const second = screen.getByRole("list")
+ sized(second, 300)
+ rerender( )
+
+ expect(second.scrollTop).toBe(300)
+ })
})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
index a6f4b5861..452faac72 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
@@ -9,7 +9,8 @@ export function LogLines({ entries }: { entries: LogEntry[] }) {
const pinned = useRef(true)
useEffect(() => {
- if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight
+ if (entries.length === 0) pinned.current = true
+ else if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight
}, [entries])
if (entries.length === 0) return No log lines for this range and filter.
From 1b4ce7433e1acbb7d5b1d5ec6a4634fe2cd1b730 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:50:33 +0700
Subject: [PATCH 038/346] fix(dashboard): keep retrying a log tail refused for
load
The proxy answers a tail over its rate or concurrent-tail cap with a
4xx, and a reconnect can arrive before the previous tail's slot is
freed. Treating that as final left Follow stopped for good, while the
existing backoff already waits long enough for the cap to clear.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../console/src/lib/observability/use-log-tail.test.tsx | 8 ++++++++
.../apps/console/src/lib/observability/use-log-tail.ts | 3 ++-
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
index 46df47970..794f30216 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
@@ -67,6 +67,14 @@ describe("useLogTail", () => {
expect(new URL(raw.mock.calls[1][0], "http://x").searchParams.get("since")).toBe(T3)
})
+ it("retries a tail refused for load instead of stopping", async () => {
+ const { wrapper } = setup(new Response("too many concurrent requests", { status: 422 }), ndjson(line(T1, "a")))
+
+ const { result } = renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper })
+
+ await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a"]))
+ })
+
it("does not connect while disabled", () => {
const { raw, wrapper } = setup()
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
index ddc835821..292f93e9e 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
@@ -14,7 +14,8 @@ interface TailSnapshot {
const FLUSH_MS = 100
const MAX_RETRY_MS = 30_000
-const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forbidden", "not-available", "limit"])
+// A 422 or 429 on tail is load (rate or concurrent-tail caps), which backoff outlasts.
+const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forbidden", "not-available"])
const sleep = (ms: number, signal: AbortSignal) =>
new Promise((resolve) => {
From ebdd74ce4e010cc347d1ddae312b2c20dfeb39fa Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:53:23 +0700
Subject: [PATCH 039/346] fix(dashboard): name the proxy's real load status for
a refused tail
The proxy caps rate and concurrent tails with 429, so the retry
reasoning and its test now say so. Leaving Follow also remounts the
log list, so the history it falls back to opens at the newest line
even when the reader had scrolled up while following.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../apps/console/src/components/observability/LogsView.tsx | 2 +-
.../apps/console/src/lib/observability/use-log-tail.test.tsx | 2 +-
.../console/apps/console/src/lib/observability/use-log-tail.ts | 3 ++-
3 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index f21f2eb86..a27ca41c5 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -129,7 +129,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
)}
{error ? : null}
- {!follow && history.isLoading ? : }
+ {!follow && history.isLoading ? : }
)
}
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
index 794f30216..0c2306613 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
@@ -68,7 +68,7 @@ describe("useLogTail", () => {
})
it("retries a tail refused for load instead of stopping", async () => {
- const { wrapper } = setup(new Response("too many concurrent requests", { status: 422 }), ndjson(line(T1, "a")))
+ const { wrapper } = setup(new Response("too many concurrent requests", { status: 429 }), ndjson(line(T1, "a")))
const { result } = renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper })
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
index 292f93e9e..53cbaf856 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
@@ -14,7 +14,8 @@ interface TailSnapshot {
const FLUSH_MS = 100
const MAX_RETRY_MS = 30_000
-// A 422 or 429 on tail is load (rate or concurrent-tail caps), which backoff outlasts.
+// Load refusals retry: the proxy caps rate and concurrent tails with 429 (422 in
+// its first builds), and a reconnect can race the previous tail's freed slot.
const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forbidden", "not-available"])
const sleep = (ms: number, signal: AbortSignal) =>
From 2b823ab79fa178f22b338ad654962bc9c334a30e Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:53:50 +0700
Subject: [PATCH 040/346] fix(dashboard): rate over at least 5m and four steps
Long metric ranges are served by the longterm store, which keeps one
sample per 5m, so a rate window narrower than that can hold no samples
and the chart comes back empty. Four steps also keep every sample
between two points inside some window.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/lib/observability/metrics.test.ts | 17 +++++++++++++----
.../console/src/lib/observability/metrics.ts | 6 +++---
2 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts
index 0803940bf..0598c0c7f 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts
@@ -32,10 +32,19 @@ describe("METRIC_PANELS", () => {
)
})
- it("widens the rate window with the step on long ranges", () => {
- const [query] = METRIC_PANELS.find((p) => p.id === "cpu")!.queries({ start: 0, end: 1, step: 2520 })
-
- expect(query.expr).toContain("[5040s]")
+ // Long ranges are served by the longterm store, deduplicated at 5m, where a
+ // narrower window can hold no samples and the rate comes back empty.
+ it.each([
+ [15, "300s"],
+ [75, "300s"],
+ [90, "360s"],
+ [2520, "10080s"],
+ ])("rates over max(5m, 4 x step) for step %is", (step, window) => {
+ for (const panel of METRIC_PANELS) {
+ for (const q of panel.queries({ start: 0, end: 1, step })) {
+ if (q.expr.includes("rate(")) expect(q.expr).toContain(`[${window}]`)
+ }
+ }
})
it("reads memory as working set per pod", () => {
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
index b23a51e27..d7182b8d6 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
@@ -23,10 +23,10 @@ export function rangeWindow(range: RangeKey, nowSeconds: number): RangeWindow {
return { start: nowSeconds - span, end: nowSeconds, step: Math.max(MIN_STEP, Math.round(span / POINTS)) }
}
-// A rate needs several scrapes inside its window, and on long ranges the window
-// must also cover the step or samples between points are skipped.
+// The longterm store keeps one sample per 5m, so a narrower window can be empty;
+// 4 x step also keeps samples between points from being skipped.
function rateWindow(step: number): string {
- return `${Math.max(300, step * 2)}s`
+ return `${Math.max(300, step * 4)}s`
}
const CONTAINERS = 'container!="",container!="POD"'
From e79946e79c595d4973f4f0062c9322575e2bbc4a Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil
Date: Wed, 4 Nov 2026 10:08:59 +0500
Subject: [PATCH 041/346] fix(platform): remove the dead registries value and
patch-containerd Secret
Nothing reads the Secret: the kubeadm certs.d mount it fed is gone and
the Talos worker config never took it. The value stayed in the Package
spec as a knob that accepts mirrors and does nothing, which is worse
than no knob for an operator in an air-gapped setup. Mirrors for tenant
workers are a per-pool field of the KubernetesNodes chart.
A Package that still sets registries keeps installing: the platform
chart has no values schema, so the key is ignored.
Assisted-By: LLM
Signed-off-by: Myasnikov Daniil
---
.../templates/containerd-registry-secret.yaml | 35 -------------------
packages/core/platform/values.yaml | 31 ----------------
2 files changed, 66 deletions(-)
delete mode 100644 packages/core/platform/templates/containerd-registry-secret.yaml
diff --git a/packages/core/platform/templates/containerd-registry-secret.yaml b/packages/core/platform/templates/containerd-registry-secret.yaml
deleted file mode 100644
index 6e9fb6408..000000000
--- a/packages/core/platform/templates/containerd-registry-secret.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-{{- if .Values.registries.mirrors }}
-apiVersion: v1
-kind: Secret
-metadata:
- name: patch-containerd
- namespace: cozy-system
-type: Opaque
-stringData:
-{{- range $registry, $mirror := .Values.registries.mirrors }}
-{{- if $mirror.endpoints }}
- {{ $registry }}.toml: |
- server = "https://{{ $registry }}"
-{{- range $endpoint := $mirror.endpoints }}
- [host."{{ $endpoint }}"]
- capabilities = ["pull", "resolve"]
-{{- $endpointConfig := index $.Values.registries.config $endpoint }}
-{{- if $endpointConfig }}
-{{- if $endpointConfig.tls }}
-{{- if $endpointConfig.tls.insecureSkipVerify }}
- skip_verify = true
-{{- end }}
-{{- end }}
-{{- if $endpointConfig.auth }}
- [host."{{ $endpoint }}".auth]
- username = "{{ $endpointConfig.auth.username }}"
- password = "{{ $endpointConfig.auth.password }}"
-{{- end }}
-{{- else }}
- skip_verify = true
-{{- end }}
-{{- end }}
-{{- end }}
-{{- end }}
-{{- end }}
-
diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml
index f2ce252ed..23ed9a448 100644
--- a/packages/core/platform/values.yaml
+++ b/packages/core/platform/values.yaml
@@ -768,37 +768,6 @@ scheduling:
# loginTheme: acme
# loginThemeImage: registry.example.com/acme-keycloak-theme:v1
branding: {}
-# Container registry mirrors configuration
-#
-# Example:
-# registries:
-# mirrors:
-# docker.io:
-# endpoints:
-# - http://10.0.0.1:8082
-# ghcr.io:
-# endpoints:
-# - http://10.0.0.1:8083
-# gcr.io:
-# endpoints:
-# - http://10.0.0.1:8084
-# registry.k8s.io:
-# endpoints:
-# - http://10.0.0.1:8085
-# quay.io:
-# endpoints:
-# - http://10.0.0.1:8086
-# cr.fluentbit.io:
-# endpoints:
-# - http://10.0.0.1:8087
-# docker-registry3.mariadb.com:
-# endpoints:
-# - http://10.0.0.1:8088
-# config:
-# "10.0.0.1:8082":
-# tls:
-# insecureSkipVerify: true
-registries: {}
# Resource allocation ratios
resources:
cpuAllocationRatio: 10
From a2d0e4cb5058092a66cb3173503850c64ba9fab0 Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil
Date: Wed, 4 Nov 2026 06:07:07 +0000
Subject: [PATCH 042/346] fix(cilium): pass the CiliumNode list to jq on stdin
in the IPAM hook
The hook fed the whole `kubectl get ciliumnodes -o json` to jq as one
argument. Cilium copies every Node label onto its CiliumNode, 22 to 30
KiB per node on Talos or with NFD and GPU labels, so the list passes the
128 KiB Linux allows a single argument within six nodes. jq then fails
to start with "Argument list too long", the hook Job fails before it
moves any node, and the cilium HelmRelease loops on UpgradeFailed with
the HelmReleases behind it stuck in DependencyNotReady.
The hook renders on every upgrade of cilium in cluster-pool mode, so a
cluster that already finished the migration hits the same failure on its
next values change.
Assisted-by: LLM
Signed-off-by: Myasnikov Daniil
---
hack/cilium-ipam-migration.bats | 27 +++++++++++++++++++
.../system/cilium/files/ipam-migration.sh | 10 ++++---
2 files changed, 33 insertions(+), 4 deletions(-)
diff --git a/hack/cilium-ipam-migration.bats b/hack/cilium-ipam-migration.bats
index c54db50ab..53053747a 100644
--- a/hack/cilium-ipam-migration.bats
+++ b/hack/cilium-ipam-migration.bats
@@ -488,6 +488,33 @@ pod_cidrs_of() {
rm -rf "$STATE"
}
+@test "a CiliumNode list larger than one argument may hold still moves every node and names the pod on an old Ingress IP" {
+ init_state
+ add_ip web-0.tenant-c ovn-default 10.244.1.135
+ add_ip web-1.tenant-c ovn-default 10.244.4.135
+ add_ip db-0.tenant-c ovn-default 10.244.9.9
+ # Cilium copies every Node label onto the CiliumNode, which takes six real
+ # ones past the 128 KiB Linux allows a single argument. One annotation stands
+ # in for the labels and keeps the list over that limit even printed compact.
+ for i in 1 2 3 4 5 6; do
+ add_node "n$i" "10.244.$i.0/24" "10.244.$i.223" "10.244.$i.135"
+ jq '.metadata.annotations.padding = ("x" * 24576)' "$STATE/ciliumnodes/n$i.json" > "$STATE/node.tmp"
+ mv "$STATE/node.tmp" "$STATE/ciliumnodes/n$i.json"
+ done
+ [ "$(kubectl get ciliumnodes -o json | jq -c . | wc -c)" -gt 131072 ]
+ "$SCRIPT" > "$STATE/out" 2>&1
+ # Each pod is named with the node whose Ingress IP it holds, and a pod on an
+ # address that is no node's Ingress IP is not named at all.
+ grep -q "web-0.tenant-c holds 10.244.1.135, the Ingress IP of n1 before the move" "$STATE/out"
+ grep -q "web-1.tenant-c holds 10.244.4.135, the Ingress IP of n4 before the move" "$STATE/out"
+ if grep -q "db-0.tenant-c" "$STATE/out"; then
+ cat "$STATE/out"
+ false
+ fi
+ [ "$(pod_cidrs_of n6)" = '["10.244.248.40/29"]' ]
+ rm -rf "$STATE"
+}
+
@test "a cluster without kube-ovn skips the exclusion and still moves its nodes" {
init_state
rm "$STATE/subnet.json"
diff --git a/packages/system/cilium/files/ipam-migration.sh b/packages/system/cilium/files/ipam-migration.sh
index 002a1bf78..9345c19f6 100755
--- a/packages/system/cilium/files/ipam-migration.sh
+++ b/packages/system/cilium/files/ipam-migration.sh
@@ -240,10 +240,12 @@ exclude_from_kube_ovn() {
fi
# A pod that holds a node's current Ingress IP keeps the address after the
# move, and that node keeps it in its ipcache as reserved:ingress, so the pod
- # stays unreachable from there until it is recreated.
- jq -r --argjson cn "$ciliumnodes" '
- ([$cn.items[] | select(.spec.ingress.ipv4) | {key: .spec.ingress.ipv4, value: .metadata.name}] | from_entries) as $ingress
- | .[] | select($ingress[.ip]) | "\(.name) \(.ip) \($ingress[.ip])"' <<<"$ipobjects" \
+ # stays unreachable from there until it is recreated. The CiliumNode list goes
+ # in on stdin, because the Node labels Cilium copies onto every CiliumNode take
+ # it past the 128 KiB a single argument may hold within a handful of nodes.
+ jq -rn 'input as $cn
+ | ([$cn.items[] | select(.spec.ingress.ipv4) | {key: .spec.ingress.ipv4, value: .metadata.name}] | from_entries) as $ingress
+ | input[] | select($ingress[.ip]) | "\(.name) \(.ip) \($ingress[.ip])"' <<<"$ciliumnodes"$'\n'"$ipobjects" \
| while read -r ipname ip node; do
if ! in_any "$POOL" "$ip"; then
log "$ipname holds $ip, the Ingress IP of $node before the move. Recreate it once the run is done, $node keeps that address as reserved:ingress"
From 7b7444feaf5508eaf62767b8c999468558732d1b Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil
Date: Wed, 4 Nov 2026 06:16:36 +0000
Subject: [PATCH 043/346] fix(kubernetes): move the OIDC users out of the
bootstrap Job's script
The bootstrap Job wrote a ClusterRoleBinding per oidc.users entry into
its script, and the script reaches the container as one `sh -c`
argument. Linux caps a single argument at 128 KiB, which about 240
users cross at roughly 530 bytes each. The container then cannot start,
the post-install and post-upgrade hook fails, and the tenant cluster's
HelmRelease stays unready. The values schema puts no bound on
oidc.users, so a shared cluster reaches that size with ordinary
per-person entries.
The bindings and the list of their names now reach the Job through a
ConfigMap it mounts, so the script is the same size for any list. Names,
labels and roles of the bindings are unchanged, so the bindings already
in a tenant cluster are updated in place and none is orphaned.
Assisted-by: LLM
Signed-off-by: Myasnikov Daniil
---
hack/kubernetes-oidc-bindings.bats | 187 +++++++++++++++++
.../templates/oidc-rbac-bindings.yaml | 42 ++++
.../kubernetes/templates/oidc-rbac-job.yaml | 57 +++---
packages/apps/kubernetes/tests/oidc_test.yaml | 193 +++++++++++++++++-
4 files changed, 439 insertions(+), 40 deletions(-)
create mode 100644 hack/kubernetes-oidc-bindings.bats
create mode 100644 packages/apps/kubernetes/templates/oidc-rbac-bindings.yaml
diff --git a/hack/kubernetes-oidc-bindings.bats b/hack/kubernetes-oidc-bindings.bats
new file mode 100644
index 000000000..f37273802
--- /dev/null
+++ b/hack/kubernetes-oidc-bindings.bats
@@ -0,0 +1,187 @@
+#!/usr/bin/env bats
+# -----------------------------------------------------------------------------
+# packages/apps/kubernetes: the OIDC bootstrap Job must not carry the users[]
+# list inside its container command.
+#
+# The Job's script reaches the container as one `sh -c` argument, and Linux caps
+# one exec argument at 128 KiB (MAX_ARG_STRLEN). With a ClusterRoleBinding per
+# user written into that script, about 240 users take it past the cap: the
+# container cannot start, the post-install/post-upgrade hook fails, and the
+# tenant cluster's HelmRelease stays unready. The bindings reach the Job through
+# a mounted ConfigMap instead, so the script is the same size for any list.
+#
+# The last three tests execute the rendered script against a fake kubectl, since
+# no assertion on the script's text says what the prune step does with the list:
+# a condition forced either way passes every helm-unittest case.
+#
+# Requires: helm, yq (mikefarah v4+), jq, like hack/admin-kubeconfig-invariant.bats.
+# -----------------------------------------------------------------------------
+
+load test_helper
+
+USERS=300
+
+# render_users renders the chart in System mode with
+# generated users and leaves one JSON document per object in /rendered.json.
+render_users() {
+ jq -n --argjson n "$2" '
+ {oidc: {users: [range($n) | {email: "user\(.)@example.org", role: (if . % 7 == 0 then "admin" else "view" end)}]}}' \
+ > "$1/users.yaml"
+ helm template bindings packages/apps/kubernetes \
+ --namespace tenant-test \
+ --values packages/apps/kubernetes/tests/values/oidc-system.yaml \
+ --values "$1/users.yaml" \
+ > "$1/rendered.yaml"
+ yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json"
+}
+
+# render_job renders the chart in CustomConfig mode and unpacks
+# what the bootstrap Job runs and mounts into : script.sh is its `sh -c`
+# script, bindings/ holds the two files of the mounted ConfigMap. -j keeps an
+# empty value an empty file, as the kubelet writes it.
+render_job() {
+ jq -n --argjson users "$2" '
+ {_cluster: {"oidc-enabled": "false"},
+ oidc: {mode: "CustomConfig",
+ customConfig: {config: "apiVersion: apiserver.config.k8s.io/v1beta1\nkind: AuthenticationConfiguration\njwt: []\n"},
+ users: $users}}' > "$1/values.yaml"
+ helm template bindings packages/apps/kubernetes \
+ --namespace tenant-test \
+ --values packages/apps/kubernetes/tests/values/oidc-system.yaml \
+ --values "$1/values.yaml" \
+ > "$1/rendered.yaml"
+ yq --output-format=json eval-all '.' "$1/rendered.yaml" > "$1/rendered.json"
+ jq -s -j '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap")
+ | .spec.template.spec.containers[0].command[2]' "$1/rendered.json" > "$1/script.sh"
+ mkdir -p "$1/bindings"
+ jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
+ | .data["crbs.yaml"]' "$1/rendered.json" > "$1/bindings/crbs.yaml"
+ jq -s -j '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
+ | .data["desired.txt"]' "$1/rendered.json" > "$1/bindings/desired.txt"
+}
+
+# fake_kubectl puts a kubectl first on PATH that logs every call and
+# answers the script's list of existing bindings with .
+fake_kubectl() {
+ mkdir -p "$1/bin"
+ printf '%s' "$2" > "$1/existing"
+ cat > "$1/bin/kubectl" <<'KEOF'
+#!/bin/sh
+echo "$*" >> "$KLOG"
+case "$*" in
+ *"get clusterrolebindings"*) cat "$EXISTING" ;;
+esac
+KEOF
+ chmod +x "$1/bin/kubectl"
+}
+
+# run_job runs the unpacked script as the container does and leaves its
+# status in $rc and its output in /out. Run by bash, which has the pipefail
+# the script sets and which is what the unit lane runs under; the container's own
+# shell is not exercised.
+run_job() {
+ : > "$1/kubectl.log"
+ echo kubeconfig > "$1/kubeconfig"
+ rc=0
+ env PATH="$1/bin:$PATH" KLOG="$1/kubectl.log" EXISTING="$1/existing" HOME="$1" \
+ KUBECONFIG_TENANT="$1/kubeconfig" RELEASE=bindings RELEASE_NAMESPACE=tenant-test \
+ OIDC_MODE=CustomConfig OIDC_BINDINGS_DIR="$1/bindings" \
+ bash "$1/script.sh" > "$1/out" 2>&1 || rc=$?
+}
+
+@test "the OIDC bootstrap Job script stays under the exec argument cap and carries no user" {
+ dir=$(mktemp -d)
+ render_users "$dir" "$USERS"
+ jq -s -r '.[] | select(.kind == "Job" and .metadata.name == "bindings-oidc-bootstrap")
+ | .spec.template.spec.containers[0].command[2]' "$dir/rendered.json" > "$dir/script"
+ size=$(wc -c < "$dir/script")
+ [ "$size" -gt 0 ]
+ [ "$size" -lt 131072 ]
+ if grep -q -E 'user[0-9]+@example\.org' "$dir/script"; then
+ echo "the Job script still carries users[] entries" >&2
+ false
+ fi
+ rm -rf "$dir"
+}
+
+@test "every user keeps one ClusterRoleBinding with its old name and role in the bindings ConfigMap" {
+ dir=$(mktemp -d)
+ render_users "$dir" "$USERS"
+ jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
+ | .data["crbs.yaml"]' "$dir/rendered.json" > "$dir/crbs.yaml"
+ jq -s -r '.[] | select(.kind == "ConfigMap" and .metadata.name == "bindings-oidc-bindings")
+ | .data["desired.txt"]' "$dir/rendered.json" > "$dir/desired.txt"
+ [ -s "$dir/crbs.yaml" ]
+ yq --output-format=json eval-all '.' "$dir/crbs.yaml" | jq -s '.' > "$dir/crbs.json"
+
+ [ "$(jq 'length' "$dir/crbs.json")" -eq "$USERS" ]
+ [ "$(jq -r '[.[].subjects[0].name] | unique | length' "$dir/crbs.json")" -eq "$USERS" ]
+ [ "$(jq -r '[.[] | select(.roleRef.name == "cluster-admin")] | length' "$dir/crbs.json")" -eq $(( (USERS + 6) / 7 )) ]
+
+ # The name is what the prune step and every binding already in a tenant cluster
+ # go by, so it must not move: cozystack-oidc- plus the first 16 hex digits of
+ # sha256("-").
+ for i in 0 1 6 7; do
+ email="user$i@example.org"
+ name="cozystack-oidc-$(printf '%s' "bindings-$email" | sha256sum | cut -c1-16)"
+ [ "$(jq -r --arg e "$email" '.[] | select(.subjects[0].name == $e) | .metadata.name' "$dir/crbs.json")" = "$name" ]
+ grep -q -x -F "$name" "$dir/desired.txt"
+ done
+
+ # The list the prune step compares against is exactly the applied names.
+ [ "$(jq -r '.[].metadata.name' "$dir/crbs.json" | sort)" = "$(grep . "$dir/desired.txt" | sort)" ]
+ rm -rf "$dir"
+}
+
+@test "the Job applies the mounted bindings and deletes only the bindings no user has" {
+ dir=$(mktemp -d)
+ render_job "$dir" '[{"email":"alice@example.org","role":"admin"},{"email":"bob@example.org","role":"view"}]'
+ alice="cozystack-oidc-$(printf '%s' 'bindings-alice@example.org' | sha256sum | cut -c1-16)"
+ bob="cozystack-oidc-$(printf '%s' 'bindings-bob@example.org' | sha256sum | cut -c1-16)"
+ fake_kubectl "$dir" "$alice cozystack-oidc-000000000000dead $bob cozystack-oidc-000000000000beef"
+ run_job "$dir"
+ if [ "$rc" -ne 0 ]; then
+ cat "$dir/out"
+ false
+ fi
+ grep -q -F -- "apply --filename $dir/bindings/crbs.yaml" "$dir/kubectl.log"
+ grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000dead --ignore-not-found" "$dir/kubectl.log"
+ grep -q -F -- "delete clusterrolebinding cozystack-oidc-000000000000beef --ignore-not-found" "$dir/kubectl.log"
+ if grep -F -e "delete clusterrolebinding $alice" -e "delete clusterrolebinding $bob" "$dir/kubectl.log"; then
+ echo "the Job deleted a binding a user still has" >&2
+ false
+ fi
+ [ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ]
+ rm -rf "$dir"
+}
+
+@test "a release with no users applies nothing and deletes every binding it left behind" {
+ dir=$(mktemp -d)
+ render_job "$dir" '[]'
+ fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef"
+ run_job "$dir"
+ if [ "$rc" -ne 0 ]; then
+ cat "$dir/out"
+ false
+ fi
+ if grep -q -F -- ' apply ' "$dir/kubectl.log"; then
+ echo "the Job ran kubectl apply with no bindings to apply" >&2
+ false
+ fi
+ [ "$(grep -c 'delete clusterrolebinding' "$dir/kubectl.log")" -eq 2 ]
+ rm -rf "$dir"
+}
+
+@test "the Job stops before deleting anything when the desired list is not mounted" {
+ dir=$(mktemp -d)
+ render_job "$dir" '[{"email":"alice@example.org","role":"admin"}]'
+ fake_kubectl "$dir" "cozystack-oidc-000000000000dead cozystack-oidc-000000000000beef"
+ rm "$dir/bindings/desired.txt"
+ run_job "$dir"
+ [ "$rc" -ne 0 ]
+ if grep -q -E 'delete clusterrolebinding| apply ' "$dir/kubectl.log"; then
+ cat "$dir/kubectl.log"
+ false
+ fi
+ rm -rf "$dir"
+}
diff --git a/packages/apps/kubernetes/templates/oidc-rbac-bindings.yaml b/packages/apps/kubernetes/templates/oidc-rbac-bindings.yaml
new file mode 100644
index 000000000..a8517da4f
--- /dev/null
+++ b/packages/apps/kubernetes/templates/oidc-rbac-bindings.yaml
@@ -0,0 +1,42 @@
+{{- /*
+ The ClusterRoleBindings the OIDC bootstrap Job (oidc-rbac-job.yaml) applies
+ inside the tenant cluster, one per `oidc.users` entry, and the list of their
+ names it prunes against. They travel in a ConfigMap the Job mounts and not as
+ text inside its script: the script reaches the container as a single `sh -c`
+ argument, Linux caps one argument at 128 KiB, and a binding per user crosses
+ that cap at about 240 users, after which the container cannot start.
+
+ A release-owned object, not a hook resource, so it exists before the
+ post-install and post-upgrade hooks run. It is rendered only where users are
+ bound; at `mode: None` the Job takes its early branch and the volume is
+ optional.
+*/}}
+{{- if ne .Values.oidc.mode "None" }}
+{{- $desired := "" }}
+{{- $bindings := "" }}
+{{- range .Values.oidc.users }}
+{{- $name := printf "cozystack-oidc-%s" (printf "%s-%s" $.Release.Name .email | sha256sum | trunc 16) }}
+{{- $role := "view" }}
+{{- if eq .role "admin" }}
+{{- $role = "cluster-admin" }}
+{{- end }}
+{{- $desired = printf "%s%s\n" $desired $name }}
+{{- $binding := dict
+ "apiVersion" "rbac.authorization.k8s.io/v1"
+ "kind" "ClusterRoleBinding"
+ "metadata" (dict "name" $name "labels" (dict "app.kubernetes.io/managed-by" "cozystack-oidc" "app.kubernetes.io/instance" $.Release.Name))
+ "roleRef" (dict "apiGroup" "rbac.authorization.k8s.io" "kind" "ClusterRole" "name" $role)
+ "subjects" (list (dict "kind" "User" "apiGroup" "rbac.authorization.k8s.io" "name" .email)) }}
+{{- $bindings = printf "%s---\n%s\n" $bindings (toYaml $binding) }}
+{{- end }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ .Release.Name }}-oidc-bindings
+ namespace: {{ .Release.Namespace }}
+ labels:
+ {{- include "kubernetes.labels" . | nindent 4 }}
+data:
+ {{- dict "crbs.yaml" $bindings "desired.txt" $desired | toYaml | nindent 2 }}
+{{- end }}
diff --git a/packages/apps/kubernetes/templates/oidc-rbac-job.yaml b/packages/apps/kubernetes/templates/oidc-rbac-job.yaml
index 8672ce124..8556d092b 100644
--- a/packages/apps/kubernetes/templates/oidc-rbac-job.yaml
+++ b/packages/apps/kubernetes/templates/oidc-rbac-job.yaml
@@ -25,7 +25,9 @@
- `mode: System | CustomConfig`:
- Apply one ClusterRoleBinding per `users[]` entry inside the tenant
cluster (`admin` → `cluster-admin`, `view` → `view`, subject = the
- literal email matched against the `email` claim).
+ literal email matched against the `email` claim). The bindings are
+ rendered by oidc-rbac-bindings.yaml into a ConfigMap this Job mounts,
+ not into the script below.
- Prune stale CRBs labelled by this release.
- `mode: System`: patch the pre-rendered `-oidc-kubeconfig`
Secret with a real kubeconfig wired to `kubectl oidc-login`
@@ -168,6 +170,8 @@ spec:
value: {{ $issuer | quote }}
- name: OIDC_SERVER_URL
value: {{ printf "https://%s:443" $externalHost | quote }}
+ - name: OIDC_BINDINGS_DIR
+ value: /etc/oidc-bindings
resources:
requests:
cpu: 10m
@@ -254,46 +258,32 @@ spec:
sleep 5
done
+ # A missing list must stop the Job here: grep on a file that is not
+ # there exits 2, `! grep` is then true, and the prune below would
+ # delete every binding of the release.
+ BINDINGS=$OIDC_BINDINGS_DIR
+ if [ ! -f "$BINDINGS/desired.txt" ]; then
+ echo "$BINDINGS/desired.txt is not mounted (ConfigMap $RELEASE-oidc-bindings); refusing to prune against a list that is not there" >&2
+ exit 1
+ fi
+
echo "Applying ${OIDC_MODE} ClusterRoleBindings to the tenant cluster..."
- DESIRED_FILE=$(mktemp)
- {{- range .Values.oidc.users }}
- {{- $hash := printf "%s-%s" $.Release.Name .email | sha256sum | trunc 16 }}
- {{- $crbName := printf "cozystack-oidc-%s" $hash }}
- {{- $role := "view" }}
- {{- if eq .role "admin" }}
- {{- $role = "cluster-admin" }}
- {{- end }}
- echo "{{ $crbName }}" >> "$DESIRED_FILE"
- cat <<'EOF' | kubectl --kubeconfig "$KUBECONFIG_TENANT" apply --filename -
- apiVersion: rbac.authorization.k8s.io/v1
- kind: ClusterRoleBinding
- metadata:
- name: {{ $crbName }}
- labels:
- app.kubernetes.io/managed-by: cozystack-oidc
- app.kubernetes.io/instance: {{ $.Release.Name }}
- roleRef:
- apiGroup: rbac.authorization.k8s.io
- kind: ClusterRole
- name: {{ $role }}
- subjects:
- - kind: User
- apiGroup: rbac.authorization.k8s.io
- name: {{ .email | quote }}
- EOF
- {{- end }}
+ # kubectl apply fails on a file with no objects, and a release with
+ # no users has none.
+ if [ -s "$BINDINGS/crbs.yaml" ]; then
+ kubectl --kubeconfig "$KUBECONFIG_TENANT" apply --filename "$BINDINGS/crbs.yaml"
+ fi
echo "Pruning stale ClusterRoleBindings labelled by this release..."
EXISTING=$(kubectl --kubeconfig "$KUBECONFIG_TENANT" get clusterrolebindings \
--selector=app.kubernetes.io/managed-by=cozystack-oidc,app.kubernetes.io/instance=$RELEASE \
--output=jsonpath='{.items[*].metadata.name}')
for crb in $EXISTING; do
- if ! grep -x -q -F "$crb" "$DESIRED_FILE"; then
+ if ! grep -x -q -F "$crb" "$BINDINGS/desired.txt"; then
echo " removing $crb"
kubectl --kubeconfig "$KUBECONFIG_TENANT" delete clusterrolebinding "$crb" --ignore-not-found
fi
done
- rm -f "$DESIRED_FILE"
if [ "$OIDC_MODE" = "System" ]; then
echo "Patching the OIDC kubeconfig Secret on the management cluster..."
@@ -349,6 +339,9 @@ spec:
- name: kubeconfig
mountPath: /etc/kubernetes/kubeconfig
readOnly: true
+ - name: oidc-bindings
+ mountPath: /etc/oidc-bindings
+ readOnly: true
- name: tmp
mountPath: /tmp
volumes:
@@ -356,6 +349,10 @@ spec:
secret:
secretName: {{ .Release.Name }}-admin-kubeconfig
optional: true
+ - name: oidc-bindings
+ configMap:
+ name: {{ .Release.Name }}-oidc-bindings
+ optional: true
- name: tmp
emptyDir: {}
{{- /*
diff --git a/packages/apps/kubernetes/tests/oidc_test.yaml b/packages/apps/kubernetes/tests/oidc_test.yaml
index 0623063e7..8511e392c 100644
--- a/packages/apps/kubernetes/tests/oidc_test.yaml
+++ b/packages/apps/kubernetes/tests/oidc_test.yaml
@@ -2561,7 +2561,73 @@ tests:
value: "true"
documentIndex: 3
- - it: bootstrap Job command builds a CRB for each users[] entry
+ - it: the bindings ConfigMap holds a CRB for each users[] entry
+ release:
+ name: kubernetes-app
+ namespace: tenant-foo
+ template: templates/oidc-rbac-bindings.yaml
+ set:
+ _cluster.oidc-enabled: "true"
+ _cluster.root-host: example.org
+ oidc.mode: System
+ oidc.users:
+ - email: alice@example.org
+ role: admin
+ - email: bob@example.org
+ role: view
+ asserts:
+ - isKind:
+ of: ConfigMap
+ - equal:
+ path: metadata.name
+ value: kubernetes-app-oidc-bindings
+ - matchRegex:
+ path: data["crbs.yaml"]
+ pattern: 'name: alice@example.org'
+ - matchRegex:
+ path: data["crbs.yaml"]
+ pattern: 'name: bob@example.org'
+ - matchRegex:
+ path: data["crbs.yaml"]
+ pattern: 'app.kubernetes.io/managed-by: cozystack-oidc'
+ - matchRegex:
+ path: data["crbs.yaml"]
+ pattern: 'app.kubernetes.io/instance: kubernetes-app'
+
+ - it: the binding names and roles stay what the Job script used to render
+ # The prune step and every binding already in a tenant cluster go by these
+ # names: cozystack-oidc- plus the first 16 hex digits of
+ # sha256("-"). A different name leaves the old binding
+ # behind next to the new one.
+ release:
+ name: kubernetes-app
+ namespace: tenant-foo
+ template: templates/oidc-rbac-bindings.yaml
+ set:
+ _cluster.oidc-enabled: "true"
+ _cluster.root-host: example.org
+ oidc.mode: System
+ oidc.users:
+ - email: alice@example.org
+ role: admin
+ - email: bob@example.org
+ role: view
+ asserts:
+ - equal:
+ path: data["desired.txt"]
+ value: "cozystack-oidc-0001a28136ca1a02\ncozystack-oidc-ec8dfb1cbc75e699\n"
+ - matchRegex:
+ path: data["crbs.yaml"]
+ pattern: 'name: cozystack-oidc-0001a28136ca1a02\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: cluster-admin\nsubjects:\n- apiGroup: rbac.authorization.k8s.io\n kind: User\n name: alice@example.org\n'
+ - matchRegex:
+ path: data["crbs.yaml"]
+ pattern: 'name: cozystack-oidc-ec8dfb1cbc75e699\nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: ClusterRole\n name: view\nsubjects:\n- apiGroup: rbac.authorization.k8s.io\n kind: User\n name: bob@example.org\n'
+
+ - it: the bootstrap Job mounts the bindings ConfigMap and carries no users[] entry in its script
+ # The script is one `sh -c` argument and Linux caps one argument at 128 KiB:
+ # a binding per user in it stops the container from starting at about 240
+ # users. hack/kubernetes-oidc-bindings.bats renders 300 users to hold the
+ # size itself.
release:
name: kubernetes-app
namespace: tenant-foo
@@ -2575,23 +2641,130 @@ tests:
role: admin
- email: bob@example.org
role: view
+ asserts:
+ - contains:
+ path: spec.template.spec.volumes
+ content:
+ name: oidc-bindings
+ configMap:
+ name: kubernetes-app-oidc-bindings
+ optional: true
+ documentIndex: 3
+ - contains:
+ path: spec.template.spec.containers[0].volumeMounts
+ content:
+ name: oidc-bindings
+ mountPath: /etc/oidc-bindings
+ readOnly: true
+ documentIndex: 3
+ # The script reads the directory from the environment, which is also how
+ # hack/kubernetes-oidc-bindings.bats points it at a scratch directory.
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: OIDC_BINDINGS_DIR
+ value: /etc/oidc-bindings
+ documentIndex: 3
+ - notMatchRegex:
+ path: spec.template.spec.containers[0].command[2]
+ pattern: 'alice@example\.org'
+ documentIndex: 3
+ - notMatchRegex:
+ path: spec.template.spec.containers[0].command[2]
+ pattern: 'bob@example\.org'
+ documentIndex: 3
+
+ - it: the bootstrap Job stops before pruning when the desired list is not mounted
+ # `! grep -F` on a file that is not there is true, so a missing mount would
+ # delete every binding of the release.
+ release:
+ name: kubernetes-app
+ namespace: tenant-foo
+ template: templates/oidc-rbac-job.yaml
+ set:
+ _cluster.oidc-enabled: "true"
+ _cluster.root-host: example.org
+ oidc.mode: System
asserts:
- matchRegex:
path: spec.template.spec.containers[0].command[2]
- pattern: 'name: "alice@example.org"'
+ pattern: '(?s)if \[ ! -f "\$BINDINGS/desired\.txt" \]; then.*?exit 1\n\s*fi.*Pruning stale ClusterRoleBindings'
documentIndex: 3
+
+ - it: the bootstrap Job applies the bindings file only when it has objects in it
+ # kubectl apply exits non-zero on a file with no objects, and a release
+ # with no users has none.
+ release:
+ name: kubernetes-app
+ namespace: tenant-foo
+ template: templates/oidc-rbac-job.yaml
+ set:
+ _cluster.oidc-enabled: "true"
+ _cluster.root-host: example.org
+ oidc.mode: System
+ asserts:
- matchRegex:
path: spec.template.spec.containers[0].command[2]
- pattern: 'name: "bob@example.org"'
+ pattern: 'if \[ -s "\$BINDINGS/crbs\.yaml" \]; then\n\s+kubectl --kubeconfig "\$KUBECONFIG_TENANT" apply --filename "\$BINDINGS/crbs\.yaml"\n\s*fi'
documentIndex: 3
+
+ - it: a release with no users renders an empty bindings ConfigMap
+ release:
+ name: kubernetes-app
+ namespace: tenant-foo
+ template: templates/oidc-rbac-bindings.yaml
+ set:
+ _cluster.oidc-enabled: "true"
+ _cluster.root-host: example.org
+ oidc.mode: System
+ oidc.users: []
+ asserts:
+ - equal:
+ path: data["crbs.yaml"]
+ value: ""
+ - equal:
+ path: data["desired.txt"]
+ value: ""
+
+ - it: CustomConfig mode renders the bindings ConfigMap for its users
+ release:
+ name: kubernetes-app
+ namespace: tenant-foo
+ template: templates/oidc-rbac-bindings.yaml
+ set:
+ _cluster.oidc-enabled: "false"
+ oidc.mode: CustomConfig
+ oidc.customConfig.config: |
+ apiVersion: apiserver.config.k8s.io/v1beta1
+ kind: AuthenticationConfiguration
+ jwt:
+ - issuer:
+ url: https://idp.example.com
+ audiences: [my-cluster]
+ oidc.users:
+ - email: alice@example.org
+ role: admin
+ asserts:
+ - isKind:
+ of: ConfigMap
- matchRegex:
- path: spec.template.spec.containers[0].command[2]
- pattern: 'name: cluster-admin'
- documentIndex: 3
- - matchRegex:
- path: spec.template.spec.containers[0].command[2]
- pattern: 'name: view'
- documentIndex: 3
+ path: data["crbs.yaml"]
+ pattern: 'name: alice@example.org'
+
+ - it: mode=None renders no bindings ConfigMap even when users are set
+ # The Job takes its early branch at mode None and never reads the file.
+ release:
+ name: kubernetes-app
+ namespace: tenant-foo
+ template: templates/oidc-rbac-bindings.yaml
+ set:
+ oidc.mode: None
+ oidc.users:
+ - email: alice@example.org
+ role: admin
+ asserts:
+ - hasDocuments:
+ count: 0
- it: mode=System aborts the Job instead of leaving the OIDC kubeconfig Secret as the install-time placeholder when the tenant CA is missing
# A placeholder kubeconfig cannot authenticate, so the release must not go Ready with it.
From 47b497194eab5187e09ecbebc1ca862ae229808d Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil
Date: Wed, 4 Nov 2026 11:10:42 +0000
Subject: [PATCH 044/346] fix(backups): restore the ClickHouse archive a Backup
was taken as
The Altinity strategy named each archive inside its Pod with date -u
and recorded nothing on the Backup, so a restore could only guess: it
listed the remote archives under the application's prefix and took the
newest. A RestoreJob of an older Backup therefore restored whichever
backup had run last.
The controller now names the archive before it creates the backup Job,
from the BackupJob's persisted StartedAt, so every reconcile and every
retried Pod of the Job writes the same one. The name is recorded in the
Backup's driverMetadata next to the strategy parameters, and a restore
renders it into the strategy Pod, which restores exactly that archive
and fails, naming it, when remote storage no longer holds it. The name
is recorded only when the Job's Pod was given it, so a Job created by
an earlier controller, or a strategy that names its own archives,
leaves the Backup without one instead of pointing it at an archive
that does not exist.
Because a retried Pod reuses the name, it waits on an attempt the
sidecar is still running instead of being refused with 423, and asks
for resume, so a local or partial remote backup an earlier attempt left
is completed rather than refused as existing.
Backups taken before names were recorded stay restorable: the Pod picks
the newest archive whose name is stamped no later than the Backup's
takenAt, which is set only after the backup Job finished, and logs a
warning that the choice is a best effort.
Assisted-by: LLM
Signed-off-by: Myasnikov Daniil
---
hack/altinity-backup-strategy.bats | 209 +++++++++++++++++
.../altinitystrategy_controller.go | 69 +++++-
.../altinitystrategy_controller_test.go | 210 +++++++++++++++++-
.../templates/strategy-altinity-default.yaml | 57 ++++-
4 files changed, 523 insertions(+), 22 deletions(-)
create mode 100644 hack/altinity-backup-strategy.bats
diff --git a/hack/altinity-backup-strategy.bats b/hack/altinity-backup-strategy.bats
new file mode 100644
index 000000000..234ef0b8b
--- /dev/null
+++ b/hack/altinity-backup-strategy.bats
@@ -0,0 +1,209 @@
+#!/usr/bin/env bats
+# Behavioural tests for the shipped Altinity (ClickHouse) backup/restore
+# strategy script, extracted from the rendered cozy-default-altinity Strategy
+# CR and EXECUTED against a stubbed clickhouse-backup HTTP API, with the real
+# jq doing the archive selection. Which archive a restore reads back is
+# decided by that selection over list/remote, so it is exercised against
+# several archives rather than string-matched.
+#
+# The script runs with `bash`, not `sh`, for the reason given in
+# hack/redis-backup-strategy.bats: it opens `set -euo pipefail`, which the
+# production image's busybox ash supports and the CI runner's dash rejects.
+
+load test_helper
+
+CHART=packages/system/backupstrategy-controller
+
+# Render the shipped strategy's in-Pod script into $1 and substitute the
+# controller-render placeholders the way the driver fills them: $2 mode, $3
+# app (release and source application), $4 the archive name recorded on the
+# Backup (empty for a Backup that records none), $5 the Backup name, $6 its
+# takenAt in the archive-name layout. An empty render fails early, as does a
+# placeholder left unsubstituted.
+render_script() {
+ out=$1; mode=$2; app=$3; archive=$4; backup=$5; takenat=$6
+ helm template t "$CHART" -n cozy-backup-controller \
+ --set backupStorage.bucketNameOverride=test-bucket 2>/dev/null \
+ | yq eval 'select(.kind == "Altinity") | .spec.template.spec.containers[0].args[0]' - \
+ | sed -e "s#{{ .Mode }}#${mode}#g" \
+ -e "s#{{ .Release.Name }}#${app}#g" \
+ -e "s#{{ .Release.Namespace }}#tenant-x#g" \
+ -e "s#{{ .Backup.ApplicationRef.Name }}#${app}#g" \
+ -e "s#{{ .Backup.Name }}#${backup}#g" \
+ -e "s#{{ .ArchiveName | default \"\" }}#${archive}#g" \
+ -e "s#{{ .Backup.TakenAt | default \"\" }}#${takenat}#g" > "$out"
+ [ -s "$out" ] || return 1
+ if grep -q '{{' "$out"; then
+ echo "rendered script still holds a controller placeholder" >&2
+ return 1
+ fi
+}
+
+# Stub the clickhouse-backup API behind curl, first on PATH. list/remote
+# serves $state/remote; actions serves $state/actions plus one success row per
+# operation the script has issued, and after its first read swaps in
+# $state/actions.next when a test seeds one (an operation that finishes while
+# the script waits). Every request is logged to $state/curl.log. sleep is
+# stubbed to fail after a few calls, so a script that never sees its
+# operation finish fails the test instead of hanging it.
+make_stubs() {
+ bin=$1; state=$2
+ mkdir -p "$bin"
+ : > "$state/remote"
+ : > "$state/actions"
+ cat > "$bin/curl" <> "$state/curl.log"
+case "\$method \$url" in
+ "GET "*/backup/status) echo '{}' ;;
+ "GET "*/backup/list/remote) cat "$state/remote" ;;
+ "GET "*/backup/actions)
+ cat "$state/actions"
+ if [ -f "$state/actions.done" ]; then cat "$state/actions.done"; fi
+ if [ -f "$state/actions.next" ]; then mv "$state/actions.next" "$state/actions"; fi
+ ;;
+ "POST "*/backup/create_remote*)
+ name=\${url#*name=}; name=\${name%%&*}
+ printf '{"command":"create_remote --resume %s","status":"success","start":"2099-01-01 00:00:00"}\n' "\$name" >> "$state/actions.done"
+ ;;
+ "POST "*/backup/restore_remote/*)
+ printf '{"command":"restore_remote %s","status":"success","start":"2099-01-01 00:00:00"}\n' "\${url##*/}" >> "$state/actions.done"
+ ;;
+ *) echo "unexpected request: \$method \$url" >&2; exit 22 ;;
+esac
+STUB
+ cat > "$bin/sleep" </dev/null || echo 0)
+echo \$((n + 1)) > "$state/sleeps"
+if [ "\$n" -ge 3 ]; then echo "sleep budget spent: the operation never finished" >&2; exit 1; fi
+STUB
+ chmod 0755 "$bin/curl" "$bin/sleep"
+ export API_USERNAME=u API_PASSWORD=p
+}
+
+# One list/remote row per argument, "name" or "name|desc"; created mirrors
+# the name's timestamp, which is the order the pre-fix selection sorted by.
+remote_rows() {
+ for row in "$@"; do
+ name=${row%%|*}; desc="tar, regular"
+ if [ "$row" != "$name" ]; then desc=${row#*|}; fi
+ ts=${name##*-}
+ printf '{"name":"%s","created":"%s","location":"remote","required":"","desc":"%s"}\n' "$name" "$ts" "$desc"
+ done
+}
+
+@test "an empty render fails the fixture instead of passing as a no-op" {
+ tmp=$(mktemp -d)
+ CHART=$tmp/no-such-chart
+ if render_script "$tmp/s.sh" restore myapp "" bkp 20260304T050622Z 2>/dev/null; then
+ echo "FAIL: render_script reported success with no chart"
+ false
+ fi
+}
+
+@test "restore reads back the archive the Backup recorded, not the newest one" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp"
+ render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z
+ remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \
+ clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/remote"
+ PATH="$tmp/bin:$PATH" bash "$tmp/s.sh"
+ grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/curl.log"
+ [ "$(grep -c '/backup/restore_remote/' "$tmp/curl.log")" -eq 1 ]
+}
+
+@test "restore fails naming the recorded archive when remote storage lacks it" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp"
+ render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z
+ remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T060000Z > "$tmp/remote"
+ out=""
+ if out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1); then
+ echo "FAIL: restore succeeded without the recorded archive: $out"
+ false
+ fi
+ printf '%s\n' "$out" | grep -qF 'archive clickhouse-myapp-20260304T050607Z recorded on Backup bkp-a is not in remote storage'
+ if grep -qF '/backup/restore_remote/' "$tmp/curl.log"; then
+ echo "FAIL: restored another archive: $(cat "$tmp/curl.log")"
+ false
+ fi
+}
+
+@test "restore of a Backup without a recorded archive takes the newest one stamped by its takenAt, and warns" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp"
+ render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z
+ # Each decoy sorts after the expected archive and is ruled out by one
+ # filter: broken, not a bare timestamp after the prefix, or stamped after
+ # takenAt.
+ remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \
+ "clickhouse-myapp-20260304T050700Z|broken (can't stat metadata.json)" \
+ clickhouse-myapp-20260304T050710Z-copy \
+ clickhouse-myapp-20260304T060000Z > "$tmp/remote"
+ out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1)
+ grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/curl.log"
+ [ "$(grep -c '/backup/restore_remote/' "$tmp/curl.log")" -eq 1 ]
+ printf '%s\n' "$out" | grep -qF 'WARNING: Backup bkp-a records no archive name'
+}
+
+@test "restore of a Backup without a recorded archive fails when no archive is stamped by its takenAt" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp"
+ render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z
+ remote_rows clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/remote"
+ out=""
+ if out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1); then
+ echo "FAIL: restore succeeded with every archive newer than takenAt: $out"
+ false
+ fi
+ printf '%s\n' "$out" | grep -qF 'no archive of myapp stamped no later than 20260304T050800Z'
+ if grep -qF '/backup/restore_remote/' "$tmp/curl.log"; then
+ echo "FAIL: restored an archive newer than takenAt: $(cat "$tmp/curl.log")"
+ false
+ fi
+}
+
+@test "backup writes the archive under the name the controller chose" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp"
+ render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" ""
+ PATH="$tmp/bin:$PATH" bash "$tmp/s.sh"
+ grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/curl.log"
+}
+
+@test "a retried backup waits on the attempt the sidecar is still running" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp"
+ render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" ""
+ printf '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"in progress","start":"2026-03-04 05:06:08"}\n' > "$tmp/actions"
+ printf '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-03-04 05:06:08"}\n' > "$tmp/actions.next"
+ PATH="$tmp/bin:$PATH" bash "$tmp/s.sh"
+ # The sidecar answers a second operation with 423 while one runs.
+ if grep -qF 'POST ' "$tmp/curl.log"; then
+ echo "FAIL: started another operation over the running one: $(cat "$tmp/curl.log")"
+ false
+ fi
+}
+
+@test "a retried backup after a failed attempt resumes the same archive" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp"
+ render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" ""
+ printf '{"command":"create_remote clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-03-04 05:06:08","error":"S3 PutObject 403"}\n' > "$tmp/actions"
+ PATH="$tmp/bin:$PATH" bash "$tmp/s.sh"
+ grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/curl.log"
+}
+
+@test "backup falls back to a timestamped name when the controller supplies none" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp"
+ render_script "$tmp/s.sh" backup myapp "" "" ""
+ PATH="$tmp/bin:$PATH" bash "$tmp/s.sh"
+ grep -qE '^POST http://[^ ]*/backup/create_remote\?name=clickhouse-myapp-[0-9]{8}T[0-9]{6}Z&resume=true$' "$tmp/curl.log"
+}
diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go
index b96d571f0..f7f60bde8 100644
--- a/internal/backupcontroller/altinitystrategy_controller.go
+++ b/internal/backupcontroller/altinitystrategy_controller.go
@@ -2,7 +2,9 @@
package backupcontroller
import (
+ "bytes"
"context"
+ "encoding/json"
"fmt"
"maps"
"strings"
@@ -43,6 +45,18 @@ const (
// were in effect at backup time.
altinityParamPrefix = "altinity.strategy.backups.cozystack.io/parameter/"
+ // altinityArchiveMetaKey records, on the Backup, the clickhouse-backup
+ // archive its Job wrote, as named by altinityArchiveName. A restore reads
+ // back exactly that archive: the newest one under the release's prefix
+ // may belong to a later Backup.
+ altinityArchiveMetaKey = "altinity.strategy.backups.cozystack.io/archive"
+
+ // altinityArchiveTimeLayout is the UTC timestamp that ends an archive
+ // name. It is fixed-width, so names sharing a prefix sort chronologically
+ // as plain strings, which is what a restore of a Backup that records no
+ // archive relies on.
+ altinityArchiveTimeLayout = "20060102T150405Z"
+
// Polling cadence for the Altinity Job lifecycle. Mirrors the CNPG /
// Velero strategy cadence so behaviour is uniform across drivers.
altinityPollInterval = 5 * time.Second
@@ -102,14 +116,37 @@ func backupParameters(b *backupsv1alpha1.Backup) map[string]string {
return out
}
+// altinityArchiveName mints the archive name for one BackupJob from its
+// persisted StartedAt, so every reconcile and every Pod retry of the Job
+// writes the same archive. It keeps the clickhouse--
+// shape, because a Backup that records no archive is restored by the
+// timestamp in that shape.
+func altinityArchiveName(j *backupsv1alpha1.BackupJob) string {
+ return fmt.Sprintf("clickhouse-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveTimeLayout))
+}
+
+// altinityJobCarriesArchive reports whether the Job's Pod was rendered with
+// the archive name. A Job created by a controller that predates archive
+// names, or rendered from a strategy that names its own archives, wrote an
+// archive under another name, and recording this one would point the Backup
+// at an archive that does not exist.
+func altinityJobCarriesArchive(job *batchv1.Job, archive string) bool {
+ spec, err := json.Marshal(job.Spec.Template.Spec)
+ if err != nil {
+ return false
+ }
+ return bytes.Contains(spec, []byte(archive))
+}
+
// renderAltinityTemplate runs the strategy's PodTemplateSpec through the
// repository's text/template engine with the same context shape as the CNPG
// driver: every string field is templated against the application object,
-// the release shorthand, the run mode, and the resolved parameters.
+// the release shorthand, the run mode, the archive name, and the resolved
+// parameters.
func renderAltinityTemplate(
tmpl corev1.PodTemplateSpec,
app map[string]any,
- releaseName, releaseNamespace, mode string,
+ releaseName, releaseNamespace, mode, archiveName string,
parameters map[string]string,
backup *backupsv1alpha1.Backup,
) (*corev1.PodTemplateSpec, error) {
@@ -119,8 +156,9 @@ func renderAltinityTemplate(
"Name": releaseName,
"Namespace": releaseNamespace,
},
- "Mode": mode,
- "Parameters": parameters,
+ "Mode": mode,
+ "ArchiveName": archiveName,
+ "Parameters": parameters,
}
if backup != nil {
// Expose ApplicationRef so restore strategies can reference the
@@ -143,6 +181,9 @@ func renderAltinityTemplate(
"Kind": backup.Spec.ApplicationRef.Kind,
"Name": backup.Spec.ApplicationRef.Name,
},
+ // In the archive-name layout, so a strategy can compare it with
+ // the timestamp an archive name ends in.
+ "TakenAt": backup.Spec.TakenAt.UTC().Format(altinityArchiveTimeLayout),
}
}
return template.Template(&tmpl, ctxMap)
@@ -209,12 +250,14 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv
return ctrl.Result{}, err
}
+ archive := altinityArchiveName(j)
rendered, err := renderAltinityTemplate(
strategy.Spec.Template,
app,
j.Spec.ApplicationRef.Name,
j.Namespace,
altinityModeBackup,
+ archive,
resolved.Parameters,
nil,
)
@@ -239,7 +282,10 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv
if j.Status.BackupRef != nil {
return ctrl.Result{}, nil
}
- artifact, err := r.createAltinityBackupArtifact(ctx, j, resolved)
+ if !altinityJobCarriesArchive(batchJob, archive) {
+ archive = ""
+ }
+ artifact, err := r.createAltinityBackupArtifact(ctx, j, resolved, archive)
if err != nil {
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to create Backup artifact: %v", err))
}
@@ -411,16 +457,20 @@ func jobFailureMessage(j *batchv1.Job) string {
}
// createAltinityBackupArtifact materialises a Cozystack Backup carrying the
-// strategy reference and the BackupClassStrategy parameters in effect at
-// backup time. Parameters round-trip via DriverMetadata under
-// altinityParamPrefix so a later RestoreJob can re-render the strategy
-// template against the same values.
+// strategy reference, the archive the Job wrote (when known) and the
+// BackupClassStrategy parameters in effect at backup time. Parameters
+// round-trip via DriverMetadata under altinityParamPrefix so a later
+// RestoreJob can re-render the strategy template against the same values.
func (r *BackupJobReconciler) createAltinityBackupArtifact(
ctx context.Context,
j *backupsv1alpha1.BackupJob,
resolved *ResolvedBackupConfig,
+ archive string,
) (*backupsv1alpha1.Backup, error) {
driverMD := map[string]string{}
+ if archive != "" {
+ driverMD[altinityArchiveMetaKey] = archive
+ }
for k, v := range resolved.Parameters {
driverMD[altinityParamPrefix+k] = v
}
@@ -616,6 +666,7 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res
targetAppName,
targetNamespace,
altinityModeRestore,
+ backup.Spec.DriverMetadata[altinityArchiveMetaKey],
backupParameters(backup),
backup,
)
diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go
index 68f18fa28..7797842c0 100644
--- a/internal/backupcontroller/altinitystrategy_controller_test.go
+++ b/internal/backupcontroller/altinitystrategy_controller_test.go
@@ -4,6 +4,7 @@ package backupcontroller
import (
"context"
"testing"
+ "time"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
@@ -697,6 +698,7 @@ func TestBackupParameters_RoundTrip(t *testing.T) {
DriverMetadata: map[string]string{
altinityParamPrefix + "bucketName": "b1",
altinityParamPrefix + "endpoint": "https://s3",
+ altinityArchiveMetaKey: "clickhouse-ch-test-20260304T050607Z",
"job.batch/name": "ignored",
"": "ignored",
},
@@ -706,7 +708,213 @@ func TestBackupParameters_RoundTrip(t *testing.T) {
if got["bucketName"] != "b1" || got["endpoint"] != "https://s3" {
t.Errorf("backupParameters = %v", got)
}
- if _, ok := got["job.batch/name"]; ok {
+ if len(got) != 2 {
t.Errorf("non-parameter key leaked through: %v", got)
}
}
+
+// newAltinityArchiveStrategy renders the archive name and the Backup's
+// takenAt where a test can read them back off the Job.
+func newAltinityArchiveStrategy(name string) *strategyv1alpha1.Altinity {
+ s := newAltinityStrategy(name)
+ s.Spec.Template.Spec.Containers[0].Args = []string{
+ "--mode={{ .Mode }}",
+ "--archive={{ .ArchiveName }}",
+ "--taken-at={{ .Backup.TakenAt }}",
+ "--bucket={{ .Parameters.bucketName }}",
+ }
+ return s
+}
+
+func TestAltinityArchiveName_FromStartedAtInUTC(t *testing.T) {
+ startedAt := metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 7, 890000000, time.FixedZone("UTC+3", 3*3600)))
+ j := &backupsv1alpha1.BackupJob{
+ Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")},
+ Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt},
+ }
+ if got, want := altinityArchiveName(j), "clickhouse-ch-test-20260304T050607Z"; got != want {
+ t.Errorf("altinityArchiveName = %q, want %q", got, want)
+ }
+}
+
+// TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven drives one BackupJob
+// through Job creation and completion in separate reconciles. The archive
+// name has to come out of the persisted StartedAt both times: the Pod is
+// told it on the first, the Backup records it on the second, and a restore
+// later reads back exactly that archive.
+func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) {
+ app := newAltinityClickHouseApp("ch-test", "tenant-test")
+ strategy := newAltinityArchiveStrategy("clickhouse-strategy")
+ startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC))
+ backupJob := &backupsv1alpha1.BackupJob{
+ ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
+ Spec: backupsv1alpha1.BackupJobSpec{
+ ApplicationRef: newAltinityClickHouseRef("ch-test"),
+ BackupClassName: "clickhouse-backup",
+ },
+ Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning},
+ }
+ resolved := &ResolvedBackupConfig{
+ StrategyRef: corev1.TypedLocalObjectReference{
+ APIGroup: new(strategyv1alpha1.GroupVersion.Group),
+ Kind: strategyv1alpha1.AltinityStrategyKind,
+ Name: "clickhouse-strategy",
+ },
+ Parameters: map[string]string{"bucketName": "ch-bucket"},
+ }
+ r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy))
+ ctx := context.Background()
+ const want = "clickhouse-ch-test-20260304T050607Z"
+
+ if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
+ t.Fatalf("reconcileAltinity() creating the Job: %v", err)
+ }
+ job := &batchv1.Job{}
+ if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil {
+ t.Fatalf("get backup Job: %v", err)
+ }
+ if got := job.Spec.Template.Spec.Containers[0].Args[1]; got != "--archive="+want {
+ t.Fatalf("backup Pod was rendered with %q, want --archive=%s", got, want)
+ }
+
+ job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}
+ if err := r.Status().Update(ctx, job); err != nil {
+ t.Fatalf("mark backup Job complete: %v", err)
+ }
+ if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil {
+ t.Fatalf("refresh BackupJob: %v", err)
+ }
+ if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
+ t.Fatalf("reconcileAltinity() on the completed Job: %v", err)
+ }
+
+ created := &backupsv1alpha1.Backup{}
+ if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil {
+ t.Fatalf("get Backup: %v", err)
+ }
+ if got := created.Spec.DriverMetadata[altinityArchiveMetaKey]; got != want {
+ t.Errorf("Backup records archive %q, want %q", got, want)
+ }
+ if got := created.Spec.DriverMetadata[altinityParamPrefix+"bucketName"]; got != "ch-bucket" {
+ t.Errorf("expected driverMetadata[parameter/bucketName]=ch-bucket, got %q", got)
+ }
+}
+
+// TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven covers a Job whose
+// Pod never carried the driver's archive name: one created before the
+// upgrade that introduced the names, or rendered from a strategy that names
+// its own archives. Recording the name would send the restore after an
+// archive nobody wrote; recording none leaves it the fallback for Backups
+// without a name.
+func TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven(t *testing.T) {
+ app := newAltinityClickHouseApp("ch-test", "tenant-test")
+ strategy := newAltinityArchiveStrategy("clickhouse-strategy")
+ startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC))
+ backupJob := &backupsv1alpha1.BackupJob{
+ ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
+ Spec: backupsv1alpha1.BackupJobSpec{
+ ApplicationRef: newAltinityClickHouseRef("ch-test"),
+ BackupClassName: "clickhouse-backup",
+ },
+ Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning},
+ }
+ completedK8sJob := &batchv1.Job{
+ ObjectMeta: metav1.ObjectMeta{Name: jobNameForBackupJob(backupJob), Namespace: backupJob.Namespace},
+ Spec: batchv1.JobSpec{
+ Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{
+ RestartPolicy: corev1.RestartPolicyNever,
+ Containers: []corev1.Container{{
+ Name: "clickhouse-backup",
+ Image: "altinity/clickhouse-backup:test",
+ Args: []string{`target="clickhouse-ch-test-$(date -u +%Y%m%dT%H%M%SZ)"`},
+ }},
+ }},
+ },
+ Status: batchv1.JobStatus{
+ Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}},
+ },
+ }
+ resolved := &ResolvedBackupConfig{
+ StrategyRef: corev1.TypedLocalObjectReference{
+ APIGroup: new(strategyv1alpha1.GroupVersion.Group),
+ Kind: strategyv1alpha1.AltinityStrategyKind,
+ Name: "clickhouse-strategy",
+ },
+ Parameters: map[string]string{"bucketName": "ch-bucket"},
+ }
+ r, _, _ := newAltinityTestEnv(t, app,
+ clientfake.NewClientBuilder().WithObjects(backupJob, strategy, completedK8sJob))
+ ctx := context.Background()
+
+ if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil {
+ t.Fatalf("reconcileAltinity() error = %v", err)
+ }
+ created := &backupsv1alpha1.Backup{}
+ if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil {
+ t.Fatalf("get Backup: %v", err)
+ }
+ if got, ok := created.Spec.DriverMetadata[altinityArchiveMetaKey]; ok {
+ t.Errorf("Backup records archive %q its Job was never given", got)
+ }
+ if got := created.Spec.DriverMetadata[altinityParamPrefix+"bucketName"]; got != "ch-bucket" {
+ t.Errorf("expected driverMetadata[parameter/bucketName]=ch-bucket, got %q", got)
+ }
+}
+
+func TestReconcileAltinityRestore_RendersTheBackupsArchive(t *testing.T) {
+ cases := []struct {
+ name string
+ archive string // recorded on the Backup; empty for a Backup taken before names were recorded
+ wantArchive string
+ }{
+ {name: "recorded archive", archive: "clickhouse-ch-test-20260304T050607Z", wantArchive: "--archive=clickhouse-ch-test-20260304T050607Z"},
+ {name: "no recorded archive", archive: "", wantArchive: "--archive="},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ targetApp := newAltinityClickHouseApp("ch-test", "tenant-test")
+ strategy := newAltinityArchiveStrategy("clickhouse-strategy")
+ driverMD := map[string]string{altinityParamPrefix + "bucketName": "ch-bucket"}
+ if tc.archive != "" {
+ driverMD[altinityArchiveMetaKey] = tc.archive
+ }
+ now := metav1.Now()
+ backup := &backupsv1alpha1.Backup{
+ ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"},
+ Spec: backupsv1alpha1.BackupSpec{
+ ApplicationRef: newAltinityClickHouseRef("ch-test"),
+ StrategyRef: corev1.TypedLocalObjectReference{
+ APIGroup: new(strategyv1alpha1.GroupVersion.Group),
+ Kind: strategyv1alpha1.AltinityStrategyKind,
+ Name: "clickhouse-strategy",
+ },
+ TakenAt: metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 22, 0, time.FixedZone("UTC+3", 3*3600))),
+ DriverMetadata: driverMD,
+ },
+ }
+ restoreJob := &backupsv1alpha1.RestoreJob{
+ ObjectMeta: metav1.ObjectMeta{Name: "rj", Namespace: "tenant-test"},
+ Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}},
+ Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
+ }
+ _, rr, _ := newAltinityTestEnv(t, targetApp,
+ clientfake.NewClientBuilder().WithObjects(strategy, backup, restoreJob))
+ ctx := context.Background()
+
+ if _, err := rr.reconcileAltinityRestore(ctx, restoreJob, backup); err != nil {
+ t.Fatalf("reconcileAltinityRestore: %v", err)
+ }
+ job := &batchv1.Job{}
+ if err := rr.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForRestoreJob(restoreJob)}, job); err != nil {
+ t.Fatalf("get restore Job: %v", err)
+ }
+ args := job.Spec.Template.Spec.Containers[0].Args
+ want := []string{"--mode=restore", tc.wantArchive, "--taken-at=20260304T050622Z", "--bucket=ch-bucket"}
+ for i, w := range want {
+ if i >= len(args) || args[i] != w {
+ t.Errorf("rendered restore args[%d]: want %q, got %q", i, w, args)
+ }
+ }
+ })
+ }
+}
diff --git a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml
index 3397d6d0d..bb9917094 100644
--- a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml
+++ b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml
@@ -3,8 +3,8 @@
1. Helm renders this file at chart install time. Anything inside
`{{ ... }}` is interpreted by Helm.
2. The strategy controller renders the resulting Strategy CR
- per-BackupJob, expanding `{{ .Application | .Release | .Mode }}`
- against the live application object.
+ per-BackupJob, expanding `{{ .Application | .Release | .Mode |
+ .ArchiveName | .Backup }}` against the live application object.
To emit a literal "{{ .Foo }}" that survives Helm, we pass it as the
string argument of `printf` — Helm does not interpret inner template
delimiters inside printf's quoted strings. Do NOT simplify by inlining
@@ -51,23 +51,56 @@ spec:
host="chi-clickhouse-{{ printf "{{ .Release.Name }}" }}-clickhouse-0-0.{{ printf "{{ .Release.Namespace }}" }}.svc"
api="http://${host}:7171"
auth="-u ${API_USERNAME}:${API_PASSWORD}"
+ # The archive the controller named for this BackupJob and records
+ # on its Backup. The default renders it empty, rather than as
+ # text/template's missing-key marker, when a controller that
+ # predates archive names renders this template during an upgrade.
+ archive="{{ printf `{{ .ArchiveName | default "" }}` }}"
for _ in $(seq 1 60); do
if curl -fsS ${auth} --max-time 2 "${api}/backup/status" >/dev/null 2>&1; then break; fi
sleep 5
done
if [ "{{ printf "{{ .Mode }}" }}" = "backup" ]; then
prefix="clickhouse-{{ printf "{{ .Release.Name }}" }}-"
- target="${prefix}$(date -u +%Y%m%dT%H%M%SZ)"
- echo "create_remote ${target}"
- curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}" >/dev/null
+ # Only that older controller leaves the name empty, and it
+ # records no name on the Backup either.
+ target="${archive:-${prefix}$(date -u +%Y%m%dT%H%M%SZ)}"
+ # A retried Pod gets the same name. Wait on an attempt the
+ # sidecar is still running, which would answer a new one with
+ # 423, and ask for resume, so a local or partial remote backup
+ # an earlier attempt left is resumed rather than refused as
+ # existing, whatever use_resumable_state the sidecar runs with.
+ state=$(curl -fsS ${auth} "${api}/backup/actions" \
+ | jq -rs --arg suffix "${target}" \
+ '[.[] | select((.command // "") | endswith($suffix))] | sort_by(.start) | last | .status // ""')
+ if [ "${state}" != "in progress" ]; then
+ echo "create_remote ${target}"
+ curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}&resume=true" >/dev/null
+ fi
else
- prefix="clickhouse-{{ printf "{{ .Backup.ApplicationRef.Name }}" }}-"
- target=$(curl -fsS ${auth} "${api}/backup/list/remote" \
- | jq -rs --arg prefix "${prefix}" \
- '[.[] | select((.name // "") | startswith($prefix))] | sort_by(.created // "") | last.name // ""')
- if [ -z "${target}" ] || [ "${target}" = "null" ]; then
- echo "no remote backup found" >&2
- exit 1
+ app="{{ printf "{{ .Backup.ApplicationRef.Name }}" }}"
+ backup="{{ printf "{{ .Backup.Name }}" }}"
+ prefix="clickhouse-${app}-"
+ remote=$(curl -fsS ${auth} "${api}/backup/list/remote")
+ if [ -n "${archive}" ]; then
+ if ! printf '%s\n' "${remote}" | jq -es --arg name "${archive}" 'any(.[]; .name == $name)' >/dev/null; then
+ echo "archive ${archive} recorded on Backup ${backup} is not in remote storage; not restoring any other archive" >&2
+ exit 1
+ fi
+ target="${archive}"
+ else
+ # takenAt is set after the backup Job finished, so the
+ # archive that Job wrote is stamped no later than it.
+ notafter="{{ printf `{{ .Backup.TakenAt | default "" }}` }}"
+ echo "WARNING: Backup ${backup} records no archive name; restoring the newest archive of ${app} stamped no later than its takenAt (${notafter}), a best-effort choice that may not be the archive this Backup was taken as" >&2
+ target=$(printf '%s\n' "${remote}" | jq -rs --arg prefix "${prefix}" --arg notafter "${notafter}" '
+ [.[] | select((.desc // "") | startswith("broken") | not) | .name // ""
+ | select(startswith($prefix) and (ltrimstr($prefix) | test("^[0-9]{8}T[0-9]{6}Z$")) and ltrimstr($prefix) <= $notafter)]
+ | sort | last // ""')
+ if [ -z "${target}" ]; then
+ echo "no archive of ${app} stamped no later than ${notafter} in remote storage" >&2
+ exit 1
+ fi
fi
echo "restore_remote ${target}"
curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null
From 3b1302ba8e8d44a5294b28302a775f805bd40e2a Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:45:36 +0700
Subject: [PATCH 045/346] feat(dashboard): add an accessible Select to the
console
The console has no select primitive: form widgets fall back to the
native element, and the shared Dropdown has no listbox roles, no
keyboard navigation and no label association. Base UI, already a
dependency, ships one; this wraps it the same way dialog.tsx wraps the
Base UI dialog so pickers get a labelled, keyboard-driven listbox that
matches the console's buttons.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../console/src/components/ui/select.test.tsx | 64 +++++++++++
.../apps/console/src/components/ui/select.tsx | 107 ++++++++++++++++++
2 files changed, 171 insertions(+)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx
new file mode 100644
index 000000000..fb5b84e6f
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx
@@ -0,0 +1,64 @@
+import { describe, it, expect, vi } from "vitest"
+import { render, screen } from "@testing-library/react"
+import userEvent from "@testing-library/user-event"
+import { SelectItem, SelectLabel, SelectPopup, SelectRoot, SelectTrigger, SelectValue } from "./select.tsx"
+
+const items = [
+ { value: "", label: "All" },
+ { value: "db-0", label: "db-0" },
+ { value: "db-1", label: "db-1" },
+]
+
+function PodSelect({ value = "", onValueChange = () => {} }: { value?: string; onValueChange?: (v: string) => void }) {
+ return (
+ onValueChange(v ?? "")}>
+ Pod
+
+
+
+
+ {items.map((i) => (
+
+ {i.label}
+
+ ))}
+
+
+ )
+}
+
+describe("Select", () => {
+ it("names the trigger by its label and shows the chosen item", () => {
+ render( )
+
+ expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("db-1")
+ })
+
+ it("shows the label of an empty-string value", () => {
+ render( )
+
+ expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("All")
+ })
+
+ it("lists the items and reports the one clicked", async () => {
+ const onValueChange = vi.fn()
+ render( )
+
+ await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
+ await userEvent.click(await screen.findByRole("option", { name: "db-1" }))
+
+ expect(onValueChange).toHaveBeenCalledWith("db-1")
+ })
+
+ it("can be driven from the keyboard", async () => {
+ const onValueChange = vi.fn()
+ render( )
+
+ screen.getByRole("combobox", { name: "Pod" }).focus()
+ await userEvent.keyboard("{ArrowDown}")
+ expect(await screen.findByRole("listbox")).toBeInTheDocument()
+ await userEvent.keyboard("{ArrowDown}{Enter}")
+
+ expect(onValueChange).toHaveBeenCalledWith("db-0")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx
new file mode 100644
index 000000000..f256dba45
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx
@@ -0,0 +1,107 @@
+import { Select as SelectPrimitive } from "@base-ui/react/select"
+import { Check, ChevronDown } from "lucide-react"
+import { cn } from "@cozystack/ui"
+
+const SelectRoot = SelectPrimitive.Root
+
+function SelectLabel({
+ className,
+ ...props
+}: SelectPrimitive.Label.Props) {
+ return (
+
+ )
+}
+
+function SelectTrigger({
+ className,
+ children,
+ ...props
+}: SelectPrimitive.Trigger.Props) {
+ return (
+
+ {children}
+
+
+
+
+ )
+}
+
+function SelectValue({
+ className,
+ ...props
+}: SelectPrimitive.Value.Props) {
+ return (
+
+ )
+}
+
+function SelectPopup({
+ className,
+ children,
+ ...props
+}: SelectPrimitive.Popup.Props) {
+ return (
+
+
+
+ {children}
+
+
+
+ )
+}
+
+function SelectItem({
+ className,
+ children,
+ ...props
+}: SelectPrimitive.Item.Props) {
+ return (
+
+
+
+
+ {children}
+
+ )
+}
+
+export {
+ SelectRoot,
+ SelectLabel,
+ SelectTrigger,
+ SelectValue,
+ SelectPopup,
+ SelectItem,
+}
From f500aa955ac65384b527bf61167e77bf5dd02a3e Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:46:15 +0700
Subject: [PATCH 046/346] feat(dashboard): add a text Input matching the
console select
Every text field in the console restyles a bare input by hand, each a
little differently. A toolbar that mixes a text filter with the new
Select needs the two at the same height, border and focus ring, so the
styling lives in one place next to the Select instead of being copied.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../console/src/components/ui/input.test.tsx | 21 +++++++++++++++++++
.../apps/console/src/components/ui/input.tsx | 17 +++++++++++++++
2 files changed, 38 insertions(+)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx
new file mode 100644
index 000000000..e65a70a3a
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx
@@ -0,0 +1,21 @@
+import { describe, it, expect, vi } from "vitest"
+import { render, screen } from "@testing-library/react"
+import userEvent from "@testing-library/user-event"
+import { Input } from "./input.tsx"
+
+describe("Input", () => {
+ it("is a labelled text box that reports what is typed", async () => {
+ const onValueChange = vi.fn()
+ render( )
+
+ await userEvent.type(screen.getByRole("textbox", { name: "Filter" }), "x")
+
+ expect(onValueChange).toHaveBeenCalledWith("x", expect.anything())
+ })
+
+ it("keeps the caller's classes next to its own", () => {
+ render( )
+
+ expect(screen.getByRole("textbox", { name: "Filter" })).toHaveClass("font-mono", "rounded-lg")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx
new file mode 100644
index 000000000..b079e6b35
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx
@@ -0,0 +1,17 @@
+import { Input as InputPrimitive } from "@base-ui/react/input"
+import { cn } from "@cozystack/ui"
+
+function Input({ className, ...props }: InputPrimitive.Props) {
+ return (
+
+ )
+}
+
+export { Input }
From b88ad09678a234ff351173a1cc0674b9721d795f Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:47:25 +0700
Subject: [PATCH 047/346] fix(dashboard): keep the log console on screen when
it has no lines
With no matching lines, while loading, and while Follow waits for the
first line, the dark log console vanished and left a line of small
grey text or a bare spinner, so the page jumped and an empty result
looked like a broken view. The console frame now keeps a fixed height
in every state and says inside it what is going on.
The pod and container pickers move to the console Select and the
filter to the console Input, so the toolbar is keyboard and screen
reader friendly and a long pod name truncates instead of widening it.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/EmptyState.test.tsx | 17 +++++
.../components/observability/EmptyState.tsx | 25 ++++++++
.../observability/LogLines.test.tsx | 35 ++++++++---
.../src/components/observability/LogLines.tsx | 44 +++++++------
.../observability/LogsView.test.tsx | 50 ++++++++++++++-
.../src/components/observability/LogsView.tsx | 62 +++++++++++++------
6 files changed, 184 insertions(+), 49 deletions(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx
new file mode 100644
index 000000000..8872f4d43
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx
@@ -0,0 +1,17 @@
+import { describe, it, expect } from "vitest"
+import { render, screen } from "@testing-library/react"
+import { EmptyState } from "./EmptyState.tsx"
+
+describe("EmptyState", () => {
+ it("announces its message as a status", () => {
+ render( )
+
+ expect(screen.getByRole("status")).toHaveTextContent("No data for this range.Pick a longer range.")
+ })
+
+ it("marks itself busy while something is loading", () => {
+ render( )
+
+ expect(screen.getByRole("status")).toHaveAttribute("aria-busy", "true")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx
new file mode 100644
index 000000000..0cf58224b
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx
@@ -0,0 +1,25 @@
+import type { ReactNode } from "react"
+import { cn, Spinner } from "@cozystack/ui"
+
+interface EmptyStateProps {
+ title: ReactNode
+ hint?: ReactNode
+ busy?: boolean
+ tone?: "light" | "dark"
+ className?: string
+}
+
+export function EmptyState({ title, hint, busy, tone = "light", className }: EmptyStateProps) {
+ const dark = tone === "dark"
+ return (
+
+ {busy &&
}
+
{title}
+ {hint &&
{hint}
}
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
index 7110fe464..dd8c503eb 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx
@@ -10,40 +10,59 @@ function sized(list: HTMLElement, scrollHeight: number) {
}
describe("LogLines", () => {
+ it("keeps the console frame and shows the placeholder inside it when there are no lines", () => {
+ render( )
+
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(frame).toHaveTextContent("Waiting for new log lines…")
+ expect(screen.queryByRole("list")).toBeNull()
+ })
+
+ it("puts the lines in the same frame once they arrive", () => {
+ const { rerender } = render( )
+ const frame = screen.getByRole("region", { name: "Log lines" })
+
+ rerender( )
+
+ expect(screen.getByRole("region", { name: "Log lines" })).toBe(frame)
+ expect(frame).toContainElement(screen.getByRole("list"))
+ expect(frame).not.toHaveTextContent("Waiting")
+ })
+
it("keeps following the newest line while the reader is at the bottom", () => {
- const { rerender } = render( )
+ const { rerender } = render( )
const list = screen.getByRole("list")
sized(list, 500)
- rerender( )
+ rerender( )
expect(list.scrollTop).toBe(500)
})
it("leaves the position alone once the reader scrolls up", () => {
- const { rerender } = render( )
+ const { rerender } = render( )
const list = screen.getByRole("list")
sized(list, 500)
list.scrollTop = 120
fireEvent.scroll(list)
- rerender( )
+ rerender( )
expect(list.scrollTop).toBe(120)
})
it("follows again once the list is cleared for a new run", () => {
- const { rerender } = render( )
+ const { rerender } = render( )
const first = screen.getByRole("list")
sized(first, 500)
first.scrollTop = 120
fireEvent.scroll(first)
- rerender( )
- rerender( )
+ rerender( )
+ rerender( )
const second = screen.getByRole("list")
sized(second, 300)
- rerender( )
+ rerender( )
expect(second.scrollTop).toBe(300)
})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
index 452faac72..14b941ab2 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx
@@ -1,10 +1,10 @@
-import { useEffect, useRef } from "react"
+import { useEffect, useRef, type ReactNode } from "react"
import { CONTAINER_FIELD, POD_FIELD } from "../../lib/observability/logs.ts"
import type { LogEntry } from "../../lib/observability/proxy.ts"
const BOTTOM_SLACK_PX = 24
-export function LogLines({ entries }: { entries: LogEntry[] }) {
+export function LogLines({ entries, placeholder }: { entries: LogEntry[]; placeholder: ReactNode }) {
const ref = useRef(null)
const pinned = useRef(true)
@@ -13,23 +13,31 @@ export function LogLines({ entries }: { entries: LogEntry[] }) {
else if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight
}, [entries])
- if (entries.length === 0) return No log lines for this range and filter.
return (
- {
- const el = e.currentTarget
- pinned.current = el.scrollHeight - el.scrollTop - el.clientHeight <= BOTTOM_SLACK_PX
- }}
- className="max-h-[60vh] overflow-auto rounded-md bg-slate-950 p-3 font-mono text-xs leading-5 text-slate-100"
+
- {entries.map((e, i) => (
-
- {new Date(e._time).toLocaleTimeString()} {" "}
- {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`} {" "}
- {e._msg}
-
- ))}
-
+ {entries.length === 0 ? (
+ placeholder
+ ) : (
+ {
+ const el = e.currentTarget
+ pinned.current = el.scrollHeight - el.scrollTop - el.clientHeight <= BOTTOM_SLACK_PX
+ }}
+ className="h-full overflow-auto p-3"
+ >
+ {entries.map((e, i) => (
+
+ {new Date(e._time).toLocaleTimeString()} {" "}
+ {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`} {" "}
+ {e._msg}
+
+ ))}
+
+ )}
+
)
}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 526efeb23..31c6fac0e 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -1,5 +1,5 @@
import { describe, it, expect, vi } from "vitest"
-import { screen, waitFor } from "@testing-library/react"
+import { screen, waitFor, within } from "@testing-library/react"
import userEvent from "@testing-library/user-event"
import { K8sClient } from "@cozystack/k8s-client"
import { renderWithK8sProvider } from "../../test-utils/render.tsx"
@@ -47,9 +47,9 @@ describe("LogsView", () => {
it("narrows to the picked pod with a stream filter", async () => {
const { client, raw } = clientWith(defaultHandler)
renderWithK8sProvider( , { client })
- await screen.findByRole("option", { name: "db-1" })
- await userEvent.selectOptions(screen.getByLabelText("Pod"), "db-1")
+ await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
+ await userEvent.click(await screen.findByRole("option", { name: "db-1" }))
await waitFor(() =>
expect(
@@ -58,6 +58,50 @@ describe("LogsView", () => {
)
})
+ it("offers All next to every pod and starts on it", async () => {
+ const { client } = clientWith(defaultHandler)
+ renderWithK8sProvider( , { client })
+ const pod = screen.getByRole("combobox", { name: "Pod" })
+ expect(pod).toHaveTextContent("All")
+
+ await userEvent.click(pod)
+
+ expect(await screen.findByRole("option", { name: "db-1" })).toBeInTheDocument()
+ expect(screen.getAllByRole("option").map((o) => o.textContent)).toEqual(["All", "db-0", "db-1"])
+ })
+
+ it("keeps the log console and says so inside it when nothing matches", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query") ? new Response("") : defaultHandler(endpoint, params),
+ )
+
+ renderWithK8sProvider( , { client })
+
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(await within(frame).findByText("No log lines for this range and filter.")).toBeInTheDocument()
+ })
+
+ it("shows loading inside the log console", () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query") ? new Promise(() => {}) : defaultHandler(endpoint, params),
+ )
+
+ renderWithK8sProvider( , { client })
+
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(within(frame).getByRole("status")).toHaveAttribute("aria-busy", "true")
+ })
+
+ it("says it is waiting while Follow has no line yet", async () => {
+ const { client } = clientWith(defaultHandler)
+ renderWithK8sProvider( , { client })
+
+ await userEvent.click(screen.getByRole("button", { name: "Follow" }))
+
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(within(frame).getByText("Waiting for new log lines…")).toBeInTheDocument()
+ })
+
it("applies the LogsQL filter on submit", async () => {
const { client, raw } = clientWith(defaultHandler)
renderWithK8sProvider( , { client })
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index a27ca41c5..d2a000010 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -1,7 +1,9 @@
import { useState } from "react"
import { useQuery } from "@tanstack/react-query"
import { useK8sClient } from "@cozystack/k8s-client"
-import { Button, Spinner } from "@cozystack/ui"
+import { Button } from "@cozystack/ui"
+import { Input } from "../ui/input.tsx"
+import { SelectItem, SelectLabel, SelectPopup, SelectRoot, SelectTrigger, SelectValue } from "../ui/select.tsx"
import {
CONTAINER_FIELD,
LOG_RANGES,
@@ -20,6 +22,7 @@ import { useLogTail } from "../../lib/observability/use-log-tail.ts"
import { ProxyNotice } from "./ProxyNotice.tsx"
import { RangePicker } from "./RangePicker.tsx"
import { LogLines } from "./LogLines.tsx"
+import { EmptyState } from "./EmptyState.tsx"
const HISTORY_LIMIT = 500
const FOLLOW_CAP = 1000
@@ -30,27 +33,42 @@ function windowFor(range: LogRangeKey) {
return { start: new Date(end - LOG_RANGES[range] * 1000).toISOString(), end: new Date(end).toISOString() }
}
+const ALL = ""
+
function Picker({ label, value, options, onChange }: { label: string; value: string; options: string[]; onChange: (v: string) => void }) {
+ const items = [{ value: ALL, label: "All" }, ...options.map((o) => ({ value: o, label: o }))]
return (
-
- {label}
- onChange(e.target.value)}
- className="rounded-md border border-slate-200 bg-white px-2 py-1 text-xs text-slate-900"
- >
- All
- {options.map((o) => (
-
- {o}
-
+ onChange(v ?? ALL)}>
+
+ {label}
+
+
+
+
+
+ {items.map((item) => (
+
+ {item.label}
+
))}
-
-
+
+
)
}
+function placeholderFor({ follow, stopped, loading, failed }: { follow: boolean; stopped: boolean; loading: boolean; failed: boolean }) {
+ if (follow) {
+ return stopped ? (
+
+ ) : (
+
+ )
+ }
+ if (loading) return
+ if (failed) return
+ return
+}
+
export function LogsView({ scope }: { scope: ObservabilityScope }) {
const client = useK8sClient()
const root = logsRoot(scope)
@@ -107,12 +125,12 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
setFilter(draft)
}}
>
- setDraft(e.target.value)}
+ onValueChange={setDraft}
placeholder='LogsQL filter, e.g. error or "connection refused"'
- className="w-full rounded-md border border-slate-200 px-2.5 py-1.5 font-mono text-xs"
+ className="font-mono"
/>
{ setPod(v); setContainer("") }} />
@@ -129,7 +147,11 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
)}
{error ? : null}
- {!follow && history.isLoading ? : }
+
)
}
From f5ebb43c4bdce908887029568065eb27785ff833 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:48:43 +0700
Subject: [PATCH 048/346] fix(dashboard): match the range picker to the log
toolbar controls
Next to the new Select and Input the range picker was shorter, lighter
bordered and had no focus ring, so the toolbar looked uneven and a
keyboard user could not see which range had focus. It now shares their
height, border and focus ring, and its buttons are grouped under one
accessible name.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/RangePicker.test.tsx | 23 +++++++++++++++++++
.../components/observability/RangePicker.tsx | 4 ++--
2 files changed, 25 insertions(+), 2 deletions(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx
new file mode 100644
index 000000000..9787963b2
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx
@@ -0,0 +1,23 @@
+import { describe, it, expect, vi } from "vitest"
+import { render, screen, within } from "@testing-library/react"
+import userEvent from "@testing-library/user-event"
+import { RangePicker } from "./RangePicker.tsx"
+
+describe("RangePicker", () => {
+ it("groups the ranges under one name and marks the chosen one pressed", () => {
+ render( {}} />)
+
+ const group = screen.getByRole("group", { name: "Time range" })
+ expect(within(group).getByRole("button", { name: "6h" })).toHaveAttribute("aria-pressed", "true")
+ expect(within(group).getByRole("button", { name: "1h" })).toHaveAttribute("aria-pressed", "false")
+ })
+
+ it("reports the range clicked", async () => {
+ const onChange = vi.fn()
+ render( )
+
+ await userEvent.click(screen.getByRole("button", { name: "1h" }))
+
+ expect(onChange).toHaveBeenCalledWith("1h")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
index 22030c706..95d245176 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
@@ -10,7 +10,7 @@ export function RangePicker({
onChange: (value: K) => void
}) {
return (
-
+
{options.map((key) => (
({
aria-pressed={value === key}
onClick={() => onChange(key)}
className={cn(
- "rounded px-2.5 py-1 text-xs",
+ "h-full rounded-md px-2.5 text-xs outline-none focus-visible:ring-2 focus-visible:ring-slate-400",
value === key ? "bg-slate-900 text-white" : "text-slate-600 hover:bg-slate-100",
)}
>
From 05b0ea0c4558c5363cc38d79b7549548de7fe504 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:49:39 +0700
Subject: [PATCH 049/346] fix(dashboard): hold each metrics chart's place while
empty or loading
A metrics panel with no series, or still loading, shrank to one line of
small text or a bare spinner, so the tab jumped by the chart's height
when data arrived and an empty range looked like a broken panel. Both
states now fill the chart's slot and say what is going on, the same way
the log console does.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../observability/MetricsView.test.tsx | 16 +++++++++++++++-
.../src/components/observability/MetricsView.tsx | 9 ++++++---
2 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
index f09e15201..b4c7fc3a4 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
@@ -22,7 +22,7 @@ function matrix(pod: string) {
})
}
-function clientWith(handler: (url: URL) => Response) {
+function clientWith(handler: (url: URL) => Response | Promise) {
const client = new K8sClient()
const raw = vi.spyOn(client, "raw").mockImplementation(async (path) => handler(new URL(path, "http://x")))
return { client, raw }
@@ -84,5 +84,19 @@ describe("MetricsView", () => {
renderWithK8sProvider( , { client })
await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(3))
+ for (const status of screen.getAllByRole("status")) expect(status).toHaveClass("h-[230px]")
+ })
+
+ it("holds each chart's place while it loads", () => {
+ const { client } = clientWith(() => new Promise(() => {}))
+
+ renderWithK8sProvider( , { client })
+
+ const statuses = screen.getAllByRole("status")
+ expect(statuses).toHaveLength(3)
+ for (const status of statuses) {
+ expect(status).toHaveAttribute("aria-busy", "true")
+ expect(status).toHaveClass("h-[230px]")
+ }
})
})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
index 35c8e8417..34d3b33a0 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
@@ -1,7 +1,7 @@
import { useState } from "react"
import { useQueries } from "@tanstack/react-query"
import { useK8sClient } from "@cozystack/k8s-client"
-import { Section, Spinner } from "@cozystack/ui"
+import { Section } from "@cozystack/ui"
import {
METRIC_PANELS,
RANGES,
@@ -13,8 +13,11 @@ import { isScopeWide, metricsRoot, queryRange, type ObservabilityScope } from ".
import { ProxyNotice } from "./ProxyNotice.tsx"
import { TimeSeriesChart } from "./TimeSeriesChart.tsx"
import { RangePicker } from "./RangePicker.tsx"
+import { EmptyState } from "./EmptyState.tsx"
const REFRESH_MS = 60_000
+// The uPlot canvas plus its one-row legend, so a placeholder takes the chart's place without a jump.
+const CHART_SLOT = "h-[230px] rounded-md border border-dashed border-slate-200 bg-slate-50"
export function MetricsView({ scope }: { scope: ObservabilityScope }) {
const client = useK8sClient()
@@ -57,9 +60,9 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
{error ? (
) : isLoading || !data ? (
-
+
) : data.series.length === 0 ? (
- No data for this range.
+
) : (
)}
From 6740ed445899fdd689ef7676840bc2d7a74936bc Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:53:47 +0700
Subject: [PATCH 050/346] fix(dashboard): offer All once when a log stream has
no pod name
VictoriaLogs can return an empty value for the pod or container field
when some streams lack the label. That empty value is the picker's own
"All" sentinel, so the list showed two selected "All" entries with the
same key. Empty values are dropped from the field values.
The tests now also pin what the log console says when Follow stops
before any line and when the history query fails, so those messages
cannot silently swap with the empty-result one.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../components/observability/LogsView.test.tsx | 18 ++++++++++++++++++
.../src/components/observability/LogsView.tsx | 2 +-
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 31c6fac0e..47b6b4ef9 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -135,6 +135,8 @@ describe("LogsView", () => {
expect(await screen.findByText(/Stopped/)).toBeInTheDocument()
expect(screen.getByText("pipe not allowed in tail")).toBeInTheDocument()
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(within(frame).getByText("Follow stopped before any line arrived.")).toBeInTheDocument()
})
it("renders a cluster without the proxy as not available", async () => {
@@ -165,6 +167,22 @@ describe("LogsView", () => {
expect(await screen.findByText("cannot parse query")).toBeInTheDocument()
expect(screen.getByLabelText("LogsQL filter")).toBeInTheDocument()
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(within(frame).getByText("No log lines to show.")).toBeInTheDocument()
+ })
+
+ it("offers All once when some lines carry no pod name", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("stream_field_values") && params.get("field") === "kubernetes_pod_name"
+ ? values("", "db-0")
+ : defaultHandler(endpoint, params),
+ )
+ renderWithK8sProvider( , { client })
+
+ await userEvent.click(screen.getByRole("combobox", { name: "Pod" }))
+
+ await screen.findByRole("option", { name: "db-0" })
+ expect(screen.getAllByRole("option").map((o) => o.textContent)).toEqual(["All", "db-0"])
})
it("names the source pod and container of each line", async () => {
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index d2a000010..1976c7dfc 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -36,7 +36,7 @@ function windowFor(range: LogRangeKey) {
const ALL = ""
function Picker({ label, value, options, onChange }: { label: string; value: string; options: string[]; onChange: (v: string) => void }) {
- const items = [{ value: ALL, label: "All" }, ...options.map((o) => ({ value: o, label: o }))]
+ const items = [{ value: ALL, label: "All" }, ...options.filter((o) => o !== ALL).map((o) => ({ value: o, label: o }))]
return (
onChange(v ?? ALL)}>
From 554dad87bb18ece7adf5afd02c050a3520cf759a Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:53:58 +0700
Subject: [PATCH 051/346] fix(dashboard): give the range picker the toolbar's
focus ring offset
The range buttons drew their focus ring flush against the button while
the Select and Input next to them offset it, so keyboard focus looked
different from one control to the next. The empty metrics hint also
repeated its title; it now suggests the one thing that can help.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../apps/console/src/components/observability/MetricsView.tsx | 2 +-
.../apps/console/src/components/observability/RangePicker.tsx | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
index 34d3b33a0..ce7bcbe38 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
@@ -62,7 +62,7 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
) : isLoading || !data ? (
) : data.series.length === 0 ? (
-
+
) : (
)}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
index 95d245176..49abd8b1d 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
@@ -18,7 +18,7 @@ export function RangePicker({
aria-pressed={value === key}
onClick={() => onChange(key)}
className={cn(
- "h-full rounded-md px-2.5 text-xs outline-none focus-visible:ring-2 focus-visible:ring-slate-400",
+ "h-full rounded-md px-2.5 text-xs outline-none focus-visible:ring-2 focus-visible:ring-slate-400 focus-visible:ring-offset-2",
value === key ? "bg-slate-900 text-white" : "text-slate-600 hover:bg-slate-100",
)}
>
From c9a15cb8aed81e3b72d4e703c381474c6dc4db2b Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 18:56:01 +0700
Subject: [PATCH 052/346] fix(dashboard): suggest a longer metrics range only
when there is one
With the longest range already picked, the empty panel still told the
reader to try a longer range, which the picker cannot offer.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/components/observability/MetricsView.test.tsx | 11 +++++++++++
.../src/components/observability/MetricsView.tsx | 6 ++++--
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
index b4c7fc3a4..d3f1fc1dd 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
@@ -87,6 +87,17 @@ describe("MetricsView", () => {
for (const status of screen.getAllByRole("status")) expect(status).toHaveClass("h-[230px]")
})
+ it("suggests a longer range only while one exists", async () => {
+ const { client } = clientWith(() => Response.json({ status: "success", data: { resultType: "matrix", result: [] } }))
+ renderWithK8sProvider( , { client })
+ await waitFor(() => expect(screen.getAllByText("Try a longer range.")).toHaveLength(3))
+
+ await userEvent.click(screen.getByRole("button", { name: "7d" }))
+
+ await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(3))
+ expect(screen.queryByText("Try a longer range.")).toBeNull()
+ })
+
it("holds each chart's place while it loads", () => {
const { client } = clientWith(() => new Promise(() => {}))
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
index ce7bcbe38..fac2a6b91 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
@@ -23,6 +23,8 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
const client = useK8sClient()
const root = metricsRoot(scope)
const [range, setRange] = useState("1h")
+ const rangeKeys = Object.keys(RANGES) as RangeKey[]
+ const longestRange = range === rangeKeys[rangeKeys.length - 1]
const results = useQueries({
queries: METRIC_PANELS.map((panel) => ({
@@ -48,7 +50,7 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
Resource consumption
-
+
{wide ? (
@@ -62,7 +64,7 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) {
) : isLoading || !data ? (
) : data.series.length === 0 ? (
-
+
) : (
)}
From fca5ca78de81666be249769a8496d4f0567cc022 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 19:04:29 +0700
Subject: [PATCH 053/346] fix(dashboard): treat an open log tail as live before
its first line
The tail counted as live, cleared its error and reset its backoff only
when a line arrived. A quiet application therefore showed "Connecting"
forever, a 429 warning stuck after the retry succeeded, and every idle
close by the proxy (its tail idle timeout) raised the backoff towards
30s and reconnected without since, losing lines written in the gap.
The tail now reports when the response opens. That marks it live,
clears the error and resets the backoff; a clean close reconnects at
the base delay; and with no line seen yet the reconnect resumes from
the time the stream opened.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/lib/observability/proxy.test.ts | 18 +++++++++
.../console/src/lib/observability/proxy.ts | 2 +
.../lib/observability/use-log-tail.test.tsx | 37 +++++++++++++++++++
.../src/lib/observability/use-log-tail.ts | 19 +++++++---
4 files changed, 71 insertions(+), 5 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
index 69a8c7c8c..2809905d8 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
@@ -194,6 +194,24 @@ describe("tailLogs", () => {
expect(err).toMatchObject({ kind: "forbidden" })
})
+
+ it("says the stream opened before any line arrives", async () => {
+ const { client } = clientReturning(new Response(""))
+ const onOpen = vi.fn()
+
+ await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}, undefined, onOpen)
+
+ expect(onOpen).toHaveBeenCalledTimes(1)
+ })
+
+ it("does not say the stream opened when the tail is refused", async () => {
+ const { client } = clientReturning(new Response("", { status: 429 }))
+ const onOpen = vi.fn()
+
+ await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}, undefined, onOpen).catch(() => {})
+
+ expect(onOpen).not.toHaveBeenCalled()
+ })
})
describe("isScopeWide", () => {
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
index c9fce3d4f..3b5567f6b 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -202,8 +202,10 @@ export async function tailLogs(
params: { query: string; since?: string },
onEntry: (entry: LogEntry) => void,
signal?: AbortSignal,
+ onOpen?: () => void,
): Promise {
const res = await proxyGet(client, `${root}select/logsql/tail`, params, signal)
+ onOpen?.()
if (!res.body) return
const reader = res.body.getReader()
const decoder = new TextDecoder()
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
index 0c2306613..b5232d18e 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx
@@ -10,6 +10,10 @@ function ndjson(...lines: string[]): Response {
return new Response(lines.map((l) => `${l}\n`).join(""))
}
+function openStream(): Response {
+ return new Response(new ReadableStream({ start() {} }))
+}
+
function setup(...responses: Response[]) {
const client = new K8sClient()
const raw = vi.spyOn(client, "raw")
@@ -75,6 +79,39 @@ describe("useLogTail", () => {
await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a"]))
})
+ it("goes live and clears a load warning once the stream opens, before any line", async () => {
+ const { wrapper } = setup(new Response("too many concurrent requests", { status: 429 }), openStream())
+
+ const { result } = renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper })
+
+ await waitFor(() => expect(result.current.state).toBe("live"))
+ expect(result.current.error).toBeNull()
+ })
+
+ it("reconnects after an idle close at the base delay, from when the quiet stream opened", async () => {
+ const delays: number[] = []
+ const realSetTimeout = globalThis.setTimeout
+ const spy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((fn: () => void, ms?: number) => {
+ if (ms !== undefined && ms % 7 === 0 && ms > 0) delays.push(ms)
+ return realSetTimeout(fn, ms)
+ }) as typeof setTimeout)
+ const refused = () => new Response("", { status: 429 })
+ const { raw, wrapper } = setup(refused(), refused(), ndjson())
+
+ try {
+ renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 7 }), { wrapper })
+ await waitFor(() => expect(raw).toHaveBeenCalledTimes(4))
+ } finally {
+ spy.mockRestore()
+ }
+
+ expect(delays).toEqual([7, 14, 7])
+ expect(new URL(raw.mock.calls[2][0], "http://x").searchParams.has("since")).toBe(false)
+ const since = new URL(raw.mock.calls[3][0], "http://x").searchParams.get("since")
+ expect(since).not.toBeNull()
+ expect(new Date(since!).toISOString()).toBe(since)
+ })
+
it("does not connect while disabled", () => {
const { raw, wrapper } = setup()
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
index 53cbaf856..2e3b4d706 100644
--- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts
@@ -66,15 +66,22 @@ export function useLogTail(
}
const run = async () => {
- for (let attempt = 0; !signal.aborted; attempt++) {
+ let failures = 0
+ let openedAt: string | undefined
+ while (!signal.aborted) {
resumedFrom = lastTime
+ let delay = retryDelayMs
try {
- await tailLogs(client, root, { query, since: lastTime }, (entry) => {
- attempt = 0
+ await tailLogs(client, root, { query, since: lastTime ?? openedAt }, (entry) => {
if (!lastTime || compareTime(entry._time, lastTime) > 0) lastTime = entry._time
pending.push(entry)
timer ??= setTimeout(flush, FLUSH_MS)
- }, signal)
+ }, signal, () => {
+ failures = 0
+ // With no line yet, the next reconnect resumes from here so the gap is not lost.
+ openedAt = new Date().toISOString()
+ update({ state: "live", error: null })
+ })
flush()
} catch (err) {
flush()
@@ -84,10 +91,12 @@ export function useLogTail(
return
}
update({ error: err })
+ delay = Math.min(retryDelayMs * 2 ** failures, MAX_RETRY_MS)
+ failures++
}
if (signal.aborted) return
update({ state: "reconnecting" })
- await sleep(Math.min(retryDelayMs * 2 ** attempt, MAX_RETRY_MS), signal)
+ await sleep(delay, signal)
}
}
void run()
From 881eea524693248cdcfb4421bad190cbf7b586c8 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 19:04:58 +0700
Subject: [PATCH 054/346] fix(dashboard): suggest a longer log range only when
there is one
On 24h, the longest range the log view offers, the empty console still
told the reader to try a longer range. It now suggests only a broader
filter there, as the metrics panels already do.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/components/observability/LogsView.test.tsx | 14 ++++++++++++++
.../src/components/observability/LogsView.tsx | 11 +++++++----
2 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
index 47b6b4ef9..1b18dd208 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx
@@ -81,6 +81,20 @@ describe("LogsView", () => {
expect(await within(frame).findByText("No log lines for this range and filter.")).toBeInTheDocument()
})
+ it("suggests a longer range only while one exists", async () => {
+ const { client } = clientWith((endpoint, params) =>
+ endpoint.endsWith("query") ? new Response("") : defaultHandler(endpoint, params),
+ )
+ renderWithK8sProvider( , { client })
+ const frame = screen.getByRole("region", { name: "Log lines" })
+ expect(await within(frame).findByText("Try a longer range or a broader filter.")).toBeInTheDocument()
+
+ await userEvent.click(screen.getByRole("button", { name: "24h" }))
+
+ expect(await within(frame).findByText("Try a broader filter.")).toBeInTheDocument()
+ expect(within(frame).queryByText(/longer range/)).toBeNull()
+ })
+
it("shows loading inside the log console", () => {
const { client } = clientWith((endpoint, params) =>
endpoint.endsWith("query") ? new Promise(() => {}) : defaultHandler(endpoint, params),
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
index 1976c7dfc..a0a73cc43 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx
@@ -56,7 +56,10 @@ function Picker({ label, value, options, onChange }: { label: string; value: str
)
}
-function placeholderFor({ follow, stopped, loading, failed }: { follow: boolean; stopped: boolean; loading: boolean; failed: boolean }) {
+const RANGE_KEYS = Object.keys(LOG_RANGES) as LogRangeKey[]
+const LONGEST_RANGE = RANGE_KEYS[RANGE_KEYS.length - 1]
+
+function placeholderFor({ follow, stopped, loading, failed, longestRange }: { follow: boolean; stopped: boolean; loading: boolean; failed: boolean; longestRange: boolean }) {
if (follow) {
return stopped ? (
@@ -66,7 +69,7 @@ function placeholderFor({ follow, stopped, loading, failed }: { follow: boolean;
}
if (loading) return
if (failed) return
- return
+ return
}
export function LogsView({ scope }: { scope: ObservabilityScope }) {
@@ -135,7 +138,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
{ setPod(v); setContainer("") }} />
- {!follow && }
+ {!follow && }
setFollowRun(follow ? null : Date.now())}>
Follow
@@ -150,7 +153,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
)
From 9498c9bff520d73fe200f5043bc7efce9eb901ef Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 19:05:42 +0700
Subject: [PATCH 055/346] fix(dashboard): keep the proxy's own advice on a
limit refusal
The notice appended "Choose a shorter range." to every 422, so a
message that already ended in "use a shorter range" said it twice and
one that ended in "raise step" got advice that contradicted it. The
console's own advice is now added only when the proxy gave none.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/components/observability/ProxyNotice.test.tsx | 9 +++++++++
.../console/src/components/observability/ProxyNotice.tsx | 3 ++-
2 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
index fd001bcae..6e85ced33 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
@@ -29,6 +29,15 @@ describe("ProxyNotice", () => {
expect(screen.getByRole("alert")).toHaveTextContent(text)
})
+ it.each([
+ "the application has more than 50 pods in this window; use a shorter range",
+ "1200 points per series exceed the limit of 1000; raise step",
+ ])("does not add advice to a limit message that already carries its own: %s", (message) => {
+ render( )
+
+ expect(screen.getByRole("alert").textContent).toBe(`${message}.`)
+ })
+
it("shows the server message for a rejected query", () => {
render( )
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
index a60b831b0..ee68e4633 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
@@ -20,7 +20,8 @@ function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: bool
case "forbidden":
return { quiet: false, text: `You do not have permission to read ${lower} here.` }
case "limit":
- return { quiet: false, text: `${error.message}. Choose a shorter range.` }
+ // The proxy puts its own remedy after "; " (e.g. "use a shorter range", "raise step").
+ return { quiet: false, text: error.message.includes("; ") ? `${error.message}.` : `${error.message}. Choose a shorter range.` }
case "rate-limited":
return { quiet: false, text: "Too many requests. Try again in a moment." }
case "timeout":
From 76441871b8c3ae73f40ecf29fd454a468f1e0573 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 19:06:59 +0700
Subject: [PATCH 056/346] fix(dashboard): keep caller headers of any shape in
the k8s client
The client spread init.headers as a plain object, so a Headers
instance or a list of pairs passed by a caller was silently dropped
and the request went out without them. Headers are now built with the
Headers constructor, which accepts every shape fetch does.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../k8s-client/client.headers.test.ts | 31 +++++++++++++++++++
.../__tests__/k8s-client/client.raw.test.ts | 20 +++++++++++-
.../console/packages/k8s-client/src/client.ts | 14 +++------
3 files changed, 55 insertions(+), 10 deletions(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts
diff --git a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts
new file mode 100644
index 000000000..37985fca5
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts
@@ -0,0 +1,31 @@
+import { describe, it, expect, vi, afterEach } from "vitest"
+import { K8sClient } from "@cozystack/k8s-client"
+
+afterEach(() => {
+ vi.unstubAllGlobals()
+})
+
+describe("K8sClient request headers", () => {
+ it("lets a method's own content type replace the JSON default", async () => {
+ const fetchMock = vi.fn(async () => new Response("{}"))
+ vi.stubGlobal("fetch", fetchMock)
+ const client = new K8sClient({ getToken: async () => "t0k" })
+
+ await client.patch("apps", "v1", "deployments", "web", { spec: {} }, "default", "merge")
+
+ const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
+ const headers = new Headers(init.headers)
+ expect(headers.get("Content-Type")).toBe("application/merge-patch+json")
+ expect(headers.get("Authorization")).toBe("Bearer t0k")
+ })
+
+ it("sends JSON by default", async () => {
+ const fetchMock = vi.fn(async () => new Response("{}"))
+ vi.stubGlobal("fetch", fetchMock)
+
+ await new K8sClient().getApiGroups()
+
+ const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
+ expect(new Headers(init.headers).get("Content-Type")).toBe("application/json")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
index ab6c056f5..0ac0d75ae 100644
--- a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
+++ b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
@@ -24,7 +24,25 @@ describe("K8sClient.raw", () => {
const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
expect(url).toBe("/base/apis/x")
- expect(init.headers).toEqual({ Accept: "application/json", Authorization: "Bearer t0k" })
+ const headers = new Headers(init.headers)
+ expect(headers.get("Accept")).toBe("application/json")
+ expect(headers.get("Authorization")).toBe("Bearer t0k")
+ })
+
+ it.each([
+ ["a Headers instance", new Headers({ Accept: "application/x-ndjson" })],
+ ["a tuple list", [["Accept", "application/x-ndjson"]] as [string, string][]],
+ ])("keeps caller headers passed as %s", async (_, given) => {
+ const fetchMock = vi.fn(async () => new Response("{}"))
+ vi.stubGlobal("fetch", fetchMock)
+ const client = new K8sClient({ getToken: async () => "t0k" })
+
+ await client.raw("/x", { headers: given })
+
+ const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
+ const headers = new Headers(init.headers)
+ expect(headers.get("Accept")).toBe("application/x-ndjson")
+ expect(headers.get("Authorization")).toBe("Bearer t0k")
})
it("routes a 401 through the client's unauthorized handler", async () => {
diff --git a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
index 04c528760..64af1131d 100644
--- a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
+++ b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
@@ -50,14 +50,12 @@ export class K8sClient {
}
private async request(path: string, init?: RequestInit): Promise {
- const headers: Record = {
- "Content-Type": "application/json",
- ...(init?.headers as Record),
- }
+ const headers = new Headers(init?.headers)
+ if (!headers.has("Content-Type")) headers.set("Content-Type", "application/json")
if (this.getToken) {
const token = await this.getToken()
- headers["Authorization"] = `Bearer ${token}`
+ headers.set("Authorization", `Bearer ${token}`)
}
let res: Response
@@ -247,11 +245,9 @@ export class K8sClient {
* Kubernetes JSON, such as the observability proxy's streamed NDJSON.
*/
async raw(path: string, init?: RequestInit): Promise {
- const headers: Record = {
- ...(init?.headers as Record),
- }
+ const headers = new Headers(init?.headers)
if (this.getToken) {
- headers["Authorization"] = `Bearer ${await this.getToken()}`
+ headers.set("Authorization", `Bearer ${await this.getToken()}`)
}
const res = await fetch(`${this.baseUrl}${path}`, { ...init, headers })
if (res.status === 401) this.handleUnauthorized()
From 2a473086cf6cd1d245e014ed56ed44f9594038cd Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 19:11:05 +0700
Subject: [PATCH 057/346] fix(dashboard): advise a shorter range unless the
proxy already did
Any "; " in a 422 message was taken as the proxy's own remedy, but the
proxy passes vmselect's 422 bodies through, and their advice (raise
the -search.max* flags) is for operators. Those lost the one thing a
tenant can do. Only the remedies the proxy itself appends now replace
the console's advice.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../src/components/observability/ProxyNotice.test.tsx | 7 +++++++
.../src/components/observability/ProxyNotice.tsx | 10 ++++++++--
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
index 6e85ced33..c4b4e89c1 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
@@ -38,6 +38,13 @@ describe("ProxyNotice", () => {
expect(screen.getByRole("alert").textContent).toBe(`${message}.`)
})
+ it("still advises a shorter range for a store limit whose own advice is for operators", () => {
+ const message = "the number of matching timeseries exceeds 300000; either narrow down the search or increase -search.max* command-line flag values"
+ render( )
+
+ expect(screen.getByRole("alert").textContent).toBe(`${message}. Choose a shorter range.`)
+ })
+
it("shows the server message for a rejected query", () => {
render( )
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
index ee68e4633..1fb30c8ff 100644
--- a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
@@ -5,6 +5,10 @@ interface ProxyNoticeProps {
error: unknown
}
+// Remedies the observability proxy itself appends; vmselect's 422 text carries
+// advice for operators (raise -search.max* flags), so it still gets ours.
+const PROXY_REMEDIES = ["; use a shorter range", "; raise step"]
+
function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: boolean; text: string } {
const lower = what.toLowerCase()
if (!(error instanceof ProxyError)) {
@@ -20,8 +24,10 @@ function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: bool
case "forbidden":
return { quiet: false, text: `You do not have permission to read ${lower} here.` }
case "limit":
- // The proxy puts its own remedy after "; " (e.g. "use a shorter range", "raise step").
- return { quiet: false, text: error.message.includes("; ") ? `${error.message}.` : `${error.message}. Choose a shorter range.` }
+ return {
+ quiet: false,
+ text: PROXY_REMEDIES.some((r) => error.message.endsWith(r)) ? `${error.message}.` : `${error.message}. Choose a shorter range.`,
+ }
case "rate-limited":
return { quiet: false, text: "Too many requests. Try again in a moment." }
case "timeout":
From fe6968be7f2530f37738bf767e0cd0a3e54955ff Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 14:45:42 +0300
Subject: [PATCH 058/346] fix(postgres): fail a backup job as soon as its cnpg
Backup fails
A failed cnpg.io/Backup kept its BackupJob Running for 30 minutes as a
"transient" failure, on the premise that CloudNativePG may move a
Backup out of the failed phase. It does not: its backup controller
returns early on any Backup already failed or completed, so the wait
only delayed the verdict and hid it behind a misleading reason.
A failed Backup now fails the BackupJob at once with the error CNPG
recorded, pointing to the plugin-barman-cloud sidecar log when that
error is only the plugin's exit status. A Backup in any other phase
keeps the BackupJob waiting as before, now with a WaitingForCNPGBackup
condition naming the Backup and its phase instead of no condition at
all. It is still not capped: CloudNativePG keeps a healthy Backup
pending while another backup of the same Cluster runs or while it
restarts one, and a fixed deadline would fail those.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
.../cnpgstrategy_controller.go | 76 ++++-----
.../cnpgstrategy_controller_test.go | 152 +++++++++++++++++-
2 files changed, 188 insertions(+), 40 deletions(-)
diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go
index 99e6b03f6..27e50f4d8 100644
--- a/internal/backupcontroller/cnpgstrategy_controller.go
+++ b/internal/backupcontroller/cnpgstrategy_controller.go
@@ -175,13 +175,11 @@ const (
// against an unreachable target reproduces the same FATAL.
cnpgRecoveryMaxInspectPods = 5
- // Cap on the wall-clock time a BackupJob can spend observing a
- // cnpg.io/Backup stuck in phase=failed before the driver gives up and
- // marks the BackupJob Failed. CNPG can transition out of "failed" on
- // internal retries (e.g. transient instance-manager restart), so we
- // don't fail on the first observation. But without this cap, a
- // permanently-broken Backup would pin the BackupJob in Running forever,
- // blocking later runs and wedging the Plan-controller queue.
+ // Cap on the wall-clock time a BackupJob waits for the barman-cloud
+ // plugin rollout before the cnpg.io/Backup exists. A Backup that exists
+ // is not capped: CloudNativePG keeps a healthy one pending while another
+ // backup of the same Cluster runs, or while it restarts one on another
+ // instance, so no fixed deadline tells those apart from a stuck one.
cnpgDefaultBackupDeadline = 30 * time.Minute
cnpgClusterHealthyPhase = "Cluster in healthy state"
@@ -453,44 +451,46 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
return ctrl.Result{}, nil
case cnpgBackupPhaseFailed:
- // CNPG can transition the same Backup CR back out of "failed" when it
- // retries internally (e.g., after a transient instance-manager
- // restart), so we don't fail the BackupJob on the first observation.
- // But without a cap a permanently-broken Backup would pin the
- // BackupJob in Running forever; once StartedAt + cnpgDefaultBackupDeadline
- // has elapsed and the cnpg.io/Backup is still failed, we give up.
- if cnpgBackupDeadlineExceeded(j.Status.StartedAt) {
- final := message
- if final == "" {
- final = fmt.Sprintf(
- "cnpg.io Backup remained in phase=failed past %s deadline",
- cnpgDefaultBackupDeadline)
- }
- return r.markBackupJobFailed(ctx, j, final)
- }
- if message == "" {
- message = "cnpg.io Backup reported phase=failed; awaiting recovery or retry"
- }
- apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
- Type: "Ready",
- Status: metav1.ConditionFalse,
- Reason: "CNPGBackupTransientFailure",
- Message: message,
- })
- if err := r.Status().Update(ctx, j); err != nil {
- return ctrl.Result{}, err
- }
- return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil
+ // CloudNativePG does not reconcile a Backup out of a terminal phase,
+ // so waiting on a failed one cannot change the outcome.
+ return r.markBackupJobFailed(ctx, j, cnpgBackupFailedMessage(cnpgBackup, message))
default:
+ waiting := fmt.Sprintf("waiting for cnpg.io/Backup %s/%s, phase %q",
+ cnpgBackup.Namespace, cnpgBackup.Name, phase)
+ if apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
+ Type: "Ready",
+ Status: metav1.ConditionFalse,
+ Reason: "WaitingForCNPGBackup",
+ Message: waiting,
+ }) {
+ if err := r.Status().Update(ctx, j); err != nil {
+ return ctrl.Result{}, err
+ }
+ }
return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil
}
}
+// cnpgBackupFailedMessage carries the error CloudNativePG recorded. When the
+// plugin run itself failed that is only its exit status, and the storage
+// error behind it is logged by the plugin-barman-cloud sidecar.
+func cnpgBackupFailedMessage(b *cnpgtypes.Backup, errText string) string {
+ if errText == "" {
+ errText = "no error recorded"
+ }
+ message := fmt.Sprintf("cnpg.io/Backup %s/%s failed: %s", b.Namespace, b.Name, errText)
+ if strings.Contains(errText, "exit status") {
+ message += "; the cause is in the log of the plugin-barman-cloud sidecar of the instance that ran it"
+ }
+ return message
+}
+
// cnpgBackupDeadlineExceeded reports whether enough wall-clock time has
-// elapsed since the BackupJob started that we should stop polling a stuck
-// cnpg.io/Backup and fail the run. Returns false when StartedAt is nil so
-// the very first reconcile (which sets StartedAt) does not trip the gate.
+// elapsed since the BackupJob started that we should stop waiting for the
+// barman-cloud plugin rollout and fail the run. Returns false when
+// StartedAt is nil so the very first reconcile (which sets StartedAt) does
+// not trip the gate.
func cnpgBackupDeadlineExceeded(startedAt *metav1.Time) bool {
if startedAt == nil {
return false
diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go
index 697843d9e..551209fac 100644
--- a/internal/backupcontroller/cnpgstrategy_controller_test.go
+++ b/internal/backupcontroller/cnpgstrategy_controller_test.go
@@ -2332,6 +2332,154 @@ func TestReconcileCNPG_StartedAtReturnsEarlyToAvoidStaleResourceVersion(t *testi
}
}
+// CloudNativePG never reconciles a Backup out of a terminal phase (the
+// backup controller returns early on failed and completed), so a failed
+// Backup fails the BackupJob at once. Any other phase keeps the BackupJob
+// waiting past the backup deadline: CNPG keeps a healthy Backup pending
+// while another backup of the Cluster runs or while it restarts one.
+func TestReconcileCNPG_BackupPhaseOutcome(t *testing.T) {
+ apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup
+ strategyGroup := strategyv1alpha1.GroupVersion.Group
+ const ns, appName = "tenant", "pg"
+ clusterName := cnpgClusterNameForApp(appName)
+ expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute))
+
+ cases := []struct {
+ name string
+ phase string
+ errText string
+ startedAt time.Time
+ wantPhase backupsv1alpha1.BackupJobPhase
+ wantMessage []string
+ notInMessage []string
+ }{
+ {name: "failed fails at once", phase: cnpgBackupPhaseFailed, errText: "rpc error: code = Unknown desc = exit status 1", startedAt: time.Now(),
+ wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "exit status 1", "plugin-barman-cloud"}},
+ {name: "failed without an error text fails at once", phase: cnpgBackupPhaseFailed, startedAt: time.Now(),
+ wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "no error recorded"}, notInMessage: []string{"plugin-barman-cloud"}},
+ {name: "failed before the plugin ran points nowhere else", phase: cnpgBackupPhaseFailed, errText: "cannot backup a hibernated cluster", startedAt: time.Now(),
+ wantPhase: backupsv1alpha1.BackupJobPhaseFailed, wantMessage: []string{"bj-cnpg", "hibernated"}, notInMessage: []string{"plugin-barman-cloud"}},
+ {name: "pending keeps waiting", phase: "pending", startedAt: time.Now(),
+ wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "pending"}},
+ {name: "pending is not capped by the backup deadline", phase: "pending", startedAt: expired,
+ wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "pending"}},
+ {name: "no phase yet keeps waiting", phase: "", startedAt: expired,
+ wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg"}},
+ {name: "started keeps running", phase: "started", startedAt: expired,
+ wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "started"}},
+ {name: "walArchivingFailing keeps waiting", phase: "walArchivingFailing", startedAt: expired,
+ wantPhase: backupsv1alpha1.BackupJobPhaseRunning, wantMessage: []string{"bj-cnpg", "walArchivingFailing"}},
+ {name: "completed succeeds", phase: cnpgBackupPhaseComplete, startedAt: expired,
+ wantPhase: backupsv1alpha1.BackupJobPhaseSucceeded},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ startedAt := metav1.NewTime(tc.startedAt)
+ job := &backupsv1alpha1.BackupJob{
+ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bj"},
+ Spec: backupsv1alpha1.BackupJobSpec{
+ ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName},
+ },
+ Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt},
+ }
+ strategy := &strategyv1alpha1.CNPG{
+ ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"},
+ Spec: strategyv1alpha1.CNPGSpec{
+ Template: strategyv1alpha1.CNPGTemplate{
+ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"},
+ },
+ },
+ }
+ cluster := &cnpgtypes.Cluster{
+ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, UID: "cluster-uid"},
+ }
+ cnpgBackup := &cnpgtypes.Backup{
+ ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bj-cnpg", Labels: map[string]string{
+ backupsv1alpha1.OwningJobNameLabel: job.Name,
+ backupsv1alpha1.OwningJobNamespaceLabel: job.Namespace,
+ }},
+ Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: clusterName}},
+ Status: cnpgtypes.BackupStatus{Phase: tc.phase, Error: tc.errText},
+ }
+ c := newCNPGStrategyTestClient(t, job, strategy, newPostgresApp(appName, ns), cluster, cnpgBackup)
+ r := &BackupJobReconciler{Client: c, Interface: cnpgDynamicFor(t, cluster)}
+ resolved := &ResolvedBackupConfig{
+ StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: strategy.Name},
+ }
+
+ j := &backupsv1alpha1.BackupJob{}
+ if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), j); err != nil {
+ t.Fatalf("get BackupJob: %v", err)
+ }
+ res, err := r.reconcileCNPG(context.Background(), j, resolved)
+ if err != nil {
+ t.Fatalf("reconcileCNPG: %v", err)
+ }
+ if j.Status.Phase != tc.wantPhase {
+ t.Fatalf("BackupJob phase = %q (%s), want %q", j.Status.Phase, j.Status.Message, tc.wantPhase)
+ }
+
+ message := j.Status.Message
+ if tc.wantPhase == backupsv1alpha1.BackupJobPhaseRunning {
+ if res.RequeueAfter == 0 {
+ t.Errorf("expected a requeue while the cnpg.io/Backup is in progress, got %+v", res)
+ }
+ persisted := &backupsv1alpha1.BackupJob{}
+ if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), persisted); err != nil {
+ t.Fatalf("get BackupJob after reconcile: %v", err)
+ }
+ ready := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready")
+ if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "WaitingForCNPGBackup" {
+ t.Fatalf("expected Ready=False/WaitingForCNPGBackup while waiting, got %+v", ready)
+ }
+ message = ready.Message
+
+ // A pass that observes the same phase again must not write the
+ // BackupJob status: the driver polls every few seconds.
+ statusWrites := 0
+ r.Client = interceptor.NewClient(c.(client.WithWatch), interceptor.Funcs{
+ SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error {
+ statusWrites++
+ return cl.SubResource(sub).Update(ctx, obj, opts...)
+ },
+ SubResourcePatch: func(ctx context.Context, cl client.Client, sub string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error {
+ statusWrites++
+ return cl.SubResource(sub).Patch(ctx, obj, patch, opts...)
+ },
+ })
+ if _, err := r.reconcileCNPG(context.Background(), persisted, resolved); err != nil {
+ t.Fatalf("second reconcileCNPG: %v", err)
+ }
+ if statusWrites != 0 {
+ t.Errorf("an unchanged phase wrote the BackupJob status %d time(s)", statusWrites)
+ }
+ }
+ for _, want := range tc.wantMessage {
+ if !strings.Contains(message, want) {
+ t.Errorf("message %q does not mention %q", message, want)
+ }
+ }
+ for _, unwanted := range tc.notInMessage {
+ if strings.Contains(message, unwanted) {
+ t.Errorf("message %q mentions %q", message, unwanted)
+ }
+ }
+
+ artifacts := &backupsv1alpha1.BackupList{}
+ if err := c.List(context.Background(), artifacts, client.InNamespace(ns)); err != nil {
+ t.Fatalf("list Backups: %v", err)
+ }
+ wantArtifacts := 0
+ if tc.wantPhase == backupsv1alpha1.BackupJobPhaseSucceeded {
+ wantArtifacts = 1
+ }
+ if len(artifacts.Items) != wantArtifacts {
+ t.Errorf("got %d Backup artifacts, want %d", len(artifacts.Items), wantArtifacts)
+ }
+ })
+ }
+}
+
// cnpgRolloutFixture drives reconcileCNPG against one Postgres app whose
// Cluster and instance pods a test swaps between passes, standing in for
// CNPG rolling the instances.
@@ -2516,8 +2664,8 @@ func TestReconcileCNPG_WaitsForPluginRolloutBeforeBackup(t *testing.T) {
if !apimeta.IsStatusConditionTrue(j.Status.Conditions, backupCondPluginRolledOut) {
t.Errorf("%s condition = %+v, want True", backupCondPluginRolledOut, apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut))
}
- if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond != nil {
- t.Errorf("Ready condition = %+v left over from the wait while the backup runs", cond)
+ if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || cond.Reason != "WaitingForCNPGBackup" {
+ t.Errorf("Ready condition = %+v, want the wait on the cnpg.io/Backup rather than one left over from the rollout", cond)
}
}
From 6e36385e00fcb9cc8b24e19ec8e9a5e11056b383 Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Tue, 27 Oct 2026 18:25:19 +0300
Subject: [PATCH 059/346] feat(api): expose HelmRelease failure state on
Application status
Tenants usually read Applications but not the HelmReleases behind them,
and only Ready and Released were copied over. A client could therefore
see that a release was failing but not how often it had failed, nor
whether helm-controller had stopped retrying it.
Copy the Stalled condition, which helm-controller sets once it stops
reconciling a release (remediation retries exhausted, rollback target
missing, invalid CEL, access denied), and surface the install and
upgrade failure counters with the last attempted release action.
Stalled is dropped when it was observed for an older generation than
the HelmRelease's current one: the Application carries no generation,
so after a spec update a client would otherwise read the previous
spec's failure as terminal for the new values.
With the RetryOnFailure policy the platform generates today, a failed
release is retried rather than stalled, so the counters are the signal.
That does not hold for every state: a release missing from storage
while an install failure count is retained still stalls, and a release
on a remediation policy stalls once its retries run out.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
pkg/apis/apps/v1alpha1/types.go | 17 +++
.../apps/v1alpha1/applicationstatus.go | 38 ++++++
pkg/generated/openapi/zz_generated.openapi.go | 21 +++
pkg/registry/apps/application/rest.go | 14 +-
.../application/rest_release_failure_test.go | 123 ++++++++++++++++++
5 files changed, 211 insertions(+), 2 deletions(-)
create mode 100644 pkg/registry/apps/application/rest_release_failure_test.go
diff --git a/pkg/apis/apps/v1alpha1/types.go b/pkg/apis/apps/v1alpha1/types.go
index 15bab146f..13270b0f1 100644
--- a/pkg/apis/apps/v1alpha1/types.go
+++ b/pkg/apis/apps/v1alpha1/types.go
@@ -55,6 +55,23 @@ type ApplicationStatus struct {
// ExternalIPsCount holds the number of LoadBalancer services with assigned external IPs for Tenant applications.
// +optional
ExternalIPsCount int32 `json:"externalIPsCount,omitempty"`
+ // InstallFailures is the install failure count of the underlying HelmRelease.
+ // It counts only attempts that recorded a Helm release, so a failure before
+ // Helm records one, such as a chart that fails to render, shows only as
+ // Ready=False; a failure to apply the rendered resources is counted. It
+ // resets when the desired state (generation, chart version, values) changes,
+ // and on success only under the RetryOnFailure strategy, so a Ready release
+ // may carry it.
+ // +optional
+ InstallFailures int64 `json:"installFailures,omitempty"`
+ // UpgradeFailures is the upgrade failure count of the underlying HelmRelease,
+ // with the same counting and reset rules as InstallFailures.
+ // +optional
+ UpgradeFailures int64 `json:"upgradeFailures,omitempty"`
+ // LastAttemptedReleaseAction is the last release action (install or upgrade)
+ // the underlying HelmRelease attempted.
+ // +optional
+ LastAttemptedReleaseAction string `json:"lastAttemptedReleaseAction,omitempty"`
}
// SchedulingClass returns the scheduling class requested by this Application.
diff --git a/pkg/generated/applyconfiguration/apps/v1alpha1/applicationstatus.go b/pkg/generated/applyconfiguration/apps/v1alpha1/applicationstatus.go
index e20e08a51..77db6b0e7 100644
--- a/pkg/generated/applyconfiguration/apps/v1alpha1/applicationstatus.go
+++ b/pkg/generated/applyconfiguration/apps/v1alpha1/applicationstatus.go
@@ -34,6 +34,20 @@ type ApplicationStatusApplyConfiguration struct {
Namespace *string `json:"namespace,omitempty"`
// ExternalIPsCount holds the number of LoadBalancer services with assigned external IPs for Tenant applications.
ExternalIPsCount *int32 `json:"externalIPsCount,omitempty"`
+ // InstallFailures is the install failure count of the underlying HelmRelease.
+ // It counts only attempts that recorded a Helm release, so a failure before
+ // Helm records one, such as a chart that fails to render, shows only as
+ // Ready=False; a failure to apply the rendered resources is counted. It
+ // resets when the desired state (generation, chart version, values) changes,
+ // and on success only under the RetryOnFailure strategy, so a Ready release
+ // may carry it.
+ InstallFailures *int64 `json:"installFailures,omitempty"`
+ // UpgradeFailures is the upgrade failure count of the underlying HelmRelease,
+ // with the same counting and reset rules as InstallFailures.
+ UpgradeFailures *int64 `json:"upgradeFailures,omitempty"`
+ // LastAttemptedReleaseAction is the last release action (install or upgrade)
+ // the underlying HelmRelease attempted.
+ LastAttemptedReleaseAction *string `json:"lastAttemptedReleaseAction,omitempty"`
}
// ApplicationStatusApplyConfiguration constructs a declarative configuration of the ApplicationStatus type for use with
@@ -78,3 +92,27 @@ func (b *ApplicationStatusApplyConfiguration) WithExternalIPsCount(value int32)
b.ExternalIPsCount = &value
return b
}
+
+// WithInstallFailures sets the InstallFailures field in the declarative configuration to the given value
+// and returns the receiver, so that objects can be built by chaining "With" function invocations.
+// If called multiple times, the InstallFailures field is set to the value of the last call.
+func (b *ApplicationStatusApplyConfiguration) WithInstallFailures(value int64) *ApplicationStatusApplyConfiguration {
+ b.InstallFailures = &value
+ return b
+}
+
+// WithUpgradeFailures sets the UpgradeFailures field in the declarative configuration to the given value
+// and returns the receiver, so that objects can be built by chaining "With" function invocations.
+// If called multiple times, the UpgradeFailures field is set to the value of the last call.
+func (b *ApplicationStatusApplyConfiguration) WithUpgradeFailures(value int64) *ApplicationStatusApplyConfiguration {
+ b.UpgradeFailures = &value
+ return b
+}
+
+// WithLastAttemptedReleaseAction sets the LastAttemptedReleaseAction field in the declarative configuration to the given value
+// and returns the receiver, so that objects can be built by chaining "With" function invocations.
+// If called multiple times, the LastAttemptedReleaseAction field is set to the value of the last call.
+func (b *ApplicationStatusApplyConfiguration) WithLastAttemptedReleaseAction(value string) *ApplicationStatusApplyConfiguration {
+ b.LastAttemptedReleaseAction = &value
+ return b
+}
diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go
index 1edff5230..3c1e96f75 100644
--- a/pkg/generated/openapi/zz_generated.openapi.go
+++ b/pkg/generated/openapi/zz_generated.openapi.go
@@ -289,6 +289,27 @@ func schema_pkg_apis_apps_v1alpha1_ApplicationStatus(ref common.ReferenceCallbac
Format: "int32",
},
},
+ "installFailures": {
+ SchemaProps: spec.SchemaProps{
+ Description: "InstallFailures is the install failure count of the underlying HelmRelease. It counts only attempts that recorded a Helm release, so a failure before Helm records one, such as a chart that fails to render, shows only as Ready=False; a failure to apply the rendered resources is counted. It resets when the desired state (generation, chart version, values) changes, and on success only under the RetryOnFailure strategy, so a Ready release may carry it.",
+ Type: []string{"integer"},
+ Format: "int64",
+ },
+ },
+ "upgradeFailures": {
+ SchemaProps: spec.SchemaProps{
+ Description: "UpgradeFailures is the upgrade failure count of the underlying HelmRelease, with the same counting and reset rules as InstallFailures.",
+ Type: []string{"integer"},
+ Format: "int64",
+ },
+ },
+ "lastAttemptedReleaseAction": {
+ SchemaProps: spec.SchemaProps{
+ Description: "LastAttemptedReleaseAction is the last release action (install or upgrade) the underlying HelmRelease attempted.",
+ Type: []string{"string"},
+ Format: "",
+ },
+ },
},
},
},
diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go
index 322ef6c6d..5f618b4e3 100644
--- a/pkg/registry/apps/application/rest.go
+++ b/pkg/registry/apps/application/rest.go
@@ -1573,13 +1573,23 @@ func (r *REST) convertHelmReleaseToApplication(ctx context.Context, hr *helmv2.H
},
Spec: filteredSpec,
Status: appsv1alpha1.ApplicationStatus{
- Version: hr.Status.LastAttemptedRevision,
+ Version: hr.Status.LastAttemptedRevision,
+ InstallFailures: hr.Status.InstallFailures,
+ UpgradeFailures: hr.Status.UpgradeFailures,
+ LastAttemptedReleaseAction: string(hr.Status.LastAttemptedReleaseAction),
},
}
var conditions []metav1.Condition
for _, hrCondition := range hr.GetConditions() {
- if hrCondition.Type == "Ready" || hrCondition.Type == "Released" {
+ // The Application carries no generation, so a client cannot tell a Stalled
+ // left over from the previous spec from one about the current spec. Compare
+ // the condition's own ObservedGeneration, not Status.ObservedGeneration: the
+ // question is which spec this condition describes.
+ if hrCondition.Type == "Stalled" && hrCondition.ObservedGeneration < hr.Generation {
+ continue
+ }
+ if hrCondition.Type == "Ready" || hrCondition.Type == "Released" || hrCondition.Type == "Stalled" {
conditions = append(conditions, metav1.Condition{
LastTransitionTime: hrCondition.LastTransitionTime,
Reason: hrCondition.Reason,
diff --git a/pkg/registry/apps/application/rest_release_failure_test.go b/pkg/registry/apps/application/rest_release_failure_test.go
new file mode 100644
index 000000000..fc28f1553
--- /dev/null
+++ b/pkg/registry/apps/application/rest_release_failure_test.go
@@ -0,0 +1,123 @@
+package application
+
+import (
+ "context"
+ "testing"
+
+ helmv2 "github.com/fluxcd/helm-controller/api/v2"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestConvertReleaseFailure_StalledAndCountersExposed(t *testing.T) {
+ r := newTestRESTWithSchemes()
+ hr := makeHelmRelease("postgresql-mydb", "default")
+ hr.Generation = 2
+ hr.Status.Conditions = []metav1.Condition{
+ {Type: "Stalled", Status: metav1.ConditionTrue, Reason: "RetriesExceeded", Message: "Failed to install after 1 attempt(s)", ObservedGeneration: 2},
+ {Type: "Ready", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 2},
+ {Type: "Released", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 2},
+ }
+ hr.Status.Failures = 7
+ hr.Status.InstallFailures = 1
+ hr.Status.UpgradeFailures = 3
+ hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionInstall
+
+ app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+
+ sc := findCondition(app.GetConditions(), "Stalled")
+ if sc == nil {
+ t.Fatal("expected Stalled condition to be copied from the HelmRelease")
+ }
+ if sc.Status != metav1.ConditionTrue || sc.Reason != "RetriesExceeded" {
+ t.Errorf("expected Stalled=True/RetriesExceeded, got %s/%s", sc.Status, sc.Reason)
+ }
+ if app.Status.InstallFailures != 1 {
+ t.Errorf("expected installFailures=1, got %d", app.Status.InstallFailures)
+ }
+ if app.Status.UpgradeFailures != 3 {
+ t.Errorf("expected upgradeFailures=3, got %d", app.Status.UpgradeFailures)
+ }
+ if app.Status.LastAttemptedReleaseAction != "install" {
+ t.Errorf("expected lastAttemptedReleaseAction=install, got %q", app.Status.LastAttemptedReleaseAction)
+ }
+}
+
+// After a spec update the HelmRelease keeps the Stalled condition of the
+// previous generation until helm-controller reconciles it; reported as is, it
+// would read as a terminal failure of the new values.
+func TestConvertReleaseFailure_StalledFromOlderGenerationOmitted(t *testing.T) {
+ r := newTestRESTWithSchemes()
+ hr := makeHelmRelease("postgresql-mydb", "default")
+ hr.Generation = 2
+ hr.Status.Conditions = []metav1.Condition{
+ {Type: "Stalled", Status: metav1.ConditionTrue, Reason: "RetriesExceeded", Message: "Failed to install after 1 attempt(s)", ObservedGeneration: 1},
+ {Type: "Ready", Status: metav1.ConditionFalse, Reason: "InstallFailed", Message: "install failed", ObservedGeneration: 1},
+ }
+ hr.Status.InstallFailures = 1
+ hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionInstall
+
+ app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+
+ if findCondition(app.GetConditions(), "Stalled") != nil {
+ t.Error("expected Stalled observed at an older generation not to be copied")
+ }
+ if findCondition(app.GetConditions(), "Ready") == nil {
+ t.Error("expected Ready to still be copied")
+ }
+}
+
+// helm-controller clears the counters on success only under the RetryOnFailure
+// strategy, so a Ready release can still carry them; they must not be hidden.
+func TestConvertReleaseFailure_CountersKeptOnReadyRelease(t *testing.T) {
+ r := newTestRESTWithSchemes()
+ hr := makeHelmRelease("postgresql-mydb", "default")
+ hr.Status.Conditions = []metav1.Condition{
+ {Type: "Ready", Status: metav1.ConditionTrue, Reason: "UpgradeSucceeded", Message: "ok"},
+ {Type: "Released", Status: metav1.ConditionTrue, Reason: "UpgradeSucceeded", Message: "ok"},
+ }
+ hr.Status.InstallFailures = 2
+ hr.Status.UpgradeFailures = 1
+ hr.Status.LastAttemptedReleaseAction = helmv2.ReleaseActionUpgrade
+
+ app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+
+ if findCondition(app.GetConditions(), "Stalled") != nil {
+ t.Error("expected no Stalled condition on a release that does not carry one")
+ }
+ if app.Status.InstallFailures != 2 || app.Status.UpgradeFailures != 1 {
+ t.Errorf("expected install=2 upgrade=1, got install=%d upgrade=%d", app.Status.InstallFailures, app.Status.UpgradeFailures)
+ }
+ if app.Status.LastAttemptedReleaseAction != "upgrade" {
+ t.Errorf("expected lastAttemptedReleaseAction=upgrade, got %q", app.Status.LastAttemptedReleaseAction)
+ }
+}
+
+func TestConvertReleaseFailure_ReconcilingNotCopied(t *testing.T) {
+ r := newTestRESTWithSchemes()
+ hr := makeHelmRelease("postgresql-mydb", "default")
+ hr.Status.Conditions = []metav1.Condition{
+ {Type: "Reconciling", Status: metav1.ConditionTrue, Reason: "Progressing", Message: "Running 'install' action"},
+ {Type: "Ready", Status: metav1.ConditionUnknown, Reason: "Progressing", Message: "Running 'install' action"},
+ }
+
+ app, err := r.convertHelmReleaseToApplication(context.TODO(), hr, nil)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+
+ if findCondition(app.GetConditions(), "Reconciling") != nil {
+ t.Error("expected Reconciling not to be copied")
+ }
+ if findCondition(app.GetConditions(), "Ready") == nil {
+ t.Error("expected Ready to be copied")
+ }
+}
From c9ce4c480f6a589edfaa13539990f18ff661b28f Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 17:12:31 +0300
Subject: [PATCH 060/346] fix(clickhouse): let the backup sidecar pick up
credentials projected after it started
With backup.useSystemBucket the clickhouse-backup sidecar took its S3
settings from optional secretKeyRefs on cozy-backups-creds. The platform
projects that Secret into the namespace at the first BackupJob, usually
after the ClickHouse pod started, and a container's environment is fixed
at start, so the sidecar ran without credentials and every backup and
restore failed until the pod was restarted by hand. Rotated keys stayed
stale the same way.
The projector now also writes the s3 section of a clickhouse-backup
config into the Secret, and the chart mounts that key, optionally, as
the sidecar's config file instead of setting S3 environment variables.
clickhouse-backup reloads its config file in each API handler that
reaches S3 and kubelet rewrites a mounted Secret in place, so projected
and rotated credentials reach the running sidecar. The flat keys stay
for the Redis strategy and for ClickHouse releases still rendered by
the previous chart.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
.../backupcontroller/credentials_projector.go | 75 +++++++++++----
.../credentials_projector_test.go | 68 +++++++++++++
.../apps/clickhouse/templates/clickhouse.yaml | 95 +++++++++----------
.../apps/clickhouse/tests/backup_test.yaml | 77 ++++++++-------
4 files changed, 211 insertions(+), 104 deletions(-)
diff --git a/internal/backupcontroller/credentials_projector.go b/internal/backupcontroller/credentials_projector.go
index 4e0c3b75a..eb34d22bc 100644
--- a/internal/backupcontroller/credentials_projector.go
+++ b/internal/backupcontroller/credentials_projector.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"net/url"
+ "strconv"
"strings"
"time"
@@ -15,6 +16,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+ "sigs.k8s.io/yaml"
)
// CredentialsProjectionRequeue is the backoff between reconcile attempts
@@ -73,9 +75,8 @@ func IsTransient(err error) bool {
// BackupCredentialsConfig describes the platform-managed source Secret
// produced by the cozy-backups Bucket, the per-tenant projection target
// Secret name, and the bucket coordinates (endpoint used to format the
-// FDB blob_credentials account-host key, region carried verbatim for
-// drivers like clickhouse-backup that wire it through env, forcePathStyle
-// projected as a flat key so the chart-emitted ClickHouse sidecar can read
+// FDB blob_credentials account-host key, region carried verbatim,
+// forcePathStyle projected so the chart-emitted ClickHouse sidecar can read
// a platform value it cannot see directly). Values are wired from
// environment variables in main() and the deployment chart populates them
// from .Values.backupStorage.
@@ -87,9 +88,9 @@ type BackupCredentialsConfig struct {
Region string
// ForcePathStyle is the string "true"/"false" from
// backupStorage.forcePathStyle, projected into the credentials Secret
- // under the forcePathStyle key. The app charts cannot read
- // backupStorage.* directly, so the ClickHouse sidecar consumes it via
- // an optional secretKeyRef as S3_FORCE_PATH_STYLE.
+ // under the forcePathStyle key and into clickhouse-backup.yml. The app
+ // charts cannot read backupStorage.* directly, so the ClickHouse
+ // sidecar takes it from there.
ForcePathStyle string
}
@@ -107,10 +108,11 @@ func (c BackupCredentialsConfig) IsEnabled() bool {
// the data so every default Strategy CR can consume the same Secret:
//
// - AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY — CNPG, MariaDB, Etcd.
-// - accessKey / secretKey + bucketName / endpoint / region — ClickHouse
-// sidecar.
+// - accessKey / secretKey + bucketName / endpoint / region — Redis
+// strategy, and ClickHouse sidecars rendered before clickhouse-backup.yml.
// - cloud — Velero credentials file ([default] section).
// - blob_credentials.json — FoundationDB backup_agent.
+// - clickhouse-backup.yml — ClickHouse sidecar, mounted as its config file.
//
// The source Secret is read in two formats:
//
@@ -156,8 +158,8 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr
if creds.endpoint == "" {
// Scheme-strip the env-var fallback to match the source-Secret path
// (parseSourceSecret strips there). The projected `endpoint` key is
- // consumed verbatim as S3_ENDPOINT by the ClickHouse sidecar, which
- // expects a bare host:port — see https://cozystack.io/docs/v1/operations/services/backup-classes/.
+ // a bare host:port and each consumer adds its own scheme — see
+ // https://cozystack.io/docs/v1/operations/services/backup-classes/.
creds.endpoint = stripScheme(cfg.Endpoint)
}
if creds.endpoint == "" {
@@ -177,6 +179,13 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr
}
cloud := buildVeleroCredentialsFile(creds.accessKey, creds.secretKey)
+ clickHouseConfig, err := buildClickHouseBackupConfig(creds, cfg.ForcePathStyle)
+ if err != nil {
+ return &ProjectionError{
+ Reason: ReasonSourceMalformed,
+ Message: fmt.Sprintf("build %s: %v", clickHouseBackupConfigKey, err),
+ }
+ }
blob, err := buildFDBBlobCredentials(creds.endpoint, creds.accessKey, creds.secretKey)
if err != nil {
return &ProjectionError{
@@ -233,24 +242,25 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr
}
target.Data["AWS_ACCESS_KEY_ID"] = []byte(creds.accessKey)
target.Data["AWS_SECRET_ACCESS_KEY"] = []byte(creds.secretKey)
- // Mirror under the legacy bucket-controller key names so
- // chart-emitted sidecars (clickhouse-backup) can consume the
- // same Secret without a separate field map.
+ // Mirror under the legacy bucket-controller key names so the Redis
+ // strategy and older ClickHouse charts can consume the same Secret
+ // without a separate field map.
target.Data["accessKey"] = []byte(creds.accessKey)
target.Data["secretKey"] = []byte(creds.secretKey)
target.Data["cloud"] = []byte(cloud)
target.Data["blob_credentials.json"] = blob
+ target.Data[clickHouseBackupConfigKey] = clickHouseConfig
// Always overwrite endpoint/bucket/region (delete if empty in
// source) so a re-projection cannot leave stale values lingering
- // after the bucket-controller rotates the source. Consumers
- // (ClickHouse sidecar via secretKeyRef) must see exactly what the
- // source carries, not a half-stale Secret.
+ // after the bucket-controller rotates the source. Consumers must
+ // see exactly what the source carries, not a half-stale Secret.
setOrDelete(target.Data, "endpoint", creds.endpoint)
setOrDelete(target.Data, "bucketName", creds.bucket)
setOrDelete(target.Data, "region", creds.region)
// forcePathStyle comes from the platform config (backupStorage.
- // forcePathStyle), not the source Secret: the ClickHouse sidecar
- // reads it as S3_FORCE_PATH_STYLE because the app chart cannot see
+ // forcePathStyle), not the source Secret: ClickHouse sidecars
+ // rendered before clickhouse-backup.yml read it as
+ // S3_FORCE_PATH_STYLE because the app chart cannot see
// backupStorage.* directly.
setOrDelete(target.Data, "forcePathStyle", cfg.ForcePathStyle)
return nil
@@ -414,6 +424,35 @@ func buildFDBBlobCredentials(endpoint, accessKey, secretKey string) ([]byte, err
return json.Marshal(doc)
}
+// clickHouseBackupConfigKey holds the s3 section of a clickhouse-backup
+// config. The ClickHouse chart mounts it as the sidecar's config file
+// instead of passing the values as environment: clickhouse-backup reloads
+// the file on every API call and kubelet rewrites a mounted Secret in
+// place, so credentials projected after the pod started, or rotated since,
+// reach a running sidecar.
+const clickHouseBackupConfigKey = "clickhouse-backup.yml"
+
+// buildClickHouseBackupConfig renders that file. The projected endpoint is
+// a bare host, and clickhouse-backup's S3 client rejects one without a
+// scheme.
+func buildClickHouseBackupConfig(creds parsedCreds, forcePathStyle string) ([]byte, error) {
+ s3 := map[string]any{
+ "bucket": creds.bucket,
+ "endpoint": "https://" + creds.endpoint,
+ "region": creds.region,
+ "access_key": creds.accessKey,
+ "secret_key": creds.secretKey,
+ }
+ if forcePathStyle != "" {
+ pathStyle, err := strconv.ParseBool(forcePathStyle)
+ if err != nil {
+ return nil, fmt.Errorf("forcePathStyle %q is not a boolean", forcePathStyle)
+ }
+ s3["force_path_style"] = pathStyle
+ }
+ return yaml.Marshal(map[string]any{"s3": s3})
+}
+
// stripScheme returns the endpoint stripped of any leading scheme prefix
// (http:// or https://) and preserves the path portion if present. The
// resulting form is what `host[:port][/path]` callers (MariaDB strategy,
diff --git a/internal/backupcontroller/credentials_projector_test.go b/internal/backupcontroller/credentials_projector_test.go
index 2b4f0bb1c..c4b154dfa 100644
--- a/internal/backupcontroller/credentials_projector_test.go
+++ b/internal/backupcontroller/credentials_projector_test.go
@@ -13,6 +13,7 @@ import (
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
+ "sigs.k8s.io/yaml"
)
func newFakeClient(objs ...client.Object) client.Client {
@@ -664,3 +665,70 @@ func TestProject_HappyPath_HTTPEndpoint(t *testing.T) {
t.Errorf("blob account host not stripped: %+v", blob.Accounts)
}
}
+
+// The ClickHouse sidecar reads its S3 settings from this key as a mounted
+// file, which kubelet refreshes in place: clickhouse-backup reloads its
+// config file on every API call, while its environment is fixed at start.
+func TestProject_WritesClickHouseBackupConfig(t *testing.T) {
+ cases := []struct {
+ name string
+ forcePathStyle string
+ wantPathStyle *bool
+ }{
+ {name: "path style on", forcePathStyle: "true", wantPathStyle: ptrBool(true)},
+ {name: "path style off", forcePathStyle: "false", wantPathStyle: ptrBool(false)},
+ {name: "path style unset", forcePathStyle: ""},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ c := newFakeClient(flatSourceSecret())
+ ctx := context.Background()
+ cfg := defaultCfg()
+ cfg.ForcePathStyle = tc.forcePathStyle
+ if err := ProjectBackupCredentials(ctx, c, cfg, "tenant-acme"); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ got := &corev1.Secret{}
+ if err := c.Get(ctx, types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, got); err != nil {
+ t.Fatalf("projected Secret not found: %v", err)
+ }
+ var doc struct {
+ S3 map[string]any `json:"s3"`
+ }
+ if err := yaml.Unmarshal(got.Data[clickHouseBackupConfigKey], &doc); err != nil {
+ t.Fatalf("%s is not YAML: %v\n%s", clickHouseBackupConfigKey, err, got.Data[clickHouseBackupConfigKey])
+ }
+ for key, want := range map[string]string{
+ "bucket": "cozy-backups",
+ "endpoint": "https://seaweedfs-s3.tenant-root.svc:8333",
+ "region": "us-east-1",
+ "access_key": "AK",
+ "secret_key": "SK",
+ } {
+ if doc.S3[key] != want {
+ t.Errorf("s3.%s = %v, want %q", key, doc.S3[key], want)
+ }
+ }
+ pathStyle, set := doc.S3["force_path_style"]
+ switch {
+ case tc.wantPathStyle == nil && set:
+ t.Errorf("s3.force_path_style = %v, want it left to the clickhouse-backup default", pathStyle)
+ case tc.wantPathStyle != nil && pathStyle != *tc.wantPathStyle:
+ t.Errorf("s3.force_path_style = %v, want %v", pathStyle, *tc.wantPathStyle)
+ }
+ })
+ }
+}
+
+func TestProject_RejectsMalformedForcePathStyle(t *testing.T) {
+ c := newFakeClient(flatSourceSecret())
+ cfg := defaultCfg()
+ cfg.ForcePathStyle = "yes please"
+ err := ProjectBackupCredentials(context.Background(), c, cfg, "tenant-acme")
+ var perr *ProjectionError
+ if !errors.As(err, &perr) || perr.Reason != ReasonSourceMalformed {
+ t.Fatalf("err = %v, want a %s ProjectionError", err, ReasonSourceMalformed)
+ }
+}
+
+func ptrBool(b bool) *bool { return &b }
diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml
index 827b5d579..d8791878e 100644
--- a/packages/apps/clickhouse/templates/clickhouse.yaml
+++ b/packages/apps/clickhouse/templates/clickhouse.yaml
@@ -225,15 +225,18 @@ spec:
- name: REMOTE_STORAGE
value: s3
{{- /*
+ On the useSystemBucket flow every S3 setting, the
+ platform's backupStorage.forcePathStyle included, comes
+ from the clickhouse-backup.yml key the platform projects
+ into cozy-backups-creds, mounted below as the config file.
+ clickhouse-backup reloads that file on every API call and
+ kubelet rewrites a mounted Secret in place, so credentials
+ projected after the Pod started, or rotated since, reach
+ the running sidecar; environment would be read only at
+ start and would override the file, so none is set here.
+
S3_FORCE_PATH_STYLE selects path-style vs vhost-style S3
- addressing: SeaweedFS (the platform default) needs
- path-style, while AWS public S3 needs vhost-style in many
- regions. On the useSystemBucket flow the value is the
- platform's backupStorage.forcePathStyle, which this app
- chart cannot read directly, so it is projected into
- cozy-backups-creds and consumed via an optional
- secretKeyRef (optional: true so the Pod starts before the
- first projection). On the legacy paths (own Secret via
+ addressing. On the legacy paths (own Secret via
s3CredentialsSecret.name, or a configured endpoint) we
cannot peek inside the Secret at render time, so we keep
the historical path-style default — AWS users in that mode
@@ -241,12 +244,8 @@ spec:
clickhouse-backup config.
*/}}
{{- if .Values.backup.useSystemBucket }}
- - name: S3_FORCE_PATH_STYLE
- valueFrom:
- secretKeyRef:
- name: {{ $sidecarSecretName | quote }}
- key: forcePathStyle
- optional: true
+ - name: CLICKHOUSE_BACKUP_CONFIG
+ value: /etc/clickhouse-backup-s3/config.yml
{{- else if or (not (eq (default "" .Values.backup.s3CredentialsSecret.name) "")) .Values.backup.endpoint }}
- name: S3_FORCE_PATH_STYLE
value: "true"
@@ -282,44 +281,18 @@ spec:
*/}}
- name: S3_PATH
value: {{ $s3Path | quote }}
- {{- /*
- optional: true in the useSystemBucket path so the Pod
- starts even before the platform projector materialises
- cozy-backups-creds in this namespace (projection runs at
- first BackupJob time; the sidecar surfaces backup errors
- only when its HTTP API is exercised). Without this gate
- every fresh ClickHouse release with useSystemBucket=true
- would hang in CreateContainerConfigError until a
- BackupJob fires. Legacy path keeps the strict reference
- shape — its Secret is chart-emitted and present at
- install time.
- */}}
- {{- $secretOptional := .Values.backup.useSystemBucket }}
+ {{- if not .Values.backup.useSystemBucket }}
- name: S3_BUCKET
- valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $bucketKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } }
- {{- /*
- clickhouse-backup's aws-sdk-go-v2 client rejects a scheme-less
- S3_ENDPOINT ("was not a valid URI"). On the useSystemBucket flow
- the projected cozy-backups-creds.endpoint is a bare host, so
- prepend https:// (the platform's CNPG endpoint helper does the
- same). Legacy / BYO-Secret endpoints are tenant-supplied with
- their own scheme and are consumed verbatim.
- */}}
- {{- if .Values.backup.useSystemBucket }}
- - name: S3_ENDPOINT_HOST
- valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $endpointKey | quote }}, optional: true } }
- - name: S3_ENDPOINT
- value: "https://$(S3_ENDPOINT_HOST)"
- {{- else }}
+ valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $bucketKey | quote }} } }
- name: S3_ENDPOINT
valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $endpointKey | quote }} } }
- {{- end }}
- name: S3_REGION
- valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $regionKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } }
+ valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $regionKey | quote }} } }
- name: S3_ACCESS_KEY
- valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $accessKeyKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } }
+ valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $accessKeyKey | quote }} } }
- name: S3_SECRET_KEY
- valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $secretKeyKey | quote }}{{ if $secretOptional }}, optional: true{{ end }} } }
+ valueFrom: { secretKeyRef: { name: {{ $sidecarSecretName | quote }}, key: {{ $secretKeyKey | quote }} } }
+ {{- end }}
{{- if $endpointCAName }}
# Trust the mounted private-CA bundle for the S3 endpoint (see
# the `backup.endpointCA` field in values.yaml). SSL_CERT_DIR,
@@ -343,6 +316,11 @@ spec:
- name: backup-s3-ca
mountPath: /etc/ssl/cozy-s3-ca
readOnly: true
+{{- end }}
+{{- if .Values.backup.useSystemBucket }}
+ - name: backup-s3-config
+ mountPath: /etc/clickhouse-backup-s3
+ readOnly: true
{{- end }}
securityContext:
# clickhouse-server stores data under /var/lib/clickhouse with
@@ -366,13 +344,14 @@ spec:
runAsGroup: 0
seccompProfile:
type: RuntimeDefault
-{{- if $endpointCAName }}
- # Private-CA bundle for the backup sidecar, projected as ca.crt for
- # SSL_CERT_DIR above. Not optional on purpose: a missing Secret holds
- # the whole ClickHouse Pod in ContainerCreating (naming the Secret in
- # `kubectl describe`) rather than mounting an empty trust dir and
- # failing later inside clickhouse-backup with an opaque x509 error.
+{{- if or $endpointCAName .Values.backup.useSystemBucket }}
volumes:
+{{- if $endpointCAName }}
+ # Private-CA bundle for the backup sidecar, projected as ca.crt for
+ # SSL_CERT_DIR above. Not optional on purpose: a missing Secret holds
+ # the whole ClickHouse Pod in ContainerCreating (naming the Secret in
+ # `kubectl describe`) rather than mounting an empty trust dir and
+ # failing later inside clickhouse-backup with an opaque x509 error.
- name: backup-s3-ca
secret:
secretName: {{ $endpointCAName | quote }}
@@ -380,6 +359,18 @@ spec:
- key: {{ $endpointCAKey | quote }}
path: ca.crt
{{- end }}
+{{- if .Values.backup.useSystemBucket }}
+ # Optional: the platform projects cozy-backups-creds into this
+ # namespace at the first BackupJob, after the Pod has started.
+ - name: backup-s3-config
+ secret:
+ secretName: {{ $sidecarSecretName | quote }}
+ optional: true
+ items:
+ - key: clickhouse-backup.yml
+ path: config.yml
+{{- end }}
+{{- end }}
{{- end }}
serviceTemplates:
- name: svc-template
diff --git a/packages/apps/clickhouse/tests/backup_test.yaml b/packages/apps/clickhouse/tests/backup_test.yaml
index 66eda4dc8..7448e02a3 100644
--- a/packages/apps/clickhouse/tests/backup_test.yaml
+++ b/packages/apps/clickhouse/tests/backup_test.yaml
@@ -2,10 +2,12 @@ suite: backup sidecar S3 wiring (useSystemBucket vs legacy, path-override guard)
# Locks in the clickhouse-backup sidecar S3 wiring this PR introduced on the
# ClickHouseInstallation PodTemplate:
-# - useSystemBucket=true points every S3_* env at the platform-projected
-# cozy-backups-creds Secret with optional:true (the Pod starts before the
-# first projection) and scopes S3_PATH to /; the
-# chart-managed -backup-s3 / -backup Secrets are skipped;
+# - useSystemBucket=true mounts the clickhouse-backup.yml key of the
+# platform-projected cozy-backups-creds Secret as the sidecar's config
+# file with optional:true (the Pod starts before the first projection),
+# sets no other S3 setting in the environment, and scopes S3_PATH to
+# /; the chart-managed -backup-s3 /
+# -backup Secrets are skipped;
# - the legacy flow keeps the chart-emitted -backup-s3 Secret and a
# strict (non-optional) secretKeyRef;
# - s3PathOverride + useSystemBucket fails fast at render time.
@@ -25,7 +27,7 @@ set:
_cluster: {}
tests:
- - it: "useSystemBucket=true: sidecar reads cozy-backups-creds (optional) and scopes S3_PATH"
+ - it: "useSystemBucket=true: sidecar reads its S3 settings from the projected config file and scopes S3_PATH"
template: templates/clickhouse.yaml
documentSelector:
path: kind
@@ -45,28 +47,29 @@ tests:
- equal:
path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_PATH")].value
value: tenant-test/ch-test
+ # clickhouse-backup reloads this file on every API call and kubelet
+ # rewrites the mounted Secret in place, so credentials projected after
+ # the Pod started, or rotated since, reach the running sidecar.
- equal:
- path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")].valueFrom.secretKeyRef.name
- value: cozy-backups-creds
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="CLICKHOUSE_BACKUP_CONFIG")].value
+ value: /etc/clickhouse-backup-s3/config.yml
- equal:
- path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")].valueFrom.secretKeyRef.optional
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].volumeMounts[?(@.name=="backup-s3-config")].mountPath
+ value: /etc/clickhouse-backup-s3
+ - equal:
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].volumeMounts[?(@.name=="backup-s3-config")].readOnly
value: true
- equal:
- path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_SECRET_KEY")].valueFrom.secretKeyRef.name
- value: cozy-backups-creds
- # The projected endpoint is a bare host; clickhouse-backup needs a URI, so
- # the chart prepends https:// via env-var expansion on this flow.
- - equal:
- path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT_HOST")].valueFrom.secretKeyRef.name
- value: cozy-backups-creds
- - equal:
- path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT_HOST")].valueFrom.secretKeyRef.key
- value: endpoint
- - equal:
- path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT")].value
- value: https://$(S3_ENDPOINT_HOST)
+ path: spec.templates.podTemplates[0].spec.volumes[?(@.name=="backup-s3-config")].secret
+ value:
+ secretName: cozy-backups-creds
+ # The Pod starts before the first projection.
+ optional: true
+ items:
+ - key: clickhouse-backup.yml
+ path: config.yml
- - it: "useSystemBucket=true: S3_FORCE_PATH_STYLE is sourced from cozy-backups-creds, not hardcoded"
+ - it: "useSystemBucket=true: no S3 setting is fixed in the sidecar environment"
template: templates/clickhouse.yaml
documentSelector:
path: kind
@@ -76,18 +79,21 @@ tests:
enabled: true
useSystemBucket: true
asserts:
- - equal:
- path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].valueFrom.secretKeyRef.name
- value: cozy-backups-creds
- - equal:
- path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].valueFrom.secretKeyRef.key
- value: forcePathStyle
- - equal:
- path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].valueFrom.secretKeyRef.optional
- value: true
+ # Environment overrides the config file and is read only at start.
- notExists:
- # must no longer be the hardcoded literal value: "true"
- path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")].value
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")]
+ - notExists:
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT")]
+ - notExists:
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ENDPOINT_HOST")]
+ - notExists:
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_REGION")]
+ - notExists:
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_ACCESS_KEY")]
+ - notExists:
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_SECRET_KEY")]
+ - notExists:
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_FORCE_PATH_STYLE")]
- it: "useSystemBucket=true: chart-managed backup Secrets are skipped"
set:
@@ -240,7 +246,10 @@ tests:
path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="SSL_CERT_DIR")].value
value: /etc/ssl/cozy-s3-ca:/etc/ssl/certs
- equal:
- path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="S3_BUCKET")].valueFrom.secretKeyRef.name
+ path: spec.templates.podTemplates[0].spec.volumes[?(@.name=="backup-s3-ca")].secret.secretName
+ value: my-s3-ca
+ - equal:
+ path: spec.templates.podTemplates[0].spec.volumes[?(@.name=="backup-s3-config")].secret.secretName
value: cozy-backups-creds
# backup.enabled=false renders no sidecar at all, so the CA must not leak a
From f5f50d2a77179c823ad6d6ab1f1e61d9c49a359c Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 17:15:10 +0300
Subject: [PATCH 061/346] fix(clickhouse): hold a strategy job until projected
credentials reach the sidecar
The clickhouse-backup sidecar now reads the projected credentials from a
mounted Secret, which kubelet rewrites only after its sync period,
about 75 seconds after the change in practice. The first BackupJob
projects the Secret and started the strategy Job a second later, so its
attempts raced the volume update and the backup could still fail on a
sidecar without credentials.
The projector records when the projected content last changed, and the
ClickHouse driver holds back creating a backup or restore Job until two
minutes have passed since, with a WaitingForCredentials condition. An
unchanged re-projection keeps the time, so only the first projection
and a rotation cause a wait, and a Job already running is never held.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
.../altinitystrategy_controller.go | 74 +++++++
.../altinitystrategy_controller_test.go | 186 ++++++++++++++++++
.../backupcontroller/credentials_projector.go | 15 ++
.../credentials_projector_test.go | 56 ++++++
4 files changed, 331 insertions(+)
diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go
index b96d571f0..a99801ac6 100644
--- a/internal/backupcontroller/altinitystrategy_controller.go
+++ b/internal/backupcontroller/altinitystrategy_controller.go
@@ -71,6 +71,56 @@ func validateAltinityApplicationRef(ref corev1.TypedLocalObjectReference) error
return nil
}
+// altinityCredentialsPropagation bounds how long kubelet takes to rewrite
+// the mounted cozy-backups-creds in the ClickHouse Pod after it changes:
+// its sync period (a minute by default) plus the Secret cache. About 75s was
+// measured; the margin covers the rest.
+const altinityCredentialsPropagation = 2 * time.Minute
+
+// credentialsPropagationWait returns how long to hold back creating the
+// strategy Job named jobName in namespace, so that the clickhouse-backup
+// sidecar, which reads the projected credentials from a mounted file, sees
+// their latest content. A Job that already exists, or a namespace without
+// projected credentials, does not wait.
+func credentialsPropagationWait(ctx context.Context, c client.Client, secretName, namespace, jobName string) (time.Duration, error) {
+ if secretName == "" {
+ return 0, nil
+ }
+ if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: jobName}, &batchv1.Job{}); err == nil {
+ return 0, nil
+ } else if !apierrors.IsNotFound(err) {
+ return 0, err
+ }
+ secret := &corev1.Secret{}
+ if err := c.Get(ctx, types.NamespacedName{Namespace: namespace, Name: secretName}, secret); err != nil {
+ if apierrors.IsNotFound(err) {
+ return 0, nil
+ }
+ return 0, err
+ }
+ raw := secret.Annotations[credentialsUpdatedAtAnnotation]
+ if raw == "" {
+ return 0, nil
+ }
+ updatedAt, err := time.Parse(time.RFC3339, raw)
+ if err != nil {
+ return 0, fmt.Errorf("parse %s on Secret %s/%s: %w", credentialsUpdatedAtAnnotation, namespace, secretName, err)
+ }
+ return time.Until(updatedAt.Add(altinityCredentialsPropagation)), nil
+}
+
+// credentialsWaitCondition is the Ready condition of a job held back by
+// credentialsPropagationWait.
+func credentialsWaitCondition(secretName string, wait time.Duration) metav1.Condition {
+ return metav1.Condition{
+ Type: "Ready",
+ Status: metav1.ConditionFalse,
+ Reason: "WaitingForCredentials",
+ Message: fmt.Sprintf("waiting %s for the updated Secret %s to reach the ClickHouse backup sidecar",
+ wait.Round(time.Second), secretName),
+ }
+}
+
// jobNameForBackupJob returns the deterministic batch/v1.Job name used for
// the backup pod that drives a BackupJob. Idempotency relies on this being
// pure: a retried reconcile must hit the same Job name.
@@ -222,6 +272,18 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv
return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to template Altinity strategy: %v", err))
}
+ wait, err := credentialsPropagationWait(ctx, r.Client, r.CredentialsConfig.TargetSecretName, j.Namespace, jobNameForBackupJob(j))
+ if err != nil {
+ return ctrl.Result{}, err
+ }
+ if wait > 0 {
+ apimeta.SetStatusCondition(&j.Status.Conditions, credentialsWaitCondition(r.CredentialsConfig.TargetSecretName, wait))
+ if err := r.Status().Update(ctx, j); err != nil {
+ return ctrl.Result{}, err
+ }
+ return ctrl.Result{RequeueAfter: wait}, nil
+ }
+
batchJob, err := r.ensureAltinityJob(ctx, j, j.Namespace, jobNameForBackupJob(j),
altinityModeBackup,
map[string]string{
@@ -623,6 +685,18 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to template Altinity strategy: %v", err))
}
+ wait, err := credentialsPropagationWait(ctx, r.Client, r.CredentialsConfig.TargetSecretName, targetNamespace, jobNameForRestoreJob(restoreJob))
+ if err != nil {
+ return ctrl.Result{}, err
+ }
+ if wait > 0 {
+ apimeta.SetStatusCondition(&restoreJob.Status.Conditions, credentialsWaitCondition(r.CredentialsConfig.TargetSecretName, wait))
+ if err := r.Status().Update(ctx, restoreJob); err != nil {
+ return ctrl.Result{}, err
+ }
+ return ctrl.Result{RequeueAfter: wait}, nil
+ }
+
batchJob, err := r.ensureAltinityRestoreJob(ctx, restoreJob, targetNamespace, jobNameForRestoreJob(restoreJob),
altinityModeRestore,
map[string]string{
diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go
index 68f18fa28..75564538b 100644
--- a/internal/backupcontroller/altinitystrategy_controller_test.go
+++ b/internal/backupcontroller/altinitystrategy_controller_test.go
@@ -3,7 +3,9 @@ package backupcontroller
import (
"context"
+ "errors"
"testing"
+ "time"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
@@ -17,6 +19,7 @@ import (
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
+ "sigs.k8s.io/controller-runtime/pkg/client/interceptor"
strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
@@ -710,3 +713,186 @@ func TestBackupParameters_RoundTrip(t *testing.T) {
t.Errorf("non-parameter key leaked through: %v", got)
}
}
+
+func projectedCredentials(namespace string, updatedAt time.Time) *corev1.Secret {
+ return &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Namespace: namespace,
+ Name: "cozy-backups-creds",
+ Annotations: map[string]string{credentialsUpdatedAtAnnotation: updatedAt.UTC().Format(time.RFC3339)},
+ },
+ }
+}
+
+// The ClickHouse sidecar reads the projected credentials from a mounted
+// file that kubelet rewrites only some time after the Secret changes, so a
+// strategy Job started right after a projection would reach a sidecar with
+// no or stale credentials.
+func TestReconcileAltinity_WaitsForProjectedCredentialsToReachTheSidecar(t *testing.T) {
+ cases := []struct {
+ name string
+ secret *corev1.Secret
+ jobExists bool
+ wantWait bool
+ }{
+ {name: "credentials changed just now", secret: projectedCredentials("tenant-test", time.Now()), wantWait: true},
+ {name: "credentials changed long ago", secret: projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))},
+ {name: "no projected credentials", secret: nil},
+ {name: "the Job already runs", secret: projectedCredentials("tenant-test", time.Now()), jobExists: true},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ now := metav1.Now()
+ backupJob := &backupsv1alpha1.BackupJob{
+ ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
+ Spec: backupsv1alpha1.BackupJobSpec{
+ ApplicationRef: newAltinityClickHouseRef("ch-test"),
+ BackupClassName: "clickhouse-backup",
+ },
+ Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now},
+ }
+ objs := []client.Object{backupJob, newAltinityStrategy("clickhouse-strategy")}
+ if tc.secret != nil {
+ objs = append(objs, tc.secret)
+ }
+ if tc.jobExists {
+ objs = append(objs, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}})
+ }
+ r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), clientfake.NewClientBuilder().WithObjects(objs...))
+ r.CredentialsConfig.TargetSecretName = "cozy-backups-creds"
+ resolved := &ResolvedBackupConfig{
+ StrategyRef: corev1.TypedLocalObjectReference{
+ APIGroup: new(strategyv1alpha1.GroupVersion.Group),
+ Kind: strategyv1alpha1.AltinityStrategyKind,
+ Name: "clickhouse-strategy",
+ },
+ }
+ ctx := context.Background()
+ j := &backupsv1alpha1.BackupJob{}
+ if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil {
+ t.Fatalf("get BackupJob: %v", err)
+ }
+ res, err := r.reconcileAltinity(ctx, j, resolved)
+ if err != nil {
+ t.Fatalf("reconcileAltinity: %v", err)
+ }
+ jobs := &batchv1.JobList{}
+ if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
+ t.Fatalf("list Jobs: %v", err)
+ }
+ ready := meta.FindStatusCondition(j.Status.Conditions, "Ready")
+ if tc.wantWait {
+ if len(jobs.Items) != 0 {
+ t.Errorf("a strategy Job was created %d time(s) before the credentials could reach the sidecar", len(jobs.Items))
+ }
+ if res.RequeueAfter <= 0 || res.RequeueAfter > altinityCredentialsPropagation {
+ t.Errorf("RequeueAfter = %v, want within (0, %v]", res.RequeueAfter, altinityCredentialsPropagation)
+ }
+ if ready == nil || ready.Reason != "WaitingForCredentials" {
+ t.Errorf("Ready = %+v, want WaitingForCredentials", ready)
+ }
+ return
+ }
+ if len(jobs.Items) != 1 {
+ t.Errorf("got %d strategy Jobs, want 1", len(jobs.Items))
+ }
+ if ready != nil && ready.Reason == "WaitingForCredentials" {
+ t.Errorf("Ready = %+v, the job should not wait", ready)
+ }
+ })
+ }
+}
+
+func TestReconcileAltinityRestore_WaitsForProjectedCredentialsToReachTheSidecar(t *testing.T) {
+ now := metav1.Now()
+ backup := &backupsv1alpha1.Backup{
+ ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"},
+ Spec: backupsv1alpha1.BackupSpec{
+ ApplicationRef: newAltinityClickHouseRef("ch-test"),
+ StrategyRef: corev1.TypedLocalObjectReference{
+ APIGroup: new(strategyv1alpha1.GroupVersion.Group),
+ Kind: strategyv1alpha1.AltinityStrategyKind,
+ Name: "clickhouse-strategy",
+ },
+ TakenAt: now,
+ },
+ }
+ restoreJob := &backupsv1alpha1.RestoreJob{
+ ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"},
+ Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}},
+ Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
+ }
+ _, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"),
+ clientfake.NewClientBuilder().WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob,
+ projectedCredentials("tenant-test", time.Now())))
+ rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds"
+ ctx := context.Background()
+ rj := &backupsv1alpha1.RestoreJob{}
+ if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil {
+ t.Fatalf("get RestoreJob: %v", err)
+ }
+ res, err := rr.reconcileAltinityRestore(ctx, rj, backup)
+ if err != nil {
+ t.Fatalf("reconcileAltinityRestore: %v", err)
+ }
+ jobs := &batchv1.JobList{}
+ if err := rr.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
+ t.Fatalf("list Jobs: %v", err)
+ }
+ if len(jobs.Items) != 0 {
+ t.Errorf("a restore Job was created before the credentials could reach the sidecar")
+ }
+ if res.RequeueAfter <= 0 {
+ t.Errorf("RequeueAfter = %v, want a wait", res.RequeueAfter)
+ }
+ if ready := meta.FindStatusCondition(rj.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCredentials" {
+ t.Errorf("Ready = %+v, want WaitingForCredentials", ready)
+ }
+}
+
+// Without credentials projection configured there is no Secret to wait
+// for; client-go refuses a Get with an empty name, so it must not be asked.
+func TestReconcileAltinity_NoProjectionConfiguredDoesNotWait(t *testing.T) {
+ now := metav1.Now()
+ backupJob := &backupsv1alpha1.BackupJob{
+ ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
+ Spec: backupsv1alpha1.BackupJobSpec{
+ ApplicationRef: newAltinityClickHouseRef("ch-test"),
+ BackupClassName: "clickhouse-backup",
+ },
+ Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now},
+ }
+ builder := clientfake.NewClientBuilder().
+ WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy")).
+ WithInterceptorFuncs(interceptor.Funcs{
+ Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+ if key.Name == "" {
+ return errors.New("resource name may not be empty")
+ }
+ return cl.Get(ctx, key, obj, opts...)
+ },
+ })
+ r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"), builder)
+ resolved := &ResolvedBackupConfig{
+ StrategyRef: corev1.TypedLocalObjectReference{
+ APIGroup: new(strategyv1alpha1.GroupVersion.Group),
+ Kind: strategyv1alpha1.AltinityStrategyKind,
+ Name: "clickhouse-strategy",
+ },
+ }
+ ctx := context.Background()
+ j := &backupsv1alpha1.BackupJob{}
+ if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil {
+ t.Fatalf("get BackupJob: %v", err)
+ }
+ if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil {
+ t.Fatalf("reconcileAltinity: %v", err)
+ }
+ jobs := &batchv1.JobList{}
+ if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
+ t.Fatalf("list Jobs: %v", err)
+ }
+ if len(jobs.Items) != 1 {
+ t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items))
+ }
+}
diff --git a/internal/backupcontroller/credentials_projector.go b/internal/backupcontroller/credentials_projector.go
index eb34d22bc..99d4e182e 100644
--- a/internal/backupcontroller/credentials_projector.go
+++ b/internal/backupcontroller/credentials_projector.go
@@ -1,10 +1,12 @@
package backupcontroller
import (
+ "bytes"
"context"
"encoding/json"
"errors"
"fmt"
+ "maps"
"net/url"
"strconv"
"strings"
@@ -240,6 +242,7 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr
if target.Data == nil {
target.Data = map[string][]byte{}
}
+ before := maps.Clone(target.Data)
target.Data["AWS_ACCESS_KEY_ID"] = []byte(creds.accessKey)
target.Data["AWS_SECRET_ACCESS_KEY"] = []byte(creds.secretKey)
// Mirror under the legacy bucket-controller key names so the Redis
@@ -263,6 +266,12 @@ func ProjectBackupCredentials(ctx context.Context, c client.Client, cfg BackupCr
// S3_FORCE_PATH_STYLE because the app chart cannot see
// backupStorage.* directly.
setOrDelete(target.Data, "forcePathStyle", cfg.ForcePathStyle)
+ if !maps.EqualFunc(before, target.Data, bytes.Equal) {
+ if target.Annotations == nil {
+ target.Annotations = map[string]string{}
+ }
+ target.Annotations[credentialsUpdatedAtAnnotation] = time.Now().UTC().Format(time.RFC3339)
+ }
return nil
}); err != nil {
// If the mutator refused to overwrite an unowned Secret, surface
@@ -424,6 +433,12 @@ func buildFDBBlobCredentials(endpoint, accessKey, secretKey string) ([]byte, err
return json.Marshal(doc)
}
+// credentialsUpdatedAtAnnotation records when the projected content last
+// changed. A consumer that mounts the Secret as a volume sees a change only
+// once kubelet rewrites the volume, so it uses this to tell how fresh its
+// copy can be.
+const credentialsUpdatedAtAnnotation = "backups.cozystack.io/credentials-updated-at"
+
// clickHouseBackupConfigKey holds the s3 section of a clickhouse-backup
// config. The ClickHouse chart mounts it as the sidecar's config file
// instead of passing the values as environment: clickhouse-backup reloads
diff --git a/internal/backupcontroller/credentials_projector_test.go b/internal/backupcontroller/credentials_projector_test.go
index c4b154dfa..208ba253c 100644
--- a/internal/backupcontroller/credentials_projector_test.go
+++ b/internal/backupcontroller/credentials_projector_test.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"testing"
+ "time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -732,3 +733,58 @@ func TestProject_RejectsMalformedForcePathStyle(t *testing.T) {
}
func ptrBool(b bool) *bool { return &b }
+
+// Consumers that read the Secret through a mounted volume need to know when
+// its content last changed; an unchanged re-projection must keep the time.
+func TestProject_StampsWhenContentChanges(t *testing.T) {
+ src := flatSourceSecret()
+ c := newFakeClient(src)
+ ctx := context.Background()
+ key := types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}
+ stamp := func() string {
+ got := &corev1.Secret{}
+ if err := c.Get(ctx, key, got); err != nil {
+ t.Fatalf("projected Secret not found: %v", err)
+ }
+ return got.Annotations[credentialsUpdatedAtAnnotation]
+ }
+
+ if err := ProjectBackupCredentials(ctx, c, defaultCfg(), "tenant-acme"); err != nil {
+ t.Fatalf("first projection: %v", err)
+ }
+ first := stamp()
+ at, err := time.Parse(time.RFC3339, first)
+ if err != nil || time.Since(at) > time.Minute {
+ t.Fatalf("first projection stamp %q is not a recent RFC3339 time: %v", first, err)
+ }
+
+ old := &corev1.Secret{}
+ if err := c.Get(ctx, key, old); err != nil {
+ t.Fatalf("get projected Secret: %v", err)
+ }
+ old.Annotations[credentialsUpdatedAtAnnotation] = "2000-01-01T00:00:00Z"
+ if err := c.Update(ctx, old); err != nil {
+ t.Fatalf("age the stamp: %v", err)
+ }
+ if err := ProjectBackupCredentials(ctx, c, defaultCfg(), "tenant-acme"); err != nil {
+ t.Fatalf("unchanged re-projection: %v", err)
+ }
+ if got := stamp(); got != "2000-01-01T00:00:00Z" {
+ t.Fatalf("an unchanged re-projection moved the stamp to %q", got)
+ }
+
+ rotated := &corev1.Secret{}
+ if err := c.Get(ctx, types.NamespacedName{Namespace: src.Namespace, Name: src.Name}, rotated); err != nil {
+ t.Fatalf("get source Secret: %v", err)
+ }
+ rotated.Data["secretKey"] = []byte("SK2")
+ if err := c.Update(ctx, rotated); err != nil {
+ t.Fatalf("rotate source Secret: %v", err)
+ }
+ if err := ProjectBackupCredentials(ctx, c, defaultCfg(), "tenant-acme"); err != nil {
+ t.Fatalf("rotated re-projection: %v", err)
+ }
+ if got := stamp(); got == "2000-01-01T00:00:00Z" {
+ t.Fatalf("a rotation did not move the stamp")
+ }
+}
From 35a4f7c5d93eac727354631d5de5c0fc2bb3b294 Mon Sep 17 00:00:00 2001
From: Myasnikov Daniil
Date: Wed, 4 Nov 2026 13:40:11 +0000
Subject: [PATCH 062/346] docs(release): split the release document into one
page per topic
docs/release.md had grown into one page that mixed the release model,
the CI pipeline, changelogs, signatures, nightlies and the fires seen
at release time. A reader looking for one of them read all of it, and
an edit to one part reflowed the rest. It now indexes one page per
topic under docs/release/.
The pages already describe how fixes reach a release line through the
backport batch, which the following commits add. Workflows, scripts
and tests that named docs/release.md now point at the page that holds
the text they mean.
Assisted-By: LLM
Signed-off-by: Myasnikov Daniil
---
.github/workflows/changelog-generate.yaml | 16 +-
.github/workflows/cut-prerelease.yaml | 2 +-
.github/workflows/tags.yaml | 4 +-
docs/agents/overview.md | 4 +-
docs/release.md | 708 +---------------------
docs/release/backport-mechanics.md | 37 ++
docs/release/changelogs.md | 107 ++++
docs/release/ci-failures.md | 16 +
docs/release/ci-pipeline.md | 107 ++++
docs/release/cleanup.md | 6 +
docs/release/introduction.md | 11 +
docs/release/nightly-builds.md | 21 +
docs/release/patch-releases.md | 82 +++
docs/release/pre-release-checklist.md | 14 +
docs/release/regular-releases.md | 78 +++
docs/release/release-candidates.md | 18 +
docs/release/release-time-fires.md | 35 ++
docs/release/signatures-and-sbom.md | 29 +
docs/release/splitting-bundle-prs.md | 10 +
docs/release/stable-tags.md | 7 +
docs/release/tag-immutability.md | 18 +
hack/ci-tool-installs-contract.bats | 5 +-
hack/release-changelog-contract.bats | 38 +-
hack/sign-release-assets.bats | 4 +-
hack/sign-release-assets.sh | 7 +-
25 files changed, 663 insertions(+), 721 deletions(-)
create mode 100644 docs/release/backport-mechanics.md
create mode 100644 docs/release/changelogs.md
create mode 100644 docs/release/ci-failures.md
create mode 100644 docs/release/ci-pipeline.md
create mode 100644 docs/release/cleanup.md
create mode 100644 docs/release/introduction.md
create mode 100644 docs/release/nightly-builds.md
create mode 100644 docs/release/patch-releases.md
create mode 100644 docs/release/pre-release-checklist.md
create mode 100644 docs/release/regular-releases.md
create mode 100644 docs/release/release-candidates.md
create mode 100644 docs/release/release-time-fires.md
create mode 100644 docs/release/signatures-and-sbom.md
create mode 100644 docs/release/splitting-bundle-prs.md
create mode 100644 docs/release/stable-tags.md
create mode 100644 docs/release/tag-immutability.md
diff --git a/.github/workflows/changelog-generate.yaml b/.github/workflows/changelog-generate.yaml
index 10452fb85..ff67b4f7e 100644
--- a/.github/workflows/changelog-generate.yaml
+++ b/.github/workflows/changelog-generate.yaml
@@ -154,6 +154,7 @@ jobs:
sparse-checkout: |
hack
docs/agents
+ docs/release
persist-credentials: false
# The agent reads docs/agents/changelog.md from the WORKING TREE, which is
@@ -163,18 +164,17 @@ jobs:
# detached rc-tag checkout is exactly that, so a COMPLIANT agent would
# silently decline to generate on every promotion of an existing rc.
# Overlay the current instructions so the agent reads today's contract.
- # changelog.md sends the agent to the checks in docs/release.md and to
- # hack/validate-changelog.sh, so those are overlaid too: the rc tag's
- # copies predate any check added since. docs/release.md reaches
- # .release-tooling only because the sparse checkout above is in cone
- # mode (the action's default), which carries the files of every parent
- # directory of docs/agents.
+ # changelog.md sends the agent to the checks in docs/release/changelogs.md
+ # and to hack/validate-changelog.sh, so those are overlaid too: the rc
+ # tag's copies predate any check added since, and an rc cut before the
+ # release docs were split into docs/release/ has no such directory.
- name: Refresh agent instructions
run: |
cp .release-tooling/docs/agents/changelog.md docs/agents/changelog.md
- cp .release-tooling/docs/release.md docs/release.md
+ mkdir -p docs/release
+ cp .release-tooling/docs/release/changelogs.md docs/release/changelogs.md
cp .release-tooling/hack/validate-changelog.sh hack/validate-changelog.sh
- echo "Refreshed docs/agents/changelog.md, docs/release.md and hack/validate-changelog.sh from ${GITHUB_REF_NAME}."
+ echo "Refreshed docs/agents/changelog.md, docs/release/changelogs.md and hack/validate-changelog.sh from ${GITHUB_REF_NAME}."
# Skip-priority + validate-then-fall-through: use a pre-existing changelog
# only if it is USABLE, so an invalid higher-priority source can never
diff --git a/.github/workflows/cut-prerelease.yaml b/.github/workflows/cut-prerelease.yaml
index e740c2978..bfd159d94 100644
--- a/.github/workflows/cut-prerelease.yaml
+++ b/.github/workflows/cut-prerelease.yaml
@@ -6,7 +6,7 @@ name: Cut Pre-release Tag
# This is the single entry point for creating a pre-release. Stable vX.Y.Z tags
# are never cut here (or by hand) — they are created write-once by
# pull-requests-release.yaml finalize at a promote PR's merge commit. See
-# docs/release.md and #2677.
+# docs/release/stable-tags.md and #2677.
#
# Dispatch from the branch the tag belongs to: `main` for a new minor's
# pre-release (vX.Y.0-rc.N), or `release-X.Y` for a patch line (vX.Y.Z-rc.N).
diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml
index 8c123fdb1..f35ca8265 100644
--- a/.github/workflows/tags.yaml
+++ b/.github/workflows/tags.yaml
@@ -556,8 +556,8 @@ jobs:
tar -xzf "$syft_dir/syft.tar.gz" -C "$syft_dir" syft
rm -f "$syft_dir/syft.tar.gz"
echo "$syft_dir" >> "$GITHUB_PATH"
- # cosign is the version docs/release.md asks users for, as in
- # promote-rc.yaml. Bump with it.
+ # cosign is the version docs/release/signatures-and-sbom.md asks users
+ # for, as in promote-rc.yaml. Bump with it.
- name: Set up cosign
if: needs.build-amd64.outputs.release_exists == 'false'
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
diff --git a/docs/agents/overview.md b/docs/agents/overview.md
index a4e47b23b..9ab67c41d 100644
--- a/docs/agents/overview.md
+++ b/docs/agents/overview.md
@@ -96,7 +96,7 @@ Build environment variables:
- `LOAD=1 PUSH=0` — load images locally instead of pushing. Builds for the host architecture only, because the classic docker image store cannot load a multi-platform index.
- `PLATFORM` — the buildx platforms. Defaults to `linux/amd64,linux/arm64` for anything but `LOAD=1`, so a push from either architecture publishes an index both can run. Needs a `docker-container` builder (`BUILDER`) or a docker daemon on the containerd image store, with QEMU registered for the non-native half. Compiling builder stages run natively on `$BUILDPLATFORM` and cross-compile for the target, but every `RUN` step of a stage that runs on the target platform (package installs, `setcap`, install scripts, patch steps) still runs emulated. `packages/core/testing` pins `linux/amd64`, because it carries an x86 KVM sandbox. `packages/core/talos` builds the Talos installer and release assets for the arches in `PLATFORM` with the upstream imager, which needs no QEMU for either; its matchbox image carries the amd64 and arm64 kernel and initramfs whatever `PLATFORM` says, because it network-boots machines of either arch.
- `PUSHED_TAGS_LOG` — a file the `image-tags` macro appends every pushed `:` to, one per line. Unset by default; the release build sets it so `hack/stitch-multiarch.sh` can move every tag it pushed, versioned ones included, onto the multi-arch index. The append happens at recipe expansion, so `make -n` writes it too.
-- `CACHE_TAG` — the tag of the mode=max build cache each image reads (and writes under `WRITE_CACHE=1`). Defaults to `buildcache`. Apart from the Talos installer and matchbox in the release job, CI builds one architecture per job, and builds arm64 images only for pre-release tags and in a nightly build of main: the images are built again on native arm64 runners under `-arm64` with `CACHE_TAG=buildcache-arm64`, which only the nightly writes. The talos and testing packages get no arm64 leg: the release job builds the Talos installer and matchbox for both arches itself, and testing is amd64 only. For a pre-release, `hack/stitch-multiarch.sh` then joins each pair into one index, `hack/verify-multiarch.sh` fails the rc on any pinned image that is not an amd64+arm64 index, and the packages artifact is republished pinned on the index digests; see `docs/release.md`. The stitch skips any image whose digest is also pinned in a file it does not rewrite, which then fails that gate; the kamaji control-plane provider's gzipped manifest is the one such copy it does rewrite (see `docs/agents/image-refs.md`).
+- `CACHE_TAG` — the tag of the mode=max build cache each image reads (and writes under `WRITE_CACHE=1`). Defaults to `buildcache`. Apart from the Talos installer and matchbox in the release job, CI builds one architecture per job, and builds arm64 images only for pre-release tags and in a nightly build of main: the images are built again on native arm64 runners under `-arm64` with `CACHE_TAG=buildcache-arm64`, which only the nightly writes. The talos and testing packages get no arm64 leg: the release job builds the Talos installer and matchbox for both arches itself, and testing is amd64 only. For a pre-release, `hack/stitch-multiarch.sh` then joins each pair into one index, `hack/verify-multiarch.sh` fails the rc on any pinned image that is not an amd64+arm64 index, and the packages artifact is republished pinned on the index digests; see `docs/release/ci-pipeline.md`. The stitch skips any image whose digest is also pinned in a file it does not rewrite, which then fails that gate; the kamaji control-plane provider's gzipped manifest is the one such copy it does rewrite (see `docs/agents/image-refs.md`).
### Values schema generation
@@ -150,7 +150,7 @@ Product documentation lives in [`cozystack/website`](https://github.com/cozystac
| --- | --- |
| `packages///README.md` | Anything scoped to a single package. Prose goes **above** `## Parameters`; that section is `cozyvalues-gen` output and is never hand-edited. The website's `make update-apps` / `update-vms` / `update-networking` / `update-k8s` / `update-services` targets append the whole README onto a published page, but only for the packages named in the `APPS`, `VMS`, `NETWORKING`, `K8S` and `SERVICES` lists of the website `Makefile`, and only from `packages/apps/` and `packages/extra/`: nothing under `packages/system/` is ever imported, and neither is a package that is not on those lists. The sync runs when a release is promoted or tagged, or by hand in the website repository, not when a pull request merges here. Use this route only for a package that is on the lists; for any other the prose reaches no reader, so put it on the website instead. |
| `cozystack/website`, under `content/en/docs/` | Operator- and tenant-facing content that spans packages or is task-shaped: guides, how-tos, troubleshooting runbooks, reference tables. Which version directory a page goes into is set by the website's own [`CONTRIBUTING.md`](https://github.com/cozystack/website/blob/main/CONTRIBUTING.md); follow that file rather than a copy of its rules here. |
-| `docs/` | Documentation that a chart template, migration, script or workflow references by repo-relative path, plus this repository's own process docs (`docs/agents/`, `docs/changelogs/`, `docs/release.md`). |
+| `docs/` | Documentation that a chart template, migration, script or workflow references by repo-relative path, plus this repository's own process docs (`docs/agents/`, `docs/changelogs/`, `docs/release.md` and the pages under `docs/release/`). |
| `api///DESIGN.md`, `pkg/apis//DESIGN.md` | Design of an API group, beside the types it describes. |
| [`cozystack/community`](https://github.com/cozystack/community) | Design proposals and RFCs. Not this repository. |
diff --git a/docs/release.md b/docs/release.md
index c8620dbe9..175bd37fc 100644
--- a/docs/release.md
+++ b/docs/release.md
@@ -1,6 +1,6 @@
# Release Workflow
-This document is both the process design (what the release model looks like) and the operational playbook (what to actually do, what breaks, what to verify). Companion to [`agents/changelog.md`](./agents/changelog.md), which is the canonical changelog process.
+The release documentation is both the process design (what the release model looks like) and the operational playbook (what to actually do, what breaks, what to verify), split into one page per topic under [`release/`](./release/). Companion to [`agents/changelog.md`](./agents/changelog.md), which is the canonical changelog process.
## When to use this guide
@@ -15,692 +15,31 @@ You are about to:
If you are about to generate a changelog only, jump straight to [`agents/changelog.md`](./agents/changelog.md).
-## Introduction
-
-Cozystack uses a staged release process to ensure stability and flexibility during development.
-
-There are three types of releases:
-
-- **Release Candidates (RC)** – Preview versions (e.g., `v1.2.0-rc.1`) used for final testing and validation.
-- **Regular Releases** – Final versions (e.g., `v1.2.0`) that are feature-complete and thoroughly tested.
-- **Patch Releases** – Bugfix-only updates (e.g., `v1.2.1`) made after a stable release, based on a dedicated release branch.
-
-All three are matched by `tags.yaml`'s regex `^v\d+\.\d+\.\d+(-(alpha|beta|rc)\.\d+)?$`. The tag push is the trigger for the whole pipeline. **Pre-release tags (`-alpha` / `-beta` / `-rc`) are cut with the [`Cut Pre-release Tag`](../.github/workflows/cut-prerelease.yaml) workflow** (manual dispatch from `main` or `release-X.Y`) — it pushes the tag as the CI app at the dispatch branch's tip, so GitHub fills `base_ref` (`tags.yaml`'s `Get base branch` step refuses tags pushed without a base) and the push triggers `tags.yaml` (a tag pushed with the default `GITHUB_TOKEN` would not). Stable `vX.Y.Z` tags are never pushed by hand at all — they are cut write-once by the promote flow at a PR merge commit. Only `-rc.N` pre-releases feed the promotion flow ([`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) accepts `vX.Y.Z-rc.N` alone); `-alpha.N` / `-beta.N` tags are built and published for download and validated on demand with the [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) button, but are never promoted to a stable release.
-
-## Release Candidates
-
-Release candidates are Cozystack versions that introduce new features and are published before a stable release. Their purpose is to help validate stability before finalizing a new feature release.
-
-Release candidates are given numbers `vX.Y.0-rc.N`, for example, `v1.2.0-rc.1`. The first one is cut from `main`, once all major features for the upcoming release have merged and the release enters its testing phase.
-
-**Cutting the first rc freezes the line.** [`cut-prerelease.yaml`](../.github/workflows/cut-prerelease.yaml) creates `release-X.Y` at the tagged commit, and from that point the release's content is closed:
-
-- Every later cut for that line — `rc.2`, `rc.3`, an `alpha`/`beta`, or a patch-line rc — must be dispatched from `release-X.Y`. A dispatch from `main` is refused.
-- Fixes reach the release only by cherry-pick or backport onto `release-X.Y` (see [Backports](#backports)).
-- `main` reopens immediately for the next minor, so feature work never has to wait for the release to ship.
-
-This is what makes an rc mean something. Previously `release-X.Y` was created only when the stable release merged, so `rc.2` was cut from `main`'s tip and silently absorbed everything that had landed since `rc.1` — the shipped release could contain code that no rc had ever validated.
-
-**Un-freezing is manual and deliberate.** If an `rc.1` turns out to be unusable and the line has to be re-cut from `main`, delete `release-X.Y` by hand first; the refuse step blocks every other route back to a `main` dispatch, which is the point of it. Nothing automates the deletion, and nothing should — throwing away a frozen line is a decision worth making explicitly.
-
-Each RC contributes to a cumulative set of release notes that will be finalized when `vX.Y.0` is released. After testing, if no critical issues remain, the last good rc is **promoted** to the regular release `vX.Y.0` — the exact rc images retagged by digest, never rebuilt and never tagged from a later commit (see [Regular Releases](#regular-releases)). Promotion cuts the write-once `vX.Y.0` tag at the promote PR's merge commit, fast-forwards the `release-X.Y` maintenance branch the freeze already created, and opens the way for patch releases.
-
-## Regular Releases
-
-A regular release `vX.Y.0` is **promoted from a release-candidate that already passed e2e** — never tagged by hand and never rebuilt. The bytes shipped as `vX.Y.0` are the exact `vX.Y.0-rc.N` images, retagged by digest. We'll use `v1.2.0` as an example.
-
-```mermaid
-gitGraph
- commit id: "feature"
- commit id: "feature 2"
- commit id: "feature 3" tag: "v1.2.0-rc.1"
-```
-
-A regular release sequence runs as follows:
-
-1. Cut the first release candidate (`v1.2.0-rc.1`) from the last good commit on `main`: run the [`Cut Pre-release Tag`](../.github/workflows/cut-prerelease.yaml) workflow (manual dispatch **from `main`**) with the tag. It tags `main`'s HEAD as the CI app, then **creates `release-1.2` at that commit, freezing the line**; `tags.yaml` then builds the rc images, publishes the rc prerelease, pushes the digest-vendored `release-1.2.0-rc.N` staging branch, and runs its mandatory `rc-e2e` job by calling [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) against that published tag.
-2. Watch the rc's mandatory full E2E and wait for it to pass, then complete the other checks in [Pre-release verification checklist](#pre-release-verification-checklist). If the latest `tags.yaml` run cannot provide a green `rc-e2e`, a successful manual dispatch of [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) for the same tag is equivalent gate evidence. `main` is already open for the next minor, so anything landing there is **not** in this release: fixes for `1.2` are cherry-picked onto `release-1.2`, and a later rc is cut by dispatching the workflow **from `release-1.2`**. **Optional:** once the notes are worth reviewing, press the `Generate RC Changelog` button ([`changelog-rc.yaml`](../.github/workflows/changelog-rc.yaml)) to generate `docs/changelogs/v1.2.0.md` now and bank it on the `release-1.2.0-rc.N` staging branch — so there is time to review and hand-edit it before promotion. Skipping the button changes nothing; promotion generates the changelog itself.
-
- ```mermaid
- gitGraph
- commit id: "feature"
- commit id: "feature 2"
- commit id: "feature 3" tag: "v1.2.0-rc.1"
- branch release-1.2
- checkout main
- commit id: "feature 4 (next minor)"
- checkout release-1.2
- commit id: "cherry-picked fix" tag: "v1.2.0-rc.2"
- ```
-
-3. Once the checks are green, run the [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) workflow (manual dispatch **from `main`**, the only branch it is dispatched from, whichever line the rc belongs to) with that rc tag. Its `parse` job verifies E2E before any staging: it accepts a successful `rc-e2e` job in the latest `tags.yaml` push run for the tag, or any retained successful manual `e2e-tag.yaml` run whose expanded E2E job name targets the tag. The emergency `skip_e2e_gate` input bypasses this check, logs a loud warning, and marks the promote PR with ⚠️. Before its first registry write, promotion also proves that the selected `main` or `release-X.Y` target already contains the candidate-aware PR gate, finalizer and trusted verifier; backport this pipeline to an existing maintenance branch before promoting its next patch. Promotion **creates no stable name before merge**: dispatch does write to the registry — it publishes one `promotion-*` packages candidate, and that tag is real and publicly listable — but it creates no `vX.Y.Z` tag, moves no `latest`, and publishes no stable installer, so nothing a consumer resolves changes until the PR merges. It:
- 1. Rewrites the rc version substring to `1.2.0` and commits that exact tree, serializes its `packages/` directory under a run-specific temporary OCI tag, then adds a second commit pinning the candidate's immutable digest in the installer before pushing `release-1.2.0`. Container digests remain exactly the rc's; the two commits give the OCI revision an honest source SHA without pretending the artifact can contain its own digest.
- 2. Drafts the `v1.2.0` release and uploads the (restamped) assets.
- 3. Generates `docs/changelogs/v1.2.0.md` (in a parallel job, detached at the rc tag) — or, if you pressed the rc-time `Generate RC Changelog` button, copies the file already banked on the `release-1.2.0-rc.N` staging branch instead of regenerating — and commits it onto the `release-1.2.0` branch, so the changelog is reviewed as part of the promotion and finalize can publish the release with it already in place. Non-blocking: if generation fails the PR still opens, carrying a ⚠️ line that says the changelog is missing.
- 4. Opens the `cozystack/website` "update managed apps reference" PR (`update-docs-v1.2.0` branch), generating the docs from the `release-1.2.0` staging branch via `FETCH_REF`. Non-blocking, and **do not merge that PR until `v1.2.0` is published** — merging early breaks the live site's version routing (see [Phase 3](#phase-3--update-website-docs-now-the-backstop)).
- 5. Opens the `chore(release): promote v1.2.0-rc.N -> v1.2.0` PR into `release-1.2`, labelled `release` only. The PR body records whether E2E was verified or bypassed, plus ✅/⚠️ changelog and website-docs lines. E2E does not run on this promote PR by default because the immutable rc already passed the mandatory full suite; a maintainer may add `full-e2e` to run it again. The stable retag itself is deferred to the merge (step 5), so an abandoned promotion leaves no stable-named images and cannot wedge a re-promotion; retention protects the temporary candidate for as long as this release PR remains open, then may remove it after 30 days.
-
- The PR targets the maintenance branch, not `main`: `promote-rc.yaml` prefers `release-X.Y` whenever it exists, and since the rc freeze it always does. The digest-vendored `Prepare release` commit therefore stays on the release line instead of landing on `main`.
-
- ```mermaid
- gitGraph
- commit id: "feature 3" tag: "v1.2.0-rc.1"
- branch release-1.2
- checkout release-1.2
- commit id: "cherry-picked fix" tag: "v1.2.0-rc.2"
- branch release-1.2.0
- checkout release-1.2.0
- commit id: "Prepare release v1.2.0"
- commit id: "Pin promoted packages artifact"
- checkout release-1.2
- merge release-1.2.0 id: "Pull Request"
- ```
-
-4. Maintainer reviews the PR and the draft release against the [Phase 4 checklist](#phase-4--review-and-merge-the-promote-pr): confirm the E2E-verification line is ✅ (or explicitly accept its ⚠️ bypass), the changelog line is ✅, and note the website-docs line — the `cozystack/website` docs PR that promotion parked, which stays unmerged for now. E2E does not run on this promote PR by default. Then merge — **do not squash-merge**, the stable tag must attach to a real merge commit. GitHub removes the merged branch `release-1.2.0` (the repo has auto-delete-on-merge enabled).
-5. CI workflow triggers on merge (this is where every consumer-visible or write-once side effect happens, after the rc E2E gate and PR review):
- 1. Pulls the pinned packages candidate by digest and verifies that it matches the merged `packages/` tree apart from the artifact's impossible self-reference, carries no rc image tags, and preserves the original rc artifact's container `repo@digest` set.
- 2. Creates the tag `v1.2.0` at the newly created merge commit — write-once. The tag is published here for the first time, never moved.
- 3. Cuts the write-once `api/apps/v1alpha1/v1.2.0` Go-module tag at the same commit.
- 4. Ensures the `release-1.2` maintenance branch exists at the tag commit. Since the rc freeze created it, this is normally a no-op fast-forward: the merge commit is already its tip.
- 5. Publishes the release page (`draft` → `latest`), using `docs/changelogs/v1.2.0.md` from the merge commit as the release body.
- 6. Retags the verified packages candidate and rc container images by digest to `v1.2.0` (and `:latest` only when `v1.2.0` is the newest published stable), and publishes the stable `cozy-installer` chart — no container rebuild.
-6. Once the release is published, merge the parked `cozystack/website` docs PR (opened back at promote time, step 3.iv) so the live site's version routing flips to `v1.2.0` — its versioned docs and `openapi.json` are on GitHub now. The maintainer can then announce the release to the community.
-
-```mermaid
-gitGraph
- commit id: "feature 2"
- commit id: "feature 3" tag: "v1.2.0-rc.1"
- branch release-1.2
- checkout main
- commit id: "feature 4 (next minor)"
- checkout release-1.2
- commit id: "cherry-picked fix" tag: "v1.2.0-rc.2"
- branch release-1.2.0
- checkout release-1.2.0
- commit id: "Prepare release v1.2.0"
- checkout release-1.2
- merge release-1.2.0 id: "Release v1.2.0" tag: "v1.2.0"
-```
-
-Note that `main` never receives the release commit. It forked away at the freeze and carries the next minor's work.
-
-## Patch Releases
-
-A patch release now follows exactly the same shape as the regular release above — by the time `vX.Y.0` ships, everything is already happening on `release-X.Y`. The only difference is where the line starts:
-
-* `release-X.Y` already exists, created by the freeze at `vX.Y.0-rc.1`, so there is no branch to create.
-* Patch commits are cherry-picked onto it, and the rc is cut by dispatching **from `release-X.Y`** — the same dispatch used for `rc.2` of the original release.
-
-
-Let's assume that we've released `v1.2.0` and that development is ongoing.
-We have introduced a couple of new features and some fixes to features that we have released
-in `v1.2.0`.
-
-Once problems were found and fixed, a patch release is due.
-
-```mermaid
-gitGraph
- commit id: "feature 3" tag: "v1.2.0-rc.1"
- branch release-1.2
- checkout release-1.2
- commit id: "Release v1.2.0" tag: "v1.2.0"
- checkout main
- commit id: "feature 4"
- commit id: "patch 1"
- commit id: "feature 5"
- commit id: "patch 2"
-```
-
-
-1. The maintainer cherry-picks patch commits from `main` onto the existing `release-1.2` branch. These must be only patches to features that were present in version `v1.2.0`.
-
- Cherry-picking can be done as soon as each patch is merged into `main`, or directly before the release.
-
- The `backport` label automates this: [`backport.yaml`](../.github/workflows/backport.yaml) targets the newest existing `release-X.Y` branch, which during a freeze window is the line being stabilised rather than the last published one.
-
- ```mermaid
- gitGraph
- commit id: "feature 3" tag: "v1.2.0-rc.1"
- branch release-1.2
- checkout release-1.2
- commit id: "Release v1.2.0" tag: "v1.2.0"
- checkout main
- commit id: "feature 4"
- commit id: "patch 1"
- commit id: "feature 5"
- commit id: "patch 2"
- checkout release-1.2
- cherry-pick id: "patch 1"
- cherry-pick id: "patch 2"
- ```
-
- When all relevant patch commits are cherry-picked, the branch is ready for release. Build with `go build ./cmd/backport-audit`, then confirm with `./backport-audit release-1.2` — it exits non-zero while anything labeled for the line is still missing from it, and prints the URLs. See [Cherry-pick triage before a patch](#cherry-pick-triage-before-a-patch).
-
-2. The maintainer cuts a release candidate (`v1.2.1-rc.N`) via the [`Cut Pre-release Tag`](../.github/workflows/cut-prerelease.yaml) workflow (manual dispatch **from `release-1.2`**), which tags that branch's `HEAD`. CI builds and publishes the rc, pushes the `release-1.2.1-rc.N` staging branch, and runs the mandatory full `rc-e2e` job against the published tag.
-3. Watch `rc-e2e` and wait for it to pass, then complete the other release checks. A successful manual `e2e-tag.yaml` run for `v1.2.1-rc.N` is also valid evidence if the latest `tags.yaml` run does not carry a green `rc-e2e`. **Optional:** press the `Generate RC Changelog` button ([`changelog-rc.yaml`](../.github/workflows/changelog-rc.yaml)) to generate `docs/changelogs/v1.2.1.md` at rc time and bank it on the rc staging branch; promotion then reuses it instead of regenerating.
-4. Once the checks are green, run [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) (manual dispatch **from `main`**, as for every promotion, not from `release-1.2`) with that rc tag. It runs the same five jobs as a regular promotion: the dispatch first gates on the latest tag-push `rc-e2e` or a successful tag-correlated manual E2E run (`skip_e2e_gate` overrides it), verifies that `release-1.2` already carries the candidate-aware pipeline, then commits and publishes the rewritten packages tree under a run-specific temporary `promotion-*` tag, adds a second commit pinning the candidate on the `release-1.2.1` branch, drafts the `v1.2.1` release with the restamped assets, generates or picks up `docs/changelogs/v1.2.1.md` and commits it onto that branch, opens the parked `cozystack/website` docs PR (updating the existing `v1.2/` directory in place, since a patch does not promote `next/`), and opens the promote PR into `release-1.2` with only the `release` label. The promote PR body carries the same ✅/⚠️ E2E, changelog, and website-docs lines and runs no E2E by default; add `full-e2e` manually to force a repeat. As with a regular release every stable name is deferred to merge; retention protects the temporary candidate while this PR is open and, after it merges, while `release-1.2` still pins it — until finalize retags it stable.
-
- ```mermaid
- gitGraph
- commit id: "feature 3" tag: "v1.2.0-rc.1"
- branch release-1.2
- checkout release-1.2
- commit id: "Release v1.2.0" tag: "v1.2.0"
- checkout main
- commit id: "feature 4"
- commit id: "patch 1"
- commit id: "feature 5"
- commit id: "patch 2"
- checkout release-1.2
- cherry-pick id: "patch 1"
- cherry-pick id: "patch 2" tag: "v1.2.1-rc.1"
- branch release-1.2.1
- commit id: "Prepare release v1.2.1"
- checkout release-1.2
- merge release-1.2.1 id: "Pull request"
- ```
-
- Finally, when release is confirmed, the release sequence goes on.
-
-5. Maintainer reviews the PR against the [Phase 4 checklist](#phase-4--review-and-merge-the-promote-pr), confirms the E2E-verification and changelog lines and notes the parked website-docs line, and merges it — **do not squash-merge**. GitHub removes the merged branch `release-1.2.1` (auto-delete-on-merge is enabled).
-6. CI workflow triggers on merge (all consumer-visible or write-once side effects, after the rc E2E gate and PR review):
- 1. Verifies the pinned packages candidate against the merged tree and the original rc artifact's container digests.
- 2. Creates the tag `v1.2.1` at the newly created merge commit — write-once. The tag is published here for the first time, never moved.
- 3. Cuts the write-once `api/apps/v1alpha1/v1.2.1` Go-module tag at the same commit.
- 4. Publishes the release page (`draft` → `latest`; `latest` moves only if `v1.2.1` is the newest published stable).
- 5. Retags the verified packages candidate and rc container images by digest to `v1.2.1` and publishes the stable `cozy-installer` chart — no container rebuild.
-7. Once the release is published, merge the parked `cozystack/website` docs PR, then announce the release to the community.
-
-## What CI does during the release process
-
-The numbered process above is implemented by seven workflows, including the reusable/manual [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml), plus the reusable [`changelog-generate.yaml`](../.github/workflows/changelog-generate.yaml) that two of them share to produce the changelog. Knowing which job does what makes the failure modes much easier to diagnose.
-
-1. [`cut-prerelease.yaml`](../.github/workflows/cut-prerelease.yaml) — manual dispatch (from `main` or `release-X.Y`): the sole entry point for creating a pre-release tag. Validates the `-alpha`/`-beta`/`-rc` tag, then pushes it (write-once, at the branch tip) as the CI app so `tags.yaml` fires. Cutting the first `vX.Y.0-rc.N` also creates `release-X.Y`, freezing the line; afterwards a `vX.Y.*` dispatch from `main` is refused. Stable tags are never created here.
-2. [`tags.yaml`](../.github/workflows/tags.yaml) — fires on any pre-release tag push: `build-amd64` and `build-arm64` build the pre-release side by side, then `prepare-release` publishes it and pushes its digest-pinned staging branch, then — **only for `-rc.N` tags** — the mandatory `rc-e2e` job calls `e2e-tag.yaml` in the same run (`-alpha.N` / `-beta.N` skip it and are validated on demand with `e2e-tag.yaml`'s manual button). `generate-changelog` and `update-website-docs` also live here but are **backstops only** — both are normally produced at promotion time (see 4); on a stable tag push each self-skips or no-ops when the promote flow already did the work. These stable-tag backstops only fire because finalize pushes the stable tag with the CI app token (a `GITHUB_TOKEN` push triggers no run — see 5).
-3. [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) — reusable from `tags.yaml::rc-e2e` and manually dispatchable for a published tag. It checks that the tag's release carries its nocloud disk, resolves the digest-pinned tree and runs the full install, OpenAPI, and Chainsaw suite on Talos containers. A tag cut before the container lane existed has to be dispatched from its own ref. Its dispatch run title is the constant workflow name, so the promote gate correlates a manual run through the successful tag-bearing E2E job name rather than the title.
-4. [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) — manual dispatch, from `main` only (the signature over a stable release's assets names that ref, see [Release signatures and SBOM](#release-signatures-and-sbom)). Five jobs: `parse` validates the rc tag and requires green full-E2E evidence before any staging; `promote` rewrites and commits the rc tags, publishes that packages tree as a run-specific temporary reproducible candidate, commits its digest pin in the `release-X.Y.Z` tree, and drafts the stable release; `changelog` calls the reusable `changelog-generate.yaml` **in parallel** to produce `docs/changelogs/vX.Y.Z.md` (regenerating it, or copying an rc-time changelog already committed to the `release-X.Y.Z-rc.N` staging branch by `changelog-rc.yaml` — see 7); `website-docs` (needs `parse`, `promote`) opens the `cozystack/website` docs PR from the staging branch via `FETCH_REF` (see [Phase 3](#phase-3--update-website-docs-now-the-backstop)); `open-pr` joins them, commits the changelog onto the staging branch, and opens the `release-X.Y.Z` promote PR with the `release` label but no automatic `full-e2e`. So the changelog and website docs are reviewed as part of the promotion, the already-checked rc suite is not repeated by default, and the PR body carries ✅ verified or ⚠️ bypassed plus ✅/⚠️ changelog and website-docs lines. Both `changelog` and `website-docs` are non-blocking. No stable registry name is created at dispatch.
-5. [`.github/workflows/pull-requests.yaml`](../.github/workflows/pull-requests.yaml) verifies a promotion candidate against the prospective merge tree and original rc artifact before the required `E2E Tests` status can turn green. [`pull-requests-release.yaml`](../.github/workflows/pull-requests-release.yaml) fires after merge and repeats that verification before any stable tag, then cuts the write-once stable + Go-module tags, publishes the release **with the merged changelog as its body**, retags the candidate and rc images to stable by digest (`:latest` gated on newest-stable), and publishes the stable chart.
-6. [`update-releasenotes.yaml`](../.github/workflows/update-releasenotes.yaml) — fires on pushes to `main` that touch `docs/changelogs/v*.md`; syncs that content into the corresponding GitHub Release body. No longer the primary path (finalize sets the body directly, which is race-free and works for maintenance lines too) — this now covers later edits to a published changelog and manual re-syncs via `workflow_dispatch`.
-7. [`changelog-rc.yaml`](../.github/workflows/changelog-rc.yaml) — manual dispatch, **optional**: generate the changelog at rc time instead of waiting for promotion, while there is still time to review it. It calls the same reusable [`changelog-generate.yaml`](../.github/workflows/changelog-generate.yaml) and commits the result to the `release-X.Y.Z-rc.N` staging branch as `docs/changelogs/vX.Y.Z.md`; the promote-time `changelog` job then copies that file instead of running the AI again. Skipping this workflow changes nothing — promote generates the changelog itself, exactly as before.
-
-### Phase 1 — `build-amd64`, `build-arm64` and `prepare-release` (hard gate)
-
-On an rc tag push: builds images for both architectures, commits digest pins, creates the draft release, and pushes the `release-X.Y.Z-rc.N` staging branch. `build-amd64` holds the release-exists no-op and the rejection of a hand-pushed stable tag, and builds the amd64 images and the release assets (its `make build` also pushes the packages artifact and the installer chart, pinned on amd64 digests); `prepare-release` stitches, gates, republishes those two from the repinned tree, and publishes the release. (The stable promote PR is opened later by `promote-rc.yaml`, not here. When the stable tag itself is pushed by finalize, a draft already exists, so this job is a no-op — see [Phase 5](#phase-5--pull-requests-releaseyaml-finalize).)
-
-The commit (`Prepare release vX.Y.Z`, authored by `cozystack-ci[bot]`) is **digest pins and image tags only**:
-
-- `packages/core/{installer,platform,testing}/values.yaml`
-- `packages/system/*/values.yaml` (cozystack-api, dashboard, kamaji, linstor, kubevirt-csi-node, etc.)
-- `packages/apps/kubernetes/images/{kubevirt-csi-driver,ubuntu-container-disk-*}.tag`
-- `packages/system/dashboard/templates/configmap.yaml` (the `$tenantText` value)
-
-Things that look surprising in the diff but are normal:
-
-- **Zero-diff packages**: when buildx fully caches and upstream base images haven't moved, the digest is byte-identical to the previous release and won't appear. Verify the build step actually ran — don't assume "no diff = nothing built."
-- **Sudden churn on `ubuntu-container-disk-*` tags**: `cloud-images.ubuntu.com/noble/current/` is a moving target; these often rebuild even without code changes.
-- **Switched registry**: if a self-built image moved from `docker.io` to `ghcr.io`, the registry portion of the digest changes — that's a deliberate move, not a regression.
-- **`targetVersion` is NOT touched by the release PR.** Platform migration `targetVersion` is bumped earlier in a feature commit (e.g. `[platform] Bump migration targetVersion to 39 for migration 38`). The release PR only re-pins the `platform-migrations` image digest.
-
-#### Multi-arch images
-
-Every pre-release is published for amd64 and arm64. `tags.yaml::build-arm64` builds the images on a native arm64 runner and pushes each as `-arm64`, while `build-amd64` builds amd64, the Talos installer and matchbox for both arches, and the assets on x86 at the same time. `build-amd64` hands its stamped tree, the log of the tags it pushed and the assets to `prepare-release` as workflow artifacts; a missing one fails the upload or the download rather than publishing a partial release. `prepare-release` waits for both builds and runs only if both succeeded: when either fails, the rc is not prepared at all, so a release is never published amd64 only. It applies the stamped tree and runs [`hack/stitch-multiarch.sh`](../hack/stitch-multiarch.sh), which joins each image with its arm64 twin into one index, moves onto it every tag the build pushed for that image (the rc tag and, under `PUBLISH_VERSIONED=1`, its component version, which the build records in `PUSHED_TAGS_LOG`), and repins the tree on the index digests, then [`hack/verify-multiarch.sh`](../hack/verify-multiarch.sh), then republishes the packages artifact and the installer chart from the repinned tree. A stable tag skips `build-arm64` and builds nothing: promotion copies the rc's indexes with every platform, so the stable release is exactly as multi-arch as its rc. Because the two builds run in parallel, an rc that is not prepared (its arm64 build failed, or `prepare-release` failed before the stitch) still leaves behind what the amd64 build pushed: amd64-only images under `:` and under the component-version tags, and the `cozystack-packages:` artifact and the installer chart, both pinned on those amd64 digests. A component tag an earlier rc had moved to an index then points at an amd64-only image again, until a run that succeeds stitches it, a re-run of the same run included.
-
-`verify-multiarch.sh` is a hard gate. It resolves every ref the tree pins, third-party images included, and fails naming each one that is not an index carrying both `linux/amd64` and `linux/arm64`, or that it cannot resolve at all. The fix is to make the image multi-arch (build it for both, or pin a multi-arch digest of the same version); an image that cannot exist for arm64 goes in [`hack/multiarch-allowlist`](../hack/multiarch-allowlist) by repository, with the reason on the same line.
-
-That gate sees only what the tree pins by digest. An image a vendored chart renders from its own defaults, or one an operator launches at runtime, is invisible to it. `rc-e2e` closes most of that gap: before tearing the sandbox down, [`e2e-tag.yaml`](../.github/workflows/e2e-tag.yaml) reads every image the three Talos nodes pulled from their image stores (`hack/collect-images.sh`), each by the digest the pull resolved to, and runs `verify-multiarch.sh --refs-file` on that list with the same allowlist. A pre-release fails on any single-arch image, on any ref it cannot read or that is still rate-limited after the retries, and on a list the collection could not produce; a stable tag's run only warns, since it ships its rc's images. A tag whose tree predates the audit (it has no `hack/verify-multiarch.sh`) only warns and passes, so an rc cut before the audit existed can still be re-validated. The list and the audit output are in the run's `image-list-` artifact. Neither check covers a package that e2e never installs and whose images the tree does not pin by digest: an arm64 break there shows up only on an arm64 cluster.
-
-[`build-arm64-nightly.yaml`](../.github/workflows/build-arm64-nightly.yaml) builds `main` for arm64 every day, so an arm64 build break shows up there before the next rc. Its first step prints `verify-multiarch.sh --report` for `main`'s committed tree: `main` pins amd64 images, so its first-party refs are all listed and are expected to be, since the rc's stitch turns them into indexes. A third-party ref that an in-tree rebuild stamps over at build time, as the Harbor components and shadowbox are, is listed too and goes away on the rc. The lines that matter are the third-party refs no build replaces and refs that cannot be resolved; each of those fails the next rc until it is fixed or allowlisted. matchbox is listed too and does not matter: `build-amd64` rebuilds it as a two-arch index. The stitch names the images it skips in its own summary: an image with no `-arm64` twin, one whose digest is also pinned in a file the rewrite does not reach, and a pair that is not one amd64 and one arm64 image. matchbox (already an index) and the e2e sandbox (never built for arm64, and allowlisted) are skipped on every rc and pass the gate; any other skipped image fails it.
-
-### RC E2E — mandatory check before promotion
-
-For an rc, `prepare-release` must finish before `tags.yaml::rc-e2e` calls the published-tag lane, so the full suite tests the digest-pinned staging tree that promotion will reuse, and the release it runs against already carries its disk. `promote-rc.yaml::parse` then checks the latest `tags.yaml` push run whose `head_branch` is the rc tag and accepts it only when the reusable E2E job succeeded. GitHub exposes called-workflow job names as `E2E Release Candidate / E2E vX.Y.Z-rc.N (full suite)`, which the gate matches by the exact tag-bearing suffix.
-
-The alternate evidence path scans retained successful `workflow_dispatch` runs of `e2e-tag.yaml` and accepts one whose direct E2E job is named `E2E vX.Y.Z-rc.N (full suite)`. The workflow's run title is always `E2E Release Tag` and therefore cannot correlate the input tag; artifact names also carry the tag but expire independently, so the job name is the strongest available correlator. If Actions retention or manual deletion removes the run/job metadata, or that job is renamed without updating the matcher, the gate fails closed and tells the maintainer to run the E2E Release Tag button again. `skip_e2e_gate=true` is an emergency override, not alternate evidence: it writes a loud warning and a ⚠️ PR-body line.
-
-### Phase 2 — the changelog (generated at rc time or promotion, not here)
-
-The changelog for `vX.Y.Z` is produced by [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml)'s `changelog` job (a thin call to the reusable [`changelog-generate.yaml`](../.github/workflows/changelog-generate.yaml)) and committed onto the `release-X.Y.Z` staging branch by `open-pr`, so it rides the promote PR and is on the base branch the moment the release publishes. Finalize then uses that file verbatim as the GitHub Release body. Generating it here — on the stable tag push — would be too late by construction: that tag is created by finalize, which has already published the release. It can also be produced earlier, at rc time, via the optional [`changelog-rc.yaml`](../.github/workflows/changelog-rc.yaml) button — which runs the same reusable core and commits the file to the `release-X.Y.Z-rc.N` staging branch, from which the promote-time `changelog` job copies it (validated) instead of regenerating.
-
-The `changelog` job runs **in parallel** with `promote`, detached at the rc tag (the "promotion" configuration in §2 of [`agents/changelog.md`](./agents/changelog.md)), using a separate **read-only** GitHub App token for the AI step so the model cannot mutate the repo even with `--allow-all-tools`. Every write — the commit onto the staging branch and the PR — happens in `open-pr` under a write-scoped token.
-
-Generation is deliberately **non-blocking**: a Copilot outage or exhausted token quota must never block a release. Only the AI step carries `continue-on-error`, and `open-pr` is gated on `promote` alone (`if: !cancelled() && needs.promote.result == 'success'`) so the promote PR opens regardless. When the changelog is missing the PR body says so explicitly — that ⚠️ line is the signal to act on.
-
-Recovery, in order of preference:
-
-1. Commit `docs/changelogs/vX.Y.Z.md` to the `release-X.Y.Z` branch by hand (follow [`agents/changelog.md`](./agents/changelog.md)). `promote` preserves it across a re-dispatch, and `open-pr` will not overwrite it with generated output.
-2. Re-dispatch `promote-rc.yaml` — generation runs again; anything already on the branch wins.
-3. Merge without one and let `tags.yaml::generate-changelog` (the backstop) open a changelog PR against `main` after the fact. The release is published with the draft's body until that merges.
-
-Known failure modes:
-
-- Copilot quota exhausted (`COPILOT_GITHUB_TOKEN` 402). Refill or rotate, then re-dispatch.
-- AI step timeout (hard 30-min cap).
-- Output truncated or whitespace-only — caught by the `Verify changelog` step (`hack/validate-changelog.sh`), which accepts either header convention (`# Cozystack vX.Y.Z` for a minor or `# vX.Y.Z ()` for a patch), and requires the leading release-link comment pointing at `releases/tag/vX.Y.Z`, a `compare/...vX.Y.Z` link ending in this version, and at least one `## ` section — deliberately with no line-count floor, since a complete short patch changelog (v1.5.1 ships in 19 lines) must not be rejected as a fragment. It downgrades to the missing-changelog path rather than shipping a fragment as release notes.
-
-**`tags.yaml::generate-changelog` is the backstop.** It self-skips when the changelog is already on `main`. It earns its keep in two cases: generation failed during promotion, or the promote PR targeted `release-X.Y` so the file never reached `main`. Since the rc freeze the second case is **every** release, minor and patch alike — the promote PR always targets the release line (see [Regular Releases](#regular-releases)), which makes this backstop the only route by which a published changelog reaches `main`. In that case it **ports the reviewed file from the tag commit verbatim** rather than regenerating — a second AI pass there would spend quota and then overwrite already-published, already-reviewed release notes.
-
-### Phase 3 — `update-website-docs` (now the backstop)
-
-The `cozystack/website` "update managed apps reference" PR is normally opened **earlier, at promote time**, by [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml)'s `website-docs` job — not here. That job fetches the README content from the `release-X.Y.Z` staging branch `promote` just pushed (via the website Makefile's `FETCH_REF` variable), so the docs are ready to review alongside the promote PR while the stable tag is still unborn. It refreshes the trunk version pins from the staging ref before promoting `next/` → `vX.Y/`, so a new minor's docs inherit fresh Talos/cozystack pins instead of a stale `next.yaml`. It is **non-blocking**: if it fails (or trips its `FETCH_REF`-support guard against a website checkout that predates that Makefile variable), the promote PR opens with a ⚠️ website-docs line and this tag-time job backfills the same branch.
-
-**Merge-timing contract:** the website PR must NOT be merged before the release is published (finalize). Merging it early flips the site's latest-version pointer to a version whose versioned docs and `openapi.json` are not on GitHub yet, so every version-pin and API link for the new version 404s. Both the website PR body and the promote PR body say so; park the website PR until the GitHub release for `vX.Y.Z` is live, then merge.
-
-This tag-time `update-website-docs` job stays as the **backstop**. It runs with `if: !cancelled() && needs.prepare-release.result == 'success'` (surviving a failed/skipped changelog phase, stopping on cancel), decides whether to promote `next/` → `vX.Y/`, runs `make update-all`, stages `content hugo.yaml data/versions`, and force-pushes the same `update-docs-vX.Y.Z` branch. On the normal path — where `promote-rc.yaml::website-docs` already did the work — it no-ops into "No changes to commit". At tag time the stable tag exists, so its `BRANCH=RELEASE_TAG` fetch resolves without needing `FETCH_REF`. For it to fire at all, finalize must push the stable tag with the **CI app token** (see [Phase 5](#phase-5--pull-requests-releaseyaml-finalize)) — a tag pushed with the default `GITHUB_TOKEN` triggers no workflow run.
-
-If anyone changes the website Makefile to write somewhere else (e.g. `static/`, `i18n/`), the `git add` list must grow in **both** jobs or the PR silently drops files.
-
-### Phase 4 — review and merge the promote PR
-
-Reviewer checklist for the `chore(release): promote vX.Y.Z-rc.N -> vX.Y.Z` PR:
-
-- [ ] The PR body says `✅ RC full e2e was verified`; if it says the gate was bypassed, obtain explicit maintainer acceptance or add `full-e2e` and wait for that opt-in run.
-- [ ] `Verify release packages candidate` is green. It pulls the candidate and original rc artifact by digest, compares the candidate to GitHub's prospective merge tree, and feeds the required `E2E Tests` status even when full E2E is not repeated. Do not edit `packages/` on this PR after generation; if the guard reports drift, re-dispatch promotion to create and pin a new run-specific candidate. Note what a green here does *not* cover: GitHub fires no `pull_request` event when the base branch advances, so a `packages/` change merged to the base afterwards is never re-verified on the PR. Finalize repeats the check against the real merge commit and catches that case before any stable name exists — but it catches it after the promote PR has already merged, so a `packages/` change landing on the base while a promote PR is open is worth holding back.
-- [ ] The PR body's **website-docs line** is present: if ✅, the `cozystack/website` docs PR is open and **parked** — do NOT merge it until `vX.Y.Z` is published (see [Phase 3](#phase-3--update-website-docs-now-the-backstop)); if ⚠️, the tag-time backstop will backfill it after finalize.
-- [ ] Diff is digest pins + image tags only — nothing else.
-- [ ] No accidental config drift (a value file you don't recognize).
-- [ ] If any chart bumped, check the new digest pulls — `crane manifest ghcr.io/cozystack/cozystack/@` should return.
-- [ ] Build artifacts on the draft release page exist and match the expected count.
-- [ ] The PR includes `docs/changelogs/vX.Y.Z.md`, it has content, and the entries match the commit range (see [Changelog pre-publish verification](#changelog-pre-publish-verification)). If the PR body carries the ⚠️ "No changelog in this PR" line, resolve that before merging — finalize publishes this file as the release body, unless the PR also carries `docs/releasenotes/vX.Y.Z.md` (see [Phase 6](#phase-6--update-releasenotesyaml-sync-github-release-body)).
-
-### Phase 5 — `pull-requests-release.yaml` (finalize)
-
-Fires on merge of a PR that is merged, carries the `release` label, and is authored by `cozystack-ci[bot]` (the last guard closes the "name a branch `release-X.Y.Z`, get it labelled, merge it" hole). Head branch must match `release-X.Y.Z[-suffix]`. This is where **every consumer-visible or write-once side effect of a promotion happens** — after the rc's mandatory E2E was checked at dispatch and the PR merged, making promotion transactional at stable names. Steps:
-
-1. **Verify the packages candidate before creating any stable tag.** [`hack/verify-promoted-packages.sh`](../hack/verify-promoted-packages.sh) pulls the candidate by the digest committed in installer values, proves it matches the merged `packages/` tree after normalizing only the impossible self-reference, rejects surviving rc image tags, then follows the candidate's embedded pre-push `platformSourceRef` back to the original rc artifact and compares normalized container `repo@digest` sets. Any package edit after candidate publication or any changed container digest aborts finalize while all stable names are still absent.
-2. **Create the tag at the merge commit** (write-once). The merge commit of `Prepare release vX.Y.Z` did not exist before the PR opened, so there is nothing to move — the tag is created here for the first time. A pre-existing tag at a different commit fails the step loudly rather than being force-moved (see [Tag immutability](#tag-immutability)). The checkout uses `persist-credentials: false` so this push authenticates as the **CI app** (via `git remote set-url`), not the default `GITHUB_TOKEN` — a `GITHUB_TOKEN` push creates no workflow run, so a persisted-credential push would silently skip `tags.yaml` entirely and its stable-tag backstops (`generate-changelog`, `update-website-docs`) would never fire.
-3. **Cut the `api/apps/v1alpha1/vX.Y.Z` Go-module tag** (write-once, stable only) at the same commit, so Go consumers of `api/apps/v1alpha1` get the release. Moved here from `tags.yaml`, whose `prepare-release` body is skipped for a promoted stable (the draft already exists).
-4. **Ensure the maintenance branch `release-X.Y` exists** at the tag commit. Since the rc freeze creates it, this is normally a no-op fast-forward — the merge commit is already the branch tip. Still created if missing; updated fast-forward-only — a non-fast-forward update warns and is left for a maintainer rather than being force-updated.
-5. **Publish the draft release**, with `docs/releasenotes/vX.Y.Z.md`, or without it `docs/changelogs/vX.Y.Z.md`, from the merge commit as its body (absent or whitespace-only files warn and leave the draft's body in place, rather than publishing blank notes, and a body over 100000 bytes is published as a link, see [Phase 6](#phase-6--update-releasenotesyaml-sync-github-release-body)). `make_latest` is computed against published-non-prerelease tags: prereleases stay `false`; tags older than the current max stay `false` (and the current max is force-restored to `latest` if necessary, so an older patch tag cut after a newer minor won't downgrade `latest`).
-6. **Retag the verified packages candidate and rc container images to stable** by digest (`hack/promote-retag.sh`, no container rebuild) and **publish the stable `cozy-installer` chart**. `:latest` (on both the images and the chart) moves only when this release's `make_latest` was `true` — the same decision as step 5, so the release's `latest` and the images' `:latest` never disagree. The chart is packaged with `platformVersion` stamped into its default values so the documented `helm --install --version X.Y.Z` path reports the stable version.
-
-### Phase 6 — `update-releasenotes.yaml` (sync GitHub Release body)
-
-Fires on pushes to `main` that touch `docs/changelogs/v*.md`. Reads each `docs/changelogs/vX.Y.Z.md` and PATCHes the matching GitHub Release's `body` if it differs. Since Phase 5 now sets the body directly from the merge commit, this is no longer on the critical path — it covers later **edits** to a published changelog, the backstop PR landing a changelog that reached `main` after the fact, and manual re-syncs via `workflow_dispatch`. It only watches `main`, which is why finalize (not this job) is what populates a release promoted onto `release-X.Y` — since the freeze, that is every release rather than only a patch.
-
-The release body is `docs/releasenotes/vX.Y.Z.md` when it exists and is not blank, otherwise the changelog, which stays the full record either way, and a push to `main` touching either file fires this sync. Finalize sets the body after the stable tags exist, where a body GitHub refuses would strand the release, so at publish time a body over 100000 bytes is replaced by a one-line link to `docs/changelogs/vX.Y.Z.md` on the tag, with a warning in the run's log. This sync never writes that link: it warns and leaves such a release's body as it is. The `tags.yaml` backstop ports only the changelog to `main`, so release notes have to be added to `main` as well, otherwise the sync falls back to the changelog and replaces the release notes with it whenever the changelog fits.
-
-## Stable tags come from rc promotion
-
-A stable `vX.Y.Z` is created only by **promoting an existing release-candidate** that already passed e2e — never by rebuilding and never by a cron.
-
-- [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) is triggered manually once an rc has gone green. It rewrites the rc version substring in the vendored image tags (the container digests stay the rc's), commits and publishes the resulting declarative packages tree under a unique run-specific `promotion-*` tag, commits that digest pin, and opens the `release-X.Y.Z` staging PR. The PR gate verifies the candidate against the prospective merge and original rc artifact; finalize repeats that check before creating the write-once stable tag at the merge commit, then retags the verified packages artifact and rc's already-built, e2e-passed container digests to the stable tag **by digest** (no container rebuild — see [`hack/promote-retag.sh`](../hack/promote-retag.sh)) and publishes the release. An abandoned promotion leaves no stable-named bytes; its temporary candidate is retention-managed.
-
-So a commit on a supported `release-X.Y` line ships when a maintainer promotes the next rc for that line — not automatically within 24h. Pushing a stable `vX.Y.Z` tag by hand is **not** a supported path: `tags.yaml` fails fast on a stable tag that has no pre-existing draft ("stable tags come from promote-rc.yaml"), and even so the finalize step would refuse to move a pre-existing tag. Pre-releases (`vX.Y.Z-rc.N`, `-alpha.N`, `-beta.N`) are cut with the [`Cut Pre-release Tag`](../.github/workflows/cut-prerelease.yaml) workflow, which pushes the tag as the CI app; [`tags.yaml`](../.github/workflows/tags.yaml) then fires on that push, builds it, and publishes the pre-release. Cutting via the workflow (rather than a manual `git push`) is what lets repo admins lock `v*` tag creation to the CI app — see the tag-protection note below.
-
-## Release signatures and SBOM
-
-A release cut from a tree that has [`hack/sign-release-assets.sh`](../hack/sign-release-assets.sh) carries three files beside its other assets, which [`hack/upload-assets.sh`](../hack/upload-assets.sh) has that script write before it uploads anything. Releases cut before it have none of them, and neither does a maintenance line until the script is backported to it (see below).
-
-- `cozystack-source-sbom.spdx.json` is an SPDX SBOM, made with syft, of the committed source tree (the Go modules, npm lockfiles, GitHub Actions references and other manifests syft reads in it) and of the `cozypkg` binaries in the `cozypkg-*.tar.gz` assets. It does not describe what runs in the cluster: the container images carry their own SBOMs, see [SBOM of the container images](#sbom-of-the-container-images), and the Talos installer image, ISO, disk, kernel and initramfs images have none. Its name says what it covers, so that `cozystack-sbom` stays free for an SBOM of the release as a whole.
-- `cozystack-checksums.txt` holds the SHA-256 of every other asset and of the SBOM.
-- `cozystack-checksums.txt.sigstore.json` is a Sigstore bundle with a keyless cosign signature over the checksums. Its certificate names the workflow that published the release: `tags.yaml` for a pre-release, `promote-rc.yaml` for a stable one.
-
-One signature covers every asset through the checksums. To check a download, verify the checksums, then the file against them, with cosign v3.1.3 or later (v2.6.5 or later on the v2 line):
-
-```sh
-cosign verify-blob cozystack-checksums.txt \
- --bundle cozystack-checksums.txt.sigstore.json \
- --certificate-identity-regexp '^https://github\.com/cozystack/cozystack/\.github/workflows/(tags\.yaml@refs/tags/v|promote-rc\.yaml@refs/heads/main$)' \
- --certificate-oidc-issuer https://token.actions.githubusercontent.com &&
- sha256sum --check --ignore-missing cozystack-checksums.txt
-```
-
-Older cosign accepts a legacy-format bundle that carries a bare public key in place of a certificate and skips the identity and issuer checks for it ([GHSA-fx35-mq7g-6g98](https://github.com/sigstore/cosign/security/advisories/GHSA-fx35-mq7g-6g98)), so a bundle swapped in next to the download, by a mirror or a proxy, passes there. The bundles this repository writes are in the new format, but cosign picks the format from the file, and `--new-bundle-format` does not stop that on the v3 line (v3.1.2 still accepts such a bundle with it), so the cosign version is what protects the check.
-
-The regexp pins the ref each workflow runs from: `tags.yaml` on a `v*` tag push, and `promote-rc.yaml` dispatched from `main`, the only branch it is dispatched from. A passing check proves that the checksums were signed by one of those runs in `cozystack/cozystack`. A `promote-rc.yaml` dispatched from another branch fails this check in its own upload step, with no asset uploaded, and an edited `promote-rc.yaml` has to be merged into `main` before it can sign. Until the `v*` [tag rulesets](#repo-side-enforcement-tag-protection-rulesets) restrict tag creation to the CI app, though, anyone with write access can push a `v*` tag whose tree carries an edited `tags.yaml`, and its signature passes.
-
-The script runs the same `cosign verify-blob` before it returns, so a signature this check would reject fails the job before the first upload, and the release is not published with assets but without a signature. Both jobs that call `make upload_assets`, `tags.yaml::prepare-release` and `promote-rc.yaml::promote`, hold `id-token: write` for the signing identity and install syft and cosign v3.1.3, the version users are asked for above. They install syft from its release tarball, checked against a SHA-256 committed in the workflow, and not with `anchore/sbom-action/download-syft`, which runs an install script fetched at run time in a job that holds the release credentials; [`hack/ci-tool-installs-contract.bats`](../hack/ci-tool-installs-contract.bats) keeps it that way. A stable release gets its own three files rather than its rc's: promotion restamps several assets, so the checksums are recomputed and signed again.
-
-`promote-rc.yaml` runs the `hack/upload-assets.sh` of the rc tree, not the dispatch ref's. A stable release is therefore signed only when its rc was tagged from a tree that has `hack/sign-release-assets.sh`, and a maintenance line starts carrying signatures once that script is backported to its `release-X.Y` branch.
-
-### SBOM of the container images
-
-The container images a release pins by digest each carry an SPDX SBOM of the packages in them, which BuildKit attaches at build time as an attestation (`SBOM=1` in `tags.yaml`, scanner and flags in [`hack/common-envs.mk`](../hack/common-envs.mk)). The attestation is an entry of the image's index, so the digest the release pins names both, and promotion copies it with the image. Read it with `docker buildx imagetools inspect @ --format '{{ json .SBOM }}'`, which prints one document per platform. The attestations are not signed. A release cut before `SBOM=1` was set has none, and neither has the Talos installer image, which the imager builds and `skopeo` pushes, nor the third-party images `packages/` vendors by digest.
-
-## Nightly builds
-
-A nightly is an **installable copy of `main` on GHCR — not a rebuild and not a release**. [`build-main.yaml`](../.github/workflows/build-main.yaml) already builds every push to `main` into the CI registry (OCIR); the nightly promotes that build to the public release registry (GHCR) and proves it installs.
-
-[`nightly.yaml`](../.github/workflows/nightly.yaml) runs daily (gated by the `NIGHTLY_ENABLED` repo variable) as three jobs; build-disk and e2e both start once mirror has finished:
-
-1. **mirror** — resolve the exact commit `cozystack-packages:main` was built from, then [`hack/nightly-mirror.sh`](../hack/nightly-mirror.sh) copies every cozystack-owned component image OCIR→GHCR **by digest** (bit-for-bit, no rebuild) and re-publishes the rewritten `cozystack-packages` artifact and the `cozy-installer` chart to GHCR, tagged `0.0.0-nightly...` plus a floating `nightly`.
-2. **build-disk** — assemble the Talos `nocloud` disk from the upstream siderolabs imager (the profile references only `ghcr.io/siderolabs/*`, so this needs no cozystack image and no rebuild) and publish it as a GHCR OCI artifact (`cozystack-nocloud`) so a nightly is installable on real hardware. The nightly e2e does not boot it.
-3. **e2e** — stage the published GHCR closure (rewritten tree + the `cozy-installer` chart pinned to the GHCR packages artifact) and run the **full** app suite on Talos containers, which do not boot the disk. A nightly has no diff, so Test Impact Analysis does not apply.
-
-No GitHub release and no `api/apps/v1alpha1/*` Go-module tag are created for a nightly. [`retention.yaml`](../.github/workflows/retention.yaml) prunes old GHCR nightly versions (keeps the newest 14 per package the current nightly publishes, named from the published `cozystack-packages:nightly` tree by [`hack/retention-packages.sh`](../hack/retention-packages.sh), except packages linked to another repository, which are skipped with a warning; the floating `nightly`, release tags and untagged versions are never touched) and abandoned packages candidates. A candidate version is eligible only when all of these hold: it carries at least one tag and **every** tag on that manifest matches the strict `promotion-vX.Y.Z-from-vX.Y.Z-rc.N-run--` form, so a manifest that has been retagged stable is excluded; it was last updated more than 30 days ago; its digest is not pinned by `packages/core/installer/values.yaml` at the head of an **open** pull request that is same-repository, authored by `cozystack-ci[bot]`, on a `release-X.Y.Z` branch and carrying the `release` label; and its digest is not pinned by that same file at the tip of `main` or of any `release-X.Y` branch — which is what keeps a candidate safe between the promote PR merging and finalize retagging it. Any such PR head whose pin is not exactly one `digest=sha256:<64 hex>` stops the candidate sweep for that run and fails the job, without affecting the nightly sweep.
-
-Install a nightly:
-
-```bash
-helm upgrade --install cozystack \
- oci://ghcr.io/cozystack/cozystack/cozy-installer --version 0.0.0-nightly.20260626.15992304129.1
-# ...or --version nightly for the latest
-```
-
-The matching Talos node image is `ghcr.io/cozystack/cozystack/cozystack-nocloud:0.0.0-nightly.20260626.15992304129.1` (pull with `oras`).
-
-## Backports
-
-### The backport bot
-
-[`backport.yaml`](../.github/workflows/backport.yaml) wraps [`korthout/backport-action`](https://github.com/korthout/backport-action) and fires on a merged `main`-targeted PR carrying one of:
-
-| Label | Target branch |
-|-------|---------------|
-| `kind/backport` | the newest existing `release-X.Y` branch |
-| `kind/backport-previous` | the second-newest existing `release-X.Y` branch |
-
-Resolution is dynamic at run time, and it reads the branches themselves: the job lists the repository's branches, keeps the ones matching `release-.`, and sorts them numerically descending — so `release-1.10` ranks above `release-1.9`, which a lexicographic sort gets backwards. `backport` takes the first, `backport-previous` the second, so both name a branch that exists by construction; asking for `backport-previous` when only one line exists fails the job rather than inventing a target. Nothing is derived arithmetically — an earlier version computed the previous line as `Y-1`, which named a non-existent branch whenever a minor was skipped.
-
-**During a freeze window the newest line is the one being stabilised, not the last published one.** `release-X.Y` is created when the first `vX.Y.0-rc.1` is cut, before `vX.Y.0` is published, so from the freeze until the release `backport` targets the release being stabilised and `backport-previous` targets the last published stable. That is the intent: the frozen branch is the only way into the upcoming release, which is when backports matter most. The trade-off is that for the length of the window the line one step further back has no label pointing at it, so a fix that has to reach it before the new release publishes must be cherry-picked by hand.
-
-The bot creates a backport PR with title `[Backport release-X.Y] `. When this PR merges, the title prefix used to re-trigger the bot through `pr-labeler.yaml`, which auto-applied `backport` to any `[Backport release-X.Y]`-titled PR. Combined with the org-level `dosubot` re-applying the label, this caused recursive backports.
-
-The fix (PR #2584): both job `if:` blocks gate on `github.event.pull_request.base.ref == 'main'`. Backport PRs target `release-X.Y`, so they cannot satisfy this — architectural protection, regardless of which bot relabels them.
-
-### When the bot fails
-
-If the run triggered by the merge dies without opening a backport PR, look for a run of this workflow that has not finished before retrying — `gh run list --workflow backport.yaml --branch ` — and clear it, because the retry queues behind it rather than replacing it. Do not filter by status: a run whose jobs are waiting for a runner, or which is held behind the concurrency group, is not `in_progress`, and filtering it out is how you conclude nothing is running. `--branch` narrows the list but does not identify the PR, since head branch names are not unique across forks and `gh run list` has no PR filter, so confirm which PR a run belongs to before cancelling it or you cancel someone else's backport. If cancelling does not clear the run it holds the group until GitHub retires it and the retry keeps queueing; that case has not been exercised here. A run whose jobs have all finished can still sit `in_progress`, which looks the same from the PR as a run that died. Then remove whichever backport label the PR already carries and add it back: a label event is the only other trigger, and it qualifies on the label it carries, so re-applying it is what starts a fresh attempt.
-
-Conflicting cherry-picks produce a draft PR via `conflict_resolution: draft_commit_conflicts`. Look for the bot's comment with the merge-conflict diff. You either:
-
-- Resolve in the draft branch and undraft, or
-- Drop the bot's branch and cherry-pick manually:
-
-```bash
-git checkout release-X.Y
-git cherry-pick -x -s
-# resolve conflicts
-git commit -s
-git push origin release-X.Y # or push to a new branch and open a PR
-```
-
-To find the bot's failed comments across a batch of PRs:
-
-```bash
-for n in $(gh pr list --search "label:kind/backport label:kind/backport-previous merged:>=2026-01-01" --json number --jq '.[].number'); do
- echo "=== #$n ==="
- gh pr view $n --json comments --jq '.comments[] | select(.author.login == "github-actions" or (.author.login | contains("backport"))) | .body' | head -20
-done
-```
-
-Before [PR #3155](https://github.com/cozystack/cozystack/pull/3155), `conflict_resolution` was passed outside `experimental`, so a conflicting cherry-pick used the `fail` fallback and left only a "Backport failed" comment on the source PR. No draft backport PR was opened. When [`backport-audit`](../cmd/backport-audit/README.md) reports `MISSING`, check those comments and the workflow logs to establish why the change did not arrive.
-
-### Cherry-pick triage before a patch
-
-A patch release includes bugfixes for code that shipped in the corresponding minor `vX.Y.0`. Use:
-
-```bash
-# 1. Audit what was labeled for this line and has NOT landed on it.
-# Exits non-zero when anything is outstanding; see cmd/backport-audit/README.md.
-go build ./cmd/backport-audit
-./backport-audit release-X.Y
-
-# 2. List commits on main since the release branch diverged that are NOT yet on release-X.Y
-git merge-base origin/main origin/release-X.Y
-git log ..origin/main --grep="(#" --oneline
-
-# 3. Open PRs that may need labeling before the cut
-gh pr list --state open --base main --label kind/bug
-```
-
-Step 1 covers the currently labeled work: [`backport-audit`](../cmd/backport-audit/README.md) assigns each request to the release line selected at merge time, then checks whether its commits reached the branch. Incomplete delivery, open or dropped backports, missing changes and duplicate open claims fail the gate. Late-added labels can disagree with the bot's target, and branch-opening dates are estimated from rc tags; see the audit's documented limits before deciding what to backport. Steps 2 and 3 cover changes that were never labeled.
-
-**Include rule:**
-
-- `kind/bug` — especially destructive (data loss, crash-loop, OOM, eviction).
-- `area/security` / CVE.
-- Narrow chart fixes whose blast radius is contained (e.g. one operator's resources).
-- Patch-line-policy-matching dependency bumps (e.g. cilium 1.X.Y → 1.X.(Y+1) on a `release-A.B` shipping cilium 1.X — never a minor bump, only a patch within the line the branch already ships).
-
-**Skip rule:**
-
-- `kind/feature` — even with `lgtm`. A patch release is not a delivery vehicle for features.
-- CI-only changes (`area/ci`, test infrastructure).
-- Docs-only changes (those land via `update-website-docs` automatically).
-- Large multi-package dependency churn (Dependabot Go-deps bumps).
-- PRs whose conventional-commit type is `!` (breaking) or that mix `feat` with `fix`.
-
-**Borderline cases:**
-
-- Large refactors with `lgtm` AND an obvious bug payload — prefer extracting the bug fix into a narrower commit and backporting only that.
-- PRs labeled both `kind/bug` and `kind/feature` — split them.
-
-### Backporting code that touches monotonic counters
-
-Migrations are the canonical case. `packages/core/platform/images/migrations/run-migrations.sh` is a linear counter — it runs `seq $CURRENT_VERSION $((TARGET_VERSION - 1))` and stops if `CURRENT >= TARGET`. This creates a forward-upgrade trap. Walking through it with `release-1.2` as the maintenance branch:
-
-- `main` has `targetVersion=10` and migrations 1–9 on disk.
-- The `release-1.2` branch was cut when `targetVersion=5` (migrations 1–4 on disk).
-- You backport a new migration to `release-1.2` at slot 5, and naively bump `targetVersion` 5 → 6 to enable it.
-- Customer running 1.2.x runs the new migration, stamps `cozystack-version=6` in-cluster.
-- On upgrade to 1.3 (main's `targetVersion=10`), the runner walks `seq 6 9` and runs main's migrations 6–9 — **main's own migration at slot 5 never runs.** The backport claimed that slot on `release-1.2`; main's slot 5 holds an unrelated migration that gets silently skipped.
-
-Mitigations, in order of preference:
-
-1. **Don't bump `targetVersion` in the backport.** Just ship the migration file. Prior precedent in the repo: an earlier maintenance branch's ACME backport did exactly this — the migration shipped but ran on no clusters until a later commit raised the counter. Pair this with relaxing the chart-side `{{ fail }}` guard on the maintenance branch (chart-side coalesce) so the new value works without the migration having executed.
-2. **If you must bump,** add an idempotent duplicate of the now-skipped main migration at a higher slot. Bounded cost (one extra file per affected migration) and survives the skip window.
-3. **Audit cross-branch counter state before deciding.** Compare migration numbering across `main`, `release-X.Y`, and `release-X.(Y-1)`. If a backport would create a skip window on any forward-upgrade path, choose option 1.
-
-This rule generalizes to any monotonic-counter state: schema versions, feature-flag generations, anything that gates "have I run X yet?" with a single integer.
-
-## Changelogs
-
-### Where the canonical process lives
-
-[`agents/changelog.md`](./agents/changelog.md) is the source of truth. Read it end-to-end before generating; do not infer the process from past commits or memory. The CI runs it under Copilot via the reusable `changelog-generate.yaml` — called by `promote-rc.yaml::changelog` at promote time and by the optional `changelog-rc.yaml` at rc time — and, as a backstop, `tags.yaml::generate-changelog`; locally the same prompt is replayable by following the doc directly.
-
-### Common changelog failure modes
-
-These are mistakes that have shipped in real changelogs. Verify against each before merging the changelog PR.
-
-**1. Commits from outside the release range.** Caused by running `git log ..HEAD` from `main` while generating a patch changelog — `HEAD` on `main` contains everything that landed since the tag plus backports merged to `release-X.Y` after the tag was cut. Use `git log ..` explicitly. The doc was updated to require this after v1.3.1's changelog shipped with 14 entries for what was actually a 1-commit release.
-
-**2. Original + backport listed as two separate entries.** Each backport must coalesce with its original into a single entry of the form `(in #orig, backport #bp)`. To verify after generation:
-
-```bash
-# Every entry that mentions "backport" should also reference the original PR number
-grep -E 'backport #[0-9]+' docs/changelogs/v.md | grep -v '#[0-9]*, backport'
-```
-
-Output should be empty.
-
-**3. Wrong PR author.** The squash-merge commit's author is whoever clicked "Merge" — not the person who wrote the code. **Always** resolve via `gh pr view --json author --jq .author.login`, never `git log --format=%an`. This bites hardest for website-repo entries where the same merger handles most PRs.
-
-**4. Superseding patch, stale changelog.** Tags are write-once, so a critical fix after `vX.Y.Z` ships as the next patch `vX.Y.(Z+1)` rather than moving the tag. If a changelog PR already exists for a tag you are superseding, the changelog work transfers to the new patch. Compare `git log ..` against what the existing `changelog-vX.Y.Z` branch already documents; only add/remove the deltas. If the only new commits are CI-internal or a revert of a feature that never reached a stable tag, no changelog edit is needed.
-
-**5. PR numbers swapped inside prose.** The entry-format validator checks bullet entries but ignores Feature Highlights paragraphs and Upgrade Notes. Both have caused wrong PR references in shipped changelogs. Verify every `#NNNN` in prose with `gh pr view `.
-
-**6. Hallucinated entries.** Every entry must correspond to a commit in `git log ..`. Verify by extracting PR numbers from the file and grep-searching the commit range:
-
-```bash
-grep -oE '#[0-9]+' docs/changelogs/v.md | sort -u | while read pr; do
- n=${pr#\#}
- if ! git log .. --grep="#${n}" --oneline | grep -q .; then
- echo "PR $pr in changelog but not in commit range"
- fi
-done
-```
-
-**7. Bot accounts as human contributors.** `cozystack-ci[bot]`, `github-actions`, `dependabot`, `renovate`, and any `app/*` login must NOT appear in the `## Contributors` list. They legitimately appear in **per-entry attribution** (e.g. a Renovate-authored PR) — but not in the human roll-call.
-
-**8. Title duplicated as description.** `* **fix(foo): X**: fix(foo): X (...)` means you wrote the conventional-commit subject twice and never wrote a user-facing description. The brief and the detail must say different things — the detail explains what the change means for users.
-
-### Changelog pre-publish verification
-
-Run these checks before merging the changelog PR. Catches problems in roughly the order they occur:
-
-```bash
-PREV=v1.2.0; NEW=v1.2.1
-
-# A. Every chart bump is mentioned somewhere in the changelog
-git diff --name-only $PREV..$NEW -- 'packages/*/charts/*/Chart.yaml' 'packages/*/*/charts/*/Chart.yaml' | while read f; do
- old=$(git show "$PREV:$f" 2>/dev/null | yq '.version // ""')
- new=$(git show "$NEW:$f" 2>/dev/null | yq '.version // ""')
- if [ "$old" != "$new" ]; then
- component=$(basename "$(dirname "$f")")
- if ! grep -qi "$component" docs/changelogs/$NEW.md; then
- echo "MISSING: $component $old -> $new ($f)"
- fi
- fi
-done
-
-# B. Out-of-Chart.yaml pins (Talos, tenant K8s, vendored images)
-for path in packages/core/talos/images/talos/profiles/installer.yaml packages/core/installer/values.yaml packages/apps/kubernetes/values.yaml; do
- git diff $PREV..$NEW -- "$path" | head -40
-done
-git diff --name-only $PREV..$NEW -- 'images/*/Dockerfile'
-
-# C. Every cited #NNNN exists and is a merged PR (not an issue)
-grep -oE '#[0-9]+' docs/changelogs/$NEW.md | sort -u | sed 's/#//' | xargs -P 20 -I{} sh -c '
- state=$(gh api /repos/cozystack/cozystack/issues/{} --jq ".pull_request.merged_at // \"ISSUE\"" 2>/dev/null)
- echo "#{} $state"
-' | grep -E 'ISSUE|null|^#[0-9]+ $'
-
-# D. URL liveness
-grep -oE 'https://[^)" ]+' docs/changelogs/$NEW.md | sort -u | xargs -P 10 -I{} sh -c '
- code=$(curl -s -o /dev/null -w "%{http_code}" {})
- if [ "$code" != "200" ]; then echo "$code {}"; fi
-'
-
-# E. Website PR authors are PR authors, not commit authors
-grep -oE 'cozystack/website#[0-9]+' docs/changelogs/$NEW.md | sort -u | sed 's|cozystack/website#||' | while read n; do
- pr_author=$(gh pr view $n --repo cozystack/website --json author --jq .author.login)
- echo "website#$n -> @$pr_author"
-done
-
-# F. Working tree clean except for the changelog
-git status --porcelain | grep -v "docs/changelogs/$NEW.md"
-# Should output nothing.
-```
-
-Ship criterion: every chart bump surfaced, no issue-cited-as-PR, no 404 URLs, every website entry attributed via `gh pr view` (not commit author), `git status` shows only the changelog.
-
-### Cozystack-specific URL quirks
-
-`cozystack.io` paths are versioned: `/docs/operations/foo/` and `/docs/next/foo/` both 404. Only `/docs/vX.Y/foo/` and the unversioned canonical `/docs/foo/` serve content. AI-generated changelogs invent the unversioned form. Verify URLs with the `curl` loop above.
-
-### Platform-component coverage
-
-For a minor release (`vX.Y.0`), changelog entries should surface what an upstream chart bump actually brought in — features, breaking changes, security fixes. PR commit messages usually don't capture this; the upstream `CHANGELOG.md` or GitHub Release notes do. For each bumped `Chart.yaml`:
-
-1. Read the `sources:` field — typically a GitHub URL.
-2. Fetch release notes strictly inside `old < x <= new` via `gh release view --repo / --json body,name`.
-3. Summarize in 2–5 user-impact bullets per component (new features, breaking changes, CVEs, important deprecations). Don't reproduce upstream notes verbatim; link to them.
-4. If a PR-level entry already mentions the bump, enrich that entry with bullets — don't duplicate the entry.
-5. If 3+ bumps have no PR-level mention, introduce a `## Platform Components` section.
-
-If you can't find upstream notes for a bumped component, list it with version-only info and no bullets. Hallucinated bullets are worse than missing detail.
-
-## Release-time fires — what's gone wrong recently
-
-These are real regressions that escaped to users. The pattern in each: code passed CI, shipped in a stable release, broke customers. Read them as a working list of pre-release verification gaps.
-
-### v1.3.0 — cert-manager ingressClassName regression (fix #2562)
-
-**Bug:** A commit migrated ACME HTTP-01 to use the modern `acme.cert-manager.io/http01-ingress-ingressclassname` annotation. The matching ingress-shim code that reads that annotation was added by cert-manager upstream PR #8244 and was **never backported** to the `release-1.19` branch — but cozystack 1.3.0 shipped cert-manager v1.19.3. The annotation was silently dropped at runtime and every solver Ingress fell back to the ClusterIssuer's default class `tenant-root`. Let's Encrypt validation broke for every tenant whose `publishing.ingressName != tenant-root`.
-
-**Blast radius:** v1.3.0, v1.3.1, and v1.1.7 (the migration was cherry-picked to `release-1.1`).
-
-**Why CI missed it:** the migration commit was self-contained and `helm template` rendered cleanly. There was no E2E that creates a tenant with a non-default ingress class and watches a real Let's Encrypt cert issue. Verification had been done from the docs side ("the API has been available since cert-manager 1.12"), not against the shipped cert-manager source.
-
-**Would have caught it:** non-root-tenant E2E with LE-staging cert issuance, asserting the solver Ingress lands on the tenant's class.
-
-### v1.4.0 — kubevirt-instancetypes upgrade failure (fix #2612)
-
-**Bug:** `packages/system/kubevirt-instancetypes/Makefile` contained `sed -i '/persistent: true/d' templates/preferences.yaml`, which stripped the only child of `preferredTPM`, leaving `preferredTPM:` (null) in six Windows preferences. KubeVirt v1.6.x silently accepted it. The v1.6.3 → v1.8.2 operator bump (2026-04-27) hardened the CRD's OpenAPI schema and started rejecting null for an object-typed field. `helm upgrade` then failed on every cluster with `kubevirt-instancetypes` enabled.
-
-**Why CI missed it:** the kubevirt-operator bump and the inert `preferences.yaml` were two weeks apart in commit history. CI only tests fresh installs against the new CRDs; it doesn't run `helm upgrade` from prior versions. Schema regression on a vendored asset that hadn't been touched in 18 months.
-
-**Would have caught it:** `helm template | kubectl apply --dry-run=server -f -` against the freshly-installed CRDs of every bumped operator. Or an upgrade-from-N-1 E2E lane.
-
-### v1.4.0 — Flux 2.7 → 2.8 readiness deadlock (#2602)
-
-**Bug:** helm-controller v1.5.0 (shipped in Flux 2.8) changes the default wait strategy to kstatus polling, which polls **every applied resource including child HelmReleases**. The `packages/apps/kubernetes` umbrella has 19 child HRs that each `dependsOn: parent` (with a `lookup` guard so the dependency only activates after the parent exists). Pre-2.8: parent went Ready as soon as Helm returned. Post-2.8: parent waits for children, children wait for parent. Deadlock.
-
-**Why CI missed it:** the cycle only manifests on the second reconcile (parent exists, so the `lookup` activates the child's `dependsOn`). Fresh-install E2E dodges this — the parent-already-exists branch isn't taken on the first apply.
-
-**Would have caught it:** apply-mutate-reapply E2E (any chart bump in the kubernetes umbrella triggers a second reconcile). Plus an explicit PR-template requirement: for any flux/helm-controller bump, paste the upstream breaking-changes section into the PR body.
-
-### Patterns these share
-
-1. **Fresh-install CI is insufficient.** All three regressions passed fresh-install lanes.
-2. **Upstream breaking-defaults bumps are the common shape.** kubevirt CRD hardening, helm-controller default wait strategy, cert-manager API surface drift. Pre-merge reading of upstream changelogs would have caught two of three.
-3. **One-engineer-bitten-once is the only safety net for live verification.** The cert-manager bump PR included an explicit live-cluster verification step — because the same engineer had been bitten the prior week. There is no checklist requiring it.
-
-## Pre-release verification checklist
-
-For RCs and final releases, run this before merging the release PR. Each item is here because something has shipped without it and broken users.
-
-- [ ] **The published rc passed mandatory full E2E.** Confirm the promote PR says `✅ RC full e2e was verified`; the normal promote PR itself runs no E2E. Add `full-e2e` manually only when a repeat is required, and treat a ⚠️ bypass line as an explicit release-risk decision.
-- [ ] **Upgrade from previous patch** (`vX.Y.(Z-1)`) on a real cluster. `helm upgrade` succeeds, all HelmReleases reach Ready, no CRD schema rejections.
-- [ ] **Upgrade from previous minor** (`v(X-1).Y.0`) for RC and minor releases. Same assertions.
-- [ ] **At least one tenant with `publishing.ingressName != tenant-root`** and a real Let's Encrypt-staging cert issuance.
-- [ ] **Apply-mutate-reapply** on the kubernetes umbrella HelmRelease (catches second-reconcile bugs).
-- [ ] **For every CRD-source bump** (kubevirt, cert-manager, flux, cilium, kamaji): `helm template | kubectl apply --dry-run=server` against the freshly-installed CRDs.
-- [ ] **For every flux / helm-controller / kubevirt-operator / cert-manager bump:** paste the upstream breaking-changes section of the bump's CHANGELOG into the PR body before merge.
-- [ ] **After cutting the tag,** watch `pull-requests-release.yaml::Finalize Release`. If it fails on `Draft release for v... not found`, undraft the release manually and file the workflow regression — this has been a known recurring failure since the `Publish draft release` step lost its explicit `github-token` in refactor `66a756b6`.
-- [ ] **The promote PR carries `docs/changelogs/vX.Y.Z.md` and it was verified** per [Changelog pre-publish verification](#changelog-pre-publish-verification). Finalize publishes this file as the release body, so verifying it before merge is verifying the release notes.
-- [ ] **The `cozystack/website` docs PR stays parked until publish.** Promotion opens it early; merging it before the GitHub release for `vX.Y.Z` is live 404s the new version's docs and API links. Merge it right after finalize publishes the release (regular-release step 6, patch step 7).
-
-## CI failures release engineers commonly hit
-
-| Symptom | Class | Diagnosis | Block release? |
-|---------|-------|-----------|----------------|
-| `Draft release for vX.Y.Z not found` on every release-* merge | Workflow regression — `Publish draft release` lost its `github-token` in a past refactor; default `GITHUB_TOKEN` cannot list drafts | At `pull-requests-release.yaml:163`. Undraft manually, file the fix | No |
-| Changelog Copilot job returns 402 | Token quota — `COPILOT_GITHUB_TOKEN` premium-request quota empty | Both changelog entry points hit it: the optional rc-time `changelog-rc` dispatch and the promote-time `changelog` job (both call the reusable `changelog-generate.yaml`), plus the `tags.yaml` backstop | No — `continue-on-error: true` on the AI step, generate changelog by hand following [`agents/changelog.md`](./agents/changelog.md) |
-| `Prepare Release` skipped and the run red, `Build arm64 images` failed | The rc's arm64 build failed | Read the `build-arm64` log; `build-arm64-nightly.yaml` usually failed the same way first. Fix on the line, cut the next rc | Yes — an rc is not published amd64 only |
-| `Stitch multi-arch indexes and republish` fails listing refs | `verify-multiarch.sh` found refs that are not amd64+arm64 indexes, or that it could not check | Each listed ref names its reason. Single-platform manifest or index without one architecture: make the image multi-arch, or allowlist its repository with a reason if it cannot be. Manifest unreadable: correct the ref or digest; the allowlist does not excuse it. Rate-limited or unavailable after 3 tries: re-run once the throttle clears | Yes |
-| E2E `kubernetes-test` fails with `CSINode does not contain driver csi.kubevirt.io` | Test ordering bug | Tenant's `kubernetes-${test_name}-csi` HelmRelease still installing when NFS PVC is created. Fix: wait for `csinode/` to advertise the driver before creating the PVC | Yes — flaky and masks real CSI bugs |
-| `tenant-root` HR in perpetual upgrade-rollback loop after OIDC patch | Real bug at the intersection of #2602's readiness change and the cozystack reconcile cadence | Look for `rate: Wait(n=1) would exceed context deadline` in operator logs; multiple tenant child HRs stuck `InProgress` | Yes |
-| `cozy-dashboard` ImagePullBackOff from `cozystack-ui:latest` | Registry flake | Known noisy on some OCI mirrors; not currently in the prepull set | No — retry |
-| PR CI: `docker push` 409 on `:latest` for the same image across two PRs | Concurrent manifest race on a shared floating tag | The `:latest` tag was previously written by every `make image` regardless of PR. Resolved by the image-tag refactor (`IMAGE_TAG=pr--` in PR builds, `IMAGE_TAG=` only at release time) | No — `gh run rerun --failed` |
-
-**Heuristic:** workflow-token and Copilot-quota issues are out-of-band — release the tag, fix the workflow in a follow-up PR. Test ordering bugs and tenant-root reconcile storms are real and should block.
-
-If you find yourself doing the same manual fixup on two consecutive releases (e.g. "undraft the release"), open a workflow-regression issue. Workflow bugs with known manual workarounds rot silently for months.
-
-## Tag immutability
-
-Published tags are **write-once** — once a `vX.Y.Z` or rc tag is pushed it is never moved or deleted. (Nightlies are not git tags at all — they are GHCR OCI tags, see [Nightly builds](#nightly-builds).) Moving a tag silently poisons the Go module proxy / pkg.go.dev cache for `api/apps/v1alpha1/vX.Y.Z` and drifts SBOM/provenance toolchains, so the release flow is built so a move is impossible by construction:
-
-| File | Tag / branch handling |
-|------|------------------------|
-| [`pull-requests-release.yaml`](../.github/workflows/pull-requests-release.yaml) | Verifies the pinned packages candidate before creating a stable name. Creates `vX.Y.Z` at the PR merge commit **write-once** (create if absent, no-op if unchanged, fail loudly if it would move — the merge commit is new, so there is nothing to move). Cuts the `api/apps/v1alpha1/` Go-submodule tag write-once at the same commit, **stable only** (never rc/beta/alpha). The `release-X.Y` maintenance branch is fast-forward-only. Retags the verified packages artifact and rc container digests to the stable image tag (also write-once at the image level). |
-| [`tags.yaml`](../.github/workflows/tags.yaml) | The `release-X.Y.Z-rc.N` staging branch is a mutable staging ref (compare-before-force: skipped when unchanged, force+log only when it genuinely moves). Fails fast on a stable `vX.Y.Z` tag with no pre-existing draft — stable tags come from `promote-rc.yaml`, never a hand push. |
-| [`promote-rc.yaml`](../.github/workflows/promote-rc.yaml) | Mutates no stable tags. Before staging, verifies green full E2E from the latest rc tag-push run or a tag-correlated manual published-tag run (unless loudly bypassed). Commits and publishes the rewritten packages tree under a run-specific temporary `promotion-*` tag, adds a second commit pinning its digest on the `release-X.Y.Z` branch, generates the changelog there, and opens the promote PR with `release` but no automatic `full-e2e`; the PR gate verifies the candidate and finalize repeats that verification before the stable tag and image retag. The staging branch is force-pushed on re-dispatch, but an existing `docs/changelogs/vX.Y.Z.md` is preserved across the rebuild. |
-
-This is the immutable-tag + rc-promotion model from [#2677](https://github.com/cozystack/cozystack/issues/2677): stable `vX.Y.Z` is created only by promoting an existing rc, rc tags are write-once, and `api/apps/v1alpha1/vX.Y.Z` is created only on a stable release. The old nightly `auto-release.yaml` (which delete-recreated auto-bumped patch tags) has been removed. rc and stable tags accrete permanently; the only churning artifacts are GHCR nightlies and abandoned temporary promotion candidates, which [`retention.yaml`](../.github/workflows/retention.yaml) prunes (see [Nightly builds](#nightly-builds)) — GHCR storage is the cost vector to watch.
-
-### Repo-side enforcement (tag-protection rulesets)
-
-The guarantees above are enforced in the workflows. Repo admins back them with two GitHub **tag rulesets** targeting `v*`:
-
-- **Immutability** — *Restrict updates* + *Restrict deletions*, **no bypass**. No actor (not even the CI app) can move or delete a shipped `v*` tag. Safe to enable at any time: nothing in the release flow ever moves or deletes a `v*` tag, it only creates new ones.
-- **Creation control** — *Restrict creations* with the **CI app as the sole bypass actor**, plus *Limit branches/tags updated in a single push* = 1. This turns "don't hand-push tags" from convention into hard enforcement: every `v*` tag — pre-release **and** stable — can then be created only by CI. Enable this **only after** [`cut-prerelease.yaml`](../.github/workflows/cut-prerelease.yaml) is in use, since that workflow is what makes CI the sole creator of pre-release tags; enabling it earlier would lock maintainers out of cutting rc/alpha/beta. (The Go-module tag `api/apps/v1alpha1/*` gets an equivalent pair of rulesets.)
-
-## Splitting a release-blocking bundle PR
-
-Sometimes the work that has to land before a release is a 40-commit grab bag (CI stabilization, dependency bumps, chart fixes). Splitting it makes it reviewable and survivable. The strategy that has worked:
-
-1. **Verify what's already on `main`.** Use `git merge-base --is-ancestor` and subject grep to drop commits that have already landed.
-2. **Find existing open PRs that cover the same commits.** Force-push to the existing branch instead of opening a duplicate — preserves review threads.
-3. **Identify the "bottleneck" PR.** The one whose changes make previously-silent misconfigurations into hard errors (typically an operator-version bump). Everything else can fan out in parallel.
-4. **Squash iterative same-author bug-on-bug fixes** inside each split. Preserve authorship via `--author` on the squashed commit.
-5. **Mark the keystone PR as draft** and link in the body to the dependency PRs. Reviewers know it intentionally fails CI until the chain lands.
-6. **Beware of concurrent registry pushes.** When fanning out 5+ PR CI runs simultaneously, multiple `make image` jobs can collide on shared floating tags. The `IMAGE_TAG=pr--` refactor solved this; if you see `Conflicted with another upload of the same manifest`, `gh run rerun --failed`.
-
-## Cleanup after release
-
-- The `release-X.Y.Z` branch is deleted by GitHub when its PR merges. The `release-X.Y` maintenance branch is created by `cut-prerelease.yaml` at the rc freeze (create-only, never force-moved) and fast-forwarded afterwards by `pull-requests-release.yaml::Ensure maintenance branch`.
-- The draft release is published by the same workflow.
-- The release body is set by finalize from `docs/changelogs/vX.Y.Z.md` at publication. Afterwards, `update-releasenotes.yaml` re-syncs that file into the GitHub Release body on any push to `main` that touches it — so edits to a published changelog land on the release page the next time `main` moves.
-- Local cleanup: remove your worktree (`git worktree remove`) and prune merged release branches.
+## Pages
+
+- [Introduction](./release/introduction.md): the three release types and the tag push that drives the pipeline.
+- [Release Candidates](./release/release-candidates.md): what an rc is, and how cutting the first one freezes `release-X.Y`.
+- [Regular Releases](./release/regular-releases.md): promoting an rc to `vX.Y.0`, step by step.
+- [Patch Releases](./release/patch-releases.md): the same flow for `vX.Y.Z` on an existing `release-X.Y` line.
+- [What CI does during the release process](./release/ci-pipeline.md): which workflow does what, phase by phase, including multi-arch builds and the rc E2E gate.
+- [Stable tags come from rc promotion](./release/stable-tags.md): why a stable tag only ever comes from promoting an rc.
+- [Release signatures and SBOM](./release/signatures-and-sbom.md): the source SBOM, checksums and cosign signature on each release, the SBOM each release image carries, and how to verify and read them.
+- [Nightly builds](./release/nightly-builds.md): how a nightly mirrors `main` to GHCR, and how to install one.
+- [Backports](./release/backports.md): the policy that decides what goes into a patch release, and how a contributor gets a fix there.
+- [Backport mechanics](./release/backport-mechanics.md): cherry-pick triage before a patch, and the monotonic-counter trap.
+- [Backport tasks](./release/backport-tasks/README.md): resolving a batch conflict, deferring the original a failed batch CI comes from, and checking a pull request title, with the check each has to pass.
+- [Changelogs](./release/changelogs.md): changelog failure modes and the pre-publish checks.
+- [Release-time fires](./release/release-time-fires.md): regressions that escaped to users, and the verification gaps they expose.
+- [Pre-release verification checklist](./release/pre-release-checklist.md): what to verify before merging a release PR.
+- [CI failures release engineers commonly hit](./release/ci-failures.md): known failures and whether each one blocks a release.
+- [Tag immutability](./release/tag-immutability.md): write-once tags and the rulesets that enforce them.
+- [Splitting a release-blocking bundle PR](./release/splitting-bundle-prs.md): how to break up a grab-bag PR that blocks a release.
+- [Cleanup after release](./release/cleanup.md): what is left to clean up once a release is out.
## See also
- [`agents/changelog.md`](./agents/changelog.md) — canonical changelog generation process.
-- [`agents/contributing.md`](./agents/contributing.md) — commit/PR conventions, backport label semantics.
+- [`agents/contributing.md`](./agents/contributing.md) — commit/PR conventions.
- [`agents/releasing.md`](./agents/releasing.md) — pointer file for AI agents handling release tasks.
- [`.github/workflows/cut-prerelease.yaml`](../.github/workflows/cut-prerelease.yaml) — the sole entry point for cutting a pre-release tag as the CI app.
- [`.github/workflows/tags.yaml`](../.github/workflows/tags.yaml) — tag-push pipeline (rc build + mandatory `rc-e2e`; stable-tag changelog/website backstops).
@@ -714,5 +53,6 @@ Sometimes the work that has to land before a release is a 40-commit grab bag (CI
- [`agents/image-refs.md`](./agents/image-refs.md) — where image references live, what their tags mean, and the invariants promotion and mirroring rely on.
- [`hack/lib/image-refs.sh`](../hack/lib/image-refs.sh) — the single enumeration of ref-bearing files, shared by the promote, retag and mirror tooling.
- [`.github/workflows/retention.yaml`](../.github/workflows/retention.yaml) — GHCR nightly and abandoned promotion-candidate pruning.
-- [`.github/workflows/backport.yaml`](../.github/workflows/backport.yaml) — automatic cherry-pick bot.
+- [`cmd/backport-batch`](../cmd/backport-batch/README.md) — the backport batch that applies the [backport policy](./release/backports.md).
+- [`.github/workflows/backport-batch.yaml`](../.github/workflows/backport-batch.yaml) — runs the backport batch on Monday and Thursday for each line it runs for: the newest released lines, and a line in its rc window.
- [`.github/workflows/update-releasenotes.yaml`](../.github/workflows/update-releasenotes.yaml) — sync changelog → GitHub Release body.
diff --git a/docs/release/backport-mechanics.md b/docs/release/backport-mechanics.md
new file mode 100644
index 000000000..f713d20aa
--- /dev/null
+++ b/docs/release/backport-mechanics.md
@@ -0,0 +1,37 @@
+## Backport mechanics
+
+Under the [backport policy](./backports.md) fixes reach a release line through the batch, and nobody labels pull requests for backport. This page covers the mechanics around it: cherry-pick triage before a patch, and the trap that keeps migrations out of patch releases.
+
+### Cherry-pick triage before a patch
+
+A patch release includes bugfixes for code that shipped in the corresponding minor `vX.Y.0`. What goes in is decided by the rules in [Backports](./backports.md), not per pull request, and each batch pull request says what it picked, what it deferred and what it skipped under which rule. What is left to check before a cut:
+
+```bash
+# Exits 1 and prints the URLs while a pull request the rules take is not on
+# the line, including one only on the open batch PR, which the cut would not
+# carry; see cmd/backport-batch/README.md.
+go build ./cmd/backport-batch
+./backport-batch check --line release-X.Y --cut
+```
+
+[`backport-batch check`](../../cmd/backport-batch/README.md) applies the rules to every pull request merged into `main` since the line branched, and looks for each one they take on the line and then on the open batch pull request: by picks whose `-x` lines name its merge commit or every commit of its own, by a pick that would change nothing, and, on commits from before the policy's batch-only day, by what their text says, such as `Backport of #N`. With `--cut` a change found only on the open batch pull request fails the check too, since a cut tags the line and would not carry it: merge the batch pull request first. A conflict the batch deferred, or an original deferred to the next batch, prints as a warning without failing the check. A change that reached the line by hand counts like any other, by what reached the line.
+
+### Backporting code that touches monotonic counters
+
+The batch never takes a change like this: rule 4 in [Backports](./backports.md) holds migrations, `targetVersion`, sizeable API changes and CRDs for the next minor. What follows is why, and what a maintainer has to do when a migration must reach a release line anyway.
+
+Migrations are the canonical case. `packages/core/platform/images/migrations/run-migrations.sh` is a linear counter — it runs `seq $CURRENT_VERSION $((TARGET_VERSION - 1))` and stops if `CURRENT >= TARGET`. This creates a forward-upgrade trap. Walking through it with `release-1.2` as the maintenance branch:
+
+- `main` has `targetVersion=10` and migrations 1–9 on disk.
+- The `release-1.2` branch was cut when `targetVersion=5` (migrations 1–4 on disk).
+- You backport a new migration to `release-1.2` at slot 5, and naively bump `targetVersion` 5 → 6 to enable it.
+- Customer running 1.2.x runs the new migration, stamps `cozystack-version=6` in-cluster.
+- On upgrade to 1.3 (main's `targetVersion=10`), the runner walks `seq 6 9` and runs main's migrations 6–9 — **main's own migration at slot 5 never runs.** The backport claimed that slot on `release-1.2`; main's slot 5 holds an unrelated migration that gets silently skipped.
+
+Mitigations, in order of preference:
+
+1. **Don't bump `targetVersion` in the backport.** Just ship the migration file. Prior precedent in the repo: an earlier maintenance branch's ACME backport did exactly this — the migration shipped but ran on no clusters until a later commit raised the counter. Pair this with relaxing the chart-side `{{ fail }}` guard on the maintenance branch (chart-side coalesce) so the new value works without the migration having executed.
+2. **If you must bump,** add an idempotent duplicate of the now-skipped main migration at a higher slot. Bounded cost (one extra file per affected migration) and survives the skip window.
+3. **Audit cross-branch counter state before deciding.** Compare migration numbering across `main`, `release-X.Y`, and `release-X.(Y-1)`. If a backport would create a skip window on any forward-upgrade path, choose option 1.
+
+This rule generalizes to any monotonic-counter state: schema versions, feature-flag generations, anything that gates "have I run X yet?" with a single integer.
diff --git a/docs/release/changelogs.md b/docs/release/changelogs.md
new file mode 100644
index 000000000..f26ac56c5
--- /dev/null
+++ b/docs/release/changelogs.md
@@ -0,0 +1,107 @@
+## Changelogs
+
+### Where the canonical process lives
+
+[`agents/changelog.md`](../agents/changelog.md) is the source of truth. Read it end-to-end before generating; do not infer the process from past commits or memory. The CI runs it under Copilot via the reusable `changelog-generate.yaml` — called by `promote-rc.yaml::changelog` at promote time and by the optional `changelog-rc.yaml` at rc time — and, as a backstop, `tags.yaml::generate-changelog`; locally the same prompt is replayable by following the doc directly.
+
+### Common changelog failure modes
+
+These are mistakes that have shipped in real changelogs. Verify against each before merging the changelog PR.
+
+**1. Commits from outside the release range.** Caused by running `git log ..HEAD` from `main` while generating a patch changelog — `HEAD` on `main` contains everything that landed since the tag plus backports merged to `release-X.Y` after the tag was cut. Use `git log ..` explicitly. The doc was updated to require this after v1.3.1's changelog shipped with 14 entries for what was actually a 1-commit release.
+
+**2. Original + backport listed as two separate entries.** Each backport must coalesce with its original into a single entry of the form `(in #orig, backport #bp)`. To verify after generation:
+
+```bash
+# Every entry that mentions "backport" should also reference the original PR number
+grep -E 'backport #[0-9]+' docs/changelogs/v.md | grep -v '#[0-9]*, backport'
+```
+
+Output should be empty.
+
+**3. Wrong PR author.** The squash-merge commit's author is whoever clicked "Merge" — not the person who wrote the code. **Always** resolve via `gh pr view --json author --jq .author.login`, never `git log --format=%an`. This bites hardest for website-repo entries where the same merger handles most PRs.
+
+**4. Superseding patch, stale changelog.** Tags are write-once, so a critical fix after `vX.Y.Z` ships as the next patch `vX.Y.(Z+1)` rather than moving the tag. If a changelog PR already exists for a tag you are superseding, the changelog work transfers to the new patch. Compare `git log ..` against what the existing `changelog-vX.Y.Z` branch already documents; only add/remove the deltas. If the only new commits are CI-internal or a revert of a feature that never reached a stable tag, no changelog edit is needed.
+
+**5. PR numbers swapped inside prose.** The entry-format validator checks bullet entries but ignores Feature Highlights paragraphs and Upgrade Notes. Both have caused wrong PR references in shipped changelogs. Verify every `#NNNN` in prose with `gh pr view `.
+
+**6. Hallucinated entries.** Every entry must correspond to a commit in `git log ..`. Verify by extracting PR numbers from the file and grep-searching the commit range:
+
+```bash
+grep -oE '#[0-9]+' docs/changelogs/v.md | sort -u | while read pr; do
+ n=${pr#\#}
+ if ! git log .. --grep="#${n}" --oneline | grep -q .; then
+ echo "PR $pr in changelog but not in commit range"
+ fi
+done
+```
+
+**7. Bot accounts as human contributors.** `cozystack-ci[bot]`, `github-actions`, `dependabot`, `renovate`, and any `app/*` login must NOT appear in the `## Contributors` list. They legitimately appear in **per-entry attribution** (e.g. a Renovate-authored PR) — but not in the human roll-call.
+
+**8. Title duplicated as description.** `* **fix(foo): X**: fix(foo): X (...)` means you wrote the conventional-commit subject twice and never wrote a user-facing description. The brief and the detail must say different things — the detail explains what the change means for users.
+
+### Changelog pre-publish verification
+
+Run these checks before merging the changelog PR. Catches problems in roughly the order they occur:
+
+```bash
+PREV=v1.2.0; NEW=v1.2.1
+
+# A. Every chart bump is mentioned somewhere in the changelog
+git diff --name-only $PREV..$NEW -- 'packages/*/charts/*/Chart.yaml' 'packages/*/*/charts/*/Chart.yaml' | while read f; do
+ old=$(git show "$PREV:$f" 2>/dev/null | yq '.version // ""')
+ new=$(git show "$NEW:$f" 2>/dev/null | yq '.version // ""')
+ if [ "$old" != "$new" ]; then
+ component=$(basename "$(dirname "$f")")
+ if ! grep -qi "$component" docs/changelogs/$NEW.md; then
+ echo "MISSING: $component $old -> $new ($f)"
+ fi
+ fi
+done
+
+# B. Out-of-Chart.yaml pins (Talos, tenant K8s, vendored images)
+for path in packages/core/talos/images/talos/profiles/installer.yaml packages/core/installer/values.yaml packages/apps/kubernetes/values.yaml; do
+ git diff $PREV..$NEW -- "$path" | head -40
+done
+git diff --name-only $PREV..$NEW -- 'images/*/Dockerfile'
+
+# C. Every cited #NNNN exists and is a merged PR (not an issue)
+grep -oE '#[0-9]+' docs/changelogs/$NEW.md | sort -u | sed 's/#//' | xargs -P 20 -I{} sh -c '
+ state=$(gh api /repos/cozystack/cozystack/issues/{} --jq ".pull_request.merged_at // \"ISSUE\"" 2>/dev/null)
+ echo "#{} $state"
+' | grep -E 'ISSUE|null|^#[0-9]+ $'
+
+# D. URL liveness
+grep -oE 'https://[^)" ]+' docs/changelogs/$NEW.md | sort -u | xargs -P 10 -I{} sh -c '
+ code=$(curl -s -o /dev/null -w "%{http_code}" {})
+ if [ "$code" != "200" ]; then echo "$code {}"; fi
+'
+
+# E. Website PR authors are PR authors, not commit authors
+grep -oE 'cozystack/website#[0-9]+' docs/changelogs/$NEW.md | sort -u | sed 's|cozystack/website#||' | while read n; do
+ pr_author=$(gh pr view $n --repo cozystack/website --json author --jq .author.login)
+ echo "website#$n -> @$pr_author"
+done
+
+# F. Working tree clean except for the changelog
+git status --porcelain | grep -v "docs/changelogs/$NEW.md"
+# Should output nothing.
+```
+
+Ship criterion: every chart bump surfaced, no issue-cited-as-PR, no 404 URLs, every website entry attributed via `gh pr view` (not commit author), `git status` shows only the changelog.
+
+### Cozystack-specific URL quirks
+
+`cozystack.io` paths are versioned: `/docs/operations/foo/` and `/docs/next/foo/` both 404. Only `/docs/vX.Y/foo/` and the unversioned canonical `/docs/foo/` serve content. AI-generated changelogs invent the unversioned form. Verify URLs with the `curl` loop above.
+
+### Platform-component coverage
+
+For a minor release (`vX.Y.0`), changelog entries should surface what an upstream chart bump actually brought in — features, breaking changes, security fixes. PR commit messages usually don't capture this; the upstream `CHANGELOG.md` or GitHub Release notes do. For each bumped `Chart.yaml`:
+
+1. Read the `sources:` field — typically a GitHub URL.
+2. Fetch release notes strictly inside `old < x <= new` via `gh release view --repo / --json body,name`.
+3. Summarize in 2–5 user-impact bullets per component (new features, breaking changes, CVEs, important deprecations). Don't reproduce upstream notes verbatim; link to them.
+4. If a PR-level entry already mentions the bump, enrich that entry with bullets — don't duplicate the entry.
+5. If 3+ bumps have no PR-level mention, introduce a `## Platform Components` section.
+
+If you can't find upstream notes for a bumped component, list it with version-only info and no bullets. Hallucinated bullets are worse than missing detail.
diff --git a/docs/release/ci-failures.md b/docs/release/ci-failures.md
new file mode 100644
index 000000000..9beabe02c
--- /dev/null
+++ b/docs/release/ci-failures.md
@@ -0,0 +1,16 @@
+## CI failures release engineers commonly hit
+
+| Symptom | Class | Diagnosis | Block release? |
+|---------|-------|-----------|----------------|
+| `Draft release for vX.Y.Z not found` on every release-* merge | Workflow regression — `Publish draft release` lost its `github-token` in a past refactor; default `GITHUB_TOKEN` cannot list drafts | At `pull-requests-release.yaml:163`. Undraft manually, file the fix | No |
+| Changelog Copilot job returns 402 | Token quota — `COPILOT_GITHUB_TOKEN` premium-request quota empty | Both changelog entry points hit it: the optional rc-time `changelog-rc` dispatch and the promote-time `changelog` job (both call the reusable `changelog-generate.yaml`), plus the `tags.yaml` backstop | No — `continue-on-error: true` on the AI step, generate changelog by hand following [`agents/changelog.md`](../agents/changelog.md) |
+| `Prepare Release` skipped and the run red, `Build arm64 images` failed | The rc's arm64 build failed | Read the `build-arm64` log; `build-arm64-nightly.yaml` usually failed the same way first. Fix on the line, cut the next rc | Yes — an rc is not published amd64 only |
+| `Stitch multi-arch indexes and republish` fails listing refs | `verify-multiarch.sh` found refs that are not amd64+arm64 indexes, or that it could not check | Each listed ref names its reason. Single-platform manifest or index without one architecture: make the image multi-arch, or allowlist its repository with a reason if it cannot be. Manifest unreadable: correct the ref or digest; the allowlist does not excuse it. Rate-limited or unavailable after 3 tries: re-run once the throttle clears | Yes |
+| E2E `kubernetes-test` fails with `CSINode does not contain driver csi.kubevirt.io` | Test ordering bug | Tenant's `kubernetes-${test_name}-csi` HelmRelease still installing when NFS PVC is created. Fix: wait for `csinode/` to advertise the driver before creating the PVC | Yes — flaky and masks real CSI bugs |
+| `tenant-root` HR in perpetual upgrade-rollback loop after OIDC patch | Real bug at the intersection of #2602's readiness change and the cozystack reconcile cadence | Look for `rate: Wait(n=1) would exceed context deadline` in operator logs; multiple tenant child HRs stuck `InProgress` | Yes |
+| `cozy-dashboard` ImagePullBackOff from `cozystack-ui:latest` | Registry flake | Known noisy on some OCI mirrors; not currently in the prepull set | No — retry |
+| PR CI: `docker push` 409 on `:latest` for the same image across two PRs | Concurrent manifest race on a shared floating tag | The `:latest` tag was previously written by every `make image` regardless of PR. Resolved by the image-tag refactor (`IMAGE_TAG=pr--` in PR builds, `IMAGE_TAG=` only at release time) | No — `gh run rerun --failed` |
+
+**Heuristic:** workflow-token and Copilot-quota issues are out-of-band — release the tag, fix the workflow in a follow-up PR. Test ordering bugs and tenant-root reconcile storms are real and should block.
+
+If you find yourself doing the same manual fixup on two consecutive releases (e.g. "undraft the release"), open a workflow-regression issue. Workflow bugs with known manual workarounds rot silently for months.
diff --git a/docs/release/ci-pipeline.md b/docs/release/ci-pipeline.md
new file mode 100644
index 000000000..73f105bb7
--- /dev/null
+++ b/docs/release/ci-pipeline.md
@@ -0,0 +1,107 @@
+## What CI does during the release process
+
+The numbered process of a [regular release](./regular-releases.md#regular-releases) is implemented by seven workflows, including the reusable/manual [`e2e-tag.yaml`](../../.github/workflows/e2e-tag.yaml), plus the reusable [`changelog-generate.yaml`](../../.github/workflows/changelog-generate.yaml) that two of them share to produce the changelog. Knowing which job does what makes the failure modes much easier to diagnose.
+
+1. [`cut-prerelease.yaml`](../../.github/workflows/cut-prerelease.yaml) — manual dispatch (from `main` or `release-X.Y`): the sole entry point for creating a pre-release tag. Validates the `-alpha`/`-beta`/`-rc` tag, then pushes it (write-once, at the branch tip) as the CI app so `tags.yaml` fires. Cutting the first `vX.Y.0-rc.N` also creates `release-X.Y`, freezing the line; afterwards a `vX.Y.*` dispatch from `main` is refused. Stable tags are never created here.
+2. [`tags.yaml`](../../.github/workflows/tags.yaml) — fires on any pre-release tag push: `build-amd64` and `build-arm64` build the pre-release side by side, then `prepare-release` publishes it and pushes its digest-pinned staging branch, then — **only for `-rc.N` tags** — the mandatory `rc-e2e` job calls `e2e-tag.yaml` in the same run (`-alpha.N` / `-beta.N` skip it and are validated on demand with `e2e-tag.yaml`'s manual button). `generate-changelog` and `update-website-docs` also live here but are **backstops only** — both are normally produced at promotion time (see 4); on a stable tag push each self-skips or no-ops when the promote flow already did the work. These stable-tag backstops only fire because finalize pushes the stable tag with the CI app token (a `GITHUB_TOKEN` push triggers no run — see 5).
+3. [`e2e-tag.yaml`](../../.github/workflows/e2e-tag.yaml) — reusable from `tags.yaml::rc-e2e` and manually dispatchable for a published tag. It checks that the tag's release carries its nocloud disk, resolves the digest-pinned tree and runs the full install, OpenAPI, and Chainsaw suite on Talos containers. A tag cut before the container lane existed has to be dispatched from its own ref. Its dispatch run title is the constant workflow name, so the promote gate correlates a manual run through the successful tag-bearing E2E job name rather than the title.
+4. [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml) — manual dispatch, from `main` only (the signature over a stable release's assets names that ref, see [Release signatures and SBOM](./signatures-and-sbom.md#release-signatures-and-sbom)). Five jobs: `parse` validates the rc tag and requires green full-E2E evidence before any staging; `promote` rewrites and commits the rc tags, publishes that packages tree as a run-specific temporary reproducible candidate, commits its digest pin in the `release-X.Y.Z` tree, and drafts the stable release; `changelog` calls the reusable `changelog-generate.yaml` **in parallel** to produce `docs/changelogs/vX.Y.Z.md` (regenerating it, or copying an rc-time changelog already committed to the `release-X.Y.Z-rc.N` staging branch by `changelog-rc.yaml` — see 7); `website-docs` (needs `parse`, `promote`) opens the `cozystack/website` docs PR from the staging branch via `FETCH_REF` (see [Phase 3](#phase-3--update-website-docs-now-the-backstop)); `open-pr` joins them, commits the changelog onto the staging branch, and opens the `release-X.Y.Z` promote PR with the `release` label but no automatic `full-e2e`. So the changelog and website docs are reviewed as part of the promotion, the already-checked rc suite is not repeated by default, and the PR body carries ✅ verified or ⚠️ bypassed plus ✅/⚠️ changelog and website-docs lines. Both `changelog` and `website-docs` are non-blocking. No stable registry name is created at dispatch.
+5. [`.github/workflows/pull-requests.yaml`](../../.github/workflows/pull-requests.yaml) verifies a promotion candidate against the prospective merge tree and original rc artifact before the required `E2E Tests` status can turn green. [`pull-requests-release.yaml`](../../.github/workflows/pull-requests-release.yaml) fires after merge and repeats that verification before any stable tag, then cuts the write-once stable + Go-module tags, publishes the release **with the merged changelog as its body**, retags the candidate and rc images to stable by digest (`:latest` gated on newest-stable), and publishes the stable chart.
+6. [`update-releasenotes.yaml`](../../.github/workflows/update-releasenotes.yaml) — fires on pushes to `main` that touch `docs/changelogs/v*.md`; syncs that content into the corresponding GitHub Release body. No longer the primary path (finalize sets the body directly, which is race-free and works for maintenance lines too) — this now covers later edits to a published changelog and manual re-syncs via `workflow_dispatch`.
+7. [`changelog-rc.yaml`](../../.github/workflows/changelog-rc.yaml) — manual dispatch, **optional**: generate the changelog at rc time instead of waiting for promotion, while there is still time to review it. It calls the same reusable [`changelog-generate.yaml`](../../.github/workflows/changelog-generate.yaml) and commits the result to the `release-X.Y.Z-rc.N` staging branch as `docs/changelogs/vX.Y.Z.md`; the promote-time `changelog` job then copies that file instead of running the AI again. Skipping this workflow changes nothing — promote generates the changelog itself, exactly as before.
+
+### Phase 1 — `build-amd64`, `build-arm64` and `prepare-release` (hard gate)
+
+On an rc tag push: builds images for both architectures, commits digest pins, creates the draft release, and pushes the `release-X.Y.Z-rc.N` staging branch. `build-amd64` holds the release-exists no-op and the rejection of a hand-pushed stable tag, and builds the amd64 images and the release assets (its `make build` also pushes the packages artifact and the installer chart, pinned on amd64 digests); `prepare-release` stitches, gates, republishes those two from the repinned tree, and publishes the release. (The stable promote PR is opened later by `promote-rc.yaml`, not here. When the stable tag itself is pushed by finalize, a draft already exists, so this job is a no-op — see [Phase 5](#phase-5--pull-requests-releaseyaml-finalize).)
+
+The commit (`Prepare release vX.Y.Z`, authored by `cozystack-ci[bot]`) is **digest pins and image tags only**:
+
+- `packages/core/{installer,platform,testing}/values.yaml`
+- `packages/system/*/values.yaml` (cozystack-api, dashboard, kamaji, linstor, kubevirt-csi-node, etc.)
+- `packages/apps/kubernetes/images/{kubevirt-csi-driver,ubuntu-container-disk-*}.tag`
+- `packages/system/dashboard/templates/configmap.yaml` (the `$tenantText` value)
+
+Things that look surprising in the diff but are normal:
+
+- **Zero-diff packages**: when buildx fully caches and upstream base images haven't moved, the digest is byte-identical to the previous release and won't appear. Verify the build step actually ran — don't assume "no diff = nothing built."
+- **Sudden churn on `ubuntu-container-disk-*` tags**: `cloud-images.ubuntu.com/noble/current/` is a moving target; these often rebuild even without code changes.
+- **Switched registry**: if a self-built image moved from `docker.io` to `ghcr.io`, the registry portion of the digest changes — that's a deliberate move, not a regression.
+- **`targetVersion` is NOT touched by the release PR.** Platform migration `targetVersion` is bumped earlier in a feature commit (e.g. `[platform] Bump migration targetVersion to 39 for migration 38`). The release PR only re-pins the `platform-migrations` image digest.
+
+#### Multi-arch images
+
+Every pre-release is published for amd64 and arm64. `tags.yaml::build-arm64` builds the images on a native arm64 runner and pushes each as `-arm64`, while `build-amd64` builds amd64, the Talos installer and matchbox for both arches, and the assets on x86 at the same time. `build-amd64` hands its stamped tree, the log of the tags it pushed and the assets to `prepare-release` as workflow artifacts; a missing one fails the upload or the download rather than publishing a partial release. `prepare-release` waits for both builds and runs only if both succeeded: when either fails, the rc is not prepared at all, so a release is never published amd64 only. It applies the stamped tree and runs [`hack/stitch-multiarch.sh`](../../hack/stitch-multiarch.sh), which joins each image with its arm64 twin into one index, moves onto it every tag the build pushed for that image (the rc tag and, under `PUBLISH_VERSIONED=1`, its component version, which the build records in `PUSHED_TAGS_LOG`), and repins the tree on the index digests, then [`hack/verify-multiarch.sh`](../../hack/verify-multiarch.sh), then republishes the packages artifact and the installer chart from the repinned tree. A stable tag skips `build-arm64` and builds nothing: promotion copies the rc's indexes with every platform, so the stable release is exactly as multi-arch as its rc. Because the two builds run in parallel, an rc that is not prepared (its arm64 build failed, or `prepare-release` failed before the stitch) still leaves behind what the amd64 build pushed: amd64-only images under `:` and under the component-version tags, and the `cozystack-packages:` artifact and the installer chart, both pinned on those amd64 digests. A component tag an earlier rc had moved to an index then points at an amd64-only image again, until a run that succeeds stitches it, a re-run of the same run included.
+
+`verify-multiarch.sh` is a hard gate. It resolves every ref the tree pins, third-party images included, and fails naming each one that is not an index carrying both `linux/amd64` and `linux/arm64`, or that it cannot resolve at all. The fix is to make the image multi-arch (build it for both, or pin a multi-arch digest of the same version); an image that cannot exist for arm64 goes in [`hack/multiarch-allowlist`](../../hack/multiarch-allowlist) by repository, with the reason on the same line.
+
+That gate sees only what the tree pins by digest. An image a vendored chart renders from its own defaults, or one an operator launches at runtime, is invisible to it. `rc-e2e` closes most of that gap: before tearing the sandbox down, [`e2e-tag.yaml`](../../.github/workflows/e2e-tag.yaml) reads every image the three Talos nodes pulled from their image stores (`hack/collect-images.sh`), each by the digest the pull resolved to, and runs `verify-multiarch.sh --refs-file` on that list with the same allowlist. A pre-release fails on any single-arch image, on any ref it cannot read or that is still rate-limited after the retries, and on a list the collection could not produce; a stable tag's run only warns, since it ships its rc's images. A tag whose tree predates the audit (it has no `hack/verify-multiarch.sh`) only warns and passes, so an rc cut before the audit existed can still be re-validated. The list and the audit output are in the run's `image-list-` artifact. Neither check covers a package that e2e never installs and whose images the tree does not pin by digest: an arm64 break there shows up only on an arm64 cluster.
+
+[`build-arm64-nightly.yaml`](../../.github/workflows/build-arm64-nightly.yaml) builds `main` for arm64 every day, so an arm64 build break shows up there before the next rc. Its first step prints `verify-multiarch.sh --report` for `main`'s committed tree: `main` pins amd64 images, so its first-party refs are all listed and are expected to be, since the rc's stitch turns them into indexes. A third-party ref that an in-tree rebuild stamps over at build time, as the Harbor components and shadowbox are, is listed too and goes away on the rc. The lines that matter are the third-party refs no build replaces and refs that cannot be resolved; each of those fails the next rc until it is fixed or allowlisted. matchbox is listed too and does not matter: `build-amd64` rebuilds it as a two-arch index. The stitch names the images it skips in its own summary: an image with no `-arm64` twin, one whose digest is also pinned in a file the rewrite does not reach, and a pair that is not one amd64 and one arm64 image. matchbox (already an index) and the e2e sandbox (never built for arm64, and allowlisted) are skipped on every rc and pass the gate; any other skipped image fails it.
+
+### RC E2E — mandatory check before promotion
+
+For an rc, `prepare-release` must finish before `tags.yaml::rc-e2e` calls the published-tag lane, so the full suite tests the digest-pinned staging tree that promotion will reuse, and the release it runs against already carries its disk. `promote-rc.yaml::parse` then checks the latest `tags.yaml` push run whose `head_branch` is the rc tag and accepts it only when the reusable E2E job succeeded. GitHub exposes called-workflow job names as `E2E Release Candidate / E2E vX.Y.Z-rc.N (full suite)`, which the gate matches by the exact tag-bearing suffix.
+
+The alternate evidence path scans retained successful `workflow_dispatch` runs of `e2e-tag.yaml` and accepts one whose direct E2E job is named `E2E vX.Y.Z-rc.N (full suite)`. The workflow's run title is always `E2E Release Tag` and therefore cannot correlate the input tag; artifact names also carry the tag but expire independently, so the job name is the strongest available correlator. If Actions retention or manual deletion removes the run/job metadata, or that job is renamed without updating the matcher, the gate fails closed and tells the maintainer to run the E2E Release Tag button again. `skip_e2e_gate=true` is an emergency override, not alternate evidence: it writes a loud warning and a ⚠️ PR-body line.
+
+### Phase 2 — the changelog (generated at rc time or promotion, not here)
+
+The changelog for `vX.Y.Z` is produced by [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml)'s `changelog` job (a thin call to the reusable [`changelog-generate.yaml`](../../.github/workflows/changelog-generate.yaml)) and committed onto the `release-X.Y.Z` staging branch by `open-pr`, so it rides the promote PR and is on the base branch the moment the release publishes. Finalize then uses that file verbatim as the GitHub Release body. Generating it here — on the stable tag push — would be too late by construction: that tag is created by finalize, which has already published the release. It can also be produced earlier, at rc time, via the optional [`changelog-rc.yaml`](../../.github/workflows/changelog-rc.yaml) button — which runs the same reusable core and commits the file to the `release-X.Y.Z-rc.N` staging branch, from which the promote-time `changelog` job copies it (validated) instead of regenerating.
+
+The `changelog` job runs **in parallel** with `promote`, detached at the rc tag (the "promotion" configuration in §2 of [`agents/changelog.md`](../agents/changelog.md)), using a separate **read-only** GitHub App token for the AI step so the model cannot mutate the repo even with `--allow-all-tools`. Every write — the commit onto the staging branch and the PR — happens in `open-pr` under a write-scoped token.
+
+Generation is deliberately **non-blocking**: a Copilot outage or exhausted token quota must never block a release. Only the AI step carries `continue-on-error`, and `open-pr` is gated on `promote` alone (`if: !cancelled() && needs.promote.result == 'success'`) so the promote PR opens regardless. When the changelog is missing the PR body says so explicitly — that ⚠️ line is the signal to act on.
+
+Recovery, in order of preference:
+
+1. Commit `docs/changelogs/vX.Y.Z.md` to the `release-X.Y.Z` branch by hand (follow [`agents/changelog.md`](../agents/changelog.md)). `promote` preserves it across a re-dispatch, and `open-pr` will not overwrite it with generated output.
+2. Re-dispatch `promote-rc.yaml` — generation runs again; anything already on the branch wins.
+3. Merge without one and let `tags.yaml::generate-changelog` (the backstop) open a changelog PR against `main` after the fact. The release is published with the draft's body until that merges.
+
+Known failure modes:
+
+- Copilot quota exhausted (`COPILOT_GITHUB_TOKEN` 402). Refill or rotate, then re-dispatch.
+- AI step timeout (hard 30-min cap).
+- Output truncated or whitespace-only — caught by the `Verify changelog` step (`hack/validate-changelog.sh`), which accepts either header convention (`# Cozystack vX.Y.Z` for a minor or `# vX.Y.Z ()` for a patch), and requires the leading release-link comment pointing at `releases/tag/vX.Y.Z`, a `compare/...vX.Y.Z` link ending in this version, and at least one `## ` section — deliberately with no line-count floor, since a complete short patch changelog (v1.5.1 ships in 19 lines) must not be rejected as a fragment. It downgrades to the missing-changelog path rather than shipping a fragment as release notes.
+
+**`tags.yaml::generate-changelog` is the backstop.** It self-skips when the changelog is already on `main`. It earns its keep in two cases: generation failed during promotion, or the promote PR targeted `release-X.Y` so the file never reached `main`. Since the rc freeze the second case is **every** release, minor and patch alike — the promote PR always targets the release line (see [Regular Releases](./regular-releases.md#regular-releases)), which makes this backstop the only route by which a published changelog reaches `main`. In that case it **ports the reviewed file from the tag commit verbatim** rather than regenerating — a second AI pass there would spend quota and then overwrite already-published, already-reviewed release notes.
+
+### Phase 3 — `update-website-docs` (now the backstop)
+
+The `cozystack/website` "update managed apps reference" PR is normally opened **earlier, at promote time**, by [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml)'s `website-docs` job — not here. That job fetches the README content from the `release-X.Y.Z` staging branch `promote` just pushed (via the website Makefile's `FETCH_REF` variable), so the docs are ready to review alongside the promote PR while the stable tag is still unborn. It refreshes the trunk version pins from the staging ref before promoting `next/` → `vX.Y/`, so a new minor's docs inherit fresh Talos/cozystack pins instead of a stale `next.yaml`. It is **non-blocking**: if it fails (or trips its `FETCH_REF`-support guard against a website checkout that predates that Makefile variable), the promote PR opens with a ⚠️ website-docs line and this tag-time job backfills the same branch.
+
+**Merge-timing contract:** the website PR must NOT be merged before the release is published (finalize). Merging it early flips the site's latest-version pointer to a version whose versioned docs and `openapi.json` are not on GitHub yet, so every version-pin and API link for the new version 404s. Both the website PR body and the promote PR body say so; park the website PR until the GitHub release for `vX.Y.Z` is live, then merge.
+
+This tag-time `update-website-docs` job stays as the **backstop**. It runs with `if: !cancelled() && needs.prepare-release.result == 'success' && !contains(github.ref_name, '-')` (surviving a failed/skipped changelog phase, stopping on cancel, and skipping every pre-release tag), decides whether to promote `next/` → `vX.Y/`, runs `make update-all`, stages `content hugo.yaml data/versions`, and force-pushes the same `update-docs-vX.Y.Z` branch. On the normal path — where `promote-rc.yaml::website-docs` already did the work — it no-ops into "No changes to commit". At tag time the stable tag exists, so its `BRANCH=RELEASE_TAG` fetch resolves without needing `FETCH_REF`. For it to fire at all, finalize must push the stable tag with the **CI app token** (see [Phase 5](#phase-5--pull-requests-releaseyaml-finalize)) — a tag pushed with the default `GITHUB_TOKEN` triggers no workflow run.
+
+If anyone changes the website Makefile to write somewhere else (e.g. `static/`, `i18n/`), the `git add` list must grow in **both** jobs or the PR silently drops files.
+
+### Phase 4 — review and merge the promote PR
+
+Reviewer checklist for the `chore(release): promote vX.Y.Z-rc.N -> vX.Y.Z` PR:
+
+- [ ] The PR body says `✅ RC full e2e was verified`; if it says the gate was bypassed, obtain explicit maintainer acceptance or add `full-e2e` and wait for that opt-in run.
+- [ ] `Verify release packages candidate` is green. It pulls the candidate and original rc artifact by digest, compares the candidate to GitHub's prospective merge tree, and feeds the required `E2E Tests` status even when full E2E is not repeated. Do not edit `packages/` on this PR after generation; if the guard reports drift, re-dispatch promotion to create and pin a new run-specific candidate. Note what a green here does *not* cover: GitHub fires no `pull_request` event when the base branch advances, so a `packages/` change merged to the base afterwards is never re-verified on the PR. Finalize repeats the check against the real merge commit and catches that case before any stable name exists — but it catches it after the promote PR has already merged, so a `packages/` change landing on the base while a promote PR is open is worth holding back.
+- [ ] The PR body's **website-docs line** is present: if ✅, the `cozystack/website` docs PR is open and **parked** — do NOT merge it until `vX.Y.Z` is published (see [Phase 3](#phase-3--update-website-docs-now-the-backstop)); if ⚠️, the tag-time backstop will backfill it after finalize.
+- [ ] Diff is digest pins + image tags only — nothing else.
+- [ ] No accidental config drift (a value file you don't recognize).
+- [ ] If any chart bumped, check the new digest pulls — `crane manifest ghcr.io/cozystack/cozystack/@` should return.
+- [ ] Build artifacts on the draft release page exist and match the expected count.
+- [ ] The PR includes `docs/changelogs/vX.Y.Z.md`, it has content, and the entries match the commit range (see [Changelog pre-publish verification](./changelogs.md#changelog-pre-publish-verification)). If the PR body carries the ⚠️ "No changelog in this PR" line, resolve that before merging — finalize publishes this file as the release body, unless the PR also carries `docs/releasenotes/vX.Y.Z.md` (see [Phase 6](#phase-6--update-releasenotesyaml-sync-github-release-body)).
+
+### Phase 5 — `pull-requests-release.yaml` (finalize)
+
+Fires on merge of a PR that is merged, carries the `release` label, and is authored by `cozystack-ci[bot]` (the last guard closes the "name a branch `release-X.Y.Z`, get it labelled, merge it" hole). Head branch must match `release-X.Y.Z[-suffix]`. This is where **every consumer-visible or write-once side effect of a promotion happens** — after the rc's mandatory E2E was checked at dispatch and the PR merged, making promotion transactional at stable names. Steps:
+
+1. **Verify the packages candidate before creating any stable tag.** [`hack/verify-promoted-packages.sh`](../../hack/verify-promoted-packages.sh) pulls the candidate by the digest committed in installer values, proves it matches the merged `packages/` tree after normalizing only the impossible self-reference, rejects surviving rc image tags, then follows the candidate's embedded pre-push `platformSourceRef` back to the original rc artifact and compares normalized container `repo@digest` sets. Any package edit after candidate publication or any changed container digest aborts finalize while all stable names are still absent.
+2. **Create the tag at the merge commit** (write-once). The merge commit of `Prepare release vX.Y.Z` did not exist before the PR opened, so there is nothing to move — the tag is created here for the first time. A pre-existing tag at a different commit fails the step loudly rather than being force-moved (see [Tag immutability](./tag-immutability.md#tag-immutability)). The checkout uses `persist-credentials: false` so this push authenticates as the **CI app** (via `git remote set-url`), not the default `GITHUB_TOKEN` — a `GITHUB_TOKEN` push creates no workflow run, so a persisted-credential push would silently skip `tags.yaml` entirely and its stable-tag backstops (`generate-changelog`, `update-website-docs`) would never fire.
+3. **Cut the `api/apps/v1alpha1/vX.Y.Z` Go-module tag** (write-once, stable only) at the same commit, so Go consumers of `api/apps/v1alpha1` get the release. Moved here from `tags.yaml`, whose `prepare-release` body is skipped for a promoted stable (the draft already exists).
+4. **Ensure the maintenance branch `release-X.Y` exists** at the tag commit. Since the rc freeze creates it, this is normally a no-op fast-forward — the merge commit is already the branch tip. Still created if missing; updated fast-forward-only — a non-fast-forward update warns and is left for a maintainer rather than being force-updated.
+5. **Publish the draft release**, with `docs/releasenotes/vX.Y.Z.md`, or without it `docs/changelogs/vX.Y.Z.md`, from the merge commit as its body (absent or whitespace-only files warn and leave the draft's body in place, rather than publishing blank notes, and a body over 100000 bytes is published as a link, see [Phase 6](#phase-6--update-releasenotesyaml-sync-github-release-body)). `make_latest` is computed against published-non-prerelease tags: prereleases stay `false`; tags older than the current max stay `false` (and the current max is force-restored to `latest` if necessary, so an older patch tag cut after a newer minor won't downgrade `latest`).
+6. **Retag the verified packages candidate and rc container images to stable** by digest (`hack/promote-retag.sh`, no container rebuild) and **publish the stable `cozy-installer` chart**. `:latest` (on both the images and the chart) moves only when this release's `make_latest` was `true` — the same decision as step 5, so the release's `latest` and the images' `:latest` never disagree. The chart is packaged with `platformVersion` stamped into its default values so the documented `helm --install --version X.Y.Z` path reports the stable version.
+
+### Phase 6 — `update-releasenotes.yaml` (sync GitHub Release body)
+
+Fires on pushes to `main` that touch `docs/changelogs/v*.md`. Reads each `docs/changelogs/vX.Y.Z.md` and PATCHes the matching GitHub Release's `body` if it differs. Since Phase 5 now sets the body directly from the merge commit, this is no longer on the critical path — it covers later **edits** to a published changelog, the backstop PR landing a changelog that reached `main` after the fact, and manual re-syncs via `workflow_dispatch`. It only watches `main`, which is why finalize (not this job) is what populates a release promoted onto `release-X.Y` — since the freeze, that is every release rather than only a patch.
+
+The release body is `docs/releasenotes/vX.Y.Z.md` when it exists and is not blank, otherwise the changelog, which stays the full record either way, and a push to `main` touching either file fires this sync. Finalize sets the body after the stable tags exist, where a body GitHub refuses would strand the release, so at publish time a body over 100000 bytes is replaced by a one-line link to `docs/changelogs/vX.Y.Z.md` on the tag, with a warning in the run's log. This sync never writes that link: it warns and leaves such a release's body as it is. The `tags.yaml` backstop ports only the changelog to `main`, so release notes have to be added to `main` as well, otherwise the sync falls back to the changelog and replaces the release notes with it whenever the changelog fits.
diff --git a/docs/release/cleanup.md b/docs/release/cleanup.md
new file mode 100644
index 000000000..b192c5af5
--- /dev/null
+++ b/docs/release/cleanup.md
@@ -0,0 +1,6 @@
+## Cleanup after release
+
+- The `release-X.Y.Z` branch is deleted by GitHub when its PR merges. The `release-X.Y` maintenance branch is created by `cut-prerelease.yaml` at the rc freeze (create-only, never force-moved) and fast-forwarded afterwards by `pull-requests-release.yaml::Ensure maintenance branch`.
+- The draft release is published by the same workflow.
+- The release body is set by finalize from `docs/changelogs/vX.Y.Z.md` at publication. Afterwards, `update-releasenotes.yaml` re-syncs that file into the GitHub Release body on any push to `main` that touches it — so edits to a published changelog land on the release page the next time `main` moves.
+- Local cleanup: remove your worktree (`git worktree remove`) and prune merged release branches.
diff --git a/docs/release/introduction.md b/docs/release/introduction.md
new file mode 100644
index 000000000..78c0e1781
--- /dev/null
+++ b/docs/release/introduction.md
@@ -0,0 +1,11 @@
+## Introduction
+
+Cozystack uses a staged release process to ensure stability and flexibility during development.
+
+There are three types of releases:
+
+- **Release Candidates (RC)** – Preview versions (e.g., `v1.2.0-rc.1`) used for final testing and validation.
+- **Regular Releases** – Final versions (e.g., `v1.2.0`) that are feature-complete and thoroughly tested.
+- **Patch Releases** – Bugfix-only updates (e.g., `v1.2.1`) made after a stable release, based on a dedicated release branch.
+
+All three are matched by `tags.yaml`'s regex `^v\d+\.\d+\.\d+(-(alpha|beta|rc)\.\d+)?$`. The tag push is the trigger for the whole pipeline. **Pre-release tags (`-alpha` / `-beta` / `-rc`) are cut with the [`Cut Pre-release Tag`](../../.github/workflows/cut-prerelease.yaml) workflow** (manual dispatch from `main` or `release-X.Y`) — it pushes the tag as the CI app at the dispatch branch's tip, so GitHub fills `base_ref` (`tags.yaml`'s `Get base branch` step refuses tags pushed without a base) and the push triggers `tags.yaml` (a tag pushed with the default `GITHUB_TOKEN` would not). Stable `vX.Y.Z` tags are never pushed by hand at all — they are cut write-once by the promote flow at a PR merge commit. Only `-rc.N` pre-releases feed the promotion flow ([`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml) accepts `vX.Y.Z-rc.N` alone); `-alpha.N` / `-beta.N` tags are built and published for download and validated on demand with the [`e2e-tag.yaml`](../../.github/workflows/e2e-tag.yaml) button, but are never promoted to a stable release.
diff --git a/docs/release/nightly-builds.md b/docs/release/nightly-builds.md
new file mode 100644
index 000000000..df83de7f2
--- /dev/null
+++ b/docs/release/nightly-builds.md
@@ -0,0 +1,21 @@
+## Nightly builds
+
+A nightly is an **installable copy of `main` on GHCR — not a rebuild and not a release**. [`build-main.yaml`](../../.github/workflows/build-main.yaml) already builds every push to `main` into the CI registry (OCIR); the nightly promotes that build to the public release registry (GHCR) and proves it installs.
+
+[`nightly.yaml`](../../.github/workflows/nightly.yaml) runs daily (gated by the `NIGHTLY_ENABLED` repo variable) as three jobs; build-disk and e2e both start once mirror has finished:
+
+1. **mirror** — resolve the exact commit `cozystack-packages:main` was built from, then [`hack/nightly-mirror.sh`](../../hack/nightly-mirror.sh) copies every cozystack-owned component image OCIR→GHCR **by digest** (bit-for-bit, no rebuild) and re-publishes the rewritten `cozystack-packages` artifact and the `cozy-installer` chart to GHCR, tagged `0.0.0-nightly...` plus a floating `nightly`.
+2. **build-disk** — assemble the Talos `nocloud` disk from the upstream siderolabs imager (the profile references only `ghcr.io/siderolabs/*`, so this needs no cozystack image and no rebuild) and publish it as a GHCR OCI artifact (`cozystack-nocloud`) so a nightly is installable on real hardware. The nightly e2e does not boot it.
+3. **e2e** — stage the published GHCR closure (rewritten tree + the `cozy-installer` chart pinned to the GHCR packages artifact) and run the **full** app suite on Talos containers, which do not boot the disk. A nightly has no diff, so Test Impact Analysis does not apply.
+
+No GitHub release and no `api/apps/v1alpha1/*` Go-module tag are created for a nightly. [`retention.yaml`](../../.github/workflows/retention.yaml) prunes old GHCR nightly versions (keeps the newest 14 per package the current nightly publishes, named from the published `cozystack-packages:nightly` tree by [`hack/retention-packages.sh`](../../hack/retention-packages.sh), except packages linked to another repository, which are skipped with a warning; the floating `nightly`, release tags and untagged versions are never touched) and abandoned packages candidates. A candidate version is eligible only when all of these hold: it carries at least one tag and **every** tag on that manifest matches the strict `promotion-vX.Y.Z-from-vX.Y.Z-rc.N-run--` form, so a manifest that has been retagged stable is excluded; it was last updated more than 30 days ago; its digest is not pinned by `packages/core/installer/values.yaml` at the head of an **open** pull request that is same-repository, authored by `cozystack-ci[bot]`, on a `release-X.Y.Z` branch and carrying the `release` label; and its digest is not pinned by that same file at the tip of `main` or of any `release-X.Y` branch — which is what keeps a candidate safe between the promote PR merging and finalize retagging it. Any such PR head whose pin is not exactly one `digest=sha256:<64 hex>` stops the candidate sweep for that run and fails the job, without affecting the nightly sweep.
+
+Install a nightly:
+
+```bash
+helm upgrade --install cozystack \
+ oci://ghcr.io/cozystack/cozystack/cozy-installer --version 0.0.0-nightly.20260626.15992304129.1
+# ...or --version nightly for the latest
+```
+
+The matching Talos node image is `ghcr.io/cozystack/cozystack/cozystack-nocloud:0.0.0-nightly.20260626.15992304129.1` (pull with `oras`).
diff --git a/docs/release/patch-releases.md b/docs/release/patch-releases.md
new file mode 100644
index 000000000..9f23e23d6
--- /dev/null
+++ b/docs/release/patch-releases.md
@@ -0,0 +1,82 @@
+## Patch Releases
+
+A patch release now follows exactly the same shape as a [regular release](./regular-releases.md#regular-releases) — by the time `vX.Y.0` ships, everything is already happening on `release-X.Y`. The only difference is where the line starts:
+
+* `release-X.Y` already exists, created by the freeze at `vX.Y.0-rc.1`, so there is no branch to create.
+* Patch commits are cherry-picked onto it, and the rc is cut by dispatching **from `release-X.Y`** — the same dispatch used for `rc.2` of the original release.
+
+
+Let's assume that we've released `v1.2.0` and that development is ongoing. We have introduced a couple of new features and some fixes to features that we have released in `v1.2.0`.
+
+Once problems were found and fixed, a patch release is due.
+
+```mermaid
+gitGraph
+ commit id: "feature 3" tag: "v1.2.0-rc.1"
+ branch release-1.2
+ checkout release-1.2
+ commit id: "Release v1.2.0" tag: "v1.2.0"
+ checkout main
+ commit id: "feature 4"
+ commit id: "patch 1"
+ commit id: "feature 5"
+ commit id: "patch 2"
+```
+
+
+1. Fixes reach the existing `release-1.2` branch from `main` through the backport batch, which cherry-picks what the [backport policy](./backports.md) admits and nothing else.
+
+ The batch runs twice a week, so a fix merged into `main` reaches the line with the next batch that can pick it cleanly, without anyone asking for it.
+
+ ```mermaid
+ gitGraph
+ commit id: "feature 3" tag: "v1.2.0-rc.1"
+ branch release-1.2
+ checkout release-1.2
+ commit id: "Release v1.2.0" tag: "v1.2.0"
+ checkout main
+ commit id: "feature 4"
+ commit id: "patch 1"
+ commit id: "feature 5"
+ commit id: "patch 2"
+ checkout release-1.2
+ cherry-pick id: "patch 1"
+ cherry-pick id: "patch 2"
+ ```
+
+ Merge the open batch pull request first. Before the cut, `./backport-batch check --line release-1.2 --cut` (build it with `go build ./cmd/backport-batch`) has to exit 0: it exits 1 while a fix the rules take is not on the line, including one that is only on the open batch pull request, which the cut would not carry, and prints the URLs; it prints what the batch deferred as warnings. See [Cherry-pick triage before a patch](./backport-mechanics.md#cherry-pick-triage-before-a-patch).
+
+2. The maintainer cuts a release candidate (`v1.2.1-rc.N`) via the [`Cut Pre-release Tag`](../../.github/workflows/cut-prerelease.yaml) workflow (manual dispatch **from `release-1.2`**), which tags that branch's `HEAD`. CI builds and publishes the rc, pushes the `release-1.2.1-rc.N` staging branch, and runs the mandatory full `rc-e2e` job against the published tag.
+3. Watch `rc-e2e` and wait for it to pass, then complete the other release checks. A successful manual `e2e-tag.yaml` run for `v1.2.1-rc.N` is also valid evidence if the latest `tags.yaml` run does not carry a green `rc-e2e`. **Optional:** press the `Generate RC Changelog` button ([`changelog-rc.yaml`](../../.github/workflows/changelog-rc.yaml)) to generate `docs/changelogs/v1.2.1.md` at rc time and bank it on the rc staging branch; promotion then reuses it instead of regenerating.
+4. Once the checks are green, run [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml) (manual dispatch **from `main`**, as for every promotion, not from `release-1.2`) with that rc tag. It runs the same five jobs as a regular promotion: the dispatch first gates on the latest tag-push `rc-e2e` or a successful tag-correlated manual E2E run (`skip_e2e_gate` overrides it), verifies that `release-1.2` already carries the candidate-aware pipeline, then commits and publishes the rewritten packages tree under a run-specific temporary `promotion-*` tag, adds a second commit pinning the candidate on the `release-1.2.1` branch, drafts the `v1.2.1` release with the restamped assets, generates or picks up `docs/changelogs/v1.2.1.md` and commits it onto that branch, opens the parked `cozystack/website` docs PR (updating the existing `v1.2/` directory in place, since a patch does not promote `next/`), and opens the promote PR into `release-1.2` with only the `release` label. The promote PR body carries the same ✅/⚠️ E2E, changelog, and website-docs lines and runs no E2E by default; add `full-e2e` manually to force a repeat. As with a regular release every stable name is deferred to merge; retention protects the temporary candidate while this PR is open and, after it merges, while `release-1.2` still pins it — until finalize retags it stable.
+
+ ```mermaid
+ gitGraph
+ commit id: "feature 3" tag: "v1.2.0-rc.1"
+ branch release-1.2
+ checkout release-1.2
+ commit id: "Release v1.2.0" tag: "v1.2.0"
+ checkout main
+ commit id: "feature 4"
+ commit id: "patch 1"
+ commit id: "feature 5"
+ commit id: "patch 2"
+ checkout release-1.2
+ cherry-pick id: "patch 1"
+ cherry-pick id: "patch 2" tag: "v1.2.1-rc.1"
+ branch release-1.2.1
+ commit id: "Prepare release v1.2.1"
+ checkout release-1.2
+ merge release-1.2.1 id: "Pull request"
+ ```
+
+ Finally, when release is confirmed, the release sequence goes on.
+
+5. Maintainer reviews the PR against the [Phase 4 checklist](./ci-pipeline.md#phase-4--review-and-merge-the-promote-pr), confirms the E2E-verification and changelog lines and notes the parked website-docs line, and merges it — **do not squash-merge**. GitHub removes the merged branch `release-1.2.1` (auto-delete-on-merge is enabled).
+6. CI workflow triggers on merge (all consumer-visible or write-once side effects, after the rc E2E gate and PR review):
+ 1. Verifies the pinned packages candidate against the merged tree and the original rc artifact's container digests.
+ 2. Creates the tag `v1.2.1` at the newly created merge commit — write-once. The tag is published here for the first time, never moved.
+ 3. Cuts the write-once `api/apps/v1alpha1/v1.2.1` Go-module tag at the same commit.
+ 4. Publishes the release page (`draft` → `latest`; `latest` moves only if `v1.2.1` is the newest published stable).
+ 5. Retags the verified packages candidate and rc container images by digest to `v1.2.1` and publishes the stable `cozy-installer` chart — no container rebuild.
+7. Once the release is published, merge the parked `cozystack/website` docs PR, then announce the release to the community.
diff --git a/docs/release/pre-release-checklist.md b/docs/release/pre-release-checklist.md
new file mode 100644
index 000000000..8e6257304
--- /dev/null
+++ b/docs/release/pre-release-checklist.md
@@ -0,0 +1,14 @@
+## Pre-release verification checklist
+
+For RCs and final releases, run this before merging the release PR. Each item is here because something has shipped without it and broken users.
+
+- [ ] **The published rc passed mandatory full E2E.** Confirm the promote PR says `✅ RC full e2e was verified`; the normal promote PR itself runs no E2E. Add `full-e2e` manually only when a repeat is required, and treat a ⚠️ bypass line as an explicit release-risk decision.
+- [ ] **Upgrade from previous patch** (`vX.Y.(Z-1)`) on a real cluster. `helm upgrade` succeeds, all HelmReleases reach Ready, no CRD schema rejections.
+- [ ] **Upgrade from previous minor** (`v(X-1).Y.0`) for RC and minor releases. Same assertions.
+- [ ] **At least one tenant with `publishing.ingressName != tenant-root`** and a real Let's Encrypt-staging cert issuance.
+- [ ] **Apply-mutate-reapply** on the kubernetes umbrella HelmRelease (catches second-reconcile bugs).
+- [ ] **For every CRD-source bump** (kubevirt, cert-manager, flux, cilium, kamaji): `helm template | kubectl apply --dry-run=server` against the freshly-installed CRDs.
+- [ ] **For every flux / helm-controller / kubevirt-operator / cert-manager bump:** paste the upstream breaking-changes section of the bump's CHANGELOG into the PR body before merge.
+- [ ] **After cutting the tag,** watch `pull-requests-release.yaml::Finalize Release`. If it fails on `Draft release for v... not found`, undraft the release manually and file the workflow regression — this has been a known recurring failure since the `Publish draft release` step lost its explicit `github-token` in refactor `66a756b6`.
+- [ ] **The promote PR carries `docs/changelogs/vX.Y.Z.md` and it was verified** per [Changelog pre-publish verification](./changelogs.md#changelog-pre-publish-verification). Finalize publishes this file as the release body, so verifying it before merge is verifying the release notes.
+- [ ] **The `cozystack/website` docs PR stays parked until publish.** Promotion opens it early; merging it before the GitHub release for `vX.Y.Z` is live 404s the new version's docs and API links. Merge it right after finalize publishes the release (regular-release step 6, patch step 7).
diff --git a/docs/release/regular-releases.md b/docs/release/regular-releases.md
new file mode 100644
index 000000000..cdb007025
--- /dev/null
+++ b/docs/release/regular-releases.md
@@ -0,0 +1,78 @@
+## Regular Releases
+
+A regular release `vX.Y.0` is **promoted from a release-candidate that already passed e2e** — never tagged by hand and never rebuilt. The bytes shipped as `vX.Y.0` are the exact `vX.Y.0-rc.N` images, retagged by digest. We'll use `v1.2.0` as an example.
+
+```mermaid
+gitGraph
+ commit id: "feature"
+ commit id: "feature 2"
+ commit id: "feature 3" tag: "v1.2.0-rc.1"
+```
+
+A regular release sequence runs as follows:
+
+1. Cut the first release candidate (`v1.2.0-rc.1`) from the last good commit on `main`: run the [`Cut Pre-release Tag`](../../.github/workflows/cut-prerelease.yaml) workflow (manual dispatch **from `main`**) with the tag. It tags `main`'s HEAD as the CI app, then **creates `release-1.2` at that commit, freezing the line**; `tags.yaml` then builds the rc images, publishes the rc prerelease, pushes the digest-vendored `release-1.2.0-rc.N` staging branch, and runs its mandatory `rc-e2e` job by calling [`e2e-tag.yaml`](../../.github/workflows/e2e-tag.yaml) against that published tag.
+2. Watch the rc's mandatory full E2E and wait for it to pass, then complete the other checks in [Pre-release verification checklist](./pre-release-checklist.md#pre-release-verification-checklist). If the latest `tags.yaml` run cannot provide a green `rc-e2e`, a successful manual dispatch of [`e2e-tag.yaml`](../../.github/workflows/e2e-tag.yaml) for the same tag is equivalent gate evidence. `main` is already open for the next minor, so anything landing there is **not** in this release until it reaches `release-1.2`: a fix merged into `main` gets there through the [backport batch](./backports.md#which-lines-the-batch-runs-for), which runs for `release-1.2` from the freeze on, and a fix with no counterpart on `main` through a maintainer's pull request against `release-1.2`. A later rc is cut by dispatching the workflow **from `release-1.2`**, and only once `./backport-batch check --line release-1.2 --cut` exits 0: merge the open batch pull request first, so that the rc carries every fix the policy takes. **Optional:** once the notes are worth reviewing, press the `Generate RC Changelog` button ([`changelog-rc.yaml`](../../.github/workflows/changelog-rc.yaml)) to generate `docs/changelogs/v1.2.0.md` now and bank it on the `release-1.2.0-rc.N` staging branch — so there is time to review and hand-edit it before promotion. Skipping the button changes nothing; promotion generates the changelog itself.
+
+ ```mermaid
+ gitGraph
+ commit id: "feature"
+ commit id: "feature 2"
+ commit id: "feature 3" tag: "v1.2.0-rc.1"
+ branch release-1.2
+ checkout main
+ commit id: "feature 4 (next minor)"
+ checkout release-1.2
+ commit id: "backported fix" tag: "v1.2.0-rc.2"
+ ```
+
+3. Once the checks are green, run the [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml) workflow (manual dispatch **from `main`**, the only branch it is dispatched from, whichever line the rc belongs to) with that rc tag. Its `parse` job verifies E2E before any staging: it accepts a successful `rc-e2e` job in the latest `tags.yaml` push run for the tag, or any retained successful manual `e2e-tag.yaml` run whose expanded E2E job name targets the tag. The emergency `skip_e2e_gate` input bypasses this check, logs a loud warning, and marks the promote PR with ⚠️. Before its first registry write, promotion also proves that the selected `main` or `release-X.Y` target already contains the candidate-aware PR gate, finalizer and trusted verifier. A maintenance branch that lacks them gets them before its next patch is promoted, through the batch with `backport/include` on the pull request that brought them to `main`, or through a maintainer's pull request against the line. Promotion **creates no stable name before merge**: dispatch does write to the registry — it publishes one `promotion-*` packages candidate, and that tag is real and publicly listable — but it creates no `vX.Y.Z` tag, moves no `latest`, and publishes no stable installer, so nothing a consumer resolves changes until the PR merges. It:
+ 1. Rewrites the rc version substring to `1.2.0` and commits that exact tree, serializes its `packages/` directory under a run-specific temporary OCI tag, then adds a second commit pinning the candidate's immutable digest in the installer before pushing `release-1.2.0`. Container digests remain exactly the rc's; the two commits give the OCI revision an honest source SHA without pretending the artifact can contain its own digest.
+ 2. Drafts the `v1.2.0` release and uploads the (restamped) assets.
+ 3. Generates `docs/changelogs/v1.2.0.md` (in a parallel job, detached at the rc tag) — or, if you pressed the rc-time `Generate RC Changelog` button, copies the file already banked on the `release-1.2.0-rc.N` staging branch instead of regenerating — and commits it onto the `release-1.2.0` branch, so the changelog is reviewed as part of the promotion and finalize can publish the release with it already in place. Non-blocking: if generation fails the PR still opens, carrying a ⚠️ line that says the changelog is missing.
+ 4. Opens the `cozystack/website` "update managed apps reference" PR (`update-docs-v1.2.0` branch), generating the docs from the `release-1.2.0` staging branch via `FETCH_REF`. Non-blocking, and **do not merge that PR until `v1.2.0` is published** — merging early breaks the live site's version routing (see [Phase 3](./ci-pipeline.md#phase-3--update-website-docs-now-the-backstop)).
+ 5. Opens the `chore(release): promote v1.2.0-rc.N -> v1.2.0` PR into `release-1.2`, labelled `release` only. The PR body records whether E2E was verified or bypassed, plus ✅/⚠️ changelog and website-docs lines. E2E does not run on this promote PR by default because the immutable rc already passed the mandatory full suite; a maintainer may add `full-e2e` to run it again. The stable retag itself is deferred to the merge (step 5), so an abandoned promotion leaves no stable-named images and cannot wedge a re-promotion; retention protects the temporary candidate for as long as this release PR remains open, then may remove it after 30 days.
+
+ The PR targets the maintenance branch, not `main`: `promote-rc.yaml` prefers `release-X.Y` whenever it exists, and since the rc freeze it always does. The digest-vendored `Prepare release` commit therefore stays on the release line instead of landing on `main`.
+
+ ```mermaid
+ gitGraph
+ commit id: "feature 3" tag: "v1.2.0-rc.1"
+ branch release-1.2
+ checkout release-1.2
+ commit id: "backported fix" tag: "v1.2.0-rc.2"
+ branch release-1.2.0
+ checkout release-1.2.0
+ commit id: "Prepare release v1.2.0"
+ commit id: "Pin promoted packages artifact"
+ checkout release-1.2
+ merge release-1.2.0 id: "Pull Request"
+ ```
+
+4. Maintainer reviews the PR and the draft release against the [Phase 4 checklist](./ci-pipeline.md#phase-4--review-and-merge-the-promote-pr): confirm the E2E-verification line is ✅ (or explicitly accept its ⚠️ bypass), the changelog line is ✅, and note the website-docs line — the `cozystack/website` docs PR that promotion parked, which stays unmerged for now. E2E does not run on this promote PR by default. Then merge — **do not squash-merge**, the stable tag must attach to a real merge commit. GitHub removes the merged branch `release-1.2.0` (the repo has auto-delete-on-merge enabled).
+5. CI workflow triggers on merge (this is where every consumer-visible or write-once side effect happens, after the rc E2E gate and PR review):
+ 1. Pulls the pinned packages candidate by digest and verifies that it matches the merged `packages/` tree apart from the artifact's impossible self-reference, carries no rc image tags, and preserves the original rc artifact's container `repo@digest` set.
+ 2. Creates the tag `v1.2.0` at the newly created merge commit — write-once. The tag is published here for the first time, never moved.
+ 3. Cuts the write-once `api/apps/v1alpha1/v1.2.0` Go-module tag at the same commit.
+ 4. Ensures the `release-1.2` maintenance branch exists at the tag commit. Since the rc freeze created it, this is normally a no-op fast-forward: the merge commit is already its tip.
+ 5. Publishes the release page (`draft` → `latest`), using `docs/changelogs/v1.2.0.md` from the merge commit as the release body.
+ 6. Retags the verified packages candidate and rc container images by digest to `v1.2.0` (and `:latest` only when `v1.2.0` is the newest published stable), and publishes the stable `cozy-installer` chart — no container rebuild.
+6. Once the release is published, merge the parked `cozystack/website` docs PR (opened back at promote time, step 3.iv) so the live site's version routing flips to `v1.2.0` — its versioned docs and `openapi.json` are on GitHub now. The maintainer can then announce the release to the community.
+
+```mermaid
+gitGraph
+ commit id: "feature 2"
+ commit id: "feature 3" tag: "v1.2.0-rc.1"
+ branch release-1.2
+ checkout main
+ commit id: "feature 4 (next minor)"
+ checkout release-1.2
+ commit id: "backported fix" tag: "v1.2.0-rc.2"
+ branch release-1.2.0
+ checkout release-1.2.0
+ commit id: "Prepare release v1.2.0"
+ checkout release-1.2
+ merge release-1.2.0 id: "Release v1.2.0" tag: "v1.2.0"
+```
+
+Note that `main` never receives the release commit. It forked away at the freeze and carries the next minor's work.
diff --git a/docs/release/release-candidates.md b/docs/release/release-candidates.md
new file mode 100644
index 000000000..55f7eec61
--- /dev/null
+++ b/docs/release/release-candidates.md
@@ -0,0 +1,18 @@
+## Release Candidates
+
+Release candidates are Cozystack versions that introduce new features and are published before a stable release. Their purpose is to help validate stability before finalizing a new feature release.
+
+Release candidates are given numbers `vX.Y.0-rc.N`, for example, `v1.2.0-rc.1`. The first one is cut from `main`, once all major features for the upcoming release have merged and the release enters its testing phase.
+
+**Cutting the first rc freezes the line.** [`cut-prerelease.yaml`](../../.github/workflows/cut-prerelease.yaml) creates `release-X.Y` at the tagged commit, and from that point the release's content is closed:
+
+- Every later cut for that line — `rc.2`, `rc.3`, an `alpha`/`beta`, or a patch-line rc — must be dispatched from `release-X.Y`. A dispatch from `main` is refused.
+- Fixes reach the release only through the backport batch onto `release-X.Y`, or a maintainer's pull request against it (see [Backports](./backports.md)). The batch runs for the line from its next scheduled run on, [while it has not shipped](./backports.md#which-lines-the-batch-runs-for): it takes the fixes merged into `main` since the freeze, and an older line takes each of them only once this line's batch that carries it has merged.
+- A later rc is cut only once `backport-batch check --line release-X.Y --cut` exits 0, so that it carries every fix the batch takes: the open batch pull request merges first.
+- `main` reopens immediately for the next minor, so feature work never has to wait for the release to ship.
+
+This is what makes an rc mean something. Previously `release-X.Y` was created only when the stable release merged, so `rc.2` was cut from `main`'s tip and silently absorbed everything that had landed since `rc.1` — the shipped release could contain code that no rc had ever validated.
+
+**Un-freezing is manual and deliberate.** If an `rc.1` turns out to be unusable and the line has to be re-cut from `main`, delete `release-X.Y` by hand first; the refuse step blocks every other route back to a `main` dispatch, which is the point of it. Nothing automates the deletion, and nothing should — throwing away a frozen line is a decision worth making explicitly.
+
+Each RC contributes to a cumulative set of release notes that will be finalized when `vX.Y.0` is released. After testing, if no critical issues remain, the last good rc is **promoted** to the regular release `vX.Y.0` — the exact rc images retagged by digest, never rebuilt and never tagged from a later commit (see [Regular Releases](./regular-releases.md#regular-releases)). Promotion cuts the write-once `vX.Y.0` tag at the promote PR's merge commit, fast-forwards the `release-X.Y` maintenance branch the freeze already created, and opens the way for patch releases.
diff --git a/docs/release/release-time-fires.md b/docs/release/release-time-fires.md
new file mode 100644
index 000000000..f58fd48b7
--- /dev/null
+++ b/docs/release/release-time-fires.md
@@ -0,0 +1,35 @@
+## Release-time fires — what's gone wrong recently
+
+These are real regressions that escaped to users. The pattern in each: code passed CI, shipped in a stable release, broke customers. Read them as a working list of pre-release verification gaps.
+
+### v1.3.0 — cert-manager ingressClassName regression (fix #2562)
+
+**Bug:** A commit migrated ACME HTTP-01 to use the modern `acme.cert-manager.io/http01-ingress-ingressclassname` annotation. The matching ingress-shim code that reads that annotation was added by cert-manager upstream PR #8244 and was **never backported** to the `release-1.19` branch — but cozystack 1.3.0 shipped cert-manager v1.19.3. The annotation was silently dropped at runtime and every solver Ingress fell back to the ClusterIssuer's default class `tenant-root`. Let's Encrypt validation broke for every tenant whose `publishing.ingressName != tenant-root`.
+
+**Blast radius:** v1.3.0, v1.3.1, and v1.1.7 (the migration was cherry-picked to `release-1.1`).
+
+**Why CI missed it:** the migration commit was self-contained and `helm template` rendered cleanly. There was no E2E that creates a tenant with a non-default ingress class and watches a real Let's Encrypt cert issue. Verification had been done from the docs side ("the API has been available since cert-manager 1.12"), not against the shipped cert-manager source.
+
+**Would have caught it:** non-root-tenant E2E with LE-staging cert issuance, asserting the solver Ingress lands on the tenant's class.
+
+### v1.4.0 — kubevirt-instancetypes upgrade failure (fix #2612)
+
+**Bug:** `packages/system/kubevirt-instancetypes/Makefile` contained `sed -i '/persistent: true/d' templates/preferences.yaml`, which stripped the only child of `preferredTPM`, leaving `preferredTPM:` (null) in six Windows preferences. KubeVirt v1.6.x silently accepted it. The v1.6.3 → v1.8.2 operator bump (2026-04-27) hardened the CRD's OpenAPI schema and started rejecting null for an object-typed field. `helm upgrade` then failed on every cluster with `kubevirt-instancetypes` enabled.
+
+**Why CI missed it:** the kubevirt-operator bump and the inert `preferences.yaml` were two weeks apart in commit history. CI only tests fresh installs against the new CRDs; it doesn't run `helm upgrade` from prior versions. Schema regression on a vendored asset that hadn't been touched in 18 months.
+
+**Would have caught it:** `helm template | kubectl apply --dry-run=server -f -` against the freshly-installed CRDs of every bumped operator. Or an upgrade-from-N-1 E2E lane.
+
+### v1.4.0 — Flux 2.7 → 2.8 readiness deadlock (#2602)
+
+**Bug:** helm-controller v1.5.0 (shipped in Flux 2.8) changes the default wait strategy to kstatus polling, which polls **every applied resource including child HelmReleases**. The `packages/apps/kubernetes` umbrella has 19 child HRs that each `dependsOn: parent` (with a `lookup` guard so the dependency only activates after the parent exists). Pre-2.8: parent went Ready as soon as Helm returned. Post-2.8: parent waits for children, children wait for parent. Deadlock.
+
+**Why CI missed it:** the cycle only manifests on the second reconcile (parent exists, so the `lookup` activates the child's `dependsOn`). Fresh-install E2E dodges this — the parent-already-exists branch isn't taken on the first apply.
+
+**Would have caught it:** apply-mutate-reapply E2E (any chart bump in the kubernetes umbrella triggers a second reconcile). Plus an explicit PR-template requirement: for any flux/helm-controller bump, paste the upstream breaking-changes section into the PR body.
+
+### Patterns these share
+
+1. **Fresh-install CI is insufficient.** All three regressions passed fresh-install lanes.
+2. **Upstream breaking-defaults bumps are the common shape.** kubevirt CRD hardening, helm-controller default wait strategy, cert-manager API surface drift. Pre-merge reading of upstream changelogs would have caught two of three.
+3. **One-engineer-bitten-once is the only safety net for live verification.** The cert-manager bump PR included an explicit live-cluster verification step — because the same engineer had been bitten the prior week. There is no checklist requiring it.
diff --git a/docs/release/signatures-and-sbom.md b/docs/release/signatures-and-sbom.md
new file mode 100644
index 000000000..8aa47051e
--- /dev/null
+++ b/docs/release/signatures-and-sbom.md
@@ -0,0 +1,29 @@
+## Release signatures and SBOM
+
+A release cut from a tree that has [`hack/sign-release-assets.sh`](../../hack/sign-release-assets.sh) carries three files beside its other assets, which [`hack/upload-assets.sh`](../../hack/upload-assets.sh) has that script write before it uploads anything. Releases cut before it have none of them, and neither does a maintenance line until the script is backported to it (see below).
+
+- `cozystack-source-sbom.spdx.json` is an SPDX SBOM, made with syft, of the committed source tree (the Go modules, npm lockfiles, GitHub Actions references and other manifests syft reads in it) and of the `cozypkg` binaries in the `cozypkg-*.tar.gz` assets. It does not describe what runs in the cluster: the container images carry their own SBOMs, see [SBOM of the container images](#sbom-of-the-container-images), and the Talos installer image, ISO, disk, kernel and initramfs images have none. Its name says what it covers, so that `cozystack-sbom` stays free for an SBOM of the release as a whole.
+- `cozystack-checksums.txt` holds the SHA-256 of every other asset and of the SBOM.
+- `cozystack-checksums.txt.sigstore.json` is a Sigstore bundle with a keyless cosign signature over the checksums. Its certificate names the workflow that published the release: `tags.yaml` for a pre-release, `promote-rc.yaml` for a stable one.
+
+One signature covers every asset through the checksums. To check a download, verify the checksums, then the file against them, with cosign v3.1.3 or later (v2.6.5 or later on the v2 line):
+
+```sh
+cosign verify-blob cozystack-checksums.txt \
+ --bundle cozystack-checksums.txt.sigstore.json \
+ --certificate-identity-regexp '^https://github\.com/cozystack/cozystack/\.github/workflows/(tags\.yaml@refs/tags/v|promote-rc\.yaml@refs/heads/main$)' \
+ --certificate-oidc-issuer https://token.actions.githubusercontent.com &&
+ sha256sum --check --ignore-missing cozystack-checksums.txt
+```
+
+Older cosign accepts a legacy-format bundle that carries a bare public key in place of a certificate and skips the identity and issuer checks for it ([GHSA-fx35-mq7g-6g98](https://github.com/sigstore/cosign/security/advisories/GHSA-fx35-mq7g-6g98)), so a bundle swapped in next to the download, by a mirror or a proxy, passes there. The bundles this repository writes are in the new format, but cosign picks the format from the file, and `--new-bundle-format` does not stop that on the v3 line (v3.1.2 still accepts such a bundle with it), so the cosign version is what protects the check.
+
+The regexp pins the ref each workflow runs from: `tags.yaml` on a `v*` tag push, and `promote-rc.yaml` dispatched from `main`, the only branch it is dispatched from. A passing check proves that the checksums were signed by one of those runs in `cozystack/cozystack`. A `promote-rc.yaml` dispatched from another branch fails this check in its own upload step, with no asset uploaded, and an edited `promote-rc.yaml` has to be merged into `main` before it can sign. Until the `v*` [tag rulesets](./tag-immutability.md#repo-side-enforcement-tag-protection-rulesets) restrict tag creation to the CI app, though, anyone with write access can push a `v*` tag whose tree carries an edited `tags.yaml`, and its signature passes.
+
+The script runs the same `cosign verify-blob` before it returns, so a signature this check would reject fails the job before the first upload, and the release is not published with assets but without a signature. Both jobs that call `make upload_assets`, `tags.yaml::prepare-release` and `promote-rc.yaml::promote`, hold `id-token: write` for the signing identity and install syft and cosign v3.1.3, the version users are asked for above. They install syft from its release tarball, checked against a SHA-256 committed in the workflow, and not with `anchore/sbom-action/download-syft`, which runs an install script fetched at run time in a job that holds the release credentials; [`hack/ci-tool-installs-contract.bats`](../../hack/ci-tool-installs-contract.bats) keeps it that way. A stable release gets its own three files rather than its rc's: promotion restamps several assets, so the checksums are recomputed and signed again.
+
+`promote-rc.yaml` runs the `hack/upload-assets.sh` of the rc tree, not the dispatch ref's. A stable release is therefore signed only when its rc was tagged from a tree that has `hack/sign-release-assets.sh`, and a maintenance line starts carrying signatures once that script is backported to its `release-X.Y` branch.
+
+### SBOM of the container images
+
+The container images a release pins by digest each carry an SPDX SBOM of the packages in them, which BuildKit attaches at build time as an attestation (`SBOM=1` in `tags.yaml`, scanner and flags in [`hack/common-envs.mk`](../../hack/common-envs.mk)). The attestation is an entry of the image's index, so the digest the release pins names both, and promotion copies it with the image. Read it with `docker buildx imagetools inspect @ --format '{{ json .SBOM }}'`, which prints one document per platform. The attestations are not signed. A release cut before `SBOM=1` was set has none, and neither has the Talos installer image, which the imager builds and `skopeo` pushes, nor the third-party images `packages/` vendors by digest.
diff --git a/docs/release/splitting-bundle-prs.md b/docs/release/splitting-bundle-prs.md
new file mode 100644
index 000000000..ace989a88
--- /dev/null
+++ b/docs/release/splitting-bundle-prs.md
@@ -0,0 +1,10 @@
+## Splitting a release-blocking bundle PR
+
+Sometimes the work that has to land before a release is a 40-commit grab bag (CI stabilization, dependency bumps, chart fixes). Splitting it makes it reviewable and survivable. The strategy that has worked:
+
+1. **Verify what's already on `main`.** Use `git merge-base --is-ancestor` and subject grep to drop commits that have already landed.
+2. **Find existing open PRs that cover the same commits.** Force-push to the existing branch instead of opening a duplicate — preserves review threads.
+3. **Identify the "bottleneck" PR.** The one whose changes make previously-silent misconfigurations into hard errors (typically an operator-version bump). Everything else can fan out in parallel.
+4. **Squash iterative same-author bug-on-bug fixes** inside each split. Preserve authorship via `--author` on the squashed commit.
+5. **Mark the keystone PR as draft** and link in the body to the dependency PRs. Reviewers know it intentionally fails CI until the chain lands.
+6. **Beware of concurrent registry pushes.** When fanning out 5+ PR CI runs simultaneously, multiple `make image` jobs can collide on shared floating tags. The `IMAGE_TAG=pr--` refactor solved this; if you see `Conflicted with another upload of the same manifest`, `gh run rerun --failed`.
diff --git a/docs/release/stable-tags.md b/docs/release/stable-tags.md
new file mode 100644
index 000000000..af908d046
--- /dev/null
+++ b/docs/release/stable-tags.md
@@ -0,0 +1,7 @@
+## Stable tags come from rc promotion
+
+A stable `vX.Y.Z` is created only by **promoting an existing release-candidate** that already passed e2e — never by rebuilding and never by a cron.
+
+- [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml) is triggered manually once an rc has gone green. It rewrites the rc version substring in the vendored image tags (the container digests stay the rc's), commits and publishes the resulting declarative packages tree under a unique run-specific `promotion-*` tag, commits that digest pin, and opens the `release-X.Y.Z` staging PR. The PR gate verifies the candidate against the prospective merge and original rc artifact; finalize repeats that check before creating the write-once stable tag at the merge commit, then retags the verified packages artifact and rc's already-built, e2e-passed container digests to the stable tag **by digest** (no container rebuild — see [`hack/promote-retag.sh`](../../hack/promote-retag.sh)) and publishes the release. An abandoned promotion leaves no stable-named bytes; its temporary candidate is retention-managed.
+
+So a commit on a supported `release-X.Y` line ships when a maintainer promotes the next rc for that line — not automatically within 24h. Pushing a stable `vX.Y.Z` tag by hand is **not** a supported path: `tags.yaml` fails fast on a stable tag that has no pre-existing draft ("stable tags come from promote-rc.yaml"), and even so the finalize step would refuse to move a pre-existing tag. Pre-releases (`vX.Y.Z-rc.N`, `-alpha.N`, `-beta.N`) are cut with the [`Cut Pre-release Tag`](../../.github/workflows/cut-prerelease.yaml) workflow, which pushes the tag as the CI app; [`tags.yaml`](../../.github/workflows/tags.yaml) then fires on that push, builds it, and publishes the pre-release. Cutting via the workflow (rather than a manual `git push`) is what lets repo admins lock `v*` tag creation to the CI app — see the tag-protection note below.
diff --git a/docs/release/tag-immutability.md b/docs/release/tag-immutability.md
new file mode 100644
index 000000000..fe7418279
--- /dev/null
+++ b/docs/release/tag-immutability.md
@@ -0,0 +1,18 @@
+## Tag immutability
+
+Published tags are **write-once** — once a `vX.Y.Z` or rc tag is pushed it is never moved or deleted. (Nightlies are not git tags at all — they are GHCR OCI tags, see [Nightly builds](./nightly-builds.md#nightly-builds).) Moving a tag silently poisons the Go module proxy / pkg.go.dev cache for `api/apps/v1alpha1/vX.Y.Z` and drifts SBOM/provenance toolchains, so the release flow is built so a move is impossible by construction:
+
+| File | Tag / branch handling |
+|------|------------------------|
+| [`pull-requests-release.yaml`](../../.github/workflows/pull-requests-release.yaml) | Verifies the pinned packages candidate before creating a stable name. Creates `vX.Y.Z` at the PR merge commit **write-once** (create if absent, no-op if unchanged, fail loudly if it would move — the merge commit is new, so there is nothing to move). Cuts the `api/apps/v1alpha1/` Go-submodule tag write-once at the same commit, **stable only** (never rc/beta/alpha). The `release-X.Y` maintenance branch is fast-forward-only. Retags the verified packages artifact and rc container digests to the stable image tag (also write-once at the image level). |
+| [`tags.yaml`](../../.github/workflows/tags.yaml) | The `release-X.Y.Z-rc.N` staging branch is a mutable staging ref (compare-before-force: skipped when unchanged, force+log only when it genuinely moves). Fails fast on a stable `vX.Y.Z` tag with no pre-existing draft — stable tags come from `promote-rc.yaml`, never a hand push. |
+| [`promote-rc.yaml`](../../.github/workflows/promote-rc.yaml) | Mutates no stable tags. Before staging, verifies green full E2E from the latest rc tag-push run or a tag-correlated manual published-tag run (unless loudly bypassed). Commits and publishes the rewritten packages tree under a run-specific temporary `promotion-*` tag, adds a second commit pinning its digest on the `release-X.Y.Z` branch, generates the changelog there, and opens the promote PR with `release` but no automatic `full-e2e`; the PR gate verifies the candidate and finalize repeats that verification before the stable tag and image retag. The staging branch is force-pushed on re-dispatch, but an existing `docs/changelogs/vX.Y.Z.md` is preserved across the rebuild. |
+
+This is the immutable-tag + rc-promotion model from [#2677](https://github.com/cozystack/cozystack/issues/2677): stable `vX.Y.Z` is created only by promoting an existing rc, rc tags are write-once, and `api/apps/v1alpha1/vX.Y.Z` is created only on a stable release. The old nightly `auto-release.yaml` (which delete-recreated auto-bumped patch tags) has been removed. rc and stable tags accrete permanently; the only churning artifacts are GHCR nightlies and abandoned temporary promotion candidates, which [`retention.yaml`](../../.github/workflows/retention.yaml) prunes (see [Nightly builds](./nightly-builds.md#nightly-builds)) — GHCR storage is the cost vector to watch.
+
+### Repo-side enforcement (tag-protection rulesets)
+
+The guarantees above are enforced in the workflows. Repo admins back them with two GitHub **tag rulesets** targeting `v*`:
+
+- **Immutability** — *Restrict updates* + *Restrict deletions*, **no bypass**. No actor (not even the CI app) can move or delete a shipped `v*` tag. Safe to enable at any time: nothing in the release flow ever moves or deletes a `v*` tag, it only creates new ones.
+- **Creation control** — *Restrict creations* with the **CI app as the sole bypass actor**, plus *Limit branches/tags updated in a single push* = 1. This turns "don't hand-push tags" from convention into hard enforcement: every `v*` tag — pre-release **and** stable — can then be created only by CI. Enable this **only after** [`cut-prerelease.yaml`](../../.github/workflows/cut-prerelease.yaml) is in use, since that workflow is what makes CI the sole creator of pre-release tags; enabling it earlier would lock maintainers out of cutting rc/alpha/beta. (The Go-module tag `api/apps/v1alpha1/*` gets an equivalent pair of rulesets.)
diff --git a/hack/ci-tool-installs-contract.bats b/hack/ci-tool-installs-contract.bats
index 009911249..739f72394 100755
--- a/hack/ci-tool-installs-contract.bats
+++ b/hack/ci-tool-installs-contract.bats
@@ -111,8 +111,9 @@ one_value_of() {
@test "every use of the cosign installer is SHA-pinned at one cosign release" {
# Without cosign-release the action installs its own default, which moves
# with a digest bump of the action. The release jobs verify their signature
- # with the cosign docs/release.md asks users for, so the version is named in
- # each step, and all steps agree on it and on the action's commit.
+ # with the cosign docs/release/signatures-and-sbom.md asks users for, so the
+ # version is named in each step, and all steps agree on it and on the
+ # action's commit.
uses="$(workflow_code 'uses: sigstore/cosign-installer@')"
[ -n "$uses" ]
bad="$(printf '%s\n' "$uses" | grep --invert-match --extended-regexp 'installer@[0-9a-f]{40} # v[0-9.]+$' || true)"
diff --git a/hack/release-changelog-contract.bats b/hack/release-changelog-contract.bats
index 44d0925e9..5938b77f2 100644
--- a/hack/release-changelog-contract.bats
+++ b/hack/release-changelog-contract.bats
@@ -570,10 +570,11 @@ code_lines() { grep -vE '^[[:space:]]*#'; }
exit 1
}
- # changelog.md sends the agent to run the docs/release.md checks and the
- # validator on its own output, by their working-tree paths. Without these two
- # copies it runs the rc tag's versions, which predate every check added since.
- for f in docs/release.md hack/validate-changelog.sh; do
+ # changelog.md sends the agent to run the docs/release/changelogs.md checks
+ # and the validator on its own output, by their working-tree paths. Without
+ # these two copies it runs the rc tag's versions, which predate every check
+ # added since.
+ for f in docs/release/changelogs.md hack/validate-changelog.sh; do
printf '%s\n' "$overlay" | code_lines | grep -qF "cp .release-tooling/$f $f" || {
echo "The refresh step does not overlay $f from the tooling checkout, so the" >&2
echo "agent following docs/agents/changelog.md section 9 runs the rc tag's copy." >&2
@@ -581,23 +582,26 @@ code_lines() { grep -vE '^[[:space:]]*#'; }
}
done
- # docs/release.md sits under neither sparse-checkout path. It reaches
- # .release-tooling only because cone mode also checks out the files of every
- # parent directory of a listed one; without it the copy above fails the job.
+ # An rc tagged before the release docs were split has no docs/release/
+ # directory, and cp does not create one.
+ printf '%s\n' "$overlay" | code_lines | grep -qE '^[[:space:]]+mkdir -p docs/release$' || {
+ echo "The refresh step does not create docs/release before copying into it, so" >&2
+ echo "the copy fails on every rc tagged before the release docs were split." >&2
+ exit 1
+ }
+
+ # The tooling checkout has to list both directories the copies read from.
tooling="$(printf '%s\n' "$block" | awk '
/^ - name: Checkout release tooling from the dispatch ref$/ { inside = 1; next }
/^ - name: / { inside = 0 }
inside' | code_lines)"
- printf '%s\n' "$tooling" | grep -qE '^[[:space:]]+docs/agents$' || {
- echo "The tooling checkout no longer lists docs/agents, which is what carries" >&2
- echo "docs/release.md into .release-tooling." >&2
- exit 1
- }
- if printf '%s\n' "$tooling" | grep -qE 'sparse-checkout-cone-mode:[[:space:]]*false'; then
- echo "The tooling checkout turned cone mode off, which drops docs/release.md" >&2
- echo "from .release-tooling and fails the refresh step's copy." >&2
- exit 1
- fi
+ for d in docs/agents docs/release; do
+ printf '%s\n' "$tooling" | grep -qE "^[[:space:]]+$d\$" || {
+ echo "The tooling checkout no longer lists $d, so .release-tooling lacks the" >&2
+ echo "file the refresh step copies from it." >&2
+ exit 1
+ }
+ done
# Every job that runs release tooling against an older checkout must source it
# from the dispatch ref. A script fetched by a script that is itself missing
diff --git a/hack/sign-release-assets.bats b/hack/sign-release-assets.bats
index 9cfd25aca..0dfa06763 100644
--- a/hack/sign-release-assets.bats
+++ b/hack/sign-release-assets.bats
@@ -161,13 +161,13 @@ cleanup() {
cleanup
}
-@test "docs/release.md tells users to verify the identity the script verifies" {
+@test "docs/release/signatures-and-sbom.md tells users to verify the identity the script verifies" {
regexp="$(sed -n "s/^IDENTITY_REGEXP='\(.*\)'\$/\1/p" "$SCRIPT")"
if [ -z "$regexp" ]; then
echo "IDENTITY_REGEXP not found in $SCRIPT" >&2
exit 1
fi
- grep -qF -- "--certificate-identity-regexp '$regexp'" "$REPO_ROOT/docs/release.md"
+ grep -qF -- "--certificate-identity-regexp '$regexp'" "$REPO_ROOT/docs/release/signatures-and-sbom.md"
}
@test "fails when the signature does not verify" {
diff --git a/hack/sign-release-assets.sh b/hack/sign-release-assets.sh
index e8120feed..6549ee4f9 100755
--- a/hack/sign-release-assets.sh
+++ b/hack/sign-release-assets.sh
@@ -26,8 +26,9 @@
# copies along with its other assets, and a stable release gets its own.
#
# The bundle is verified before the script returns, with the same certificate
-# identity users are told to check in docs/release.md, so a release whose
-# signature that check would reject fails here instead of being uploaded.
+# identity users are told to check in docs/release/signatures-and-sbom.md, so a
+# release whose signature that check would reject fails here instead of being
+# uploaded.
#
# Needs syft and cosign on PATH, and `id-token: write` in the job.
set -euo pipefail
@@ -39,7 +40,7 @@ BUNDLE=cozystack-checksums.txt.sigstore.json
# The workflows that publish releases, at the refs they run from: tags.yaml for a
# pre-release, on a v* tag push, and promote-rc.yaml for a stable one, dispatched
# from main only. A run from any other ref signs, but fails the check below.
-# docs/release.md shows users the same pattern.
+# docs/release/signatures-and-sbom.md shows users the same pattern.
IDENTITY_REGEXP='^https://github\.com/cozystack/cozystack/\.github/workflows/(tags\.yaml@refs/tags/v|promote-rc\.yaml@refs/heads/main$)'
OIDC_ISSUER=https://token.actions.githubusercontent.com
From 2cc90b18f51b77f6c5a1324efcbfef8b34e4b305 Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:03:31 +0300
Subject: [PATCH 063/346] fix(seaweedfs): serve the s3 internal gRPC over mTLS
weed s3 takes its gRPC server identity from [grpc.s3] in security.toml
and serves plaintext when the section is missing, which it is in the
4.45 chart. Every peer dials that port with mTLS: the s3_lifecycle
worker with its [grpc.worker] certificate, other s3 replicas with
[grpc.client] for IAM cache propagation. Both fail the handshake, so
lifecycle deletes cannot run and identity changes do not reach the
other replicas.
The section is backported from upstream (seaweedfs/seaweedfs#11157,
first released in 4.46) as a chart patch. It uses the internal
CA-signed client certificate the s3 pods already mount, not the public
listener's tlsSecret, which internal peers would not trust.
security.toml is mounted by subPath, so a running s3 pod never sees the
new section. A pod annotation changes the s3 template in this release
so the upgrade restarts it.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
packages/system/seaweedfs/Makefile | 1 +
.../templates/shared/security-configmap.yaml | 13 ++++++++
.../seaweedfs/patches/s3-grpc-tls.patch | 24 ++++++++++++++
.../seaweedfs/tests/s3_grpc_tls_test.yaml | 32 +++++++++++++++++++
packages/system/seaweedfs/values.yaml | 8 +++++
5 files changed, 78 insertions(+)
create mode 100644 packages/system/seaweedfs/patches/s3-grpc-tls.patch
create mode 100644 packages/system/seaweedfs/tests/s3_grpc_tls_test.yaml
diff --git a/packages/system/seaweedfs/Makefile b/packages/system/seaweedfs/Makefile
index 3436e5f0f..2f7790b38 100644
--- a/packages/system/seaweedfs/Makefile
+++ b/packages/system/seaweedfs/Makefile
@@ -61,6 +61,7 @@ update:
patch --no-backup-if-mismatch -p4 < patches/s3-service-name.patch
patch --no-backup-if-mismatch -p4 < patches/s3-service-name-consumers.patch
patch --no-backup-if-mismatch -p4 < patches/cluster-scoped-names-per-namespace.patch
+ patch --no-backup-if-mismatch -p4 < patches/s3-grpc-tls.patch
#patch --no-backup-if-mismatch -p4 < patches/retention-policy-delete.yaml
$(MAKE) update-image
diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml
index a067e1de9..17b82cca0 100644
--- a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml
+++ b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml
@@ -97,6 +97,19 @@ data:
cert = "/usr/local/share/ca-certificates/worker/tls.crt"
key = "/usr/local/share/ca-certificates/worker/tls.key"
+ {{- /* S3 gRPC server identity (weed s3 -port.grpc, default httpPort+10000).
+ Without this section, LoadServerTLS("grpc.s3") returns nil and the S3
+ server serves its internal gRPC (IAM cache propagation, s3 lifecycle
+ delete RPCs) as plaintext, while peers dial it with mTLS credentials:
+ "tls: first record does not look like a TLS handshake".
+ Always uses the internal CA-signed client certificate (already mounted
+ on s3 pods by s3-deployment.yaml). Deliberately NOT s3.tlsSecret: that
+ secret is for the public HTTPS listener and may be issued by a public
+ CA which internal gRPC peers (trusting only grpc.ca) would reject. */}}
+ [grpc.s3]
+ cert = "/usr/local/share/ca-certificates/client/tls.crt"
+ key = "/usr/local/share/ca-certificates/client/tls.key"
+
# use this for any place needs a grpc client
# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
[grpc.client]
diff --git a/packages/system/seaweedfs/patches/s3-grpc-tls.patch b/packages/system/seaweedfs/patches/s3-grpc-tls.patch
new file mode 100644
index 000000000..d4d45fbd0
--- /dev/null
+++ b/packages/system/seaweedfs/patches/s3-grpc-tls.patch
@@ -0,0 +1,24 @@
+diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml
+index a067e1de9..17b82cca0 100644
+--- a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml
++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/security-configmap.yaml
+@@ -97,6 +97,19 @@ data:
+ cert = "/usr/local/share/ca-certificates/worker/tls.crt"
+ key = "/usr/local/share/ca-certificates/worker/tls.key"
+
++ {{- /* S3 gRPC server identity (weed s3 -port.grpc, default httpPort+10000).
++ Without this section, LoadServerTLS("grpc.s3") returns nil and the S3
++ server serves its internal gRPC (IAM cache propagation, s3 lifecycle
++ delete RPCs) as plaintext, while peers dial it with mTLS credentials:
++ "tls: first record does not look like a TLS handshake".
++ Always uses the internal CA-signed client certificate (already mounted
++ on s3 pods by s3-deployment.yaml). Deliberately NOT s3.tlsSecret: that
++ secret is for the public HTTPS listener and may be issued by a public
++ CA which internal gRPC peers (trusting only grpc.ca) would reject. */}}
++ [grpc.s3]
++ cert = "/usr/local/share/ca-certificates/client/tls.crt"
++ key = "/usr/local/share/ca-certificates/client/tls.key"
++
+ # use this for any place needs a grpc client
+ # i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
+ [grpc.client]
diff --git a/packages/system/seaweedfs/tests/s3_grpc_tls_test.yaml b/packages/system/seaweedfs/tests/s3_grpc_tls_test.yaml
new file mode 100644
index 000000000..afac836ed
--- /dev/null
+++ b/packages/system/seaweedfs/tests/s3_grpc_tls_test.yaml
@@ -0,0 +1,32 @@
+suite: seaweedfs s3 gRPC listener speaks mTLS
+
+templates:
+ - charts/seaweedfs/templates/shared/security-configmap.yaml
+ - charts/seaweedfs/templates/s3/s3-deployment.yaml
+
+release:
+ name: seaweedfs-system
+ namespace: tenant-root
+
+tests:
+ # weed s3 serves its internal gRPC (the LifecycleDelete RPC among it) with
+ # the [grpc.s3] identity and as plaintext without one, while the worker dials
+ # it with [grpc.worker] mTLS credentials, so every delete failed the TLS
+ # handshake. patches/s3-grpc-tls.patch backports the section from upstream.
+ - it: gives the s3 gRPC server a TLS identity signed by the internal CA
+ template: charts/seaweedfs/templates/shared/security-configmap.yaml
+ asserts:
+ - matchRegex:
+ path: data["security.toml"]
+ pattern: '\[grpc\.s3\]\s+cert = "/usr/local/share/ca-certificates/client/tls\.crt"\s+key = "/usr/local/share/ca-certificates/client/tls\.key"'
+
+ # security.toml reaches the s3 pods through a subPath mount, which a running
+ # container never re-reads, so an upgrade that only changes the ConfigMap
+ # leaves s3 serving plaintext gRPC until something restarts it. The pod
+ # template has to change in the same release.
+ - it: rolls the s3 pods so they load the new section
+ template: charts/seaweedfs/templates/s3/s3-deployment.yaml
+ asserts:
+ - equal:
+ path: spec.template.metadata.annotations["seaweedfs.cozystack.io/grpc-s3-tls"]
+ value: "true"
diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml
index bce78c47f..e1eba8a58 100644
--- a/packages/system/seaweedfs/values.yaml
+++ b/packages/system/seaweedfs/values.yaml
@@ -207,6 +207,14 @@ seaweedfs:
# weed skip startLanceServer and drops the container port and Service port
# with it.
lancePort: 0
+ # patches/s3-grpc-tls.patch adds [grpc.s3] to security.toml, which the s3
+ # pods mount by subPath and so never re-read. This annotation changes the
+ # pod template in the release that brings the section, so an upgrade
+ # restarts s3 onto it instead of leaving its gRPC in plaintext. The
+ # upstream s3 template carries no checksum of that ConfigMap, and adding
+ # one would be another vendored patch for a change made once.
+ podAnnotations:
+ seaweedfs.cozystack.io/grpc-s3-tls: "true"
# Keep S3 traffic on same-zone endpoints. Native chart support landed in 4.12
# (replaces the former s3-traffic-distribution.patch); the chart gates the
# field on Kubernetes >=1.31 and auto-converts to PreferSameZone on >=1.35.
From cd91138f3327a4886c72b9e8a35b4e54c06abad7 Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:13:55 +0300
Subject: [PATCH 064/346] fix(backups): re-render the default backup objects
when the S3 endpoint changes
The S3 endpoint the platform backups use is copied twice and neither
copy is refreshed. The COSI sidecar writes BucketInfo once per
BucketAccess, so the credentials Secret rendered from it keeps its
endpoint, and the Velero BSL and the default Strategy CRs keep whatever
endpoint the last render of backupstrategy-controller saw. The default
objects gate only acted on objects that were missing, so after a
publishing.host change every backup kept going to the old host with
nothing reporting it.
The gate now also forces a render when a copy is stale: the bucket's
-system release when the credentials Secret's endpoint differs
from the COSI Secret it is rendered from, and backupstrategy-controller
when the endpoint an object was rendered with differs from the
credentials Secret's. That endpoint sits at a different path in every
kind, in some only as a bare host, so the chart stamps the rendered
value on each object as backups.cozystack.io/endpoint and the gate
compares that. Objects rendered before the annotation existed count as
stale until one forced render has gone through, which stamps it.
The COSI Secret itself still holds the endpoint the BucketAccess was
granted with; it follows a host change only once the BucketAccess is
re-granted.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
cmd/backupstrategy-controller/main.go | 1 +
hack/backupstrategy-crd-gate.bats | 36 ++
.../backupcontroller/default_objects_gate.go | 163 ++++++-
.../default_objects_gate_test.go | 401 +++++++++++++++++-
.../templates/_helpers.tpl | 7 +-
.../templates/deployment.yaml | 7 +
.../templates/strategy-altinity-default.yaml | 2 +
.../templates/strategy-cnpg-default.yaml | 2 +
.../templates/strategy-etcd-default.yaml | 2 +
.../strategy-foundationdb-default.yaml | 2 +
.../templates/strategy-kafka-default.yaml | 2 +
.../templates/strategy-mariadb-default.yaml | 2 +
.../templates/strategy-mongodb-default.yaml | 2 +
.../templates/strategy-rabbitmq-default.yaml | 2 +
.../templates/strategy-redis-default.yaml | 2 +
.../strategy-velero-vmdisk-default.yaml | 2 +
.../strategy-velero-vminstance-default.yaml | 2 +
.../templates/velero-bsl.yaml | 2 +
.../tests/default_objects_gate_test.yaml | 60 +++
.../backupstrategy-controller/values.yaml | 11 +-
.../bucket/templates/user-credentials.yaml | 4 +-
21 files changed, 678 insertions(+), 36 deletions(-)
diff --git a/cmd/backupstrategy-controller/main.go b/cmd/backupstrategy-controller/main.go
index 625d79820..119ae3007 100644
--- a/cmd/backupstrategy-controller/main.go
+++ b/cmd/backupstrategy-controller/main.go
@@ -243,6 +243,7 @@ func main() {
Namespace: os.Getenv("BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE"),
Name: os.Getenv("BACKUP_CREDENTIALS_HELMRELEASE_NAME"),
},
+ CredentialsCOSISecretName: os.Getenv("BACKUP_CREDENTIALS_COSI_SECRET_NAME"),
}
if err := gate.SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to add DefaultObjectsGate runnable")
diff --git a/hack/backupstrategy-crd-gate.bats b/hack/backupstrategy-crd-gate.bats
index b4ad482a0..0f68a1ca4 100644
--- a/hack/backupstrategy-crd-gate.bats
+++ b/hack/backupstrategy-crd-gate.bats
@@ -133,3 +133,39 @@ $tpl: $obj does not gate on crdEstablished for $crd"
rm -rf "$tmp"
}
+
+# The controller's DefaultObjectsGate tells an object rendered with a stale S3
+# endpoint (publishing.host changed after the render) by the
+# backups.cozystack.io/endpoint annotation, because the endpoint itself sits at
+# a different path in every kind. An object the chart renders without it is
+# forced once and then never compared again, so a new template that forgets it
+# would silently keep a stale endpoint.
+@test "every rendered Strategy CR and the Velero BSL carry the rendered endpoint annotation" {
+ tmp=$(mktemp -d)
+
+ helm template t "$CHART" -n cozy-backup-controller \
+ --set backupStorage.bucketNameOverride=ext-bucket \
+ --set backupStorage.endpoint=https://s3.example.org \
+ --output-dir "$tmp/open" >/dev/null
+
+ objs=$(find "$tmp/open" -name '*.yaml' | sort | while read -r f; do
+ yq eval-all "select((.apiVersion // \"\" | test(\"^${STRATEGY_API}\")) or .kind == \"BackupStorageLocation\") | .kind + \"/\" + .metadata.name + \" \" + (.metadata.annotations[\"backups.cozystack.io/endpoint\"] // \"\")" "$f" \
+ | sed "s#^#$(basename "$f") #"
+ done)
+ if ! echo "$objs" | grep -q " BackupStorageLocation/"; then
+ echo "render produced no BSL; nothing was checked for it" >&2
+ false
+ fi
+ if ! echo "$objs" | grep -q " CNPG/"; then
+ echo "render produced no Strategy CR; nothing was checked" >&2
+ false
+ fi
+ bad=$(echo "$objs" | awk '$3 != "https://s3.example.org"')
+ if [ -n "$bad" ]; then
+ echo "objects without the rendered endpoint in backups.cozystack.io/endpoint:" >&2
+ echo "$bad" >&2
+ false
+ fi
+
+ rm -rf "$tmp"
+}
diff --git a/internal/backupcontroller/default_objects_gate.go b/internal/backupcontroller/default_objects_gate.go
index 023682036..efcd970d5 100644
--- a/internal/backupcontroller/default_objects_gate.go
+++ b/internal/backupcontroller/default_objects_gate.go
@@ -2,10 +2,12 @@ package backupcontroller
import (
"context"
+ "encoding/json"
"errors"
"fmt"
"net/http"
"sort"
+ "strings"
"time"
"github.com/go-logr/logr"
@@ -91,6 +93,11 @@ const (
// same lookup-gated template as the Strategy CRs, so it belongs to the
// same existence check.
defaultBackupStorageLocationName = "cozy-default"
+
+ // endpointAnnotation carries the S3 endpoint the chart rendered into the
+ // object. The endpoint itself sits at a different path in every kind,
+ // and in some only as a bare host, so the gate compares this instead.
+ endpointAnnotation = "backups.cozystack.io/endpoint"
)
var backupStorageLocationGVR = schema.GroupVersionResource{
@@ -157,6 +164,14 @@ type DefaultObjectsGate struct {
// then admin-managed and no release renders it, so there is nothing
// to force.
CredentialsHelmRelease types.NamespacedName
+ // CredentialsCOSISecretName is the Secret the COSI sidecar writes for the
+ // platform bucket's BucketAccess, in Config.SourceNamespace, from which
+ // CredentialsHelmRelease renders the credentials Secret. The sidecar
+ // writes it once per BucketAccess, so after a publishing.host change a
+ // re-granted BucketAccess carries the new endpoint while the credentials
+ // Secret keeps the old one; the gate compares the two. Empty disables
+ // the comparison.
+ CredentialsCOSISecretName string
// VeleroNamespace is where the cozy-default BackupStorageLocation
// lives. Empty skips the BSL check (velero.bslEnabled=false).
VeleroNamespace string
@@ -177,6 +192,11 @@ type DefaultObjectsGate struct {
// MinForceInterval for no reason.
lastForce time.Time
lastCredentialsForce time.Time
+ // renderForced records that a forced render of HelmRelease went through.
+ // That render stamps endpointAnnotation on every object the chart
+ // renders, so from then on an object without it (one rendered by a chart
+ // that predates the annotation) is not counted as stale again.
+ renderForced bool
}
// errHelmReleaseSuspended reports that the target release has
@@ -285,10 +305,10 @@ func (g *DefaultObjectsGate) checkAndLog(ctx context.Context, logger logr.Logger
// The release named here is whichever one renders what is missing:
// the credentials Secret comes from the bucket's -system
// release, everything else from this chart's own.
- logger.Info("forced a Helm upgrade to create missing default backup objects",
+ logger.Info("forced a Helm upgrade to create missing or stale default backup objects",
"helmRelease", g.forcedReleaseFor(missing).String(), "missing", missing)
case len(missing) > 0:
- logger.Info("default backup objects still missing, no force issued on this tick",
+ logger.Info("default backup objects still missing or stale, no force issued on this tick",
"helmRelease", g.forcedReleaseFor(missing).String(), "missing", missing,
"minForceInterval", g.MinForceInterval.String())
default:
@@ -302,7 +322,7 @@ func (g *DefaultObjectsGate) checkAndLog(ctx context.Context, logger logr.Logger
// resolving the bucket name is a precondition for checking those at all.
func (g *DefaultObjectsGate) forcedReleaseFor(missing []string) types.NamespacedName {
credsKey := fmt.Sprintf("Secret/%s", g.Config.SourceSecretName)
- if len(missing) == 1 && missing[0] == credsKey && g.CredentialsHelmRelease.Name != "" {
+ if len(missing) == 1 && (missing[0] == credsKey || strings.HasPrefix(missing[0], credsKey+" ")) && g.CredentialsHelmRelease.Name != "" {
return g.CredentialsHelmRelease
}
return g.HelmRelease
@@ -345,23 +365,95 @@ func (g *DefaultObjectsGate) Check(ctx context.Context) ([]string, bool, error)
return g.reconcileCredentials(ctx)
}
+ // The objects below are rendered from this Secret, so while it is stale
+ // they would only be re-rendered with the same stale endpoint.
+ stale, err := g.staleCredentials(ctx, src)
+ if err != nil {
+ defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc()
+ return nil, false, err
+ }
+ if stale != "" {
+ return g.force(ctx, g.CredentialsHelmRelease, &g.lastCredentialsForce, []string{stale})
+ }
+
backupClass := &backupsv1alpha1.BackupClass{}
if err := g.Client.Get(ctx, client.ObjectKey{Name: g.BackupClassName}, backupClass); err != nil {
defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc()
return nil, false, fmt.Errorf("get BackupClass %s: %w", g.BackupClassName, err)
}
- missing, err := g.missingObjects(ctx, backupClass)
+ // With provisionBucket=false (no bucket release) the chart renders
+ // backupStorage.endpoint rather than the Secret's, so the two may
+ // legitimately differ and there is nothing to compare.
+ want := ""
+ if g.CredentialsHelmRelease.Name != "" {
+ want = renderedEndpointFor(src.Data["endpoint"])
+ }
+ state, err := g.inspectObjects(ctx, backupClass, want)
if err != nil {
defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc()
return nil, false, err
}
- defaultObjectsMissing.WithLabelValues(g.BackupClassName).Set(float64(len(missing)))
- if len(missing) == 0 {
+ defaultObjectsMissing.WithLabelValues(g.BackupClassName).Set(float64(len(state.missing)))
+ needed := append(state.missing, state.stale...)
+ if len(needed) == 0 {
return nil, false, nil
}
+ sort.Strings(needed)
- return g.force(ctx, g.HelmRelease, &g.lastForce, missing)
+ needed, forced, err := g.force(ctx, g.HelmRelease, &g.lastForce, needed)
+ if forced {
+ g.renderForced = true
+ }
+ return needed, forced, err
+}
+
+// staleCredentials reports the credentials Secret, formatted for logs, when
+// its endpoint differs from the one in the COSI Secret it is rendered from,
+// comparing against what packages/system/bucket/templates/user-credentials.yaml
+// writes for it. The COSI Secret being absent (the BucketAccess is being
+// re-granted) leaves nothing to compare against and is not staleness.
+func (g *DefaultObjectsGate) staleCredentials(ctx context.Context, src *corev1.Secret) (string, error) {
+ if g.CredentialsCOSISecretName == "" || g.CredentialsHelmRelease.Name == "" || g.CredentialsHelmRelease.Namespace == "" {
+ return "", nil
+ }
+ cosi := &corev1.Secret{}
+ key := types.NamespacedName{Namespace: g.Config.SourceNamespace, Name: g.CredentialsCOSISecretName}
+ if err := g.Client.Get(ctx, key, cosi); err != nil {
+ if apierrors.IsNotFound(err) {
+ return "", nil
+ }
+ return "", fmt.Errorf("get COSI Secret %s: %w", key, err)
+ }
+ var info struct {
+ Spec struct {
+ SecretS3 struct {
+ Endpoint string `json:"endpoint"`
+ } `json:"secretS3"`
+ } `json:"spec"`
+ }
+ if err := json.Unmarshal(cosi.Data["BucketInfo"], &info); err != nil {
+ return "", fmt.Errorf("parse BucketInfo of COSI Secret %s: %w", key, err)
+ }
+ want := strings.TrimPrefix(strings.TrimPrefix(info.Spec.SecretS3.Endpoint, "https://"), "http://")
+ have := string(src.Data["endpoint"])
+ if have == want {
+ return "", nil
+ }
+ return fmt.Sprintf("Secret/%s (endpoint %q, COSI Secret has %q)", g.Config.SourceSecretName, have, want), nil
+}
+
+// renderedEndpointFor mirrors the "backupstrategy-controller.endpoint" helper
+// for a provisioned bucket: the credentials Secret's endpoint, trimmed, with
+// any scheme replaced by https://. It is the value the chart stamps as
+// endpointAnnotation. An empty endpoint makes the helper fall back to
+// backupStorage.endpoint, which the gate cannot see, so it returns "".
+func renderedEndpointFor(raw []byte) string {
+ host := strings.TrimSpace(string(raw))
+ if host == "" {
+ return ""
+ }
+ return "https://" + strings.TrimPrefix(strings.TrimPrefix(host, "https://"), "http://")
}
// reconcileCredentials handles the one object the gate cannot check the same
@@ -431,16 +523,42 @@ func (g *DefaultObjectsGate) force(ctx context.Context, target types.NamespacedN
return missing, true, nil
}
-// missingObjects returns a sorted, de-duplicated list of the
+// defaultObjectsState is what inspectObjects found: the objects that do not
+// exist, and the existing ones rendered with an endpoint other than the one
+// the chart would render now.
+type defaultObjectsState struct {
+ missing []string
+ stale []string
+}
+
+// staleEndpoint reports an existing object rendered with another endpoint
+// than want, formatted for logs. An empty want disables the comparison.
+func (g *DefaultObjectsGate) staleEndpoint(state *defaultObjectsState, key string, obj *unstructured.Unstructured, want string) {
+ if want == "" {
+ return
+ }
+ have, ok := obj.GetAnnotations()[endpointAnnotation]
+ switch {
+ case !ok:
+ if !g.renderForced {
+ state.stale = append(state.stale, key+" (no endpoint annotation)")
+ }
+ case have != want:
+ state.stale = append(state.stale, fmt.Sprintf("%s (endpoint %q, want %q)", key, have, want))
+ }
+}
+
+// inspectObjects returns a sorted, de-duplicated list of the
// "/" objects the BackupClass references that do not exist,
-// plus the Velero BSL when a Velero namespace is configured.
+// plus the Velero BSL when a Velero namespace is configured, and the existing
+// ones whose endpointAnnotation differs from want.
//
// An unmappable group/kind (the CRD is not installed) is NOT reported as
// missing: no Helm re-render can create an object whose CRD is absent, and
// counting it would keep the gate forcing upgrades forever.
-func (g *DefaultObjectsGate) missingObjects(ctx context.Context, backupClass *backupsv1alpha1.BackupClass) ([]string, error) {
+func (g *DefaultObjectsGate) inspectObjects(ctx context.Context, backupClass *backupsv1alpha1.BackupClass, want string) (defaultObjectsState, error) {
seen := map[string]struct{}{}
- var missing []string
+ var state defaultObjectsState
for _, strategy := range backupClass.Spec.Strategies {
name := strategy.StrategyRef.Name
@@ -472,17 +590,18 @@ func (g *DefaultObjectsGate) missingObjects(ctx context.Context, backupClass *ba
if meta.IsNoMatchError(err) {
continue
}
- return nil, fmt.Errorf("map %s/%s: %w", group, kind, err)
+ return defaultObjectsState{}, fmt.Errorf("map %s/%s: %w", group, kind, err)
}
// Strategy CRs are cluster-scoped; Namespace("") is correct for
// both scopes on the dynamic client.
- _, err = g.Resource(mapping.Resource).Get(ctx, name, metav1.GetOptions{})
+ obj, err := g.Resource(mapping.Resource).Get(ctx, name, metav1.GetOptions{})
switch {
case err == nil:
+ g.staleEndpoint(&state, key, obj, want)
case apierrors.IsNotFound(err):
- missing = append(missing, key)
+ state.missing = append(state.missing, key)
default:
- return nil, fmt.Errorf("get %s %s: %w", kind, name, err)
+ return defaultObjectsState{}, fmt.Errorf("get %s %s: %w", kind, name, err)
}
}
@@ -496,24 +615,26 @@ func (g *DefaultObjectsGate) missingObjects(ctx context.Context, backupClass *ba
mapping, err := g.RESTMapping(schema.GroupKind{Group: backupStorageLocationGVR.Group, Kind: "BackupStorageLocation"})
switch {
case err == nil:
- _, getErr := g.Resource(mapping.Resource).Namespace(g.VeleroNamespace).Get(ctx, defaultBackupStorageLocationName, metav1.GetOptions{})
+ key := fmt.Sprintf("BackupStorageLocation/%s", defaultBackupStorageLocationName)
+ obj, getErr := g.Resource(mapping.Resource).Namespace(g.VeleroNamespace).Get(ctx, defaultBackupStorageLocationName, metav1.GetOptions{})
switch {
case getErr == nil:
+ g.staleEndpoint(&state, key, obj, want)
case apierrors.IsNotFound(getErr):
- missing = append(missing, fmt.Sprintf("BackupStorageLocation/%s", defaultBackupStorageLocationName))
+ state.missing = append(state.missing, key)
default:
- return nil, fmt.Errorf("get BackupStorageLocation %s: %w", defaultBackupStorageLocationName, getErr)
+ return defaultObjectsState{}, fmt.Errorf("get BackupStorageLocation %s: %w", defaultBackupStorageLocationName, getErr)
}
case meta.IsNoMatchError(err):
// Velero API not served (e.g. Velero uninstalled after bootstrap):
// no Helm re-render can create the BSL, so it is not "missing".
default:
- return nil, fmt.Errorf("map BackupStorageLocation: %w", err)
+ return defaultObjectsState{}, fmt.Errorf("map BackupStorageLocation: %w", err)
}
}
- sort.Strings(missing)
- return missing, nil
+ sort.Strings(state.missing)
+ return state, nil
}
// forceHelmRelease stamps BOTH reconcile.fluxcd.io/forceAt and
diff --git a/internal/backupcontroller/default_objects_gate_test.go b/internal/backupcontroller/default_objects_gate_test.go
index f5cbb5489..dc1c0c9f0 100644
--- a/internal/backupcontroller/default_objects_gate_test.go
+++ b/internal/backupcontroller/default_objects_gate_test.go
@@ -2,6 +2,8 @@ package backupcontroller
import (
"context"
+ "errors"
+ "strings"
"testing"
"time"
@@ -18,6 +20,7 @@ import (
k8stesting "k8s.io/client-go/testing"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
+ "sigs.k8s.io/controller-runtime/pkg/client/interceptor"
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
)
@@ -110,14 +113,60 @@ func cozyDefaultBackupClass() *backupsv1alpha1.BackupClass {
}
}
+// renderedEndpoint is what the chart stamps on every default object for the
+// endpoint sourceSecret carries: the steady state after a render that saw it.
+const renderedEndpoint = "https://s3.example.com"
+
func strategyObject(kind, name string) *unstructured.Unstructured {
u := &unstructured.Unstructured{}
u.SetAPIVersion(strategyAPIGroup + "/v1alpha1")
u.SetKind(kind)
u.SetName(name)
+ u.SetAnnotations(map[string]string{endpointAnnotation: renderedEndpoint})
return u
}
+func withEndpoint(u *unstructured.Unstructured, endpoint string) *unstructured.Unstructured {
+ u.SetAnnotations(map[string]string{endpointAnnotation: endpoint})
+ return u
+}
+
+// unannotated is an object rendered by a chart that predates the endpoint
+// annotation.
+func unannotated(u *unstructured.Unstructured) *unstructured.Unstructured {
+ u.SetAnnotations(nil)
+ return u
+}
+
+// allDefaultObjects is every object cozy-default routes to, plus the BSL, as
+// the chart renders them for sourceSecret's endpoint.
+func allDefaultObjects() []runtime.Object {
+ return []runtime.Object{
+ strategyObject("CNPG", "cozy-default-cnpg"),
+ strategyObject("MariaDB", "cozy-default-mariadb"),
+ strategyObject("Etcd", "cozy-default-etcd"),
+ strategyObject("Altinity", "cozy-default-altinity"),
+ strategyObject("Velero", "cozy-default-velero-vminstance"),
+ strategyObject("Velero", "cozy-default-velero-vmdisk"),
+ bslObject(),
+ }
+}
+
+// cosiSecret is the Secret the COSI sidecar writes for the platform bucket's
+// BucketAccess, which user-credentials.yaml renders the credentials Secret
+// from. The BucketInfo document is one read off a live cluster, with the keys
+// and the host replaced.
+func cosiSecret(endpoint string) *corev1.Secret {
+ bucketInfo := `{"metadata": {"name": "bc-b8e30337-1821-418f-a4dd-163e8016aaee", "creationTimestamp": null}, ` +
+ `"spec": {"bucketName": "bucket-1a2b", "authenticationType": "KEY", ` +
+ `"secretS3": {"endpoint": "` + endpoint + `", "region": "", "accessKeyID": "AK", "accessSecretKey": "SK"}, ` +
+ `"secretAzure": null, "protocols": ["s3"]}}`
+ return &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bucket-cozy-backups-system"},
+ Data: map[string][]byte{"BucketInfo": []byte(bucketInfo)},
+ }
+}
+
func helmReleaseObject() *unstructured.Unstructured {
u := &unstructured.Unstructured{}
u.SetAPIVersion("helm.toolkit.fluxcd.io/v2")
@@ -163,9 +212,10 @@ func newGate(t *testing.T, ctrlObjs []client.Object, dynObjs ...runtime.Object)
Namespace: "tenant-root",
Name: "bucket-cozy-backups-system",
},
- VeleroNamespace: "cozy-velero",
- MinForceInterval: 5 * time.Minute,
- now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
+ CredentialsCOSISecretName: "bucket-cozy-backups-system",
+ VeleroNamespace: "cozy-velero",
+ MinForceInterval: 5 * time.Minute,
+ now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) },
}
g.Interface = dyn
g.RESTMapper = gateRESTMapper()
@@ -193,6 +243,7 @@ func bslObject() *unstructured.Unstructured {
u.SetKind("BackupStorageLocation")
u.SetNamespace("cozy-velero")
u.SetName(defaultBackupStorageLocationName)
+ u.SetAnnotations(map[string]string{endpointAnnotation: renderedEndpoint})
return u
}
@@ -259,7 +310,7 @@ func TestCheckForcesWhenObjectsMissing(t *testing.T) {
// the cluster.
func TestCheckNoopWhenAllPresent(t *testing.T) {
g, dyn := newGate(t,
- []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()},
+ []client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.example.com"), cozyDefaultBackupClass()},
helmReleaseObject(),
strategyObject("CNPG", "cozy-default-cnpg"),
strategyObject("MariaDB", "cozy-default-mariadb"),
@@ -550,10 +601,11 @@ func TestMissingObjectsIgnoresUnmappedKinds(t *testing.T) {
bslObject(),
)
- missing, err := g.missingObjects(context.Background(), bc)
+ state, err := g.inspectObjects(context.Background(), bc, "")
if err != nil {
- t.Fatalf("missingObjects: %v", err)
+ t.Fatalf("inspectObjects: %v", err)
}
+ missing := state.missing
if len(missing) != 0 {
t.Fatalf("missing = %v, want none (unmapped kind must be ignored)", missing)
}
@@ -579,10 +631,11 @@ func TestMissingObjectsSkipsVeleroWhenNamespaceEmpty(t *testing.T) {
)
g.VeleroNamespace = ""
- missing, err := g.missingObjects(context.Background(), bc)
+ state, err := g.inspectObjects(context.Background(), bc, "")
if err != nil {
- t.Fatalf("missingObjects: %v", err)
+ t.Fatalf("inspectObjects: %v", err)
}
+ missing := state.missing
if len(missing) != 0 {
t.Fatalf("missing = %v, want none when Velero is disabled", missing)
}
@@ -630,10 +683,11 @@ func TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent(t *testing.T) {
}
g.RESTMapper = m
- missing, err := g.missingObjects(context.Background(), bc)
+ state, err := g.inspectObjects(context.Background(), bc, "")
if err != nil {
- t.Fatalf("missingObjects: %v", err)
+ t.Fatalf("inspectObjects: %v", err)
}
+ missing := state.missing
if len(missing) != 0 {
t.Fatalf("missing = %v, want none when the Velero API is absent", missing)
}
@@ -744,3 +798,330 @@ func TestCheckToleratesAbsentHelmRelease(t *testing.T) {
t.Error("objects should still be reported as missing")
}
}
+
+// TestCheckForcesCredentialsReleaseWhenEndpointStale covers the credentials
+// Secret outliving the endpoint it was rendered from. The COSI sidecar writes
+// BucketInfo once per BucketAccess, so a re-granted BucketAccess (after a
+// publishing.host change) carries the new endpoint while the credentials
+// Secret -system rendered earlier keeps the old one: the Secret
+// exists, so the missing-Secret path never fires, and every consumer keeps
+// the dead host.
+func TestCheckForcesCredentialsReleaseWhenEndpointStale(t *testing.T) {
+ objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
+ g, dyn := newGate(t,
+ []client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.new.example.com"), cozyDefaultBackupClass()},
+ objs...,
+ )
+
+ missing, forced, err := g.Check(context.Background())
+ if err != nil {
+ t.Fatalf("Check: %v", err)
+ }
+ if !forced {
+ t.Fatal("expected the bucket release forced while its credentials Secret carries a stale endpoint")
+ }
+ if len(missing) != 1 || !strings.HasPrefix(missing[0], "Secret/bucket-cozy-backups-system-credentials") {
+ t.Fatalf("missing = %v, want the credentials Secret", missing)
+ }
+ if ann := credentialsForceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" {
+ t.Errorf("bucket release not forced: %v", ann)
+ }
+ // The objects were rendered from the stale Secret and agree with it;
+ // re-rendering them before the Secret is fixed would bake the old
+ // endpoint in again.
+ if ann := forceAnnotations(t, dyn); len(ann) != 0 {
+ t.Errorf("own HelmRelease forced while the credentials Secret is stale: %v", ann)
+ }
+}
+
+// TestCheckCredentialsEndpointComparison pins the comparison against what
+// user-credentials.yaml writes for a given BucketInfo endpoint (the scheme
+// stripped, nothing else), so a Secret the chart rendered from the current
+// BucketInfo never counts as stale and the bucket release is not forced on
+// every throttle window.
+func TestCheckCredentialsEndpointComparison(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ cosi string
+ wantForce bool
+ }{
+ {"https scheme stripped", "https://s3.example.com", false},
+ {"http scheme stripped", "http://s3.example.com", false},
+ {"bare host", "s3.example.com", false},
+ {"other host", "https://s3.new.example.com", true},
+ {"other port", "https://s3.example.com:8443", true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
+ g, dyn := newGate(t,
+ []client.Object{sourceSecret("bucket-1a2b"), cosiSecret(tc.cosi), cozyDefaultBackupClass()},
+ objs...,
+ )
+ if _, _, err := g.Check(context.Background()); err != nil {
+ t.Fatalf("Check: %v", err)
+ }
+ forced := credentialsForceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"] != ""
+ if forced != tc.wantForce {
+ t.Fatalf("bucket release forced = %v, want %v", forced, tc.wantForce)
+ }
+ })
+ }
+}
+
+// TestCheckSkipsCredentialsComparisonWithoutCOSISecret covers the window in
+// which the BucketAccess is being re-granted: the COSI Secret is gone and
+// there is nothing to compare against, which is not evidence of staleness.
+func TestCheckSkipsCredentialsComparisonWithoutCOSISecret(t *testing.T) {
+ objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
+ g, dyn := newGate(t,
+ []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()},
+ objs...,
+ )
+
+ missing, forced, err := g.Check(context.Background())
+ if err != nil || forced || len(missing) != 0 {
+ t.Fatalf("missing = %v, forced = %v, err = %v; want a no-op", missing, forced, err)
+ }
+ if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 {
+ t.Errorf("bucket release forced with no COSI Secret to compare against: %v", ann)
+ }
+}
+
+// TestCheckSkipsCredentialsComparisonWhenUnconfigured pins both switches that
+// turn the comparison off: no COSI Secret name, and no bucket release (external
+// S3, where the Secret is admin-managed and no BucketInfo backs it).
+func TestCheckSkipsCredentialsComparisonWhenUnconfigured(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ apply func(g *DefaultObjectsGate)
+ }{
+ {"no COSI Secret name", func(g *DefaultObjectsGate) { g.CredentialsCOSISecretName = "" }},
+ {"no bucket release", func(g *DefaultObjectsGate) { g.CredentialsHelmRelease = types.NamespacedName{} }},
+ {"no bucket release name", func(g *DefaultObjectsGate) { g.CredentialsHelmRelease.Name = "" }},
+ {"no bucket release namespace", func(g *DefaultObjectsGate) { g.CredentialsHelmRelease.Namespace = "" }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
+ g, dyn := newGate(t,
+ []client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.new.example.com"), cozyDefaultBackupClass()},
+ objs...,
+ )
+ tc.apply(g)
+ // The real API server rejects a Get with an empty name; the fake
+ // answers NotFound, which would hide a lookup of an unset name.
+ g.Client = interceptor.NewClient(g.Client.(client.WithWatch), interceptor.Funcs{
+ Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
+ if key.Name == "" {
+ return errors.New("resource name may not be empty")
+ }
+ return c.Get(ctx, key, obj, opts...)
+ },
+ })
+ missing, forced, err := g.Check(context.Background())
+ if err != nil || forced || len(missing) != 0 {
+ t.Fatalf("missing = %v, forced = %v, err = %v; want a no-op", missing, forced, err)
+ }
+ if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 {
+ t.Errorf("bucket release forced with the comparison disabled: %v", ann)
+ }
+ })
+ }
+}
+
+// TestCheckReportsMalformedCOSISecret pins that a COSI Secret the gate cannot
+// read is an error the check counts, not a silent "not stale".
+func TestCheckReportsMalformedCOSISecret(t *testing.T) {
+ defaultObjectsCheckErrors.Reset()
+ bad := cosiSecret("")
+ bad.Data["BucketInfo"] = []byte("{not json")
+ objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
+ g, _ := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), bad, cozyDefaultBackupClass()}, objs...)
+
+ if _, forced, err := g.Check(context.Background()); err == nil || forced {
+ t.Fatalf("forced = %v, err = %v; want an error", forced, err)
+ }
+ if got := testutil.ToFloat64(defaultObjectsCheckErrors.WithLabelValues("cozy-default")); got != 1 {
+ t.Fatalf("check errors = %v, want 1", got)
+ }
+}
+
+// TestCheckForcesOwnReleaseWhenObjectEndpointStale is the second half of a
+// publishing.host change: the credentials Secret now carries the new
+// endpoint, but the BSL and the default Strategies baked the old one at the
+// last render of this chart and nothing re-renders them, so every backup
+// keeps going to the dead host.
+func TestCheckForcesOwnReleaseWhenObjectEndpointStale(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ stale *unstructured.Unstructured
+ key string
+ }{
+ {"strategy", strategyObject("CNPG", "cozy-default-cnpg"), "CNPG/cozy-default-cnpg"},
+ {"BSL", bslObject(), "BackupStorageLocation/cozy-default"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ var objs []runtime.Object
+ for _, o := range allDefaultObjects() {
+ u := o.(*unstructured.Unstructured)
+ if u.GetKind() == tc.stale.GetKind() && u.GetName() == tc.stale.GetName() {
+ withEndpoint(u, "https://s3.old.example.com")
+ }
+ objs = append(objs, u)
+ }
+ objs = append(objs, helmReleaseObject(), credentialsHelmReleaseObject())
+ g, dyn := newGate(t,
+ []client.Object{sourceSecret("bucket-1a2b"), cosiSecret("https://s3.example.com"), cozyDefaultBackupClass()},
+ objs...,
+ )
+
+ missing, forced, err := g.Check(context.Background())
+ if err != nil {
+ t.Fatalf("Check: %v", err)
+ }
+ if !forced {
+ t.Fatal("expected this chart's release forced while an object carries a stale endpoint")
+ }
+ if len(missing) != 1 || !strings.HasPrefix(missing[0], tc.key+" ") {
+ t.Fatalf("missing = %v, want only %s", missing, tc.key)
+ }
+ if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" {
+ t.Errorf("own HelmRelease not forced: %v", ann)
+ }
+ if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 {
+ t.Errorf("bucket release forced for a stale object: %v", ann)
+ }
+ })
+ }
+}
+
+// TestCheckIgnoresObjectEndpointOnExternalS3 pins that the comparison is off
+// when no bucket release is configured: with provisionBucket=false the chart
+// renders backupStorage.endpoint, not the Secret's, so the two legitimately
+// differ and forcing would loop every MinForceInterval.
+func TestCheckIgnoresObjectEndpointOnExternalS3(t *testing.T) {
+ objs := []runtime.Object{helmReleaseObject()}
+ for _, o := range allDefaultObjects() {
+ objs = append(objs, withEndpoint(o.(*unstructured.Unstructured), "https://minio.example.internal:9000"))
+ }
+ g, dyn := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, objs...)
+ g.CredentialsHelmRelease = types.NamespacedName{}
+
+ missing, forced, err := g.Check(context.Background())
+ if err != nil || forced || len(missing) != 0 {
+ t.Fatalf("missing = %v, forced = %v, err = %v; want a no-op", missing, forced, err)
+ }
+ if ann := forceAnnotations(t, dyn); len(ann) != 0 {
+ t.Errorf("own HelmRelease forced on external S3: %v", ann)
+ }
+}
+
+// TestCheckIgnoresObjectEndpointWithoutSecretEndpoint covers a credentials
+// Secret with no endpoint key: the chart then renders its values fallback,
+// which the gate cannot reproduce, so there is nothing to compare against.
+func TestCheckIgnoresObjectEndpointWithoutSecretEndpoint(t *testing.T) {
+ src := sourceSecret("bucket-1a2b")
+ delete(src.Data, "endpoint")
+ objs := append([]runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}, allDefaultObjects()...)
+ g, dyn := newGate(t, []client.Object{src, cozyDefaultBackupClass()}, objs...)
+
+ if _, forced, err := g.Check(context.Background()); err != nil || forced {
+ t.Fatalf("forced = %v, err = %v; want a no-op", forced, err)
+ }
+ if ann := forceAnnotations(t, dyn); len(ann) != 0 {
+ t.Errorf("own HelmRelease forced with no endpoint to compare against: %v", ann)
+ }
+}
+
+// TestCheckForcesOnceForUnannotatedObjects covers objects rendered by a chart
+// that predates the endpoint annotation. Their endpoint is unknown, so they
+// are treated as stale until one forced render has gone through, which stamps
+// the annotation. A force that did not go through (throttled) does not count,
+// and once one has, an object still without the annotation (one the chart
+// does not render) must not force again every MinForceInterval.
+func TestCheckForcesOnceForUnannotatedObjects(t *testing.T) {
+ base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+ objs := []runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}
+ for _, o := range allDefaultObjects() {
+ objs = append(objs, unannotated(o.(*unstructured.Unstructured)))
+ }
+ g, dyn := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, objs...)
+ g.lastForce = base.Add(-time.Minute)
+ g.now = func() time.Time { return base }
+
+ if missing, forced, err := g.Check(context.Background()); err != nil || forced || len(missing) != 7 {
+ t.Fatalf("throttled Check: missing = %v, forced = %v, err = %v; want 7 reported, none forced", missing, forced, err)
+ }
+
+ g.now = func() time.Time { return base.Add(5 * time.Minute) }
+ if _, forced, err := g.Check(context.Background()); err != nil || !forced {
+ t.Fatalf("first unthrottled Check: forced = %v, err = %v; want forced", forced, err)
+ }
+ first := forceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"]
+
+ g.now = func() time.Time { return base.Add(time.Hour) }
+ missing, forced, err := g.Check(context.Background())
+ if err != nil || forced || len(missing) != 0 {
+ t.Fatalf("after the forced render: missing = %v, forced = %v, err = %v; want a no-op", missing, forced, err)
+ }
+ if again := forceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"]; again != first {
+ t.Errorf("forced again for unannotated objects: %q, then %q", first, again)
+ }
+}
+
+// TestCheckStillForcesStaleAfterUnannotatedForce pins that the once-only
+// treatment covers the missing annotation alone: an annotation that names the
+// wrong endpoint still forces after that.
+func TestCheckStillForcesStaleAfterUnannotatedForce(t *testing.T) {
+ objs := []runtime.Object{helmReleaseObject(), credentialsHelmReleaseObject()}
+ for _, o := range allDefaultObjects() {
+ objs = append(objs, withEndpoint(o.(*unstructured.Unstructured), "https://s3.old.example.com"))
+ }
+ g, _ := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, objs...)
+ g.renderForced = true
+
+ if _, forced, err := g.Check(context.Background()); err != nil || !forced {
+ t.Fatalf("forced = %v, err = %v; want forced", forced, err)
+ }
+}
+
+// TestRenderedEndpointMatchesTheChart pins renderedEndpointFor to the
+// "backupstrategy-controller.endpoint" helper, which the chart stamps as the
+// annotation: trim, strip a scheme, force https://. Any divergence makes the
+// gate see every object as stale and force this release forever.
+func TestRenderedEndpointMatchesTheChart(t *testing.T) {
+ for _, tc := range []struct{ in, want string }{
+ {"s3.example.com", "https://s3.example.com"},
+ {"s3.example.com\n", "https://s3.example.com"},
+ {" s3.example.com ", "https://s3.example.com"},
+ {"https://s3.example.com", "https://s3.example.com"},
+ {"http://s3.example.com:8333", "https://s3.example.com:8333"},
+ {"s3.example.com/prefix/", "https://s3.example.com/prefix/"},
+ // No endpoint in the Secret: the chart renders its values fallback,
+ // which the gate cannot reproduce, so it compares nothing.
+ {"", ""},
+ } {
+ if got := renderedEndpointFor([]byte(tc.in)); got != tc.want {
+ t.Errorf("renderedEndpointFor(%q) = %q, want %q", tc.in, got, tc.want)
+ }
+ }
+}
+
+// TestForcedReleaseForNamesTheStaleCredentialsRelease pins the release the
+// log line names: a stale credentials Secret is repaired by the bucket
+// release, like a missing one, and everything else by this chart's own.
+func TestForcedReleaseForNamesTheStaleCredentialsRelease(t *testing.T) {
+ g, _ := newGate(t, nil)
+ for _, tc := range []struct {
+ missing []string
+ want types.NamespacedName
+ }{
+ {[]string{"Secret/bucket-cozy-backups-system-credentials"}, g.CredentialsHelmRelease},
+ {[]string{`Secret/bucket-cozy-backups-system-credentials (endpoint "s3.old", COSI Secret has "s3.new")`}, g.CredentialsHelmRelease},
+ {[]string{"Secret/bucket-cozy-backups-system-credentials-other"}, g.HelmRelease},
+ {[]string{`CNPG/cozy-default-cnpg (endpoint "https://s3.old", want "https://s3.new")`}, g.HelmRelease},
+ } {
+ if got := g.forcedReleaseFor(tc.missing); got != tc.want {
+ t.Errorf("forcedReleaseFor(%v) = %v, want %v", tc.missing, got, tc.want)
+ }
+ }
+}
diff --git a/packages/system/backupstrategy-controller/templates/_helpers.tpl b/packages/system/backupstrategy-controller/templates/_helpers.tpl
index 6665cc639..f64c77b88 100644
--- a/packages/system/backupstrategy-controller/templates/_helpers.tpl
+++ b/packages/system/backupstrategy-controller/templates/_helpers.tpl
@@ -101,8 +101,11 @@
https://.
- provisionBucket: true + Secret missing (offline `helm template`/unit
render, or pre-reconcile install where lookup returns nil) → fall back
- to .Values.backupStorage.endpoint. On a live deploy Flux re-renders on
- spec.interval once the Secret exists, promoting the derived endpoint.
+ to .Values.backupStorage.endpoint. Flux does not re-render on
+ spec.interval; the controller's DefaultObjectsGate forces the render
+ once the Secret exists, and again whenever the Secret's endpoint stops
+ matching the backups.cozystack.io/endpoint annotation every default
+ Strategy CR and the Velero BSL carry with this helper's output.
Normalization of the decoded value: trim first (a trailing newline in the
Secret data would otherwise ride into the URL and break it), then strip any
diff --git a/packages/system/backupstrategy-controller/templates/deployment.yaml b/packages/system/backupstrategy-controller/templates/deployment.yaml
index f35e97661..adc09fa2a 100644
--- a/packages/system/backupstrategy-controller/templates/deployment.yaml
+++ b/packages/system/backupstrategy-controller/templates/deployment.yaml
@@ -87,6 +87,13 @@ spec:
value: {{ printf "bucket-%s-system" .Values.backupStorage.bucketName | quote }}
- name: BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE
value: {{ .Values.backupStorage.namespace | quote }}
+ # The COSI Secret that release renders the credentials Secret from
+ # (packages/system/bucket/templates/user-credentials.yaml appends
+ # "-credentials" to its name). The COSI sidecar writes it once per
+ # BucketAccess, so the gate compares the two endpoints and forces the
+ # release when the credentials Secret lags behind.
+ - name: BACKUP_CREDENTIALS_COSI_SECRET_NAME
+ value: {{ trimSuffix "-credentials" .Values.backupStorage.systemSecretName | quote }}
{{- end }}
{{- if .Values.velero.bslEnabled }}
- name: BACKUP_DEFAULT_OBJECTS_VELERO_NAMESPACE
diff --git a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml
index 3397d6d0d..6d09d4aa9 100644
--- a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml
+++ b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml
@@ -18,6 +18,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: Altinity
metadata:
name: cozy-default-altinity
+ annotations:
+ backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
spec:
template:
spec:
diff --git a/packages/system/backupstrategy-controller/templates/strategy-cnpg-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-cnpg-default.yaml
index 58dda61b1..bf34b2e0d 100644
--- a/packages/system/backupstrategy-controller/templates/strategy-cnpg-default.yaml
+++ b/packages/system/backupstrategy-controller/templates/strategy-cnpg-default.yaml
@@ -5,6 +5,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: CNPG
metadata:
name: cozy-default-cnpg
+ annotations:
+ backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
spec:
template:
{{- /*
diff --git a/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml
index f335f04cf..5183af45b 100644
--- a/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml
+++ b/packages/system/backupstrategy-controller/templates/strategy-etcd-default.yaml
@@ -5,6 +5,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: Etcd
metadata:
name: cozy-default-etcd
+ annotations:
+ backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
spec:
template:
destination:
diff --git a/packages/system/backupstrategy-controller/templates/strategy-foundationdb-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-foundationdb-default.yaml
index 061be2ffb..82f9ab6ca 100644
--- a/packages/system/backupstrategy-controller/templates/strategy-foundationdb-default.yaml
+++ b/packages/system/backupstrategy-controller/templates/strategy-foundationdb-default.yaml
@@ -8,6 +8,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: FoundationDB
metadata:
name: cozy-default-foundationdb
+ annotations:
+ backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
spec:
template:
blobStoreConfiguration:
diff --git a/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml
index c65462295..1f14465ed 100644
--- a/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml
+++ b/packages/system/backupstrategy-controller/templates/strategy-kafka-default.yaml
@@ -23,6 +23,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: Kafka
metadata:
name: cozy-default-kafka
+ annotations:
+ backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
spec:
artifactURITemplate: {{ printf "s3://%s/{{ .Release.Namespace }}/{{ .Release.Name }}/{{ .BackupName }}/kafka-metadata.txt" $bucketName | quote }}
template:
diff --git a/packages/system/backupstrategy-controller/templates/strategy-mariadb-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-mariadb-default.yaml
index ec8b35160..7615a8b0d 100644
--- a/packages/system/backupstrategy-controller/templates/strategy-mariadb-default.yaml
+++ b/packages/system/backupstrategy-controller/templates/strategy-mariadb-default.yaml
@@ -7,6 +7,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: MariaDB
metadata:
name: cozy-default-mariadb
+ annotations:
+ backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
spec:
template:
storage:
diff --git a/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml
index 7509f4bf7..1bddbde16 100644
--- a/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml
+++ b/packages/system/backupstrategy-controller/templates/strategy-mongodb-default.yaml
@@ -54,6 +54,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: MongoDB
metadata:
name: cozy-default-mongodb
+ annotations:
+ backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
spec:
template:
storageName: s3-storage
diff --git a/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml
index 9c303a6d3..9515945d2 100644
--- a/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml
+++ b/packages/system/backupstrategy-controller/templates/strategy-rabbitmq-default.yaml
@@ -26,6 +26,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: Rabbitmq
metadata:
name: cozy-default-rabbitmq
+ annotations:
+ backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
spec:
# The driver defers the run until the RabbitmqCluster reports AllReplicasReady
# (a Go-side precondition), so a not-yet-ready broker surfaces a precise
diff --git a/packages/system/backupstrategy-controller/templates/strategy-redis-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-redis-default.yaml
index 3db27f7e2..dc930f394 100644
--- a/packages/system/backupstrategy-controller/templates/strategy-redis-default.yaml
+++ b/packages/system/backupstrategy-controller/templates/strategy-redis-default.yaml
@@ -21,6 +21,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: Redis
metadata:
name: cozy-default-redis
+ annotations:
+ backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
spec:
template:
spec:
diff --git a/packages/system/backupstrategy-controller/templates/strategy-velero-vmdisk-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-velero-vmdisk-default.yaml
index 4511fbc58..1c1a7b9b5 100644
--- a/packages/system/backupstrategy-controller/templates/strategy-velero-vmdisk-default.yaml
+++ b/packages/system/backupstrategy-controller/templates/strategy-velero-vmdisk-default.yaml
@@ -5,6 +5,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: Velero
metadata:
name: cozy-default-velero-vmdisk
+ annotations:
+ backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
spec:
template:
restoreSpec:
diff --git a/packages/system/backupstrategy-controller/templates/strategy-velero-vminstance-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-velero-vminstance-default.yaml
index b1f919125..9666a7cce 100644
--- a/packages/system/backupstrategy-controller/templates/strategy-velero-vminstance-default.yaml
+++ b/packages/system/backupstrategy-controller/templates/strategy-velero-vminstance-default.yaml
@@ -5,6 +5,8 @@ apiVersion: strategy.backups.cozystack.io/v1alpha1
kind: Velero
metadata:
name: cozy-default-velero-vminstance
+ annotations:
+ backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
spec:
template:
restoreSpec:
diff --git a/packages/system/backupstrategy-controller/templates/velero-bsl.yaml b/packages/system/backupstrategy-controller/templates/velero-bsl.yaml
index e0ba05ad8..e50f3a1fc 100644
--- a/packages/system/backupstrategy-controller/templates/velero-bsl.yaml
+++ b/packages/system/backupstrategy-controller/templates/velero-bsl.yaml
@@ -12,6 +12,8 @@ kind: BackupStorageLocation
metadata:
name: cozy-default
namespace: {{ .Values.velero.namespace }}
+ annotations:
+ backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }}
spec:
provider: aws
# default=false: do not silently displace an admin-provisioned default
diff --git a/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml b/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml
index 976fee166..ed616ce79 100644
--- a/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml
+++ b/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml
@@ -11,6 +11,8 @@ suite: DefaultObjectsGate wiring — the release the gate forces, and its RBAC
templates:
- templates/deployment.yaml
- templates/rbac.yaml
+ - templates/strategy-cnpg-default.yaml
+ - templates/velero-bsl.yaml
release:
name: backupstrategy-controller
@@ -61,6 +63,16 @@ tests:
content:
name: BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE
value: tenant-root
+ # The COSI Secret the bucket release renders the credentials Secret
+ # from is the BucketAccess credentialsSecretName, -system,
+ # which is also BACKUP_STORAGE_SECRET_NAME minus "-credentials"
+ # (packages/system/bucket/templates/user-credentials.yaml). The gate
+ # compares the two endpoints.
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: BACKUP_CREDENTIALS_COSI_SECRET_NAME
+ value: bucket-cozy-backups-system
# BACKUP_STORAGE_SECRET_NAME is exactly the release name above with
# "-credentials" appended — that is what makes forcing this release the
# thing that creates that Secret.
@@ -86,6 +98,11 @@ tests:
content:
name: BACKUP_STORAGE_SECRET_NAME
value: bucket-platform-backups-system-credentials
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: BACKUP_CREDENTIALS_COSI_SECRET_NAME
+ value: bucket-platform-backups-system
# External S3: the Secret is admin-managed and no release renders it, so
# there is nothing to force. Handing the gate stale coordinates would make
@@ -105,6 +122,11 @@ tests:
content:
name: BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE
value: tenant-root
+ - notContains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: BACKUP_CREDENTIALS_COSI_SECRET_NAME
+ value: bucket-cozy-backups-system
- it: "velero.bslEnabled=false: the BSL is not rendered, so the gate must not look for it"
template: templates/deployment.yaml
@@ -154,3 +176,41 @@ tests:
apiGroups: ["velero.io"]
resources: ["backupstoragelocations"]
verbs: ["get"]
+
+ # The gate compares backups.cozystack.io/endpoint with the endpoint it
+ # derives from the credentials Secret (renderedEndpointFor in
+ # internal/backupcontroller/default_objects_gate.go, whose test uses the
+ # same input). Any divergence would make every object look stale and force
+ # this release every MinForceInterval.
+ - it: "objects carry the endpoint derived from the credentials Secret as their endpoint annotation"
+ set:
+ backupStorage.bucketNameOverride: test-bucket
+ kubernetesProvider:
+ scheme:
+ "v1/Secret":
+ gvr:
+ version: v1
+ resource: secrets
+ namespaced: true
+ objects:
+ - apiVersion: v1
+ kind: Secret
+ metadata:
+ name: bucket-cozy-backups-system-credentials
+ namespace: tenant-root
+ data:
+ # "http://s3.example.com:8333\n"
+ endpoint: aHR0cDovL3MzLmV4YW1wbGUuY29tOjgzMzMK
+ asserts:
+ - equal:
+ path: metadata.annotations["backups.cozystack.io/endpoint"]
+ value: https://s3.example.com:8333
+ template: templates/strategy-cnpg-default.yaml
+ - equal:
+ path: spec.template.barmanObjectStore.endpointURL
+ value: https://s3.example.com:8333
+ template: templates/strategy-cnpg-default.yaml
+ - equal:
+ path: metadata.annotations["backups.cozystack.io/endpoint"]
+ value: https://s3.example.com:8333
+ template: templates/velero-bsl.yaml
diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml
index 6cf39c65a..d87c978e6 100644
--- a/packages/system/backupstrategy-controller/values.yaml
+++ b/packages/system/backupstrategy-controller/values.yaml
@@ -162,8 +162,15 @@ backupStorage:
# credentials Secret is missing it forces the bucket's -system
# release; once the bucket name resolves it checks that every object
# cozy-default routes to exists and forces THIS release when any is
- # missing. Both are throttled, independently, and a no-op in the steady
- # state. A suspended release is skipped rather than re-stamped forever. The
+ # missing. It repairs a stale S3 endpoint the same way: when the
+ # credentials Secret's endpoint differs from the COSI Secret it is rendered
+ # from it forces the bucket's -system release, and when the
+ # backups.cozystack.io/endpoint annotation on the Velero BSL or a default
+ # Strategy differs from the credentials Secret's endpoint it forces THIS
+ # release. The COSI Secret itself keeps the endpoint it was granted with, so
+ # after a publishing.host change that endpoint follows only once the
+ # BucketAccess is re-granted. Both are throttled, independently, and a no-op
+ # in the steady state. A suspended release is skipped rather than re-stamped forever. The
# gate does NOT create the objects itself: their bodies are values-driven
# and Helm stays their single owner.
#
diff --git a/packages/system/bucket/templates/user-credentials.yaml b/packages/system/bucket/templates/user-credentials.yaml
index 6c2a14bd1..a45320254 100644
--- a/packages/system/bucket/templates/user-credentials.yaml
+++ b/packages/system/bucket/templates/user-credentials.yaml
@@ -27,7 +27,9 @@
So the repair lives outside the chart, where it is per-object and costs a
throttled annotation patch: backupstrategy-controller's DefaultObjectsGate
(internal/backupcontroller/default_objects_gate.go) forces this release
- when the platform bucket's credentials Secret is absent. That covers the
+ when the platform bucket's credentials Secret is absent, or when its
+ endpoint differs from the COSI Secret's (a BucketAccess re-granted after a
+ publishing.host change). That covers the
cluster-breaking case — the cozy-backups Secret feeds the credentials
projector, hence every default BackupClass strategy, Velero, and migration
50. Tenant buckets keep the pre-existing behaviour; generalising the gate
From 4f3f1a5cadab5e99c0193efd496b5d5aac673098 Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:26:23 +0300
Subject: [PATCH 065/346] fix(clickhouse): stop the backup sidecar from
publishing failed attempts
create_remote keeps its local backup when the upload fails, and with
the upstream defaults the sidecar never prunes local backups and
resumes every unfinished upload when it starts. A strategy attempt that
fails mid-upload therefore lands in the bucket on the next sidecar
restart as an archive no Backup refers to, and restore can pick it as
the newest one. The resumed upload also holds the API lock, so the
strategy Pods racing it get 423 and burn the Job's retries.
Turning the resume off keeps a failed attempt out of the bucket, and a
negative local retention leaves at most the newest local backup and
drops it once its upload succeeds, so leftovers no longer accumulate on
the data volume.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
.../apps/clickhouse/templates/clickhouse.yaml | 12 ++++++
.../apps/clickhouse/tests/backup_test.yaml | 39 +++++++++++++++++++
2 files changed, 51 insertions(+)
diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml
index 827b5d579..571469bfe 100644
--- a/packages/apps/clickhouse/templates/clickhouse.yaml
+++ b/packages/apps/clickhouse/templates/clickhouse.yaml
@@ -222,6 +222,18 @@ spec:
key: backup
- name: ALLOW_EMPTY_BACKUPS
value: "true"
+ {{- /*
+ create_remote keeps the local backup when its upload fails,
+ and the upstream defaults never prune local backups (0) and
+ resume unfinished uploads at start, which pushes a failed
+ attempt to the bucket as an archive no Backup refers to and
+ holds the API lock (423) while it does. -1 keeps only the
+ newest local backup and removes it once its upload succeeds.
+ */}}
+ - name: BACKUPS_TO_KEEP_LOCAL
+ value: "-1"
+ - name: API_COMPLETE_RESUMABLE_AFTER_RESTART
+ value: "false"
- name: REMOTE_STORAGE
value: s3
{{- /*
diff --git a/packages/apps/clickhouse/tests/backup_test.yaml b/packages/apps/clickhouse/tests/backup_test.yaml
index 66eda4dc8..90bcd1066 100644
--- a/packages/apps/clickhouse/tests/backup_test.yaml
+++ b/packages/apps/clickhouse/tests/backup_test.yaml
@@ -292,3 +292,42 @@ tests:
- notMatchRegex:
path: stringData["backup.sh"]
pattern: 'echo DB_LIST='
+
+ # A strategy attempt that fails mid-upload leaves its local backup behind,
+ # and with the upstream defaults the sidecar would keep it forever and push
+ # it to the bucket on its next start as an archive no Backup refers to. The
+ # sidecar must neither resume uploads at start nor keep local copies once an
+ # upload succeeds, on either S3 flow.
+ - it: "sidecar drops local backups after upload and does not resume uploads at start"
+ template: templates/clickhouse.yaml
+ documentSelector:
+ path: kind
+ value: ClickHouseInstallation
+ set:
+ backup:
+ enabled: true
+ useSystemBucket: true
+ asserts:
+ - equal:
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="BACKUPS_TO_KEEP_LOCAL")].value
+ value: "-1"
+ - equal:
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="API_COMPLETE_RESUMABLE_AFTER_RESTART")].value
+ value: "false"
+
+ - it: "legacy flow: sidecar drops local backups after upload and does not resume uploads at start"
+ template: templates/clickhouse.yaml
+ documentSelector:
+ path: kind
+ value: ClickHouseInstallation
+ set:
+ backup:
+ enabled: true
+ useSystemBucket: false
+ asserts:
+ - equal:
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="BACKUPS_TO_KEEP_LOCAL")].value
+ value: "-1"
+ - equal:
+ path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="API_COMPLETE_RESUMABLE_AFTER_RESTART")].value
+ value: "false"
From a25db5aeb6469d1fe5df473479108dd2a84e6911 Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:26:23 +0300
Subject: [PATCH 066/346] fix(backups): wait for an idle ClickHouse backup
sidecar before starting
The sidecar refuses a new operation with 423 while another one is in
progress, but the strategy only waited for the API to answer, and went
on to start its operation even when it never did. An operation already
running in the sidecar therefore failed the attempt within seconds.
Poll the action log until nothing is in progress, and fail the Pod
instead of proceeding when the sidecar stays busy or unreachable, so
the failure names the cause rather than surfacing as a 423 or a
connection error from the operation itself.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
hack/altinity-backup-strategy.bats | 149 ++++++++++++++++++
.../templates/strategy-altinity-default.yaml | 13 +-
2 files changed, 161 insertions(+), 1 deletion(-)
create mode 100644 hack/altinity-backup-strategy.bats
diff --git a/hack/altinity-backup-strategy.bats b/hack/altinity-backup-strategy.bats
new file mode 100644
index 000000000..61729bf00
--- /dev/null
+++ b/hack/altinity-backup-strategy.bats
@@ -0,0 +1,149 @@
+#!/usr/bin/env bats
+# Behavioural tests for the shipped ClickHouse (Altinity) strategy script,
+# extracted from the rendered cozy-default-altinity Strategy CR and EXECUTED
+# against a stubbed clickhouse-backup sidecar API.
+#
+# The contract pinned here is the start gate: the sidecar serialises its
+# operations and answers any new one with HTTP 423 while another is "in
+# progress", so a strategy Pod that starts its operation without waiting for
+# an idle sidecar burns a Job attempt in seconds. The gate has to fail the Pod
+# rather than proceed when the sidecar never becomes idle.
+#
+# Run from the repo root with make bats-unit-tests. The rendered script is run
+# with `bash`, not `sh`: it opens `set -euo pipefail`, which the production
+# image's busybox ash supports but the CI runner's /bin/sh (dash) rejects.
+
+load test_helper
+
+CHART=packages/system/backupstrategy-controller
+
+# Render the shipped strategy's in-Pod script into $1 with the controller-render
+# placeholders substituted ($2 mode, $3 application name). An empty render, or
+# a placeholder left unsubstituted, is an early failure: these tests EXECUTE it.
+render_script() {
+ out=$1; mode=$2; app=$3
+ helm template t "$CHART" -n cozy-backup-controller \
+ --set backupStorage.bucketNameOverride=test-bucket 2>/dev/null \
+ | yq eval 'select(.kind == "Altinity") | .spec.template.spec.containers[0].args[0]' - \
+ | sed -e "s#{{ .Mode }}#${mode}#g" \
+ -e "s#{{ .Release.Name }}#${app}#g" \
+ -e "s#{{ .Release.Namespace }}#tenant-x#g" \
+ -e "s#{{ .Backup.ApplicationRef.Name }}#${app}#g" > "$out"
+ [ -s "$out" ] || return 1
+ if grep -q '{{' "$out"; then
+ echo "rendered script still holds a controller placeholder" >&2
+ return 1
+ fi
+}
+
+# A curl stub standing in for the sidecar API, put first on PATH next to a
+# no-op sleep. State lives in $1:
+# unreachable present: every request fails as a refused connection (exit 7)
+# busy a number: that many GET /backup/actions answers still list an
+# operation "in progress"; while it is positive a POST is
+# refused the way the sidecar refuses it (423, curl -f exit 22)
+# posted the operation the script started, once it started one
+# requests one line per request: method and URL
+make_stubs() {
+ bin=$1; state=$2
+ mkdir -p "$bin" "$state"
+ cat > "$bin/curl" <> "\$state/requests"
+if [ -e "\$state/unreachable" ]; then exit 7; fi
+busy=\$(cat "\$state/busy" 2>/dev/null || echo 0)
+case "\$method \$url" in
+ "POST "*)
+ if [ "\$busy" -gt 0 ]; then exit 22; fi
+ echo "\${url##*/}" > "\$state/posted"
+ ;;
+ "GET "*/backup/actions)
+ echo '{"command":"create_remote clickhouse-myapp-20260101T000000Z","status":"error","start":"2026-01-01 00:00:00","error":"old"}'
+ if [ "\$busy" -gt 0 ]; then
+ echo \$((busy - 1)) > "\$state/busy"
+ echo '{"command":"upload --resumable=1 clickhouse-myapp-20260101T000000Z","status":"in progress","start":"2026-01-01 00:01:00"}'
+ elif [ -s "\$state/posted" ]; then
+ echo "{\"command\":\"\$(cat "\$state/posted")\",\"status\":\"success\",\"start\":\"2026-01-01 00:02:00\"}"
+ fi
+ ;;
+ "GET "*/backup/list/remote)
+ echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}'
+ ;;
+esac
+exit 0
+STUB
+ cat > "$bin/sleep" <<'STUB'
+#!/bin/sh
+exit 0
+STUB
+ chmod 0755 "$bin/curl" "$bin/sleep"
+}
+
+run_script() {
+ tmp=$1
+ API_USERNAME=u API_PASSWORD=p PATH="$tmp/bin:$PATH" bash "$tmp/s.sh"
+}
+
+@test "an empty render fails the fixture instead of passing as a no-op" {
+ tmp=$(mktemp -d)
+ CHART=$tmp/no-such-chart
+ if render_script "$tmp/s.sh" backup myapp 2>/dev/null; then
+ echo "FAIL: render_script reported success with no chart"
+ false
+ fi
+}
+
+@test "backup on an idle sidecar starts create_remote and waits for its success" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp/state"
+ render_script "$tmp/s.sh" backup myapp
+ run_script "$tmp"
+ grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests"
+}
+
+@test "backup waits until the running operation finishes before create_remote" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp/state"
+ echo 3 > "$tmp/state/busy"
+ render_script "$tmp/s.sh" backup myapp
+ run_script "$tmp"
+ grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests"
+ # All three busy answers were consumed before the operation was started.
+ [ "$(cat "$tmp/state/busy")" = 0 ]
+}
+
+@test "backup fails without starting create_remote when the sidecar stays busy" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp/state"
+ echo 1000 > "$tmp/state/busy"
+ render_script "$tmp/s.sh" backup myapp
+ if run_script "$tmp"; then echo "FAIL: a busy sidecar must fail the Pod"; false; fi
+ if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no operation may start on a busy sidecar"; false; fi
+}
+
+@test "backup fails without starting create_remote when the sidecar never answers" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp/state"
+ touch "$tmp/state/unreachable"
+ render_script "$tmp/s.sh" backup myapp
+ if run_script "$tmp"; then echo "FAIL: an unreachable sidecar must fail the Pod"; false; fi
+ if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no operation may start on an unreachable sidecar"; false; fi
+}
+
+@test "restore fails without starting restore_remote when the sidecar stays busy" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp/state"
+ echo 1000 > "$tmp/state/busy"
+ render_script "$tmp/s.sh" restore myapp
+ if run_script "$tmp"; then echo "FAIL: a busy sidecar must fail the Pod"; false; fi
+ if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no restore may start on a busy sidecar"; false; fi
+}
diff --git a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml
index 3397d6d0d..560b1e597 100644
--- a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml
+++ b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml
@@ -51,10 +51,21 @@ spec:
host="chi-clickhouse-{{ printf "{{ .Release.Name }}" }}-clickhouse-0-0.{{ printf "{{ .Release.Namespace }}" }}.svc"
api="http://${host}:7171"
auth="-u ${API_USERNAME}:${API_PASSWORD}"
+ # The sidecar answers 423 to a new operation while any other is
+ # in progress, so starting before it is idle burns the attempt.
+ idle=
for _ in $(seq 1 60); do
- if curl -fsS ${auth} --max-time 2 "${api}/backup/status" >/dev/null 2>&1; then break; fi
+ if actions=$(curl -fsS ${auth} --max-time 5 "${api}/backup/actions") \
+ && [ "$(printf '%s\n' "${actions}" | jq -rs '[.[] | select(.status == "in progress")] | length')" = 0 ]; then
+ idle=1
+ break
+ fi
sleep 5
done
+ if [ -z "${idle}" ]; then
+ echo "clickhouse-backup sidecar at ${api} did not become idle" >&2
+ exit 1
+ fi
if [ "{{ printf "{{ .Mode }}" }}" = "backup" ]; then
prefix="clickhouse-{{ printf "{{ .Release.Name }}" }}-"
target="${prefix}$(date -u +%Y%m%dT%H%M%SZ)"
From f36d864e99d0fcdad7b8d6433705c7d6286367c8 Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:05:58 +0300
Subject: [PATCH 067/346] fix(seaweedfs): run the worker that enforces bucket
lifecycle rules
SeaweedFS accepts and stores a bucket lifecycle configuration, but only
the s3_lifecycle job of its plugin worker acts on it, and that job is
scheduled by the admin server. Neither ran here, so a rule returned 200
and no object ever expired, on the platform backup bucket and on tenant
buckets alike, and the documented way to bound backup storage did
nothing.
Both now run in Simple and MultiZone instances, behind lifecycle.enabled
(on by default). A Client instance has no data plane of its own; its
rules are enforced where the master it connects to runs. The worker
registers for s3_lifecycle alone, so enabling lifecycle does not also
start vacuum, balancing or erasure coding. The admin keeps its job state
on a small volume (on volume.storageClass when set), and its UI gets a
generated password, because a tenant namespace admits traffic from the
whole cluster and the server is unauthenticated without one. The
data-plane chart keeps both off unless its parent turns them on, so it
never starts an admin without that password.
Existing rules start applying minutes after the upgrade: the admin's
scheduler runs the first pass within one idle cycle of its lifecycle
lane, then once a day, and objects the rules have already expired are
deleted on that first pass.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
packages/extra/seaweedfs/README.md | 2 +
.../templates/admin-credentials.yaml | 21 ++
.../extra/seaweedfs/templates/seaweedfs.yaml | 28 +++
.../extra/seaweedfs/tests/lifecycle_test.yaml | 196 ++++++++++++++++++
packages/extra/seaweedfs/values.schema.json | 12 ++
packages/extra/seaweedfs/values.yaml | 7 +
.../seaweedfs-rd/cozyrds/seaweedfs.yaml | 4 +-
.../seaweedfs/tests/lifecycle_test.yaml | 138 ++++++++++++
packages/system/seaweedfs/values.yaml | 41 ++++
9 files changed, 447 insertions(+), 2 deletions(-)
create mode 100644 packages/extra/seaweedfs/templates/admin-credentials.yaml
create mode 100644 packages/extra/seaweedfs/tests/lifecycle_test.yaml
create mode 100644 packages/system/seaweedfs/tests/lifecycle_test.yaml
diff --git a/packages/extra/seaweedfs/README.md b/packages/extra/seaweedfs/README.md
index e269992bf..c0866722b 100644
--- a/packages/extra/seaweedfs/README.md
+++ b/packages/extra/seaweedfs/README.md
@@ -78,4 +78,6 @@
| `s3.resources.cpu` | Number of CPU cores allocated. | `quantity` | `""` |
| `s3.resources.memory` | Amount of memory allocated. | `quantity` | `""` |
| `s3.resourcesPreset` | Default sizing preset used when `resources` is omitted. | `string` | `t1.small` |
+| `lifecycle` | S3 bucket lifecycle configuration. | `object` | `{}` |
+| `lifecycle.enabled` | Run the SeaweedFS admin server and a worker that applies bucket lifecycle rules, once a day. When disabled, rules are still accepted and stored, but not applied. Ignored with the Client topology, whose rules are enforced by the instance it connects to. | `bool` | `true` |
diff --git a/packages/extra/seaweedfs/templates/admin-credentials.yaml b/packages/extra/seaweedfs/templates/admin-credentials.yaml
new file mode 100644
index 000000000..4b9f6dd33
--- /dev/null
+++ b/packages/extra/seaweedfs/templates/admin-credentials.yaml
@@ -0,0 +1,21 @@
+{{- if and .Values.lifecycle.enabled (ne .Values.topology "Client") }}
+{{- $name := printf "%s-admin-credentials" .Release.Name }}
+{{- /* The admin reads these into its environment when it starts and nothing
+ restarts it on a Secret change, so a password regenerated on every render
+ would leave the Secret holding one the running server does not accept.
+ Keep the one already issued. */}}
+{{- $password := randAlphaNum 40 | b64enc }}
+{{- with lookup "v1" "Secret" .Release.Namespace $name }}
+{{- with (dig "data" "adminPassword" "" .) }}
+{{- $password = . }}
+{{- end }}
+{{- end }}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: {{ $name }}
+type: Opaque
+data:
+ adminUser: {{ "admin" | b64enc }}
+ adminPassword: {{ $password }}
+{{- end }}
diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml
index 27e5557e0..c111a80c5 100644
--- a/packages/extra/seaweedfs/templates/seaweedfs.yaml
+++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml
@@ -370,6 +370,16 @@ spec:
secretName: {{ .Release.Name }}-s3-ingress-tls
{{- end }}
resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.s3.resourcesPreset .Values.s3.resources $) | nindent 10 }}
+ admin:
+ enabled: {{ .Values.lifecycle.enabled }}
+ secret:
+ existingSecret: {{ .Release.Name }}-admin-credentials
+ {{- with .Values.volume.storageClass }}
+ data:
+ storageClass: {{ . }}
+ {{- end }}
+ worker:
+ enabled: {{ .Values.lifecycle.enabled }}
cosi:
driverName: "{{ .Release.Namespace }}.seaweedfs.objectstorage.k8s.io"
bucketClassName: "{{ .Release.Namespace }}"
@@ -502,4 +512,22 @@ spec:
app.kubernetes.io/component: s3
app.kubernetes.io/name: seaweedfs
version: {{ $.Chart.Version }}
+{{- if .Values.lifecycle.enabled }}
+{{- range $component := list "admin" "worker" }}
+---
+apiVersion: cozystack.io/v1alpha1
+kind: WorkloadMonitor
+metadata:
+ name: {{ $.Release.Name }}-{{ $component }}
+spec:
+ replicas: 1
+ minReplicas: 1
+ kind: seaweedfs
+ type: {{ $component }}
+ selector:
+ app.kubernetes.io/component: {{ $component }}
+ app.kubernetes.io/name: seaweedfs
+ version: {{ $.Chart.Version }}
+{{- end }}
+{{- end }}
{{- end }}
diff --git a/packages/extra/seaweedfs/tests/lifecycle_test.yaml b/packages/extra/seaweedfs/tests/lifecycle_test.yaml
new file mode 100644
index 000000000..a040e0ae6
--- /dev/null
+++ b/packages/extra/seaweedfs/tests/lifecycle_test.yaml
@@ -0,0 +1,196 @@
+suite: seaweedfs lifecycle switch reaches the data plane
+
+# lifecycle.enabled decides whether the data-plane release runs the admin
+# server and the worker that enforce bucket lifecycle rules. Both flags are
+# sent either way, so the data-plane chart's own default never decides.
+
+templates:
+ - templates/seaweedfs.yaml
+ - templates/admin-credentials.yaml
+
+release:
+ name: seaweedfs
+ namespace: tenant-root
+
+tests:
+ - it: runs the lifecycle admin and worker by default
+ template: templates/seaweedfs.yaml
+ set: &platform
+ _namespace:
+ host: example.org
+ ingress: tenant-root
+ _cluster:
+ issuer-name: letsencrypt-prod
+ solver: http01
+ documentSelector:
+ path: metadata.name
+ value: seaweedfs-system
+ asserts:
+ - equal:
+ path: spec.values.seaweedfs.admin.enabled
+ value: true
+ - equal:
+ path: spec.values.seaweedfs.worker.enabled
+ value: true
+ - equal:
+ path: spec.values.seaweedfs.admin.secret.existingSecret
+ value: seaweedfs-admin-credentials
+
+ - it: turns both off when lifecycle is disabled
+ template: templates/seaweedfs.yaml
+ set:
+ <<: *platform
+ lifecycle:
+ enabled: false
+ documentSelector:
+ path: metadata.name
+ value: seaweedfs-system
+ asserts:
+ - equal:
+ path: spec.values.seaweedfs.admin.enabled
+ value: false
+ - equal:
+ path: spec.values.seaweedfs.worker.enabled
+ value: false
+
+ # The admin's volume follows volume.storageClass when the instance sets one.
+ # Zone storage classes are not followed: the admin is not pinned to a zone.
+ - it: puts the admin volume on the instance's storage class
+ template: templates/seaweedfs.yaml
+ set:
+ <<: *platform
+ volume:
+ storageClass: replicated
+ documentSelector:
+ path: metadata.name
+ value: seaweedfs-system
+ asserts:
+ - equal:
+ path: spec.values.seaweedfs.admin.data.storageClass
+ value: replicated
+
+ - it: leaves the admin volume on the cluster default when the instance names none
+ template: templates/seaweedfs.yaml
+ set: *platform
+ documentSelector:
+ path: metadata.name
+ value: seaweedfs-system
+ asserts:
+ - notExists:
+ path: spec.values.seaweedfs.admin.data
+
+ - it: monitors the admin and the worker while lifecycle is on
+ template: templates/seaweedfs.yaml
+ set: *platform
+ asserts:
+ - containsDocument:
+ kind: WorkloadMonitor
+ apiVersion: cozystack.io/v1alpha1
+ name: seaweedfs-admin
+ any: true
+ - containsDocument:
+ kind: WorkloadMonitor
+ apiVersion: cozystack.io/v1alpha1
+ name: seaweedfs-worker
+ any: true
+ - equal:
+ path: spec.selector
+ value:
+ app.kubernetes.io/component: worker
+ app.kubernetes.io/name: seaweedfs
+ documentSelector:
+ path: metadata.name
+ value: seaweedfs-worker
+ - equal:
+ path: spec.selector
+ value:
+ app.kubernetes.io/component: admin
+ app.kubernetes.io/name: seaweedfs
+ documentSelector:
+ path: metadata.name
+ value: seaweedfs-admin
+
+ - it: does not monitor them when lifecycle is off
+ template: templates/seaweedfs.yaml
+ set:
+ <<: *platform
+ lifecycle:
+ enabled: false
+ asserts:
+ - containsDocument:
+ kind: WorkloadMonitor
+ apiVersion: cozystack.io/v1alpha1
+ name: seaweedfs-admin
+ any: true
+ not: true
+ - containsDocument:
+ kind: WorkloadMonitor
+ apiVersion: cozystack.io/v1alpha1
+ name: seaweedfs-worker
+ any: true
+ not: true
+
+ - it: generates admin credentials for a new instance
+ template: templates/admin-credentials.yaml
+ set: *platform
+ asserts:
+ - isKind:
+ of: Secret
+ - equal:
+ path: metadata.name
+ value: seaweedfs-admin-credentials
+ - equal:
+ path: data.adminUser
+ value: YWRtaW4=
+ - matchRegex:
+ path: data.adminPassword
+ pattern: '^.{40,}$'
+
+ # A new password on every render would leave the running admin on the old
+ # one, since it reads the Secret into its environment once at start.
+ - it: keeps the password of an existing Secret
+ template: templates/admin-credentials.yaml
+ set: *platform
+ kubernetesProvider:
+ scheme:
+ "v1/Secret":
+ gvr:
+ version: v1
+ resource: secrets
+ namespaced: true
+ objects:
+ - apiVersion: v1
+ kind: Secret
+ metadata:
+ name: seaweedfs-admin-credentials
+ namespace: tenant-root
+ data:
+ adminUser: YWRtaW4=
+ adminPassword: a2VwdA==
+ asserts:
+ - equal:
+ path: data.adminPassword
+ value: a2VwdA==
+
+ - it: renders no credentials when lifecycle is off
+ template: templates/admin-credentials.yaml
+ set:
+ <<: *platform
+ lifecycle:
+ enabled: false
+ asserts:
+ - hasDocuments:
+ count: 0
+
+ # A Client instance runs no data plane of its own; its rules are enforced
+ # where the master it points at runs.
+ - it: renders no credentials for a Client instance
+ template: templates/admin-credentials.yaml
+ set:
+ <<: *platform
+ topology: Client
+ filer:
+ grpcHost: seaweedfs-filer.tenant-other.svc
+ asserts:
+ - hasDocuments:
+ count: 0
diff --git a/packages/extra/seaweedfs/values.schema.json b/packages/extra/seaweedfs/values.schema.json
index ab9b953ac..9ce4b184a 100644
--- a/packages/extra/seaweedfs/values.schema.json
+++ b/packages/extra/seaweedfs/values.schema.json
@@ -868,6 +868,18 @@
]
}
}
+ },
+ "lifecycle": {
+ "description": "S3 bucket lifecycle configuration.",
+ "type": "object",
+ "default": {},
+ "properties": {
+ "enabled": {
+ "description": "Run the SeaweedFS admin server and a worker that applies bucket lifecycle rules, once a day. When disabled, rules are still accepted and stored, but not applied. Ignored with the Client topology, whose rules are enforced by the instance it connects to.",
+ "type": "boolean",
+ "default": true
+ }
+ }
}
}
}
diff --git a/packages/extra/seaweedfs/values.yaml b/packages/extra/seaweedfs/values.yaml
index 0aa569bc4..4f47c4fcc 100644
--- a/packages/extra/seaweedfs/values.yaml
+++ b/packages/extra/seaweedfs/values.yaml
@@ -192,3 +192,10 @@ s3:
replicas: 2
resources: {}
resourcesPreset: "t1.small"
+
+## @typedef {struct} Lifecycle - Enforcement of S3 bucket lifecycle rules.
+## @field {bool} [enabled] - Run the SeaweedFS admin server and a worker that applies bucket lifecycle rules, once a day. When disabled, rules are still accepted and stored, but not applied. Ignored with the Client topology, whose rules are enforced by the instance it connects to.
+
+## @param {Lifecycle} [lifecycle] - S3 bucket lifecycle configuration.
+lifecycle:
+ enabled: true
diff --git a/packages/system/seaweedfs-rd/cozyrds/seaweedfs.yaml b/packages/system/seaweedfs-rd/cozyrds/seaweedfs.yaml
index 1fa5ad336..ddf5834e0 100644
--- a/packages/system/seaweedfs-rd/cozyrds/seaweedfs.yaml
+++ b/packages/system/seaweedfs-rd/cozyrds/seaweedfs.yaml
@@ -8,7 +8,7 @@ spec:
singular: seaweedfs
plural: seaweedfses
openAPISchema: |-
- {"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access SeaweedFS externally (defaults to 's3' subdomain for the tenant host).","type":"string","default":""},"topology":{"description":"The topology of the SeaweedFS cluster.","type":"string","default":"Simple","enum":["Simple","MultiZone","Client"]},"replicationFactor":{"description":"Replication factor: number of replicas for each volume in the SeaweedFS cluster.","type":"integer","default":2},"db":{"description":"Database configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of database replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""}}},"master":{"description":"Master service configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of master replicas.","type":"integer","default":3},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"volumeSizeLimitMB":{"description":"Size in MB at which a volume stops taking writes, and the divisor that sets how many buckets a volume server can serve. Lower it to fit more buckets on the same disk, at the cost of more volume files for the same stored data; on a large volume it is worth raising instead. Changing it on an instance that already holds data costs something either way: lowering it stops writes to every bucket whose volumes all reach the new value until that bucket can grow, and raising it takes the room new buckets grow into, so run the check in docs/operations/seaweedfs-lowering-the-volume-size-limit.md first. Must be between 1 and 30000. A value close to or above `volume.size` leaves each volume server a single slot rather than failing, since the server floors its slot count at one.","type":"integer","default":400,"maximum":30000,"minimum":1}}},"filer":{"description":"Filer service configuration.","type":"object","default":{},"properties":{"grpcHost":{"description":"The hostname used to expose or access the filer service externally.","type":"string","default":""},"grpcPort":{"description":"The port used to access the filer service externally.","type":"integer","default":443},"replicas":{"description":"Number of filer replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"whitelist":{"description":"A list of IP addresses or CIDR ranges that are allowed to access the filer service.","type":"array","default":[],"items":{"type":"string"}}}},"volume":{"description":"Volume service configuration.","type":"object","default":{},"properties":{"diskType":{"description":"SeaweedFS disk type tag for the default volume servers (e.g., \"hdd\", \"ssd\").","type":"string","default":""},"pools":{"description":"A map of storage pools. Each pool creates a separate Volume StatefulSet with its own disk type.","type":"object","default":{},"additionalProperties":{"type":"object","required":["diskType"],"properties":{"diskType":{"description":"SeaweedFS disk type tag (e.g., \"ssd\", \"hdd\", \"nvme\").","type":"string"},"replicas":{"description":"Number of volume replicas. Defaults to volume.replicas (Simple) or zone.replicas/volume.replicas (MultiZone).","type":"integer"},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted. Defaults to volume.resourcesPreset.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size. Defaults to volume.size (Simple) or zone.size/volume.size (MultiZone).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"Kubernetes StorageClass for the pool. Defaults to volume.storageClass.","type":"string"}}}},"replicas":{"description":"Number of volume replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"zones":{"description":"A map of zones for MultiZone topology. Each zone can have its own number of replicas and size.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"dataCenter":{"description":"SeaweedFS data center name for this zone. Defaults to the zone name.","type":"string"},"nodeSelector":{"description":"YAML nodeSelector for this zone (default: topology.kubernetes.io/zone: ).","type":"string"},"pools":{"description":"A map of storage pools for this zone. Each pool creates a separate Volume StatefulSet per zone.","type":"object","additionalProperties":{"type":"object","required":["diskType"],"properties":{"diskType":{"description":"SeaweedFS disk type tag (e.g., \"ssd\", \"hdd\", \"nvme\").","type":"string"},"replicas":{"description":"Number of volume replicas. Defaults to volume.replicas (Simple) or zone.replicas/volume.replicas (MultiZone).","type":"integer"},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted. Defaults to volume.resourcesPreset.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size. Defaults to volume.size (Simple) or zone.size/volume.size (MultiZone).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"Kubernetes StorageClass for the pool. Defaults to volume.storageClass.","type":"string"}}}},"replicas":{"description":"Number of replicas in the zone.","type":"integer"},"size":{"description":"Zone storage size.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store zone data. Defaults to volume.storageClass.","type":"string"}}}}}},"s3":{"description":"S3 service configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of S3 replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}}
+ {"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access SeaweedFS externally (defaults to 's3' subdomain for the tenant host).","type":"string","default":""},"topology":{"description":"The topology of the SeaweedFS cluster.","type":"string","default":"Simple","enum":["Simple","MultiZone","Client"]},"replicationFactor":{"description":"Replication factor: number of replicas for each volume in the SeaweedFS cluster.","type":"integer","default":2},"db":{"description":"Database configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of database replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""}}},"master":{"description":"Master service configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of master replicas.","type":"integer","default":3},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"volumeSizeLimitMB":{"description":"Size in MB at which a volume stops taking writes, and the divisor that sets how many buckets a volume server can serve. Lower it to fit more buckets on the same disk, at the cost of more volume files for the same stored data; on a large volume it is worth raising instead. Changing it on an instance that already holds data costs something either way: lowering it stops writes to every bucket whose volumes all reach the new value until that bucket can grow, and raising it takes the room new buckets grow into, so run the check in docs/operations/seaweedfs-lowering-the-volume-size-limit.md first. Must be between 1 and 30000. A value close to or above `volume.size` leaves each volume server a single slot rather than failing, since the server floors its slot count at one.","type":"integer","default":400,"maximum":30000,"minimum":1}}},"filer":{"description":"Filer service configuration.","type":"object","default":{},"properties":{"grpcHost":{"description":"The hostname used to expose or access the filer service externally.","type":"string","default":""},"grpcPort":{"description":"The port used to access the filer service externally.","type":"integer","default":443},"replicas":{"description":"Number of filer replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"whitelist":{"description":"A list of IP addresses or CIDR ranges that are allowed to access the filer service.","type":"array","default":[],"items":{"type":"string"}}}},"volume":{"description":"Volume service configuration.","type":"object","default":{},"properties":{"diskType":{"description":"SeaweedFS disk type tag for the default volume servers (e.g., \"hdd\", \"ssd\").","type":"string","default":""},"pools":{"description":"A map of storage pools. Each pool creates a separate Volume StatefulSet with its own disk type.","type":"object","default":{},"additionalProperties":{"type":"object","required":["diskType"],"properties":{"diskType":{"description":"SeaweedFS disk type tag (e.g., \"ssd\", \"hdd\", \"nvme\").","type":"string"},"replicas":{"description":"Number of volume replicas. Defaults to volume.replicas (Simple) or zone.replicas/volume.replicas (MultiZone).","type":"integer"},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted. Defaults to volume.resourcesPreset.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size. Defaults to volume.size (Simple) or zone.size/volume.size (MultiZone).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"Kubernetes StorageClass for the pool. Defaults to volume.storageClass.","type":"string"}}}},"replicas":{"description":"Number of volume replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"zones":{"description":"A map of zones for MultiZone topology. Each zone can have its own number of replicas and size.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"dataCenter":{"description":"SeaweedFS data center name for this zone. Defaults to the zone name.","type":"string"},"nodeSelector":{"description":"YAML nodeSelector for this zone (default: topology.kubernetes.io/zone: ).","type":"string"},"pools":{"description":"A map of storage pools for this zone. Each pool creates a separate Volume StatefulSet per zone.","type":"object","additionalProperties":{"type":"object","required":["diskType"],"properties":{"diskType":{"description":"SeaweedFS disk type tag (e.g., \"ssd\", \"hdd\", \"nvme\").","type":"string"},"replicas":{"description":"Number of volume replicas. Defaults to volume.replicas (Simple) or zone.replicas/volume.replicas (MultiZone).","type":"integer"},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted. Defaults to volume.resourcesPreset.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume size. Defaults to volume.size (Simple) or zone.size/volume.size (MultiZone).","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"Kubernetes StorageClass for the pool. Defaults to volume.storageClass.","type":"string"}}}},"replicas":{"description":"Number of replicas in the zone.","type":"integer"},"size":{"description":"Zone storage size.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store zone data. Defaults to volume.storageClass.","type":"string"}}}}}},"s3":{"description":"S3 service configuration.","type":"object","default":{},"properties":{"replicas":{"description":"Number of S3 replicas.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.small","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"lifecycle":{"description":"S3 bucket lifecycle configuration.","type":"object","default":{},"properties":{"enabled":{"description":"Run the SeaweedFS admin server and a worker that applies bucket lifecycle rules, once a day. When disabled, rules are still accepted and stored, but not applied. Ignored with the Client topology, whose rules are enforced by the instance it connects to.","type":"boolean","default":true}}}}}
release:
prefix: ""
labels:
@@ -26,7 +26,7 @@ spec:
description: Seaweedfs
module: true
icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82NzlfMTkxMCkiLz4KPHBhdGggZD0iTTEzOC42ODUgMTIxLjA1N0MxMzguNjg1IDEyNi42NTIgMTM2LjQ2MiAxMzIuMDE3IDEzMi41MDQgMTM1Ljk3M0MxMjguNTQ3IDEzOS45MjkgMTIzLjE3OSAxNDIuMTUxIDExNy41ODIgMTQyLjE1MUMxMTEuOTg1IDE0Mi4xNTEgMTA2LjYxOCAxMzkuOTI5IDEwMi42NiAxMzUuOTczQzk4LjcwMjggMTMyLjAxNyA5Ni40Nzk1IDEyNi42NTIgOTYuNDc5NSAxMjEuMDU3Qzk2LjQ3OTUgMTE4LjI4NyA5Ny4wMjUzIDExNS41NDQgOTguMDg1OCAxMTIuOTg1Qzk5LjE0NjMgMTEwLjQyNSAxMDAuNzAxIDEwOC4xIDEwMi42NiAxMDYuMTQxQzEwNC42MiAxMDQuMTgyIDEwNi45NDYgMTAyLjYyOSAxMDkuNTA3IDEwMS41NjlDMTEyLjA2NyAxMDAuNTA5IDExNC44MTEgOTkuOTYyOSAxMTcuNTgyIDk5Ljk2MjlDMTIwLjM1MyA5OS45NjI5IDEyMy4wOTggMTAwLjUwOSAxMjUuNjU4IDEwMS41NjlDMTI4LjIxOCAxMDIuNjI5IDEzMC41NDQgMTA0LjE4MiAxMzIuNTA0IDEwNi4xNDFDMTM0LjQ2NCAxMDguMSAxMzYuMDE4IDExMC40MjUgMTM3LjA3OSAxMTIuOTg1QzEzOC4xMzkgMTE1LjU0NCAxMzguNjg1IDExOC4yODcgMTM4LjY4NSAxMjEuMDU3WiIgZmlsbD0idXJsKCNwYWludDFfcmFkaWFsXzY3OV8xOTEwKSIvPgo8cGF0aCBkPSJNMTEwLjMzNiAxMjYuMTQ3SDEyMy42OFYxMzYuODIzSDExMC4zMzZWMTI2LjE0N1oiIGZpbGw9IiMwRjVFOUMiLz4KPHBhdGggZD0iTTExOS4yNyAxMTIuMjk0QzExOS4yNyAxMTIuNzE0IDExOS4xNzggMTEzLjEzMSAxMTkgMTEzLjUxOUMxMTguODIxIDExMy45MDggMTE4LjU2IDExNC4yNjEgMTE4LjIzIDExNC41NThDMTE3LjkwMSAxMTQuODU1IDExNy41MDkgMTE1LjA5MSAxMTcuMDc5IDExNS4yNTJDMTE2LjY0OCAxMTUuNDEzIDExNi4xODYgMTE1LjQ5NiAxMTUuNzIgMTE1LjQ5NkMxMTQuNzc4IDExNS40OTYgMTEzLjg3NSAxMTUuMTU4IDExMy4yMSAxMTQuNTU4QzExMi41NDQgMTEzLjk1NyAxMTIuMTcgMTEzLjE0MyAxMTIuMTcgMTEyLjI5NEMxMTIuMTcgMTExLjQ0NSAxMTIuNTQ0IDExMC42MyAxMTMuMjEgMTEwLjAzQzExMy44NzUgMTA5LjQyOSAxMTQuNzc4IDEwOS4wOTIgMTE1LjcyIDEwOS4wOTJDMTE2LjE4NiAxMDkuMDkyIDExNi42NDggMTA5LjE3NSAxMTcuMDc5IDEwOS4zMzZDMTE3LjUwOSAxMDkuNDk2IDExNy45MDEgMTA5LjczMiAxMTguMjMgMTEwLjAzQzExOC41NiAxMTAuMzI3IDExOC44MjEgMTEwLjY4IDExOSAxMTEuMDY4QzExOS4xNzggMTExLjQ1NyAxMTkuMjcgMTExLjg3MyAxMTkuMjcgMTEyLjI5NFoiIGZpbGw9IiM1OTY4NkYiLz4KPHBhdGggZD0iTTEyOC4xMSAxMTQuMTAzQzEyOC4xMSAxMTQuOTUzIDEyNy43MzYgMTE1Ljc2NyAxMjcuMDcgMTE2LjM2OEMxMjYuNDA0IDExNi45NjggMTI1LjUwMSAxMTcuMzA1IDEyNC41NiAxMTcuMzA1QzEyNC4wOTQgMTE3LjMwNSAxMjMuNjMyIDExNy4yMjMgMTIzLjIwMSAxMTcuMDYyQzEyMi43NzEgMTE2LjkwMSAxMjIuMzc5IDExNi42NjUgMTIyLjA1IDExNi4zNjhDMTIxLjcyIDExNi4wNyAxMjEuNDU4IDExNS43MTcgMTIxLjI4IDExNS4zMjlDMTIxLjEwMiAxMTQuOTQgMTIxLjAxIDExNC41MjQgMTIxLjAxIDExNC4xMDNDMTIxLjAxIDExMy42ODMgMTIxLjEwMiAxMTMuMjY2IDEyMS4yOCAxMTIuODc4QzEyMS40NTggMTEyLjQ5IDEyMS43MiAxMTIuMTM3IDEyMi4wNSAxMTEuODM5QzEyMi4zNzkgMTExLjU0MiAxMjIuNzcxIDExMS4zMDYgMTIzLjIwMSAxMTEuMTQ1QzEyMy42MzIgMTEwLjk4NCAxMjQuMDk0IDExMC45MDEgMTI0LjU2IDExMC45MDFDMTI1LjUwMSAxMTAuOTAxIDEyNi40MDQgMTExLjIzOSAxMjcuMDcgMTExLjgzOUMxMjcuNzM2IDExMi40NCAxMjguMTEgMTEzLjI1NCAxMjguMTEgMTE0LjEwM1oiIGZpbGw9IiM1OTY4NkYiLz4KPHBhdGggZD0iTTEyMi4zMzMgMTE4Ljk3NkMxMjIuMzMzIDExOS44MjYgMTIxLjk1OCAxMjAuNjQgMTIxLjI5MyAxMjEuMjQxQzEyMC42MjcgMTIxLjg0MSAxMTkuNzI0IDEyMi4xNzggMTE4Ljc4MiAxMjIuMTc4QzExOC4zMTYgMTIyLjE3OCAxMTcuODU1IDEyMi4wOTYgMTE3LjQyNCAxMjEuOTM1QzExNi45OTMgMTIxLjc3NCAxMTYuNjAyIDEyMS41MzggMTE2LjI3MiAxMjEuMjQxQzExNS45NDMgMTIwLjk0MyAxMTUuNjgxIDEyMC41OSAxMTUuNTAzIDEyMC4yMDJDMTE1LjMyNCAxMTkuODEzIDExNS4yMzIgMTE5LjM5NyAxMTUuMjMyIDExOC45NzZDMTE1LjIzMiAxMTguNTU2IDExNS4zMjQgMTE4LjE0IDExNS41MDMgMTE3Ljc1MUMxMTUuNjgxIDExNy4zNjMgMTE1Ljk0MyAxMTcuMDEgMTE2LjI3MiAxMTYuNzEyQzExNi42MDIgMTE2LjQxNSAxMTYuOTkzIDExNi4xNzkgMTE3LjQyNCAxMTYuMDE4QzExNy44NTUgMTE1Ljg1NyAxMTguMzE2IDExNS43NzQgMTE4Ljc4MiAxMTUuNzc0QzExOS43MjQgMTE1Ljc3NCAxMjAuNjI3IDExNi4xMTIgMTIxLjI5MyAxMTYuNzEyQzEyMS45NTggMTE3LjMxMyAxMjIuMzMzIDExOC4xMjcgMTIyLjMzMyAxMTguOTc2WiIgZmlsbD0iIzU5Njg2RiIvPgo8cGF0aCBkPSJNMTE1LjMwOCAxMjEuOTA1QzExMy43MzUgMTIxLjQyNiAxMTUuNzA3IDEyMC42OCAxMTUuNDI5IDEyMC41NzNDMTE0LjY1MyAxMjAuMjc2IDExNS43MyAxMTkuMzMzIDExNi43NCAxMTguNTM5QzExNy42MjggMTE3Ljg0MSAxMTcuNjU5IDExNy44MzkgMTE3Ljg5MiAxMTguNDY4QzExOC4wMjQgMTE4LjgyMyAxMTguMzcxIDExOS4yMDYgMTE4LjY2NSAxMTkuMzE5QzExOS4yODkgMTE5LjU1OCAxMjAuOTUxIDExOS4yNjkgMTIwLjk1MSAxMTguODM1QzEyMC45NTEgMTE4LjIyMyAxMjEuNDE1IDExOC42OTkgMTIxLjc5NCAxMTkuNTMxQzEyMi40NTcgMTIwLjk4NyAxMjIuNDM3IDEyMi40NzIgMTIxLjQ1IDEyMi40NzJDMTIwLjg1OSAxMjIuNDcyIDEyMC4zMSAxMjIuNjkxIDEyMC4xOSAxMjMuMTQ3QzExNS4wNDYgMTI0LjYgMTE2LjQ3MSAxMjMuMjg0IDExNS4zMDggMTIxLjkwNVpNMTIzLjA5NCAxMTguMDU0QzEyMS42MDkgMTE3LjQ2NiAxMjAuNTQ3IDExNC44MzggMTIwLjU0NyAxMTMuMzA5QzEyMC41NDcgMTExLjMyMyAxMjIuNTQxIDEwOS4wOTUgMTI0LjMxNyAxMDkuMDk1QzEyOC4zMTUgMTA5LjA5NSAxMzAuNjg0IDExMi4yNjEgMTI4LjgzOCAxMTguNDA5QzEyOC4zODUgMTE5LjkxOSAxMjMuOTMzIDExOC4zODcgMTIzLjA5NCAxMTguMDU0Wk0xMjUuODY2IDExNS42NDlDMTI3LjU0NCAxMTQuNDc0IDEyNS44NTcgMTExLjc1NSAxMjMuOTgxIDExMi42MUMxMjIuNDc5IDExMy4yOTQgMTIzLjExOSAxMTYuMTE1IDEyNC43NyAxMTYuMTY0QzEyNC45NjEgMTE2LjE2OSAxMjUuNDU0IDExNS45MzggMTI1Ljg2NiAxMTUuNjQ5Wk0xMjQuMzM5IDExNC41NzlDMTIzLjc3MSAxMTQuMzgxIDEyMy44MDMgMTEzLjI1NiAxMjQuMzgzIDExMy4wMzRDMTI0LjkwMiAxMTIuODM1IDEyNS42MDQgMTEzLjM5OCAxMjUuNjA0IDExNC4wMTRDMTI1LjYwNCAxMTQuNDg5IDEyNC45MzYgMTE0Ljc4NyAxMjQuMzM5IDExNC41NzlaTTExMi4zMTkgMTE2Ljk4QzEwOS45NiAxMTUuNjcgMTEwLjI4NiAxMTEuMDA2IDExMi40MTcgMTA4Ljk2NUMxMTQuMzk2IDEwNy4wNjkgMTE4Ljc3OCAxMDcuMTAzIDExOS43NjQgMTA5LjQ2MkMxMjAuNDE1IDExMS4wMjEgMTIwLjIyOCAxMTIuNiAxMTkuMzk4IDExNC4wNzdDMTE4LjE5NCAxMTYuMjE5IDExNC4yNDIgMTE4LjA0OSAxMTIuMzE5IDExNi45OFpNMTE2LjU2IDExMy41OTNDMTE3LjMyNSAxMTIuOTAxIDExNy4yOTcgMTEyLjA2IDExNi41NzIgMTExLjI1OUMxMTUuNzc3IDExMC4zNzkgMTE0LjY5MiAxMTAuMzQ1IDExNC4wMzUgMTExLjM0N0MxMTMuNTI0IDExMi4xMjcgMTEzLjQzMSAxMTIuNTI4IDExMy45NDMgMTEzLjMwOUMxMTQuNTggMTE0LjI4IDExNS42NyAxMTQuMzk5IDExNi41NiAxMTMuNTkzWk0xMTQuMzc0IDExMi4zNEMxMTQuMTI4IDExMS42OTggMTE0Ljk4NSAxMTAuOTgxIDExNS42OCAxMTEuMjQ3QzExNS45NzQgMTExLjM2IDExNi4xNjQgMTExLjcxOCAxMTYuMTAyIDExMi4wNDRDMTE1Ljk2MSAxMTIuNzg1IDExNC42MzIgMTEzLjAxMiAxMTQuMzc0IDExMi4zNFoiIGZpbGw9IiNEM0Q2REEiLz4KPHBhdGggZD0iTTExOC40NjMgMTIxLjAwOEwxMTcuOTQ1IDEyMC43OEMxMTcuNTEgMTIwLjY4NSAxMTcuMjMxIDEyMS4xMDcgMTE3LjEzNiAxMjEuNTQzTDExNi44ODEgMTIyLjcwOUMxMTYuNzg2IDEyMy4xNDUgMTE3LjA2MSAxMjMuNTcxIDExNy40OTYgMTIzLjY2NkwxMTguNDQ3IDEyMy44NzRDMTE4Ljg4MiAxMjMuOTY5IDExOS4zMTEgMTIzLjY5NCAxMTkuNDA2IDEyMy4yNTlMMTE5LjY4NSAxMjEuNTU2QzExOS43OCAxMjEuMTIgMTE5LjQ3OSAxMjEuMjI4IDExOS4wNDMgMTIxLjEzM0wxMTguNjI4IDEyMS4wNDNDMTE4LjU3MyAxMjEuMzIxIDExOC41MTYgMTIxLjYxMyAxMTguNDcxIDEyMS44MzNDMTE4LjQzOCAxMjEuOTk4IDExOC40MDcgMTIyLjE0OCAxMTguMzc5IDEyMi4yODJDMTE4LjM1MSAxMjIuNDE1IDExOC4zMjcgMTIyLjUzMyAxMTguMzA1IDEyMi42MzVDMTE4LjI4MiAxMjIuNzM4IDExOC4yNjIgMTIyLjgyNSAxMTguMjQ1IDEyMi44OTdDMTE4LjIyOCAxMjIuOTY5IDExOC4yMTQgMTIzLjAyNSAxMTguMjAyIDEyMy4wNjhDMTE4LjE5NiAxMjMuMDg5IDExOC4xOTMgMTIzLjEwNyAxMTguMTg4IDEyMy4xMjFDMTE4LjE4MyAxMjMuMTM1IDExOC4xNzggMTIzLjE0NSAxMTguMTc1IDEyMy4xNTJDMTE4LjE3MyAxMjMuMTU1IDExOC4xNzIgMTIzLjE1NiAxMTguMTcxIDEyMy4xNThDMTE4LjE3IDEyMy4xNTkgMTE4LjE2OCAxMjMuMTYgMTE4LjE2NyAxMjMuMTZMMTE4LjE2NSAxMjMuMTU4QzExOC4xNjQgMTIzLjE1NiAxMTguMTYzIDEyMy4xNTIgMTE4LjE2MyAxMjMuMTQ4QzExOC4xNjIgMTIzLjE0IDExOC4xNjIgMTIzLjEyOSAxMTguMTYzIDEyMy4xMTVDMTE4LjE2MyAxMjMuMSAxMTguMTYzIDEyMy4wODMgMTE4LjE2NSAxMjMuMDYxQzExOC4xNjggMTIzLjAxOCAxMTguMTc1IDEyMi45NjEgMTE4LjE4MyAxMjIuODkxQzExOC4xOTIgMTIyLjgyIDExOC4yMDMgMTIyLjczNiAxMTguMjE2IDEyMi42NEMxMTguMjI5IDEyMi41NDMgMTE4LjI0NiAxMjIuNDMxIDExOC4yNjQgMTIyLjMwOEMxMTguMjgxIDEyMi4xODUgMTE4LjMwMSAxMjIuMDUxIDExOC4zMjMgMTIxLjkwM0MxMTguMzQ2IDEyMS43NTUgMTE4LjM3IDEyMS41OTIgMTE4LjM5NyAxMjEuNDE5QzExOC40MTYgMTIxLjI5OSAxMTguNDQyIDEyMS4xNCAxMTguNDYzIDEyMS4wMDhaIiBmaWxsPSIjOThDNkQ4Ii8+CjxwYXRoIGQ9Ik0xMDMuNTgxIDExNi4zNjdDMTAzLjQ3OSAxMTYuMTAzIDEwMy42MjIgMTE1LjY5OSAxMDMuODk4IDExNS40NjlDMTA0LjMwMyAxMTUuMTMzIDEwNC41MDQgMTE1LjE1NSAxMDQuOTMgMTE1LjU4MUMxMDUuMjIgMTE1Ljg3MSAxMDUuMzU1IDExNi4yNzYgMTA1LjIzIDExNi40NzlDMTA0LjkwMiAxMTcuMDA5IDEwMy43OTggMTE2LjkzNCAxMDMuNTgxIDExNi4zNjdaIiBmaWxsPSIjOThDNkQ4Ii8+CjxwYXRoIGQ9Ik0xMDYuMDM4IDExMi4yODFDMTA1LjY4IDExMS44NDkgMTA1LjcwMiAxMTEuNjYgMTA2LjE2NSAxMTEuMTk3QzEwNi42ODkgMTEwLjY3NCAxMDYuNzY0IDExMC42NzQgMTA3LjI4NyAxMTEuMTk3QzEwNy43NTEgMTExLjY2IDEwNy43NzMgMTExLjg0OSAxMDcuNDE1IDExMi4yODFDMTA3LjE3NiAxMTIuNTY4IDEwNi44NjYgMTEyLjgwMyAxMDYuNzI2IDExMi44MDNDMTA2LjU4NiAxMTIuODAzIDEwNi4yNzcgMTEyLjU2OCAxMDYuMDM4IDExMi4yODFaIiBmaWxsPSIjOThDNkQ4Ii8+CjxwYXRoIGQ9Ik0xMDYuMDM4IDEwNy4yMjRDMTA1LjY4MiAxMDYuNzk1IDEwNS42OTQgMTA2LjYxMiAxMDYuMTA2IDEwNi4yMDFDMTA2LjY1IDEwNS42NTcgMTA3LjczOCAxMDUuODggMTA3LjczOCAxMDYuNTM2QzEwNy43MzggMTA2Ljk3IDEwNy4wNzIgMTA3Ljc0NyAxMDYuNyAxMDcuNzQ3QzEwNi41NzUgMTA3Ljc0NyAxMDYuMjc3IDEwNy41MTIgMTA2LjAzOCAxMDcuMjI0WiIgZmlsbD0iIzk4QzZEOCIvPgo8cGF0aCBkPSJNMTE3Ljk1NSAxMzYuNTQxQzExNC41NzggMTM2LjExMSAxMTAuMDg3IDEzNC44ODEgMTA5LjQxIDEzNC4yNzRDMTA4LjEyMyAxMzMuMTE5IDEwOC45NDIgMTI3Ljg3OSAxMTAuNDY2IDEyNi41NEMxMTEuMzg3IDEyNS43MzEgMTE1LjAzOSAxMjQuNjYzIDExNy4xODIgMTI1LjE2OUMxMTkuNjQyIDEyNS43NDkgMTIzLjkzMiAxMjguNjQ4IDEyNC4yNSAxMjkuNTA4QzEyNC42MjUgMTMwLjUyMiAxMjQuMDQ4IDEzNC41NDEgMTIzLjMyMyAxMzUuNTM2QzEyMi42OTQgMTM2LjM5OSAxMjAuMzI1IDEzNi44NDMgMTE3Ljk1NSAxMzYuNTQxWk0xMjAuNTQgMTM0LjA4NEMxMjEuMjk3IDEzMy4zOTkgMTIxLjM0MiAxMzEuODQyIDEyMC42MjcgMTMxLjEyNkMxMTkuNDkgMTI5Ljk4OSAxMTcuMTEyIDEzMC44NjkgMTE3LjExMiAxMzIuNDI4QzExNy4xMTIgMTM0LjMwNCAxMTkuMTg5IDEzNS4zMDcgMTIwLjU0IDEzNC4wODRaTTExOC41ODIgMTMzLjEyNUMxMTguMzExIDEzMi40MTkgMTE4LjY4IDEzMS42MDggMTE5LjI3MiAxMzEuNjA4QzEyMCAxMzEuNjA4IDEyMC4yMjggMTMyLjE3MyAxMTkuODEyIDEzMi45NDlDMTE5LjM4MSAxMzMuNzU1IDExOC44NTMgMTMzLjgzIDExOC41ODIgMTMzLjEyNVpNMTE2LjQyMiAxMzMuMTQzQzExNy4wOTQgMTMyLjMzMyAxMTYuNzExIDEzMS4xNzYgMTE1LjY4MSAxMzAuOTAyQzExNC41ODEgMTMwLjYxIDExNC4wNTIgMTMxLjE4MyAxMTQuODY5IDEzMS43OEMxMTUuNjgzIDEzMi4zNzUgMTE1LjU1MSAxMzIuNjc5IDExNC41MzMgMTMyLjU1N0MxMTMuMzI3IDEzMi40MTMgMTEyLjgzMyAxMzEuNTY2IDExMy4yNTQgMTMwLjM2MkMxMTMuNjgxIDEyOS4xNDIgMTE1LjE5NyAxMjguODYzIDExNS43NTMgMTI5LjkwMkMxMTYuMTkzIDEzMC43MjUgMTE2LjYyNyAxMzAuNzkzIDExNi45IDEzMC4wODNDMTE3LjIyOSAxMjkuMjI0IDExNS45MTQgMTI4LjIzNyAxMTQuNDQgMTI4LjIzN0MxMTEuOTkzIDEyOC4yMzcgMTEwLjgxNiAxMzEuMDc0IDExMi41NDYgMTMyLjgwM0MxMTMuNSAxMzMuNzU4IDExNS43NDkgMTMzLjk1NSAxMTYuNDIyIDEzMy4xNDNaIiBmaWxsPSIjN0JBOUI5Ii8+CjxwYXRoIGQ9Ik0xMjAuMTE4IDEzOC41OTJDMTE4Ljc1MSAxMzcuMjk3IDEyMS45MTYgMTM2LjA5NiAxMjMuNjA2IDEzOC4yODNDMTI0LjQ2OSAxMzkuMzIyIDEyMi44NTMgMTM5Ljk3NiAxMjAuMTE4IDEzOC41OTJaTTEwNi4xMiAxMzUuNjU4QzEwNC43NTEgMTM0LjI4OSAxMDYuOTYzIDEzMy45MDQgMTA4LjU4MSAxMzUuMjNMMTA5Ljk0MSAxMzUuOTJMMTA4LjA1OSAxMzYuMDYxQzEwNy4yMTUgMTM2LjA2MiAxMDYuMzQzIDEzNS44ODEgMTA2LjEyIDEzNS42NThWMTM1LjY1OFpNMTI1LjM2MyAxMjcuODY2QzEyNS4wODIgMTI3LjQxIDEyOC4zMzMgMTI2LjI2NyAxMjguNjg4IDEyNi40ODZDMTI4Ljg0NiAxMjYuNTg0IDEyOC45NzUgMTI3LjMzNyAxMjguOTc1IDEyOC4xNjFDMTI4Ljk3NSAxMjkuMjM5IDEyOC44NCAxMjkuNjU4IDEyOC40OSAxMjkuNjU4QzEyOC4yMjIgMTI5LjY1OCAxMjUuNDc2IDEyOC4wNDcgMTI1LjM2MyAxMjcuODY2Wk0xMTguNjQxIDEyMS4wOTVDMTE0Ljg3MSAxMjAuNzUzIDExNS4zNzggMTE5LjMyNyAxMTYuMzYxIDExOC40OTlDMTE2LjkyOCAxMTguMDIxIDExNy4zNCAxMTcuNzc5IDExNy45MTMgMTE3LjY2N0MxMTcuOTEzIDExNy42NjcgMTE3Ljg3MiAxMTkuMDQ2IDExOC42NjYgMTE5LjMxOUMxMTkuMjk3IDExOS41MzYgMTIwLjI3OCAxMTkuMDY0IDEyMC40NzEgMTE4LjY3NUMxMjIuMTg4IDExNS4yMDEgMTIxLjQxNSAxMTguNjk5IDEyMS43OTQgMTE5LjUzMkMxMjMuMjM4IDEyNC4wMDcgMTIwLjE4OCAxMjEuNzMgMTE4LjY0MSAxMjEuMDk1Wk0xMDQuOTEyIDEyMS4yM0MxMDQuMjQyIDEyMC40ODcgMTAzLjY5MyAxMTkuNzI5IDEwMy42OTMgMTE5LjU0NEMxMDMuNjkzIDExOS4xMTYgMTA0Ljc0NiAxMTkuMTEyIDEwNS44NjMgMTE5LjUzN0MxMDYuNTkzIDExOS44MTQgMTEwLjM0NyAxMjAuMDA2IDExMC4zNDcgMTIxLjE1M0MxMTAuMzQ3IDEyMS44OTkgMTA4Ljg5IDEyMy43NjIgMTA4LjcyNiAxMjMuNzYyQzEwOC41NjMgMTIzLjc2MiAxMDUuNTgzIDEyMS45NzIgMTA0LjkxMiAxMjEuMjNaIiBmaWxsPSIjQTZCM0MyIi8+CjxwYXRoIGQ9Ik0xMTguNDYzIDEyMS4wMDhDMTE4LjQ0MiAxMjEuMTQgMTE4LjQxNiAxMjEuMjk5IDExOC4zOTcgMTIxLjQxOUMxMTguMzcgMTIxLjU5MiAxMTguMzQ1IDEyMS43NTUgMTE4LjMyMyAxMjEuOTAzQzExOC4zIDEyMi4wNTEgMTE4LjI4MSAxMjIuMTg1IDExOC4yNjMgMTIyLjMwOEMxMTguMjQ1IDEyMi40MzEgMTE4LjIyOSAxMjIuNTQyIDExOC4yMTYgMTIyLjYzOUMxMTguMjAzIDEyMi43MzYgMTE4LjE5MSAxMjIuODIgMTE4LjE4MyAxMjIuODlDMTE4LjE3NSAxMjIuOTYxIDExOC4xNjggMTIzLjAxOCAxMTguMTY1IDEyMy4wNjFDMTE4LjE2MyAxMjMuMDgzIDExOC4xNjQgMTIzLjEgMTE4LjE2MyAxMjMuMTE1QzExOC4xNjIgMTIzLjEyOSAxMTguMTYyIDEyMy4xNCAxMTguMTYzIDEyMy4xNDhDMTE4LjE2MyAxMjMuMTUyIDExOC4xNjQgMTIzLjE1NiAxMTguMTY1IDEyMy4xNThMMTE4LjE2NyAxMjMuMTZDMTE4LjE2OCAxMjMuMTYgMTE4LjE3IDEyMy4xNTkgMTE4LjE3MSAxMjMuMTU4QzExOC4xNzIgMTIzLjE1NyAxMTguMTczIDEyMy4xNTUgMTE4LjE3NSAxMjMuMTUyQzExOC4xNzggMTIzLjE0NSAxMTguMTgyIDEyMy4xMzUgMTE4LjE4NyAxMjMuMTIxQzExOC4xOTIgMTIzLjEwNyAxMTguMTk2IDEyMy4wODkgMTE4LjIwMiAxMjMuMDY3QzExOC4yMTMgMTIzLjAyNSAxMTguMjI4IDEyMi45NjkgMTE4LjI0NSAxMjIuODk3QzExOC4yNjIgMTIyLjgyNSAxMTguMjgyIDEyMi43MzggMTE4LjMwNSAxMjIuNjM1QzExOC4zMjcgMTIyLjUzMyAxMTguMzUxIDEyMi40MTUgMTE4LjM3OSAxMjIuMjgxQzExOC40MDYgMTIyLjE0OCAxMTguNDM4IDEyMS45OTggMTE4LjQ3MSAxMjEuODMzQzExOC41MTYgMTIxLjYxMyAxMTguNTczIDEyMS4zMjEgMTE4LjYyOCAxMjEuMDQzTDExOC40NjMgMTIxLjAwOFoiIGZpbGw9IiM0QzlDQkIiLz4KPHBhdGggZD0iTTMwLjk5ODYgMTQyLjY3M0MyNi40NDg5IDE0MC45MTYgMjEuNDY2NCAxMzYuODA1IDE4Ljc5MjEgMTMyLjYwM0MxNS40MDUzIDEyNy4yODEgMTQuNDA1MiAxMjMuNDE5IDE1LjA2MzYgMTE4LjIwM0MxNS42ODA2IDExMy4zMTUgMTcuMzU1OCAxMTAuNTU3IDIyLjIyMzIgMTA2LjQxM0MyOS4wNDE3IDEwMC42MDggMzQuNDg0NiA5Ny45Nzk5IDQwLjU5NDcgOTcuNTQxNUM0OS44MjEyIDk2Ljg3OTQgNTUuMDk3OCA5OS44MTA5IDU3LjY0NjUgMTA3LjAxNUM1OC41MDQzIDEwOS40MzkgNTguNjg1NiAxMTUuNTcgNTguMDU0NCAxMjAuODA5QzU3LjgzMTEgMTIyLjY2MyA1Ny40Nzg5IDEyNi4wNTUgNTcuMjcxOCAxMjguMzQ4QzU2LjU0NjcgMTM2LjM3NSA1NC40MTQ3IDE0MC40MTcgNDkuNjI1MyAxNDIuODQzQzQ4LjI5NzkgMTQzLjUxNiA0Ny43MTk5IDE0My41NjEgNDAuNjkyNCAxNDMuNTM5QzMzLjUxNTEgMTQzLjUxNiAzMy4wODEgMTQzLjQ3OCAzMC45OTg2IDE0Mi42NzNaTTAuMDc5NTc2NSAxMDQuOTY1QzAuMDgxNTcyMyAxMDMuODUzIDAuMTQ0NDAzIDEwMy40MzggMC4yMTk0MjcgMTA0LjA0NEMwLjI5NDQ1MSAxMDQuNjUgMC4yOTI4OTkgMTA1LjU2IDAuMjE1OTk2IDEwNi4wNjdDMC4xMzkwNzkgMTA2LjU3MyAwLjA3NzY4MzggMTA2LjA3OCAwLjA3OTU3NjUgMTA0Ljk2NVpNMC4wMDE3MzM1MSAxMDIuMjRDMC4wMTc4OTkyIDEwMS44NDggMC4wOTc3ODk3IDEwMS43NjggMC4yMDUzOTggMTAyLjAzN0MwLjMwMjc3MiAxMDIuMjggMC4yOTA3OTcgMTAyLjU3MSAwLjE3ODc4NyAxMDIuNjgzQzAuMDY2Nzc0OCAxMDIuNzk1IC0wLjAxMjkwMjEgMTAyLjU5NiAwLjAwMTczMzUxIDEwMi4yNFpNMC4wMjgzNDQ0IDc1LjEzMjZDMC4wMjgzNDQ0IDc0LjY2OTEgMC4xMDQ4NTcgNzQuNDc5NSAwLjE5ODM2OSA3NC43MTEyQzAuMjkxODg0IDc0Ljk0MyAwLjI5MTg4NCA3NS4zMjIyIDAuMTk4MzY5IDc1LjU1MzlDMC4xMDQ4NTcgNzUuNzg1NyAwLjAyODM0NDQgNzUuNTk2MSAwLjAyODM0NDQgNzUuMTMyNloiIGZpbGw9InVybCgjcGFpbnQyX3JhZGlhbF82NzlfMTkxMCkiLz4KPHBhdGggZD0iTTM1LjMxMzMgMTEyLjU0NEwzNy45IDExMS4wOTFMMzkuMzUxNyAxMjEuNDZMMzkuMjQ4IDEyOC45MjZMMzUuMzcwMyAxMzAuODUxTDMyLjkyMjkgMTI2LjQzN0wzNS4zMTMzIDExMi41NDRaIiBmaWxsPSIjMzA2MEFEIi8+CjxwYXRoIGQ9Ik0zOC43Mjk1IDExNi42OUwzOS4zNTE2IDEyNC41NzFMNDMuMjkxOSAxMjUuOTE5TDQzLjM5NTYgMTA5LjIyNUw0MC4xODEyIDExMC4wNTRMMzguNzI5NSAxMTYuNjlaIiBmaWxsPSIjNjA2MzY4Ii8+CjxwYXRoIGQ9Ik0zNS4zNjk5IDEzMC44NTFDMzMuMDMyNCAxMjkuNzMgMjkuNzMwMSAxMjYuNDA4IDI4LjY2MjEgMTI0LjEwM0MyOC4wMzMyIDEyMi43NDUgMjcuNjM5NyAxMjIuMzEgMjYuODkyMyAxMjIuMTQ2QzI1Ljc4NzkgMTIxLjkwMyAyNS44MDg0IDEyMS45NzEgMjYuMzIxOCAxMjAuMjU4QzI2LjU5MzYgMTE5LjM1IDI2LjkwMzIgMTE4Ljk1NCAyNy4zNDA2IDExOC45NTRDMjcuNjg2MiAxMTguOTU0IDI5LjE1OTcgMTE3Ljg0MSAzMC42MTQ5IDExNi40OEMzMi4wNzAyIDExNS4xMTkgMzMuNzIyNiAxMTMuNjc3IDM0LjI4NjkgMTEzLjI3NUwzNS4zMTI5IDExMi41NDRMMzUuNjc2NyAxMTQuMjQxQzM3LjA0NDMgMTIwLjYxOCAzNy4wMjcgMTI0LjI2NiAzNS42MTk4IDEyNi4yNDNDMzQuNTUyMSAxMjcuNzQ0IDM0Ljg0MTcgMTI4LjM5MyAzNi41NzkxIDEyOC4zOTNDMzcuMzk3IDEyOC4zOTMgMzcuODIyMiAxMjguMTg1IDM4LjEzMDUgMTI3LjYzNEMzOC42NzQ4IDEyNi42NjMgMzguNjMxNSAxMjAuOTY4IDM4LjA0NTQgMTE2LjQyNkMzNy4zNDQ2IDExMC45OTUgMzcuMzEyNyAxMTEuMTY3IDM5LjE1NzggMTEwLjM5OEM0MC4wMzYgMTEwLjAzMSA0MC44MTggMTA5Ljc5NyA0MC44OTU1IDEwOS44NzdDNDAuOTczMSAxMDkuOTU2IDQwLjgxOTEgMTExLjIzNSA0MC41NTM0IDExMi43MThDMzkuNjA2NyAxMTguMDAzIDM5LjkzMTUgMTIzLjIzOCA0MS4yNzUyIDEyNC4zNTNDNDEuNjg1MSAxMjQuNjkzIDQxLjcyMzMgMTIzLjk2OSA0MS41NDUyIDExOS4yMzJDNDEuMzE1NiAxMTMuMTIzIDQxLjUxNzQgMTEwLjkxOSA0Mi40MDc1IDEwOS44MkM0Mi45MTk4IDEwOS4xODggNDMuMjEzIDEwOS4xMTYgNDQuNDk1OSAxMDkuMzFDNDYuOTU2NiAxMDkuNjgyIDQ2Ljk1MDYgMTA5LjY2OCA0Ni4yMDk5IDExMy4yNTRDNDUuODQ5NiAxMTQuOTk5IDQ1LjQ0ODMgMTE3LjIyNCA0NS4zMTggMTE4LjJMNDUuMDgxMiAxMTkuOTczTDQ1Ljk0NDYgMTE5LjY0NUM0Ni40MTk1IDExOS40NjUgNDcuNDQxMyAxMTguOTk0IDQ4LjIxNTMgMTE4LjU5OUw0OS42MjI2IDExNy44ODFMNDkuMzg5MyAxMTguNjdDNDkuMjYxIDExOS4xMDUgNDkuMDY3MiAxMjAuNzAxIDQ4Ljk1ODcgMTIyLjIxOEw0OC43NjE0IDEyNC45NzZMNDcuMTA5NSAxMjQuMzI1QzQ2LjIwMSAxMjMuOTY3IDQ1LjMxMTYgMTIzLjY3NCA0NS4xMzMxIDEyMy42NzRDNDQuOTQxIDEyMy42NzQgNDQuODA4NSAxMjUuNTMyIDQ0LjgwODUgMTI4LjIyNEM0NC44MDg1IDEzMS44OTEgNDQuNzE3MSAxMzIuNzc1IDQ0LjMzODYgMTMyLjc3NUM0MS4wMTk4IDEzMi4yOTUgMzguNzg0MiAxMzIuNDU4IDM1LjM2OTkgMTMwLjg1MVoiIGZpbGw9IiNFRUMyM0IiLz4KPHBhdGggZD0iTTQxLjE2MzkgMTMyLjQxOEM0MS4xNjM5IDEzMi4yODMgNDEuMzQwOCAxMzAuNTE1IDQxLjY2OTIgMTI4LjE0NUM0MS45OTc3IDEyNS43NzUgNDIuMzU3NyAxMjIuOTYxIDQyLjQ2OTIgMTIxLjg5M0M0Mi41ODA4IDEyMC44MjQgNDIuNzk2OSAxMjAuMDI3IDQyLjk0OTUgMTIwLjEyMUM0My42NDQzIDEyMC41NTEgNDQuMjkzOCAxMjkuMzc1IDQ0LjA0NzQgMTMxLjQyQzQzLjg5MDkgMTMyLjcxOSA0My42NTk1IDEzMi43MzcgNDMuMzAxOSAxMzIuNjc3QzQyLjY2MDIgMTMyLjU3IDQxLjkxMDEgMTMyLjUxMyA0MS4xNjM5IDEzMi40MThaTTMyLjU5NjkgMTIyLjczNUMzMS42ODEzIDEyMi41ODMgMzAuODI0MiAxMjEuNDQyIDMwLjgyNDIgMTIwLjM3NEMzMC44MjQyIDExOS40MDcgMzIuMjM4MSAxMTcuOTM3IDMzLjE2ODUgMTE3LjkzN0MzNC42NjY2IDExNy45MzcgMzUuODM5MiAxMjAuNTEzIDM0Ljk0ODQgMTIxLjg0N0MzNC41NTE3IDEyMi40NDEgMzMuNDE0NyAxMjIuODcgMzIuNTk2OSAxMjIuNzM1WiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTExNC4wMjEgMTM3LjQ5OEMxMDcuMzE4IDEzNS43NCAxMDYuMjczIDEzNC4xMDQgMTA2LjYwOSAxMjUuODkxQzEwNi43MTYgMTIzLjI4MSAxMDYuODg1IDEyMC41MzkgMTA2Ljk4NiAxMTkuNzk3QzEwNy41NDUgMTE1LjY2NCAxMDguMzQyIDExMi4xNzMgMTA5LjA4NiAxMTAuNTk5QzEwOS43NDQgMTA5LjIwNiAxMDkuODQyIDEwOC43MTIgMTA5LjUzNSAxMDguMzQyQzEwOC40NSAxMDcuMDM1IDExMC4zNzEgMTA0Ljk5NSAxMTEuOTEgMTA1LjgxOUMxMTIuNDU3IDEwNi4xMTEgMTEyLjkxOCAxMDYuMDc3IDExMy45NDUgMTA1LjY2NUMxMTcuMDYyIDEwNC40MTkgMTIzLjExNyAxMDUuNTcxIDEyNi44MTQgMTA4LjExNUMxMjguMTQ5IDEwOS4wMzMgMTI4LjQ4NSAxMDkuMTM2IDEyOS4wMzggMTA4Ljc5MUMxMzAuNTIzIDEwNy44NjMgMTMyLjQ1MSAxMTAuNTczIDEzMS4yNSAxMTEuOUMxMzAuNzg3IDExMi40MTIgMTMwLjc1NSAxMTIuNzA1IDEzMS4wNjUgMTEzLjU5NkMxMzEuODUgMTE1Ljg0NiAxMzEuNzk0IDExNi4xMDQgMTI4LjAwNiAxMjcuNzE5QzEyNS41OCAxMzUuMTYgMTI0LjU0MyAxMzcuMzY1IDEyMy4yNjQgMTM3LjgxMUMxMjEuNjUzIDEzOC4zNzMgMTE2LjcyMyAxMzguMjA2IDExNC4wMjEgMTM3LjQ5OFpNMTIyLjU0MSAxMzUuMjI5QzEyMy4wMDMgMTM0LjU0MSAxMjMuMzU5IDEzMy4zMTMgMTIzLjUxMyAxMzEuODc4QzEyMy43OTQgMTI5LjI1IDEyMy44MTkgMTI5LjI4MyAxMjAuMzE3IDEyNy42MDFDMTE3LjY4NSAxMjYuMzM4IDExMy44MzMgMTI1Ljk3MiAxMTEuODU1IDEyNi43OThDMTEwLjgyOSAxMjcuMjI3IDExMC41OTUgMTI3LjU1NiAxMTAuMDU5IDEyOS4zMjdDMTA5LjExNiAxMzIuNDQzIDEwOS43MDMgMTM0LjMwOCAxMTEuODMyIDEzNC45NThDMTEzLjEzOSAxMzUuMzU3IDEyMC4wOSAxMzYuNDMgMTIwLjk4IDEzNi4zN0MxMjEuNTA0IDEzNi4zMzUgMTIyLjA4MiAxMzUuOTEyIDEyMi41NDEgMTM1LjIyOVpNMTE5LjQ1MSAxMjIuNDJDMTE5LjYzIDEyMi4wODEgMTE5LjY4NSAxMjEuNTU2IDExOS42ODUgMTIxLjU1NkMxMjAuMTU2IDEyMS42NTQgMTIwLjQ4IDEyMS42ODMgMTIwLjcyNiAxMjEuNzE5QzEyMS4xMDUgMTIxLjc3NCAxMjEuNTY0IDEyMS41ODcgMTIxLjc0NSAxMjEuMzAzQzEyMi4xNDkgMTIwLjY3MSAxMjAuNDc0IDExNy43MDEgMTE5LjU0OCAxMTcuNDA4QzExOC42NjggMTE3LjEyOCAxMTUuOTM1IDExOC44MjcgMTE1LjkzNSAxMTkuNjUzQzExNS45MzUgMTIwLjAyMiAxMTYuNTUzIDEyMC40MTEgMTE2LjgzMSAxMjAuNTE4QzExNi45ODIgMTIwLjU3NiAxMTcuMjUyIDEyMC43MzggMTE3LjU3NiAxMjAuOTM4QzExNy4zOCAxMjEuMzUxIDExNy4zNDYgMTIxLjQ3OCAxMTcuMzEyIDEyMS43MjRDMTE3LjIwNCAxMjIuNDk4IDExNy40MzUgMTIyLjY4NiAxMTguNDE2IDEyMi45MTJDMTE5LjE1MSAxMjIuOTkxIDExOS4yNSAxMjIuODQ1IDExOS40NTEgMTIyLjQyWk0xMjcuMjY1IDExNi44MzZDMTMwLjkzMyAxMTMuNDc1IDEyNy4wNzQgMTA3LjQxIDEyMi43MDUgMTA5LjY3QzEyMS42MjYgMTEwLjIyOCAxMjAuNjU0IDExMi4wMTkgMTIwLjY1NCAxMTMuNDQ5QzEyMC42NTQgMTE1LjY2NCAxMjIuODMxIDExNy45MzcgMTI0Ljk1NyAxMTcuOTQxQzEyNS43MzMgMTE3Ljk0MyAxMjYuNDE0IDExNy42MTcgMTI3LjI2NSAxMTYuODM2Wk0xMTcuMTQxIDExNS42MjJDMTE5LjAzNSAxMTQuNDY4IDExOS44MDQgMTExLjc5NSAxMTguODAyIDEwOS44NTdDMTE4LjA4MiAxMDguNDY0IDExNy4wMDggMTA3LjgzIDExNS4zNjYgMTA3LjgzQzExMS41MDggMTA3LjgzIDEwOS40ODUgMTEzLjIzNCAxMTIuNDY1IDExNS41NzhDMTEzLjU3IDExNi40NDggMTE1Ljc1NCAxMTYuNDY4IDExNy4xNDEgMTE1LjYyMloiIGZpbGw9IiMwOTk2RDEiLz4KPHBhdGggb3BhY2l0eT0iMC45IiBkPSJNMzMuMTc1NyAyNkwzMi4zODEgMjcuMjE5QzMxLjk0NDkgMjcuODg5IDMxLjI4MzYgMjkuMzI0OCAzMC45MDkxIDMwLjQwODhDMzAuNTM0NyAzMS40OTI4IDI5Ljk5MjIgMzIuOTcwMSAyOS43MDQzIDMzLjY5MjhDMjkuNDE2MyAzNC40MTU0IDI5LjE3MzMgMzYuMTE1IDI5LjE2MzggMzcuNDdDMjkuMTQzNiA0MC4zNTMgMjguNzQ5NCA0MS44ODg4IDI3LjYzNjQgNDMuNDIyNUMyNi45MjEyIDQ0LjQwOCAyNi42NzUxIDQ0LjUyMjMgMjUuNDUzMiA0NC40NDFDMjMuODk5NSA0NC4zMzc3IDIyLjgxNTMgNDUuMDgxNSAyMS4yMTcgNDcuMzQ4MUMyMC43MTY1IDQ4LjA1OCAyMC4xMzY3IDQ4LjYzOTMgMTkuOTI4OSA0OC42MzkzQzE5LjcyMTEgNDguNjM5MyAxOS4xODA1IDQ4LjIzMTkgMTguNzI2MiA0Ny43MzUxQzE4LjI3MTkgNDcuMjM4MiAxNy45OTQ0IDQ3LjAxMTUgMTguMTExIDQ3LjIyOThDMTguNDkgNDcuOTM5NiAxNy41OTA0IDUzLjI2MTUgMTYuODM3OCA1NC43NjQyQzE1Ljk2NyA1Ni41MDI4IDE0LjM4NzIgNTcuNzIzMyAxMi41MzU0IDU4LjA4NDNDMTEuMDA1OSA1OC4zODI2IDExLjIyODMgNTguMDkgOC43MTc5NSA2My4wOTQ2QzYuNjYzMzQgNjcuMTkwNyA1Ljk2OTM1IDY3LjgyNjQgMi42NTUzMyA2OC42NTQyQzEuNDk3MTcgNjguOTQzNCAwLjQ1ODc1OSA2OS43MTQyIDAgNzAuNTI4N1Y3Mi44MjYzQzAuMzU2MTkzIDcyLjc0NzMgMC44Mjg3MjggNzIuNjIwMyAxLjYxNDk5IDcyLjM4NzNDMi45OTQ4NiA3MS45NzgyIDQuODIzNDggNzEuNDgwNiA1LjY3ODEgNzEuMjgwNkM3LjcyOTM0IDcwLjgwMDYgOS4zOTM1MSA2OS4yMTk1IDkuOTIyNzkgNjcuMjQ2N0MxMC4yNDI0IDY2LjA1NTQgMTAuNzAzMyA2NS40MjQxIDExLjk5OTIgNjQuNDA1OEMxMy4yODQgNjMuMzk2MiAxMy43NTY1IDYyLjc1NDYgMTQuMDY3MSA2MS41OTY5QzE0LjM3MzkgNjAuNDUzMiAxNC43MDExIDYwLjAwNCAxNS40NDA3IDU5LjcwNjNDMTcuMDE1MyA1OS4wNzI0IDIwLjY0NDQgNTUuODY1NSAyMS4yMzYzIDU0LjU4NThDMjEuNTM3MyA1My45MzQ5IDIyLjExOTggNTIuNTM5OSAyMi41Mjg3IDUxLjQ4NjJDMjMuMjYzNCA0OS41OTMyIDI1LjAxOTUgNDcuMzI2MSAyNS43NTAxIDQ3LjMyNjFDMjUuOTU3NCA0Ny4zMjYxIDI3LjEwOSA0Ni43MjQ4IDI4LjMwOTMgNDUuOTkwOEMzMS4yMzc2IDQ0LjIwMDMgMzEuNTk5MSA0My40MTM2IDMxLjU5OTEgMzguODYxNEMzMS41OTkxIDM1LjY4MTQgMzEuNjg3IDM1LjE1NTcgMzIuMzQ2OCAzNC4zNjg0QzMzLjAwMzcgMzMuNTg0NyAzMy4wOTk3IDMzLjAyNDkgMzMuMTM1MSAyOS43MzkxTDMzLjE3NTcgMjZaTTI0LjMzMzggNjMuNTk1OEMyNC4yMDM3IDYzLjYxMzUgMjQuMDczOCA2My42NjE4IDIzLjkyNTggNjMuNzM2MkMyMi45Mzk3IDY0LjIzMTUgMjIuNjEzNiA2NC45MTAyIDIxLjU5NzMgNjguNTg2QzIxLjAxNzkgNzAuNjgxNSAyMC40MjEyIDcyLjA4MTkgMTkuODc5OCA3Mi42MTc4QzE5LjQyMzUgNzMuMDY5NSAxOC45MzgzIDczLjg4MjMgMTguODAxIDc0LjQyNDJDMTguMTgxIDc2Ljg3MTUgMTYuNzQ1NCA3OC42MDczIDE0LjQ0OTUgNzkuNjg3MUMxMi40NTU1IDgwLjYyNDkgMTAuNjQwNyA4Mi4xMTc5IDkuMTExMDIgODQuMDc5OEM4LjMyMTA5IDg1LjA5MjkgNy40ODc1MiA4Ni4wMjkgNy4yNTY3OCA4Ni4xNjA5QzYuNzYxNjQgODYuNDQ0IDUuNzA4OTkgODguODk0MSA0LjYwNzg1IDkyLjMyOEMzLjcxMDkyIDk1LjEyNSAxLjcxMDY1IDk3LjQ2ODIgMC40Nzg1MTQgOTcuMTY1OEMwLjI1NDkzMSA5Ny4xMTA5IDAuMTA2OTExIDk3LjExMzcgMCA5Ny4yOTAxVjEwNC40OTJDMC41NTY0MTEgMTAzLjY2NCAxLjIzNDIgMTAyLjYyOSAyLjU2MzQ3IDEwMC41NEM0LjM0NTExIDk3LjczOTcgNi40NjM4NSA5NC40ODY3IDcuMjcxNzMgOTMuMzEyNEM4LjA3OTYgOTIuMTM4MSA5LjM1Njg3IDkwLjE0ODggMTAuMTA4NiA4OC44OTE2QzExLjIzNDIgODcuMDA5MiAxMi4xMDE2IDg2LjEyNDMgMTUuMDI4NCA4My44ODMzTDE4LjU4MzEgODEuMTYyN0wyMC40MDUzIDc1Ljk4ODFDMjMuMTg5NSA2OC4wODE3IDIzLjk1ODUgNjcuMTk5NyAyNy43NjI0IDY3LjU1MTVDMjkuNDYwNSA2Ny43MDg1IDMwLjYzMzQgNjcuNjE5NiAzMi4wNzM0IDY3LjIyNjdDMzQuMTgyNiA2Ni42NTEyIDM0LjQ3OTkgNjYuMTg3MiAzMy4zNTUxIDY1LjIzMThDMzIuNjU1NCA2NC42Mzc2IDMxLjMxNTMgNjQuNjQgMjguMjM4OCA2NS4yNDE4QzI3LjMzOTcgNjUuNDE3NyAyNi45MzQzIDY1LjI3NTcgMjUuODkzMiA2NC40MTc4QzI1LjEwNTEgNjMuNzY4NCAyNC43MjQxIDYzLjU0MjYgMjQuMzMzOCA2My41OTU4Wk0zMS43MTY2IDgzLjI1OThDMzEuNjYwNSA4My4yNzQ5IDMxLjU5MjEgODMuMzEyNiAzMS41MTE1IDgzLjM3MjFDMzEuMjcwOSA4My41NSAzMC42NTM2IDgzLjc5IDMwLjEzNzkgODMuOTA1NEMyOS4zODc2IDg0LjA3MzQgMjkuMDQ5OCA4NC40OTMyIDI4LjQ0ODIgODYuMDA0NUMyOC4wMzQ3IDg3LjA0MzIgMjcuNDkxNSA4OC4xMTUzIDI3LjI0MTIgODguMzg2M0MyNi45OTA5IDg4LjY1NzMgMjYuNDc1IDg5LjU1MSAyNi4wOTQxIDkwLjM3MzJDMjQuOTQyMiA5Mi44NTkyIDIzLjUyODIgOTQuMDYxOCAxOS45MDExIDk1LjY0MjFDMTYuNjEwNyA5Ny4wNzU2IDE2LjU2NyA5Ny4xMDg2IDE1LjY2NzEgOTguOTgwMkMxNS4xMDI3IDEwMC4xNTQgMTQuNjc5MiAxMDEuNzU3IDE0LjU0OTkgMTAzLjIwNUMxNC4yNzMxIDEwNi4zMDIgMTMuNTk2MiAxMDcuMDMxIDkuNTIxMTggMTA4LjYxNkM1LjkxNDk3IDExMC4wMTkgMy41NzcwMiAxMTEuNzQ1IDMuNDE1ODMgMTEzLjEyM0MzLjI0MDQxIDExNC42MjMgMi41OTgxMyAxMTUuMDg0IDAuOTI5MjYgMTE0LjkwN0MwLjUzOTUyOSAxMTQuODY2IDAuMjQ0MzMyIDExNC44NTEgMCAxMTQuODgxVjEyMUMxLjAyNjU4IDExOS42NTggMi43MTkyMiAxMTguMzYxIDYuMjI0OTcgMTE2LjIwNkMxMC4zNjA0IDExMy42NjUgMTIuNzA2NSAxMTEuNTc1IDEzLjU3NzkgMTA5LjY1NEMxMy45MjYgMTA4Ljg4NyAxNC42NTQ3IDEwNy4zOTMgMTUuMTk3MSAxMDYuMzM0QzE1LjczOTUgMTA1LjI3NSAxNi4yODQ1IDEwMy45MDEgMTYuNDA4NCAxMDMuMjgxQzE2LjczMjQgMTAxLjY2IDE4LjMwMzIgMTAwLjM0OSAyMC42NjU5IDk5LjcyNkMyNS4xNzcgOTguNTM3MiAyNS43NDYxIDk4LjAxNjMgMjYuMzUyNSA5NC41MzU0QzI2Ljg1NjEgOTEuNjQ1MiAyOC42MTIgODguMDY2MSAzMC41MjY3IDg2LjAyNDZDMzEuMzA5NSA4NS4xOSAzMS45NDk1IDg0LjE4IDMxLjk0OTUgODMuNzc5MUMzMS45NDk1IDgzLjM3NjYgMzEuODg0OSA4My4yMTQ2IDMxLjcxNjYgODMuMjU5OFoiIGZpbGw9IiMzNTkxMzYiLz4KPHBhdGggZD0iTTguMDgxNzcgNzQuNDA4M0M2LjAzNDE5IDczLjg3MDUgNS44OTQ0MyA3My43MzAyIDYuMDQyNzEgNzIuMzYxNUM2LjEzMDI1IDcxLjU1MzMgNi4yNDAxNyA3MC44NTc1IDYuMjg2OTYgNzAuODE1NEM2LjMzMzc2IDcwLjc3MzMgNy43MDQ0MyA3MS4wNjg3IDkuMzMyOTIgNzEuNDcxOUMxMS45NzY4IDcyLjEyNjQgMTIuNDg2NiA3Mi4xMjM4IDE0LjA5NDcgNzEuNDQ3NEMxNi40MDQxIDcwLjQ3NiAxNy4yMzgzIDY4Ljc3NTMgMTYuNDAyMiA2Ni43NDMxQzE1Ljk0MjQgNjUuNjI1NyAxNC45MjA0IDY0LjgyODEgMTIuMjM4NCA2My40OTRDNy42NDE3IDYxLjIwNzUgNi41MzA2MiA1OS45OTkgNi4yNzQ2NSA1Ny4wMDc2QzUuODkzMzcgNTIuNTUxOSA4Ljg0Njg5IDQ5Ljk4MzYgMTQuMzUyMiA0OS45ODM2QzE1Ljk3MzMgNDkuOTgzNiAxNy44NDg3IDUwLjE5MzcgMTguNTE5NiA1MC40NTA1QzE5LjYzNjggNTAuODc4MSAxOS42OTM3IDUxLjAzOTkgMTkuMTk0NSA1Mi4zNzEyQzE4LjcxODIgNTMuNjQxNiAxOC41MDI3IDUzLjc2ODggMTcuNDg2MSA1My4zNzk2QzE1LjUxOTkgNTIuNjI3IDEyLjE5NDggNTIuODU1NCAxMS4wMDU4IDUzLjgyNDdDOS43OTA1OCA1NC44MTUzIDkuNTU3MzYgNTYuOTI1MiAxMC41MzI0IDU4LjEwNzlDMTAuODcyNyA1OC41MjA3IDEyLjkwODcgNTkuNzc5IDE1LjA1NjkgNjAuOTA0MUMxOC40NjI4IDYyLjY4ODEgMTkuMDgyIDYzLjIyMTQgMTkuODk1MSA2NS4wNzE5QzIxLjExNTIgNjcuODQ4NyAyMC42Nzg4IDcwLjM0MzcgMTguNjA3OCA3Mi40MzIzQzE2LjE2MDcgNzQuOTAwMiAxMi41NDEzIDc1LjU3OTYgOC4wODE3NyA3NC40MDgzWk0yOC45MzI4IDc0LjM4MjNDMjUuMjM2NSA3Mi45OTU4IDIzLjcxMzggNzAuNTQwNSAyMy43MTM4IDY1Ljk2NjdDMjMuNzEzOCA2Mi41MDM0IDI0LjU1ODggNjAuNDY5NyAyNi44MzMzIDU4LjQ1OTNDMjguMzk2NCA1Ny4wNzc2IDI4Ljk4NjcgNTYuODY4NiAzMS4zMjY4IDU2Ljg2ODZDMzIuODE2OCA1Ni44Njg2IDM0LjY4MjEgNTcuMjEyOSAzNS41MDA1IDU3LjYzODlDMzcuMzE2OSA1OC41ODQ2IDM4LjgzMTkgNjEuNzQyNiAzOC44NDY2IDY0LjYxNDNMMzguODU3MyA2Ni43MDQ0SDMyLjk0MjNIMjcuMDI3M0wyNy4zNDIgNjguMDU2OUMyNy41MTUgNjguODAwNyAyOC4zMzU0IDcwLjAzOTEgMjkuMTY1IDcwLjgwOUMzMC41NzczIDcyLjExOTYgMzAuODk0OSA3Mi4xOTc5IDM0LjE1NDcgNzIuMDM4NUMzNy4zOTM4IDcxLjg4MDEgMzcuNjQ2MyA3MS45NDA5IDM3Ljc4NCA3Mi45MTIyQzM3Ljk2MjYgNzQuMTcyNSAzNy4xODgyIDc0LjUyMzYgMzMuNTI0MiA3NC44NDM0QzMxLjgwODUgNzQuOTkzMiAzMC4xMDU2IDc0LjgyMjIgMjguOTMyOCA3NC4zODIzWk0zNS4wODgxIDYyLjUyNDJDMzQuOTU1OCA2MC42NzMgMzMuMzY0NCA1OS4zMjc2IDMxLjMwNzIgNTkuMzI3NkMyOS43MDg2IDU5LjMyNzYgMjguNzI4MSA2MC4wOTA5IDI3LjY2MiA2Mi4xNjU1QzI2Ljc0NTcgNjMuOTQ4NiAyNy40MTIgNjQuMjk2NSAzMS40NDg4IDY0LjE0MjVMMzUuMTkzNiA2My45OTk2TDM1LjA4ODEgNjIuNTI0MlpNNDQuMzkxNCA3NC4zNzA4QzQyLjU3MTEgNzMuNTQ2OCA0MS43MDU5IDcyLjEyODYgNDEuNjQyMSA2OS44NjQ3QzQxLjU0MTYgNjYuMjk0NCA0NC4xMTEgNjQuMTQ2OCA0OS4yNTc2IDYzLjQ5OTRDNTEuODQxNCA2My4xNzQ0IDUyLjA0OTUgNjMuMDU3NyA1MS43NTkzIDYyLjA5NjlDNTAuOTY1IDU5LjQ2NjYgNDguMjQyMyA1OC41NjQ0IDQ0LjkyNzYgNTkuODMzMUM0My40MzAzIDYwLjQwNjMgNDMuNDg2OCA2MC40MzEzIDQzLjA1MTQgNTkuMDAzMkM0Mi43ODAxIDU4LjExMzUgNDIuOTk4MiA1Ny44NTExIDQ0LjM5NSA1Ny4zODdDNDYuOTM2NCA1Ni41NDI3IDUxLjM3NDUgNTYuNjM0MiA1Mi43Nzc5IDU3LjU2QzU0LjkzOTIgNTguOTg1NyA1NS4zNzUxIDYwLjYwMTkgNTUuNTQ4NyA2Ny44MzI4TDU1LjcxMDYgNzQuNTc0NUw1NC4xMjMgNzQuNTc0QzUzLjAzNzcgNzQuNTczMyA1Mi41MzUzIDc0LjMzOTYgNTIuNTM1MyA3My44MzU0QzUyLjUzNTMgNzIuOTA2NiA1Mi4wNTI4IDcyLjkwMjQgNTAuNzUyIDczLjgxOTZDNDkuMjAxMyA3NC45MTMgNDYuMTY5NyA3NS4xNzU3IDQ0LjM5MTQgNzQuMzcwOFpNNTAuOTQ3NyA3MC44MTc1QzUxLjcwOTkgNjkuOTczNSA1Mi4wNDY4IDY4Ljk4NiA1Mi4wNDY4IDY3LjU5NjZWNjUuNTkyOUw0OS44MTgyIDY1Ljg0NThDNDUuNzE5IDY2LjMxMSA0My44NzQ5IDY5LjUxMTggNDYuNTA1OCA3MS41OTUzQzQ4LjAzNzEgNzIuODA3OSA0OS4zNTk0IDcyLjU3NjMgNTAuOTQ3NyA3MC44MTc1Wk05Mi4wMDIgNzQuNTc4OEM4OC40MDI0IDczLjQ4NjQgODYuMjQxOSA3MC4yNDY2IDg2LjI0MTkgNjUuOTQxQzg2LjI0MTkgNjAuNDA0MiA4OS41MjExIDU2LjgzMjUgOTQuNjQxIDU2Ljc5MjVDOTguNjk4OSA1Ni43NjA5IDEwMS4yNjMgNTkuNDY3OSAxMDEuNTAzIDY0LjAzNjZMMTAxLjYzIDY2LjQ1ODVMOTYuMjU2MiA2Ni41ODg1QzkzLjMwMDcgNjYuNjYgOTAuNjE1OCA2Ni44MTYgOTAuMjg5NyA2Ni45MzUxQzg5LjM4MTggNjcuMjY2OCA5MC40Nzg4IDY5Ljk0NyA5Mi4wMzA2IDcxLjE4ODZDOTMuMTU5OSA3Mi4wOTIxIDkzLjc3MSA3Mi4xOTEyIDk2LjgxNzggNzEuOTY1QzEwMC4xOTMgNzEuNzE0NCAxMDAuMzE4IDcxLjc0NDkgMTAwLjUzMSA3Mi44Njg0QzEwMC43MyA3My45MTMzIDEwMC41MDkgNzQuMDgzNSA5OC4zNTc5IDc0LjU0OEM5NS42MTAzIDc1LjE0MTIgOTMuODgyOSA3NS4xNDk2IDkyLjAwMiA3NC41Nzg4Wk05Ny45NjU5IDYzLjA4MjNDOTcuOTY1OSA2MS4xMjMxIDk2LjE4NTggNTkuMzI3NiA5NC4yNDM0IDU5LjMyNzZDOTIuMjQ5NCA1OS4zMjc2IDkwLjE0OTkgNjEuMjU2MiA5MC4xNDk5IDYzLjA4NzhDOTAuMTQ5OSA2NC4yMDcgOTAuMjc5NyA2NC4yNDU1IDk0LjA1NzkgNjQuMjQ1NUM5Ny44NDE0IDY0LjI0NTUgOTcuOTY1OSA2NC4yMDg0IDk3Ljk2NTkgNjMuMDgyM1pNMTEwLjA0MiA3NC40N0MxMDYuMTQ5IDczLjQzIDEwMy45MyA2OS40MDI5IDEwNC41NCA2NC40ODYyQzEwNS4xNjYgNTkuNDQ0MiAxMDcuOTUxIDU2Ljg3MzYgMTEyLjc5MiA1Ni44NzA1QzExNS4zNzQgNTYuODY5IDExNS44MDUgNTcuMDI0OSAxMTcuMjYgNTguNDlDMTE4Ljg3OCA2MC4xMTg1IDExOS45NDEgNjIuODM0MyAxMTkuOTQ2IDY1LjM1MkwxMTkuOTQ5IDY2LjcwNDRIMTE0LjA4N0MxMDguNDg4IDY2LjcwNDQgMTA4LjIyNSA2Ni43NTAxIDEwOC4yMjUgNjcuNzIwM0MxMDguMjI1IDcwLjcxMjIgMTEwLjk1MiA3Mi4zNDQyIDExNS4zNTcgNzEuOTg4MkMxMTguMTM2IDcxLjc2MzcgMTE4LjQyMyA3MS44MzE1IDExOC42NjcgNzIuNzY3NEMxMTguOTk4IDc0LjA0NDMgMTE4Ljk2OCA3NC4wNzIgMTE2LjY5IDc0LjYwNDNDMTE0LjM2NyA3NS4xNDcgMTEyLjQzIDc1LjEwNzggMTEwLjA0MiA3NC40N1pNMTE2LjExOSA2Mi43NzAxQzExNS44NjQgNjAuODc4NyAxMTQuMTU5IDU5LjMyNzYgMTEyLjMzNSA1OS4zMjc2QzExMC41NjYgNTkuMzI3NiAxMDguMjI0IDYxLjYwMTQgMTA4LjIyNCA2My4zMTk1QzEwOC4yMjQgNjQuMjEyOCAxMDguNTI3IDY0LjI3NDYgMTEyLjI1NSA2NC4xNDI0QzExNi4yNDEgNjQuMDAxMSAxMTYuMjgzIDYzLjk4NjIgMTE2LjExOSA2Mi43NzAxWk0xMjYuODk0IDc0LjI0N0MxMjQuODI2IDczLjIyMDggMTIzLjUzMiA3MS4zNTg5IDEyMi44OTQgNjguNDg4OUMxMjEuOTc1IDY0LjM2MDUgMTIzLjA3NCA2MC40OTE0IDEyNS44MyA1OC4xNTY3QzEyNy44MjcgNTYuNDY1NyAxMzEuODM4IDU2LjMzMDUgMTMzLjgwNSA1Ny44ODc5TDEzNS4wOTIgNTguOTA3MlY1My45NTM2VjQ5SDEzNy4wNDZIMTM5VjYxLjc4NjVWNzQuNTczMUgxMzcuMjlDMTM1Ljc2MiA3NC41NzMxIDEzNS41OCA3NC40MzMxIDEzNS41OCA3My4yNTc5VjcxLjk0MjdMMTM0LjQ4MSA3Mi45NzkyQzEzMi4yMzMgNzUuMDk5OCAxMjkuNTI1IDc1LjU1MjMgMTI2Ljg5NCA3NC4yNDdaTTEzMy42NjQgNzAuNjc2NkMxMzQuOTc2IDY5LjM1NTcgMTM1LjA5MiA2OC45NTM3IDEzNS4wOTIgNjUuNzIwOUMxMzUuMDkyIDYyLjQ1NzIgMTM0Ljk4NSA2Mi4wOTQ1IDEzMy42MDcgNjAuNzA3NkMxMzIuMzY1IDU5LjQ1NzUgMTMxLjg2NSA1OS4yNjIyIDEzMC41NTQgNTkuNTE1NEMxMjYuMTQ0IDYwLjM2NzIgMTI0LjgzNSA2OC4xNDIzIDEyOC41OTggNzEuMTIyOUMxMzAuMzQ4IDcyLjUwODEgMTMxLjk4OCA3Mi4zNjM1IDEzMy42NjQgNzAuNjc2NlpNNjEuNDEwNyA2Ni4wNTg0QzYwLjAyMiA2MS4zNzU0IDU4Ljg4NTkgNTcuMzkxOSA1OC44ODU5IDU3LjIwNjJDNTguODg1OSA1Ny4wMjA2IDU5LjcyNzQgNTYuODY4NiA2MC43NTU5IDU2Ljg2ODZINjIuNjI1OUw2NC4yNTUxIDYzLjg3NjZDNjUuMTUxMiA2Ny43MzEgNjUuOTU2IDcwLjY2MzMgNjYuMDQzNyA3MC4zOTI5QzY2LjEzMTMgNzAuMTIyNCA2Ny4wNTU3IDY3LjAyNDEgNjguMDk3OCA2My41MDc4TDY5Ljk5MjYgNTcuMTE0NUg3MS41OTY5SDczLjIwMTJMNzQuODg2NSA2My4yNjE5Qzc1LjgxMzQgNjYuNjQzIDc2LjY4MzIgNjkuNzQxMiA3Ni44MTk0IDcwLjE0N0M3Ni45NTU3IDcwLjU1MjcgNzcuODQxOSA2Ny43MzEgNzguNzg5IDYzLjg3NjZMODAuNTEwOSA1Ni44Njg2SDgyLjQ1MTdDODMuOTUxNyA1Ni44Njg2IDg0LjMzMDMgNTcuMDMxNiA4NC4xMTkgNTcuNTg2QzgzLjk2ODYgNTcuOTgwNiA4Mi42OTU2IDYxLjk2NDEgODEuMjkwMiA2Ni40MzgyTDc4LjczNDkgNzQuNTczMUg3Ni45NjM1SDc1LjE5MjFMNzMuNTgxMiA2OC43OTQ1QzcyLjY5NTIgNjUuNjE2MyA3MS44NTg1IDYyLjY4NDEgNzEuNzIxNyA2Mi4yNzgzQzcxLjQ0NjUgNjEuNDYxNiA3MS41OTExIDYxLjA3MzMgNjkuMTExNCA2OS4yODYzTDY3LjUxNTIgNzQuNTczMUg2NS43MjU0SDYzLjkzNTZMNjEuNDEwNyA2Ni4wNTg0WiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTE5Ljk3NDYgMTE5LjI3NEMxOS4wMTU4IDExOC4xMTkgMjAuMzM4MSAxMTYuMTY2IDIxLjU2MDQgMTE2LjkzMkMyMi43NjMyIDExNy42ODYgMjIuMzMwNSAxMTkuNzYxIDIwLjk3MDQgMTE5Ljc2MUMyMC42NDUgMTE5Ljc2MSAyMC4xOTY5IDExOS41NDIgMTkuOTc0NiAxMTkuMjc0WiIgZmlsbD0idXJsKCNwYWludDNfcmFkaWFsXzY3OV8xOTEwKSIvPgo8cGF0aCBkPSJNMjMuODM2IDExMi42M0MyMi4xMTUyIDExMS42ODkgMjEuODAyOSAxMDkuNzk3IDIzLjE0NjUgMTA4LjQ1M0MyNC4wODU3IDEwNy41MTQgMjUuMjEgMTA3LjM5NCAyNi4yNTA3IDEwOC4xMjNDMjcuODU3MSAxMDkuMjQ5IDI3LjQ2NTUgMTEyLjAxMyAyNS42MDc5IDExMi42NkMyNS42MDc5IDExMi42NiAyNC40NzQ1IDExMi45NzMgMjMuODM2IDExMi42M1oiIGZpbGw9InVybCgjcGFpbnQ0X3JhZGlhbF82NzlfMTkxMCkiLz4KPHBhdGggZD0iTTMzLjc4NTUgMTIwLjAzNkMzMy43ODU1IDEyMC4xODggMzMuNzU1NyAxMjAuMzM4IDMzLjY5NzYgMTIwLjQ3OUMzMy42Mzk1IDEyMC42MTkgMzMuNTU0NCAxMjAuNzQ2IDMzLjQ0NzEgMTIwLjg1M0MzMy4zMzk4IDEyMC45NjEgMzMuMjEyNCAxMjEuMDQ2IDMzLjA3MjMgMTIxLjEwNEMzMi45MzIxIDEyMS4xNjIgMzIuNzgxOCAxMjEuMTkyIDMyLjYzMDEgMTIxLjE5MkMzMi40NzgzIDEyMS4xOTIgMzIuMzI4MSAxMjEuMTYyIDMyLjE4NzkgMTIxLjEwNEMzMi4wNDc3IDEyMS4wNDYgMzEuOTIwMyAxMjAuOTYxIDMxLjgxMyAxMjAuODUzQzMxLjcwNTcgMTIwLjc0NiAzMS42MjA2IDEyMC42MTkgMzEuNTYyNiAxMjAuNDc5QzMxLjUwNDUgMTIwLjMzOCAzMS40NzQ2IDEyMC4xODggMzEuNDc0NiAxMjAuMDM2QzMxLjQ3NDYgMTE5LjczIDMxLjU5NjMgMTE5LjQzNiAzMS44MTMgMTE5LjIxOUMzMi4wMjk3IDExOS4wMDMgMzIuMzIzNiAxMTguODgxIDMyLjYzMDEgMTE4Ljg4MUMzMi45MzY1IDExOC44ODEgMzMuMjMwNCAxMTkuMDAzIDMzLjQ0NzEgMTE5LjIxOUMzMy42NjM4IDExOS40MzYgMzMuNzg1NSAxMTkuNzMgMzMuNzg1NSAxMjAuMDM2WiIgZmlsbD0iYmxhY2siLz4KPHBhdGggZD0iTTEyMC43MjEgMTE4LjI3NkMxMjAuNzIxIDExOC42MjMgMTIwLjU2NSAxMTguOTU2IDEyMC4yODcgMTE5LjIwMUMxMjAuMDA5IDExOS40NDYgMTE5LjYzMiAxMTkuNTg0IDExOS4yMzkgMTE5LjU4NEMxMTguODQ2IDExOS41ODQgMTE4LjQ2OSAxMTkuNDQ2IDExOC4xOTEgMTE5LjIwMUMxMTcuOTEzIDExOC45NTYgMTE3Ljc1NyAxMTguNjIzIDExNy43NTcgMTE4LjI3NkMxMTcuNzU3IDExOC4xMDUgMTE3Ljc5NSAxMTcuOTM1IDExNy44NyAxMTcuNzc2QzExNy45NDQgMTE3LjYxNyAxMTguMDUzIDExNy40NzMgMTE4LjE5MSAxMTcuMzUyQzExOC4zMjkgMTE3LjIzIDExOC40OTIgMTE3LjEzNCAxMTguNjcyIDExNy4wNjhDMTE4Ljg1MSAxMTcuMDAzIDExOS4wNDQgMTE2Ljk2OSAxMTkuMjM5IDExNi45NjlDMTE5LjQzMyAxMTYuOTY5IDExOS42MjYgMTE3LjAwMyAxMTkuODA2IDExNy4wNjhDMTE5Ljk4NiAxMTcuMTM0IDEyMC4xNDkgMTE3LjIzIDEyMC4yODcgMTE3LjM1MkMxMjAuNDI0IDExNy40NzMgMTIwLjUzMyAxMTcuNjE3IDEyMC42MDggMTE3Ljc3NkMxMjAuNjgyIDExNy45MzUgMTIwLjcyMSAxMTguMTA1IDEyMC43MjEgMTE4LjI3NloiIGZpbGw9IiM1OTY4NkYiLz4KPGRlZnM+CjxsaW5lYXJHcmFkaWVudCBpZD0icGFpbnQwX2xpbmVhcl82NzlfMTkxMCIgeDE9IjExIiB5MT0iLTQuNzc3NjhlLTA3IiB4Mj0iMTY0LjUiIHkyPSIxNTAuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNjRCN0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzAwNURBRCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8cmFkaWFsR3JhZGllbnQgaWQ9InBhaW50MV9yYWRpYWxfNjc5XzE5MTAiIGN4PSIwIiBjeT0iMCIgcj0iMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiIGdyYWRpZW50VHJhbnNmb3JtPSJ0cmFuc2xhdGUoMTI0LjkwNCAxMjguNjM2KSByb3RhdGUoLTAuMDQ4NTUyKSBzY2FsZSgzNi4yODAyIDMxLjQwMzIpIj4KPHN0b3Agc3RvcC1jb2xvcj0iIzExNzdDRSIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiM3RUM3RkYiIHN0b3Atb3BhY2l0eT0iMCIvPgo8L3JhZGlhbEdyYWRpZW50Pgo8cmFkaWFsR3JhZGllbnQgaWQ9InBhaW50Ml9yYWRpYWxfNjc5XzE5MTAiIGN4PSIwIiBjeT0iMCIgcj0iMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiIGdyYWRpZW50VHJhbnNmb3JtPSJ0cmFuc2xhdGUoMzkuMzUxMiAxMjEuNDYpIHJvdGF0ZSgtMC4wNTczKSBzY2FsZSg1MC4yMjc2IDUxLjMwOTEpIj4KPHN0b3Agc3RvcC1jb2xvcj0iIzExNzdDRSIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiM3RUM3RkYiIHN0b3Atb3BhY2l0eT0iMCIvPgo8L3JhZGlhbEdyYWRpZW50Pgo8cmFkaWFsR3JhZGllbnQgaWQ9InBhaW50M19yYWRpYWxfNjc5XzE5MTAiIGN4PSIwIiBjeT0iMCIgcj0iMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiIGdyYWRpZW50VHJhbnNmb3JtPSJ0cmFuc2xhdGUoMjAuOTU0OCAxMTguMjYpIHNjYWxlKDEuMzA5MjkgMS41MDEyNykiPgo8c3RvcCBzdG9wLWNvbG9yPSIjODhBM0QwIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzVEODNCRiIvPgo8L3JhZGlhbEdyYWRpZW50Pgo8cmFkaWFsR3JhZGllbnQgaWQ9InBhaW50NF9yYWRpYWxfNjc5XzE5MTAiIGN4PSIwIiBjeT0iMCIgcj0iMSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiIGdyYWRpZW50VHJhbnNmb3JtPSJ0cmFuc2xhdGUoMjQuNzg0MyAxMTAuMjIxKSBzY2FsZSgyLjQ3Mjc3IDIuNTY5NTQpIj4KPHN0b3Agc3RvcC1jb2xvcj0iIzg4QTNEMCIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiM1RDgzQkYiLz4KPC9yYWRpYWxHcmFkaWVudD4KPC9kZWZzPgo8L3N2Zz4K
- keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "topology"], ["spec", "replicationFactor"], ["spec", "db"], ["spec", "db", "replicas"], ["spec", "db", "size"], ["spec", "db", "storageClass"], ["spec", "db", "resources"], ["spec", "db", "resourcesPreset"], ["spec", "master"], ["spec", "master", "replicas"], ["spec", "master", "resources"], ["spec", "master", "resourcesPreset"], ["spec", "master", "volumeSizeLimitMB"], ["spec", "filer"], ["spec", "filer", "replicas"], ["spec", "filer", "resources"], ["spec", "filer", "resourcesPreset"], ["spec", "filer", "grpcHost"], ["spec", "filer", "grpcPort"], ["spec", "filer", "whitelist"], ["spec", "volume"], ["spec", "volume", "replicas"], ["spec", "volume", "size"], ["spec", "volume", "storageClass"], ["spec", "volume", "diskType"], ["spec", "volume", "resources"], ["spec", "volume", "resourcesPreset"], ["spec", "volume", "zones"], ["spec", "volume", "pools"], ["spec", "s3"], ["spec", "s3", "replicas"], ["spec", "s3", "resources"], ["spec", "s3", "resourcesPreset"]]
+ keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "topology"], ["spec", "replicationFactor"], ["spec", "db"], ["spec", "db", "replicas"], ["spec", "db", "size"], ["spec", "db", "storageClass"], ["spec", "db", "resources"], ["spec", "db", "resourcesPreset"], ["spec", "master"], ["spec", "master", "replicas"], ["spec", "master", "resources"], ["spec", "master", "resourcesPreset"], ["spec", "master", "volumeSizeLimitMB"], ["spec", "filer"], ["spec", "filer", "replicas"], ["spec", "filer", "resources"], ["spec", "filer", "resourcesPreset"], ["spec", "filer", "grpcHost"], ["spec", "filer", "grpcPort"], ["spec", "filer", "whitelist"], ["spec", "volume"], ["spec", "volume", "replicas"], ["spec", "volume", "size"], ["spec", "volume", "storageClass"], ["spec", "volume", "diskType"], ["spec", "volume", "resources"], ["spec", "volume", "resourcesPreset"], ["spec", "volume", "zones"], ["spec", "volume", "pools"], ["spec", "s3"], ["spec", "s3", "replicas"], ["spec", "s3", "resources"], ["spec", "s3", "resourcesPreset"], ["spec", "lifecycle"], ["spec", "lifecycle", "enabled"]]
secrets:
exclude: []
include: []
diff --git a/packages/system/seaweedfs/tests/lifecycle_test.yaml b/packages/system/seaweedfs/tests/lifecycle_test.yaml
new file mode 100644
index 000000000..84680cb5f
--- /dev/null
+++ b/packages/system/seaweedfs/tests/lifecycle_test.yaml
@@ -0,0 +1,138 @@
+suite: seaweedfs runs the S3 lifecycle worker
+
+# SeaweedFS stores a bucket lifecycle configuration on PUT and acts on it only
+# from the s3_lifecycle job of the plugin worker, which the admin server
+# schedules (weed/worker/tasks/s3_lifecycle/handler.go). Without an admin and a
+# worker in the release, a rule is accepted and nothing ever expires.
+
+templates:
+ - charts/seaweedfs/templates/admin/admin-statefulset.yaml
+ - charts/seaweedfs/templates/worker/worker-deployment.yaml
+
+release:
+ name: seaweedfs-system
+ namespace: tenant-root
+
+tests:
+ - it: runs the admin server that schedules lifecycle jobs
+ template: charts/seaweedfs/templates/admin/admin-statefulset.yaml
+ set: &enabled
+ seaweedfs:
+ admin:
+ enabled: true
+ secret:
+ existingSecret: seaweedfs-admin-credentials
+ worker:
+ enabled: true
+ asserts:
+ - isKind:
+ of: StatefulSet
+ - equal:
+ path: metadata.name
+ value: seaweedfs-admin
+
+ # The admin keeps its job state and any change an operator makes in its UI
+ # in -dataDir; on an emptyDir a restart would silently undo them.
+ - it: keeps the admin data directory on a persistent volume
+ template: charts/seaweedfs/templates/admin/admin-statefulset.yaml
+ set: *enabled
+ asserts:
+ - matchRegex:
+ path: spec.template.spec.containers[0].command[2]
+ pattern: '-dataDir=/data'
+ - equal:
+ path: spec.volumeClaimTemplates[0].metadata.name
+ value: admin-data
+ - equal:
+ path: spec.volumeClaimTemplates[0].spec.resources.requests.storage
+ value: 1Gi
+
+ - it: runs the worker against the admin of the same release
+ template: charts/seaweedfs/templates/worker/worker-deployment.yaml
+ set: *enabled
+ asserts:
+ - isKind:
+ of: Deployment
+ - matchRegex:
+ path: spec.template.spec.containers[0].command[2]
+ pattern: '-admin=seaweedfs-admin\.tenant-root:23646\.33646 '
+
+ # The worker executes every job type it registers for. Lifecycle is the only
+ # one this release asks for: vacuum, balance and erasure coding move data and
+ # are not something enabling lifecycle should start doing as a side effect.
+ - it: registers the worker for the s3_lifecycle job only
+ template: charts/seaweedfs/templates/worker/worker-deployment.yaml
+ set: *enabled
+ asserts:
+ - matchRegex:
+ path: spec.template.spec.containers[0].command[2]
+ pattern: '-jobType=s3_lifecycle '
+
+ - it: runs no Lance worker container, since the Lance catalog is off
+ template: charts/seaweedfs/templates/worker/worker-deployment.yaml
+ set: *enabled
+ asserts:
+ - lengthEqual:
+ path: spec.template.spec.containers
+ count: 1
+
+ - it: gives the admin and the worker explicit resources
+ set: *enabled
+ asserts:
+ - exists:
+ path: spec.template.spec.containers[0].resources.requests.memory
+ template: charts/seaweedfs/templates/admin/admin-statefulset.yaml
+ - exists:
+ path: spec.template.spec.containers[0].resources.limits.memory
+ template: charts/seaweedfs/templates/admin/admin-statefulset.yaml
+ - equal:
+ path: spec.template.spec.containers[0].resources.requests.memory
+ value: 64Mi
+ template: charts/seaweedfs/templates/worker/worker-deployment.yaml
+
+ # The admin serves an unauthenticated UI unless a password is set, and a
+ # tenant namespace admits ingress from the whole cluster. The credentials come
+ # from a Secret the parent release keeps; the values here only wire them in.
+ - it: reads the admin credentials from the named Secret
+ template: charts/seaweedfs/templates/admin/admin-statefulset.yaml
+ set: *enabled
+ asserts:
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: WEED_ADMIN_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: seaweedfs-admin-credentials
+ key: adminPassword
+ - contains:
+ path: spec.template.spec.containers[0].env
+ content:
+ name: WEED_ADMIN_USER
+ valueFrom:
+ secretKeyRef:
+ name: seaweedfs-admin-credentials
+ key: adminUser
+
+ - it: renders neither the admin nor the worker unless the parent turns them on
+ asserts:
+ - hasDocuments:
+ count: 0
+ template: charts/seaweedfs/templates/admin/admin-statefulset.yaml
+ - hasDocuments:
+ count: 0
+ template: charts/seaweedfs/templates/worker/worker-deployment.yaml
+
+ # Every other component here logs to stderr; the upstream default for these
+ # two writes to a file in an emptyDir that kubectl logs never shows.
+ - it: logs the admin and the worker to stderr
+ set: *enabled
+ asserts:
+ - matchRegex:
+ path: spec.template.spec.containers[0].command[2]
+ pattern: '-logtostderr=true'
+ template: charts/seaweedfs/templates/admin/admin-statefulset.yaml
+ - matchRegex:
+ path: spec.template.spec.containers[0].command[2]
+ pattern: '-logtostderr=true'
+ template: charts/seaweedfs/templates/worker/worker-deployment.yaml
diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml
index e1eba8a58..1fce951cf 100644
--- a/packages/system/seaweedfs/values.yaml
+++ b/packages/system/seaweedfs/values.yaml
@@ -319,6 +319,47 @@ seaweedfs:
region: ""
sidecar:
image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.0@sha256:4d665becd3399c2fe1e8a66e15d14d16e4a5a75395fb0ae65cff46679a8989f1"
+ # A bucket lifecycle rule is stored on PUT and acted on only by the
+ # s3_lifecycle job of the plugin worker, scheduled by the admin server once a
+ # day (weed/worker/tasks/s3_lifecycle/handler.go at 4.45). The parent chart
+ # sends both enabled flags from its lifecycle.enabled, together with the
+ # admin password below; off here, so a render without the parent never
+ # starts an admin without one.
+ admin:
+ enabled: false
+ # Without a password weed admin serves its UI unauthenticated, and a tenant
+ # namespace admits ingress from the whole cluster. The parent chart names
+ # the Secret it keeps; these are the keys it writes.
+ secret:
+ userKey: adminUser
+ pwKey: adminPassword
+ # Job state and anything changed in the admin UI live in -dataDir.
+ data:
+ type: "persistentVolumeClaim"
+ size: "1Gi"
+ logs:
+ type: ""
+ resources:
+ requests:
+ cpu: "50m"
+ memory: "64Mi"
+ limits:
+ cpu: "500m"
+ memory: "256Mi"
+ worker:
+ enabled: false
+ # Vacuum, balance and erasure coding move data; enabling lifecycle must
+ # not start them.
+ jobType: "s3_lifecycle"
+ logs:
+ type: ""
+ resources:
+ requests:
+ cpu: "50m"
+ memory: "64Mi"
+ limits:
+ cpu: "1"
+ memory: "512Mi"
certificates:
commonName: "SeaweedFS CA"
ipAddresses: []
From 27e0aa288079c1715608ae0286bcaa59c87c03ce Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:19:01 +0300
Subject: [PATCH 068/346] docs(backups): say who applies a bucket lifecycle
rule
The notes that send a tenant to a bucket lifecycle rule for a backup
object Cozystack leaves behind read as if some policy were already in
place. None is: the rule has to be set on the bucket, and it is applied
only by an object store that implements lifecycle, which the bundled
SeaweedFS now does once a day while lifecycle.enabled is on.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
examples/backups/kafka/92-scenario-user-restore.md | 5 +----
examples/backups/nats/92-scenario-user-restore.md | 5 +----
internal/backupcontroller/psmdbstrategy_controller.go | 7 ++++---
3 files changed, 6 insertions(+), 11 deletions(-)
diff --git a/examples/backups/kafka/92-scenario-user-restore.md b/examples/backups/kafka/92-scenario-user-restore.md
index 3215fe056..5416500ed 100644
--- a/examples/backups/kafka/92-scenario-user-restore.md
+++ b/examples/backups/kafka/92-scenario-user-restore.md
@@ -71,9 +71,6 @@ The script:
had already replayed. The Pod log names each topic before its replay and
again once it landed; empty or delete every topic it shows as started, not
only the one the refusal names, then re-run.
-- The tarball in object storage is not managed by the Cozystack `Backup`
- lifecycle: deleting the `Backup` CR removes the reference but leaves the S3
- object in place. Tenants who need to reclaim space should delete the object
- (or set a bucket lifecycle policy).
+- The tarball in object storage is not managed by the Cozystack `Backup` lifecycle: deleting the `Backup` CR removes the reference but leaves the S3 object in place. Tenants who need to reclaim space should delete the object, or set a lifecycle rule on the bucket. The Cozystack SeaweedFS applies such rules once a day while its `lifecycle.enabled` is on, which is the default; another object store applies them only if it implements bucket lifecycle.
- Consumer-group offsets, topic configs, ACLs and users are not part of the
backup; consumers start from the restored topic's fresh offsets.
diff --git a/examples/backups/nats/92-scenario-user-restore.md b/examples/backups/nats/92-scenario-user-restore.md
index 9f0e7c02a..98c51a5cc 100644
--- a/examples/backups/nats/92-scenario-user-restore.md
+++ b/examples/backups/nats/92-scenario-user-restore.md
@@ -55,9 +55,6 @@ The script:
- Both flows rely on the strategy template branching on `.Mode`. See
`01-create-strategy.sh`.
-- The tarball in object storage is not managed by the Cozystack `Backup`
- lifecycle: deleting the `Backup` CR removes the reference but leaves the S3
- object in place. Tenants who need to reclaim space should delete the object
- (or set a bucket lifecycle policy).
+- The tarball in object storage is not managed by the Cozystack `Backup` lifecycle: deleting the `Backup` CR removes the reference but leaves the S3 object in place. Tenants who need to reclaim space should delete the object, or set a lifecycle rule on the bucket. The Cozystack SeaweedFS applies such rules once a day while its `lifecycle.enabled` is on, which is the default; another object store applies them only if it implements bucket lifecycle.
- `nats stream restore` recreates the stream's configuration from the snapshot;
consumers are restored as captured at backup time.
diff --git a/internal/backupcontroller/psmdbstrategy_controller.go b/internal/backupcontroller/psmdbstrategy_controller.go
index 0ead1bc58..819274f71 100644
--- a/internal/backupcontroller/psmdbstrategy_controller.go
+++ b/internal/backupcontroller/psmdbstrategy_controller.go
@@ -1376,9 +1376,10 @@ func (r *BackupJobReconciler) createMongoDBBackupArtifact(
// annotation (honoured before any apiserver read, so it still frees a Backup
// whose operator CR has become unreadable) and a going-away namespace (the psmdb
// delete-backup exec cannot run there) each make the driver strip its own
-// finalizer and release, leaving the object for a bucket lifecycle policy. This
-// bounds only what the driver contributes: a Get that fails with something other
-// than NotFound/NoMatchError still requeues (below), and the psmdb operator's own
+// finalizer and release, leaving the object to a lifecycle rule on the bucket,
+// if one is set, or to manual cleanup. This bounds only what the driver
+// contributes: a Get that fails with something other than
+// NotFound/NoMatchError still requeues (below), and the psmdb operator's own
// release-lock finalizer keeps the CR Terminating while the operator is down,
// whatever the driver strips. A down operator in a live namespace is likewise
// not auto-released: the Backup stays Terminating as a visible signal until an
From 3a10f956ccefac3990c3319cdf5c6889ee63dcde Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:30:54 +0300
Subject: [PATCH 069/346] test(backups): cover a restore that starts on an idle
ClickHouse sidecar
The strategy suite only exercised restore against a busy sidecar, so a
gate that wrongly blocked restore on an idle one would have stayed
green.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
hack/altinity-backup-strategy.bats | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/hack/altinity-backup-strategy.bats b/hack/altinity-backup-strategy.bats
index 61729bf00..4703a861c 100644
--- a/hack/altinity-backup-strategy.bats
+++ b/hack/altinity-backup-strategy.bats
@@ -117,10 +117,17 @@ run_script() {
render_script "$tmp/s.sh" backup myapp
run_script "$tmp"
grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests"
- # All three busy answers were consumed before the operation was started.
[ "$(cat "$tmp/state/busy")" = 0 ]
}
+@test "restore on an idle sidecar starts restore_remote of the listed archive" {
+ tmp=$(mktemp -d)
+ make_stubs "$tmp/bin" "$tmp/state"
+ render_script "$tmp/s.sh" restore myapp
+ run_script "$tmp"
+ grep -q '^POST .*/backup/restore_remote/clickhouse-myapp-20260101T000000Z$' "$tmp/state/requests"
+}
+
@test "backup fails without starting create_remote when the sidecar stays busy" {
tmp=$(mktemp -d)
make_stubs "$tmp/bin" "$tmp/state"
From e7529520471c41053970f708993186f2eeff48ae Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:31:29 +0300
Subject: [PATCH 070/346] test(clickhouse): back up and restore a running
release through the system bucket
A ClickHouse release on the system bucket starts before the platform
projects cozy-backups-creds into its namespace, and no suite exercised
that order: the round-trips bring their own Bucket and endpoint, so the
sidecar never had to pick up credentials that arrived after it started.
The new Test starts the release without the projected Secret and runs a
backup and an in-place restore through cozy-default against the bucket
as advertised. In CI that host neither resolves nor has a certificate,
so the step makes it reachable from tenant-root for its own duration
and proves the served certificate before any job starts, which keeps a
DNS or TLS problem from reading as a backup failure.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
.../clickhouse/chainsaw-test.yaml | 176 +++++++++++++++++
.../clickhouse/clickhouse-system-bucket.yaml | 28 +++
.../clickhouse/system-bucket-roundtrip.sh | 185 ++++++++++++++++++
.../clickhouse/system-bucket-tls.yaml | 42 ++++
4 files changed, 431 insertions(+)
create mode 100644 hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml
create mode 100755 hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh
create mode 100644 hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml
diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml
index 939e8f69a..f5b444a84 100644
--- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml
+++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml
@@ -377,3 +377,179 @@ spec:
set -eu
cd ../../..
NAMESPACE="$NAMESPACE" examples/backups/clickhouse/cleanup.sh
+---
+# Backup and in-place restore through the shipped cozy-default BackupClass and
+# the platform system bucket, for a ClickHouse whose Pod started before the
+# platform projected cozy-backups-creds into its namespace. The sidecar has to
+# pick those credentials up while running; until it could, the first BackupJob
+# of such a release failed with "no EC2 IMDS role found" and kept failing until
+# the Pod was restarted.
+#
+# Unlike clickhouse-2-backup-roundtrip this targets the system bucket as
+# advertised, https://s3.example.org, so system-bucket-roundtrip.sh makes that
+# name reachable from tenant-root for the duration of its own step and undoes
+# it on exit: see its header for the two shared objects it touches and the
+# cert-manager behaviour that relies on. Runs in tenant-root, where the system
+# bucket's SeaweedFS and its Ingress controller sit.
+#
+# The precondition step deletes tenant-root's projected cozy-backups-creds,
+# which the mongodb and rabbitmq round-trips leave there after their
+# cozy-default BackupJobs. Nothing reads it between suites, and the projector
+# writes it again on the next BackupJob, which is the moment under test.
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+ name: clickhouse-3-system-bucket-roundtrip
+spec:
+ namespace: tenant-root
+ catch:
+ - events: {}
+ - describe:
+ apiVersion: helm.toolkit.fluxcd.io/v2
+ kind: HelmRelease
+ name: clickhouse-sysbucket
+ - describe:
+ apiVersion: cert-manager.io/v1
+ kind: Certificate
+ name: e2e-system-bucket-s3
+ - describe:
+ apiVersion: cert-manager.io/v1
+ kind: Certificate
+ name: seaweedfs-s3-ingress-tls
+ - describe:
+ apiVersion: networking.k8s.io/v1
+ kind: Ingress
+ name: ingress-seaweedfs-s3
+ - describe:
+ apiVersion: backups.cozystack.io/v1alpha1
+ kind: BackupJob
+ name: sysbucket-backup
+ - describe:
+ apiVersion: backups.cozystack.io/v1alpha1
+ kind: RestoreJob
+ name: sysbucket-restore
+ - podLogs:
+ selector: app.kubernetes.io/instance=clickhouse-sysbucket
+ container: clickhouse-backup
+ tail: 200
+ - podLogs:
+ selector: app=clickhouse-sysbucket-keeper
+ tail: 100
+ steps:
+ - name: preconditions
+ try:
+ - script:
+ timeout: 2m
+ content: |
+ set -eu
+ if kubectl -n "$NAMESPACE" get secret seaweedfs-s3-ingress-tls -o name >/dev/null 2>&1; then
+ echo "Secret $NAMESPACE/seaweedfs-s3-ingress-tls already exists; this Test would have to replace it" >&2
+ exit 1
+ fi
+ if kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF 'BEGIN e2e-clickhouse-system-bucket'; then
+ echo "kube-system/coredns still carries the e2e-clickhouse-system-bucket block from an earlier run" >&2
+ exit 1
+ fi
+ # Fixed names in a persistent namespace: a leftover Succeeded job
+ # would satisfy the waits in the run step without a new attempt.
+ kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s
+ kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s
+ kubectl -n "$NAMESPACE" delete secret cozy-backups-creds --ignore-not-found --wait=true --timeout=30s
+ check:
+ ($error == null): true
+
+ - name: create-certificate-and-probe
+ try:
+ - apply:
+ file: system-bucket-tls.yaml
+ - assert:
+ timeout: 2m
+ resource:
+ apiVersion: cert-manager.io/v1
+ kind: Certificate
+ metadata:
+ name: e2e-system-bucket-s3
+ status:
+ (conditions[?type == 'Ready']):
+ - status: "True"
+ - assert:
+ timeout: 3m
+ resource:
+ apiVersion: v1
+ kind: Pod
+ metadata:
+ name: e2e-system-bucket-probe
+ status:
+ (conditions[?type == 'Ready']):
+ - status: "True"
+
+ - name: create-clickhouse
+ try:
+ - apply:
+ file: clickhouse-system-bucket.yaml
+ # 11m: above the 600s install timeout cozystack-api gives this release;
+ # see "Sizing an HR-Ready budget" in docs/agents/e2e-testing.md.
+ - assert:
+ timeout: 11m
+ resource:
+ apiVersion: helm.toolkit.fluxcd.io/v2
+ kind: HelmRelease
+ metadata:
+ name: clickhouse-sysbucket
+ status:
+ (conditions[?type == 'Ready']):
+ - status: "True"
+ - assert:
+ timeout: 5m
+ resource:
+ apiVersion: apps/v1
+ kind: StatefulSet
+ metadata:
+ name: chi-clickhouse-sysbucket-clickhouse-0-0
+ status:
+ readyReplicas: 1
+ # The Pod is running with no projected credentials: the state the first
+ # BackupJob of a fresh release meets.
+ - error:
+ timeout: 30s
+ resource:
+ apiVersion: v1
+ kind: Secret
+ metadata:
+ name: cozy-backups-creds
+
+ - name: run
+ try:
+ # Inner budgets of system-bucket-roundtrip.sh:
+ # 240s probe: CoreDNS picking up the rewrite and the Ingress serving
+ # the Test certificate
+ # 480s BackupJob to Succeeded, including the driver's 2m hold
+ # 480s RestoreJob to Succeeded, same hold
+ # 90s the EXIT trap: Corefile replace plus a 60s Secret delete
+ # 1290s (21m30s)
+ # 25m leaves 3m30s for the SQL and kubectl calls between those waits, so
+ # an inner wait is what fires and names what stalled.
+ - script:
+ timeout: 25m
+ content: |
+ set -eu
+ cd ../../..
+ NAMESPACE="$NAMESPACE" hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh
+ check:
+ ($error == null): true
+ finally:
+ # Chainsaw did not apply these: the jobs come from the script and the
+ # Backup from the controller, and deleting a BackupJob reclaims no Backup.
+ # The Certificate's Secret outlives the Certificate Chainsaw deletes.
+ - description: system-bucket round-trip cleanup
+ script:
+ timeout: 3m
+ content: |
+ set -eu
+ backup=$(kubectl -n "$NAMESPACE" get backupjob sysbucket-backup --ignore-not-found -o jsonpath='{.status.backupRef.name}')
+ kubectl -n "$NAMESPACE" delete restorejob sysbucket-restore --ignore-not-found --wait=true --timeout=30s
+ kubectl -n "$NAMESPACE" delete backupjob sysbucket-backup --ignore-not-found --wait=true --timeout=30s
+ if [ -n "$backup" ]; then
+ kubectl -n "$NAMESPACE" delete backup "$backup" --ignore-not-found --wait=true --timeout=60s
+ fi
+ kubectl -n "$NAMESPACE" delete secret e2e-system-bucket-s3-tls --ignore-not-found --wait=true --timeout=30s
diff --git a/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml b/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml
new file mode 100644
index 000000000..f7327b46d
--- /dev/null
+++ b/hack/e2e-chainsaw/clickhouse/clickhouse-system-bucket.yaml
@@ -0,0 +1,28 @@
+# ClickHouse on the platform system bucket for the
+# clickhouse-3-system-bucket-roundtrip Test. No S3 coordinates are given: the
+# sidecar reads them from the clickhouse-backup.yml key the platform projects
+# into cozy-backups-creds at the first BackupJob, after this Pod has started.
+# backup.endpointCA trusts the SeaweedFS CA that signs the certificate the Test
+# serves for s3.example.org.
+apiVersion: apps.cozystack.io/v1alpha1
+kind: ClickHouse
+metadata:
+ name: sysbucket
+spec:
+ size: 5Gi
+ logStorageSize: 1Gi
+ shards: 1
+ replicas: 1
+ resourcesPreset: nano
+ resources: {}
+ backup:
+ enabled: true
+ useSystemBucket: true
+ endpointCA:
+ name: e2e-system-bucket-s3-tls
+ key: ca.crt
+ clickhouseKeeper:
+ enabled: true
+ replicas: 1
+ size: 1Gi
+ resourcesPreset: micro
diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh
new file mode 100755
index 000000000..4ca52024b
--- /dev/null
+++ b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh
@@ -0,0 +1,185 @@
+#!/bin/bash
+# Backup and in-place restore of the `sysbucket` ClickHouse through the
+# cozy-default BackupClass, against the platform system bucket. Run from the
+# repository root by the clickhouse-3-system-bucket-roundtrip Test, which has
+# already started the release with no cozy-backups-creds in its namespace.
+#
+# The system bucket is addressed as https://s3.example.org, the S3 Ingress host
+# the COSI driver advertises. In CI that name does not resolve and its ACME
+# certificate is never issued, so for the duration of this script, and only of
+# this script, two shared objects are changed and the EXIT trap below puts both
+# back:
+#
+# - the S3 Ingress TLS Secret seaweedfs-s3-ingress-tls is created from the
+# Test's own certificate (SeaweedFS CA, s3.example.org). It relies on
+# cert-manager leaving a Secret's data alone while the Certificate that
+# names it has no completed issuance, verified with cert-manager v1.20.2:
+# a fresh ACME order would replace this certificate, and ingress-nginx
+# would then serve one the sidecar does not trust, which fails the probe
+# below rather than the backup.
+# - the host CoreDNS Corefile gets one marked `rewrite name exact` line for
+# s3.example.org. The host CoreDNS is Talos-managed and imports no snippet
+# files, so the block goes into the live Corefile the way ouroboros writes
+# its own; ouroboros is off in CI and no other suite uses this name.
+set -euo pipefail
+
+source examples/backups/clickhouse/00-helpers.sh
+
+: "${NAMESPACE:?}"
+CLICKHOUSE_NAME=sysbucket
+BACKUPJOB_NAME=sysbucket-backup
+RESTOREJOB_NAME=sysbucket-restore
+S3_HOST=s3.example.org
+INGRESS_SECRET=seaweedfs-s3-ingress-tls
+CERT_SECRET=e2e-system-bucket-s3-tls
+PROBE_POD=e2e-system-bucket-probe
+OWNER_LABEL=cozystack.io/e2e-owner=clickhouse-system-bucket
+DNS_BEGIN=' # === BEGIN e2e-clickhouse-system-bucket ==='
+DNS_END=' # === END e2e-clickhouse-system-bucket ==='
+DNS_TARGET="root-ingress-controller.${NAMESPACE}.svc.cozy.local"
+# Kubelet refreshes a ConfigMap volume within about a minute and CoreDNS's
+# reload plugin polls every 30s; the rest is the 30s answer cache.
+PROBE_TIMEOUT=240
+# Each covers the driver's 2m hold after the credentials change plus the
+# strategy Job itself.
+JOB_TIMEOUT=480
+
+workdir=$(mktemp -d)
+
+corefile_has_block() {
+ kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' | grep -qF -- "$DNS_BEGIN"
+}
+
+remove_dns_block() {
+ kubectl -n kube-system get configmap coredns -o json >"$workdir/coredns.json"
+ local begins
+ begins=$(jq -r '.data.Corefile' "$workdir/coredns.json" | grep -cF -- "$DNS_BEGIN" || true)
+ if [ "$begins" = 0 ]; then
+ return 0
+ fi
+ if [ "$begins" != 1 ] || ! jq -r '.data.Corefile' "$workdir/coredns.json" | grep -qF -- "$DNS_END"; then
+ log_error "kube-system/coredns holds ${begins} '${DNS_BEGIN# *}' markers or no END marker; reconcile the Corefile by hand"
+ return 1
+ fi
+ jq --arg b "$DNS_BEGIN" --arg e "$DNS_END" \
+ '.data.Corefile |= (split("\n") | (index($b)) as $i | (index($e)) as $j | .[:$i] + .[$j+1:] | join("\n"))' \
+ "$workdir/coredns.json" >"$workdir/coredns-clean.json"
+ kubectl replace -f "$workdir/coredns-clean.json" >/dev/null
+ log_success "Removed the ${S3_HOST} rewrite from kube-system/coredns"
+}
+
+restore_shared_state() {
+ local rc=$?
+ set +e
+ remove_dns_block || rc=1
+ if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -l "$OWNER_LABEL" -o name 2>/dev/null | grep -q .; then
+ kubectl -n "$NAMESPACE" delete secret "$INGRESS_SECRET" --wait=true --timeout=60s || rc=1
+ fi
+ rm -rf "$workdir"
+ exit "$rc"
+}
+
+if kubectl -n "$NAMESPACE" get secret "$INGRESS_SECRET" -o name >/dev/null 2>&1; then
+ log_error "Secret ${NAMESPACE}/${INGRESS_SECRET} already exists; this Test serves its own certificate under that name and will not overwrite one it did not create"
+ exit 1
+fi
+if corefile_has_block; then
+ log_error "kube-system/coredns already carries the ${S3_HOST} rewrite block, left by an earlier run; remove it by hand"
+ exit 1
+fi
+trap restore_shared_state EXIT
+
+log_step "Serving the Test certificate for ${S3_HOST} through the platform S3 Ingress"
+kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o json \
+ | jq --arg name "$INGRESS_SECRET" '{apiVersion, kind, type,
+ metadata: {name: $name, labels: {"cozystack.io/e2e-owner": "clickhouse-system-bucket"}},
+ data: {"tls.crt": .data["tls.crt"], "tls.key": .data["tls.key"], "ca.crt": .data["ca.crt"]}}' \
+ | kubectl -n "$NAMESPACE" create -f -
+
+log_step "Rewriting ${S3_HOST} to ${DNS_TARGET} in the host CoreDNS"
+kubectl -n kube-system get configmap coredns -o json >"$workdir/coredns.json"
+jq -e '.data.Corefile | startswith(".:53 {\n")' "$workdir/coredns.json" >/dev/null \
+ || { log_error "kube-system/coredns Corefile does not open with '.:53 {'; refusing to edit it"; exit 1; }
+# The resourceVersion read above travels with the replace, so a concurrent
+# writer makes it fail instead of being overwritten.
+jq --arg block "${DNS_BEGIN}"$'\n'" rewrite name exact ${S3_HOST} ${DNS_TARGET}"$'\n'"${DNS_END}" \
+ '.data.Corefile |= sub("^\\.:53 \\{\n"; ".:53 {\n" + $block + "\n")' \
+ "$workdir/coredns.json" >"$workdir/coredns-new.json"
+kubectl replace -f "$workdir/coredns-new.json" >/dev/null
+
+log_step "Checking that ${S3_HOST} serves the Test certificate from inside ${NAMESPACE}"
+kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.ca\.crt}' | base64 -d >"$workdir/ca.crt"
+want=$(kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.tls\.crt}' | base64 -d \
+ | openssl x509 -noout -fingerprint -sha256)
+elapsed=0
+while true; do
+ # --cacert makes curl verify both the chain and the host name; %{certs}
+ # prints the chain it was served, leaf first.
+ if served=$(kubectl -n "$NAMESPACE" exec -i "$PROBE_POD" -- sh -c \
+ "cat >/tmp/ca.crt && curl -sS -o /dev/null --max-time 10 --cacert /tmp/ca.crt -w '%{certs}' https://${S3_HOST}/" \
+ <"$workdir/ca.crt" 2>"$workdir/probe.err"); then
+ got=$(printf '%s\n' "$served" | sed -n '/BEGIN CERTIFICATE/,/END CERTIFICATE/p;/END CERTIFICATE/q' \
+ | openssl x509 -noout -fingerprint -sha256)
+ if [ "$got" = "$want" ]; then
+ log_success "${S3_HOST} resolves in ${NAMESPACE} and serves the Test certificate"
+ break
+ fi
+ echo "served ${got}, want ${want}" >"$workdir/probe.err"
+ fi
+ if [ "$elapsed" -ge "$PROBE_TIMEOUT" ]; then
+ log_error "${S3_HOST} did not serve the Test certificate within ${PROBE_TIMEOUT}s: $(cat "$workdir/probe.err")"
+ kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' >&2 || true
+ kubectl -n "$NAMESPACE" describe ingress ingress-seaweedfs-s3 >&2 || true
+ exit 1
+ fi
+ sleep 10
+ elapsed=$((elapsed + 10))
+done
+
+log_step "Writing the sentinel row"
+clickhouse_query "$CLICKHOUSE_NAME" "CREATE TABLE IF NOT EXISTS default.sentinel (id UInt32, name String) ENGINE = MergeTree ORDER BY id"
+clickhouse_query "$CLICKHOUSE_NAME" "INSERT INTO default.sentinel VALUES (1, 'before-backup')"
+
+log_step "Backing up through cozy-default"
+kubectl apply -f - <= 1 )) || { log_error "sentinel row count after restore is ${count}; expected >= 1"; exit 1; }
+log_success "System-bucket round-trip verified: ${count} sentinel row(s) restored"
diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml b/hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml
new file mode 100644
index 000000000..06203a299
--- /dev/null
+++ b/hack/e2e-chainsaw/clickhouse/system-bucket-tls.yaml
@@ -0,0 +1,42 @@
+# Certificate and probe Pod for the clickhouse-3-system-bucket-roundtrip Test.
+#
+# The certificate is signed by the SeaweedFS CA already present in tenant-root,
+# so its Secret carries that CA as ca.crt: the ClickHouse release trusts it via
+# backup.endpointCA, and the round-trip script copies the same key pair under
+# the name the platform's S3 Ingress serves for s3.example.org.
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+ name: e2e-system-bucket-s3
+spec:
+ secretName: e2e-system-bucket-s3-tls
+ dnsNames:
+ - s3.example.org
+ issuerRef:
+ kind: Issuer
+ name: seaweedfs-ca-issuer
+---
+# Resolves and fetches s3.example.org from inside tenant-root, where the
+# ClickHouse sidecar runs, so the script can prove the DNS rewrite and the
+# served certificate before it spends minutes on a BackupJob. The image is the
+# one hack/e2e-install-cozystack.bats pre-pulls; it carries curl but no openssl.
+apiVersion: v1
+kind: Pod
+metadata:
+ name: e2e-system-bucket-probe
+spec:
+ restartPolicy: Never
+ terminationGracePeriodSeconds: 0
+ securityContext:
+ runAsNonRoot: true
+ runAsUser: 65534
+ seccompProfile:
+ type: RuntimeDefault
+ containers:
+ - name: probe
+ image: docker.io/alpine/k8s:1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47
+ command: [sleep, "3600"]
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop: [ALL]
From 2e37aaec5f46a88c01f65764788ab8e58ab7ac7d Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:31:59 +0300
Subject: [PATCH 071/346] docs(backups): drop the claim that a region change
needs a ClickHouse restart
The ClickHouse sidecar on the system bucket no longer reads S3 settings
from environment set at start, so a re-projected region reaches it
without a Pod rollout, and the contracts fixture no longer depends on
optional Secret keys to start its Pod.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml | 4 ++--
packages/system/backupstrategy-controller/values.yaml | 4 ----
2 files changed, 2 insertions(+), 6 deletions(-)
diff --git a/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml b/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml
index b88e7dd6d..fb0979146 100644
--- a/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml
+++ b/hack/e2e-chainsaw/clickhouse/clickhouse-backup.yaml
@@ -8,8 +8,8 @@
# backup.enabled=true with placeholder S3 coordinates is deliberate: the
# sidecar runs `clickhouse-backup server`, which starts its HTTP API without
# validating S3 (ALLOW_EMPTY_BACKUPS=true, lazy connect — the same reason the
-# platform useSystemBucket flow marks the S3 secret keys optional so the Pod
-# starts before any bucket is projected). So the Pod becomes Ready and the
+# platform useSystemBucket flow can start the Pod before any bucket is
+# projected). So the Pod becomes Ready and the
# chart<->strategy contract is assertable with no reachable object store. The
# real S3 backup/restore round-trip lives in clickhouse-2-backup-roundtrip,
# which runs in tenant-root against the in-cluster SeaweedFS.
diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml
index 6cf39c65a..2a4152597 100644
--- a/packages/system/backupstrategy-controller/values.yaml
+++ b/packages/system/backupstrategy-controller/values.yaml
@@ -105,10 +105,6 @@ backupStorage:
endpoint: http://seaweedfs-s3.tenant-root.svc.cozy.local:8333
# region is re-projected into cozy-backups-creds on the next reconcile
# (1-minute tick for system namespaces, on-demand for tenant namespaces).
- # However, charts that consume the region via secretKeyRef in env vars —
- # the chart-emitted ClickHouse sidecar on `useSystemBucket: true` is the
- # only such consumer today — only pick up the new value on Pod restart.
- # Plan a rollout of those Pods when you change this value.
region: us-east-1
forcePathStyle: true
# systemSecretName — the human-friendly Secret produced by the Bucket
From 38b00730ff072b6b03360e8ae12b047d7568c4ac Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:40:01 +0300
Subject: [PATCH 072/346] fix(clickhouse): keep the clickhouse-backup default
region when none is configured
The environment the sidecar used to read had no S3_REGION when the
platform carried no region, which left clickhouse-backup on its own
us-east-1. Writing an empty region into the config file replaces that
default with an empty one, and requests are then signed for no region.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
.../backupcontroller/credentials_projector.go | 6 ++++-
.../credentials_projector_test.go | 25 +++++++++++++++++++
2 files changed, 30 insertions(+), 1 deletion(-)
diff --git a/internal/backupcontroller/credentials_projector.go b/internal/backupcontroller/credentials_projector.go
index 99d4e182e..ddf5ff8ef 100644
--- a/internal/backupcontroller/credentials_projector.go
+++ b/internal/backupcontroller/credentials_projector.go
@@ -454,10 +454,14 @@ func buildClickHouseBackupConfig(creds parsedCreds, forcePathStyle string) ([]by
s3 := map[string]any{
"bucket": creds.bucket,
"endpoint": "https://" + creds.endpoint,
- "region": creds.region,
"access_key": creds.accessKey,
"secret_key": creds.secretKey,
}
+ // An empty region in the file would override clickhouse-backup's own
+ // us-east-1 default instead of leaving it in place.
+ if creds.region != "" {
+ s3["region"] = creds.region
+ }
if forcePathStyle != "" {
pathStyle, err := strconv.ParseBool(forcePathStyle)
if err != nil {
diff --git a/internal/backupcontroller/credentials_projector_test.go b/internal/backupcontroller/credentials_projector_test.go
index 208ba253c..78ffb6c94 100644
--- a/internal/backupcontroller/credentials_projector_test.go
+++ b/internal/backupcontroller/credentials_projector_test.go
@@ -721,6 +721,31 @@ func TestProject_WritesClickHouseBackupConfig(t *testing.T) {
}
}
+// clickhouse-backup defaults the S3 region to us-east-1; an empty value in
+// its config file would replace that default rather than fall back to it.
+func TestProject_ClickHouseBackupConfigOmitsAnEmptyRegion(t *testing.T) {
+ c := newFakeClient(flatSourceSecret())
+ ctx := context.Background()
+ cfg := defaultCfg()
+ cfg.Region = ""
+ if err := ProjectBackupCredentials(ctx, c, cfg, "tenant-acme"); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ got := &corev1.Secret{}
+ if err := c.Get(ctx, types.NamespacedName{Namespace: "tenant-acme", Name: "cozy-backups-creds"}, got); err != nil {
+ t.Fatalf("projected Secret not found: %v", err)
+ }
+ var doc struct {
+ S3 map[string]any `json:"s3"`
+ }
+ if err := yaml.Unmarshal(got.Data[clickHouseBackupConfigKey], &doc); err != nil {
+ t.Fatalf("%s is not YAML: %v", clickHouseBackupConfigKey, err)
+ }
+ if region, set := doc.S3["region"]; set {
+ t.Errorf("s3.region = %q, want it left to the clickhouse-backup default", region)
+ }
+}
+
func TestProject_RejectsMalformedForcePathStyle(t *testing.T) {
c := newFakeClient(flatSourceSecret())
cfg := defaultCfg()
From 5f636ce62469aa313a3f1a67cba4e72dbf8f4d87 Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:43:14 +0300
Subject: [PATCH 073/346] fix(clickhouse): do not hold strategy jobs on a
credentials stamp that cannot expire
The projector rewrites the credentials-updated-at stamp only when the
projected content changes, so a stamp the hold cannot use stays put.
An unreadable one failed every reconcile with no condition on the job,
and one far in the future held each job again on every reconcile; in
both cases every ClickHouse backup and restore in the namespace stopped
for good. Such a stamp is now logged and ignored. A stamp only slightly
ahead, as a skewed clock writes it, still waits, and that wait ends.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
.../altinitystrategy_controller.go | 10 ++++-
.../altinitystrategy_controller_test.go | 41 +++++++++++++++++++
2 files changed, 50 insertions(+), 1 deletion(-)
diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go
index a99801ac6..c3c609350 100644
--- a/internal/backupcontroller/altinitystrategy_controller.go
+++ b/internal/backupcontroller/altinitystrategy_controller.go
@@ -102,9 +102,17 @@ func credentialsPropagationWait(ctx context.Context, c client.Client, secretName
if raw == "" {
return 0, nil
}
+ // The projector rewrites the stamp only when the content changes, so a
+ // stamp this hold cannot use stays as it is: holding on it would block
+ // every Job in the namespace for good.
updatedAt, err := time.Parse(time.RFC3339, raw)
if err != nil {
- return 0, fmt.Errorf("parse %s on Secret %s/%s: %w", credentialsUpdatedAtAnnotation, namespace, secretName, err)
+ getLogger(ctx).Error(err, "ignoring unreadable credentials stamp", "secret", namespace+"/"+secretName, "annotation", credentialsUpdatedAtAnnotation)
+ return 0, nil
+ }
+ if updatedAt.After(time.Now().Add(altinityCredentialsPropagation)) {
+ getLogger(ctx).Error(nil, "ignoring a credentials stamp further ahead than clock skew explains", "secret", namespace+"/"+secretName, "annotation", credentialsUpdatedAtAnnotation, "value", raw)
+ return 0, nil
}
return time.Until(updatedAt.Add(altinityCredentialsPropagation)), nil
}
diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go
index 75564538b..717b425d1 100644
--- a/internal/backupcontroller/altinitystrategy_controller_test.go
+++ b/internal/backupcontroller/altinitystrategy_controller_test.go
@@ -896,3 +896,44 @@ func TestReconcileAltinity_NoProjectionConfiguredDoesNotWait(t *testing.T) {
t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items))
}
}
+
+func projectedCredentialsStamped(namespace, stamp string) *corev1.Secret {
+ return &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Namespace: namespace,
+ Name: "cozy-backups-creds",
+ Annotations: map[string]string{credentialsUpdatedAtAnnotation: stamp},
+ },
+ }
+}
+
+// The stamp is only ever rewritten when the projected content changes, so a
+// value the hold cannot use must not hold the job: an unreadable one would
+// fail every reconcile, and one far in the future would wait again on each.
+// A stamp a little ahead, as a skewed clock writes it, still waits, but only
+// until that stamp plus the propagation time.
+func TestCredentialsPropagationWait_UnusableStampsDoNotHoldTheJob(t *testing.T) {
+ cases := []struct {
+ name string
+ stamp string
+ wantMin time.Duration
+ wantMax time.Duration
+ }{
+ {name: "unreadable", stamp: "garbage"},
+ {name: "far in the future", stamp: "3000-01-01T00:00:00Z"},
+ {name: "slightly ahead", stamp: time.Now().Add(30 * time.Second).UTC().Format(time.RFC3339),
+ wantMin: altinityCredentialsPropagation, wantMax: altinityCredentialsPropagation + 31*time.Second},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ c := clientfake.NewClientBuilder().WithObjects(projectedCredentialsStamped("tenant-test", tc.stamp)).Build()
+ wait, err := credentialsPropagationWait(context.Background(), c, "cozy-backups-creds", "tenant-test", "job")
+ if err != nil {
+ t.Fatalf("credentialsPropagationWait: %v", err)
+ }
+ if wait < tc.wantMin || wait > tc.wantMax {
+ t.Errorf("wait = %v, want within [%v, %v]", wait, tc.wantMin, tc.wantMax)
+ }
+ })
+ }
+}
From 6345b939d2eef42156d2bbcafd59e859be307326 Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:43:15 +0300
Subject: [PATCH 074/346] fix(clickhouse): stop reporting a credentials wait
once the strategy job runs
The hold sets Ready=False/WaitingForCredentials with a countdown, and
nothing removed it once the Job was created, so a backup or restore
that ran for minutes kept saying it was still waiting. The condition is
dropped when the Job runs; a Ready condition set for another reason is
left alone.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
.../altinitystrategy_controller.go | 24 +++-
.../altinitystrategy_controller_test.go | 104 ++++++++++++++++++
2 files changed, 127 insertions(+), 1 deletion(-)
diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go
index c3c609350..c72e30719 100644
--- a/internal/backupcontroller/altinitystrategy_controller.go
+++ b/internal/backupcontroller/altinitystrategy_controller.go
@@ -117,13 +117,25 @@ func credentialsPropagationWait(ctx context.Context, c client.Client, secretName
return time.Until(updatedAt.Add(altinityCredentialsPropagation)), nil
}
+// clearCredentialsWait drops the Ready condition credentialsWaitCondition
+// left behind, once the job no longer waits. It reports whether it did.
+func clearCredentialsWait(conditions *[]metav1.Condition) bool {
+ ready := apimeta.FindStatusCondition(*conditions, "Ready")
+ if ready == nil || ready.Reason != credentialsWaitReason {
+ return false
+ }
+ return apimeta.RemoveStatusCondition(conditions, "Ready")
+}
+
+const credentialsWaitReason = "WaitingForCredentials"
+
// credentialsWaitCondition is the Ready condition of a job held back by
// credentialsPropagationWait.
func credentialsWaitCondition(secretName string, wait time.Duration) metav1.Condition {
return metav1.Condition{
Type: "Ready",
Status: metav1.ConditionFalse,
- Reason: "WaitingForCredentials",
+ Reason: credentialsWaitReason,
Message: fmt.Sprintf("waiting %s for the updated Secret %s to reach the ClickHouse backup sidecar",
wait.Round(time.Second), secretName),
}
@@ -336,6 +348,11 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv
return r.markBackupJobFailed(ctx, j, message)
default:
+ if clearCredentialsWait(&j.Status.Conditions) {
+ if err := r.Status().Update(ctx, j); err != nil {
+ return ctrl.Result{}, err
+ }
+ }
return ctrl.Result{RequeueAfter: altinityPollInterval}, nil
}
}
@@ -741,6 +758,11 @@ func (r *RestoreJobReconciler) reconcileAltinityRestore(ctx context.Context, res
return r.markRestoreJobFailed(ctx, restoreJob, message)
default:
+ if clearCredentialsWait(&restoreJob.Status.Conditions) {
+ if err := r.Status().Update(ctx, restoreJob); err != nil {
+ return ctrl.Result{}, err
+ }
+ }
return ctrl.Result{RequeueAfter: altinityPollInterval}, nil
}
}
diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go
index 717b425d1..e521d2d46 100644
--- a/internal/backupcontroller/altinitystrategy_controller_test.go
+++ b/internal/backupcontroller/altinitystrategy_controller_test.go
@@ -907,6 +907,100 @@ func projectedCredentialsStamped(namespace, stamp string) *corev1.Secret {
}
}
+func waitingForCredentials() []metav1.Condition {
+ return []metav1.Condition{credentialsWaitCondition("cozy-backups-creds", time.Minute)}
+}
+
+// Once the hold is over and the Job runs, a Ready condition still saying the
+// job is waiting for credentials describes a state the job has left.
+func TestReconcileAltinity_ClearsTheCredentialsWaitOnceTheJobRuns(t *testing.T) {
+ now := metav1.Now()
+ backupJob := &backupsv1alpha1.BackupJob{
+ ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"},
+ Spec: backupsv1alpha1.BackupJobSpec{
+ ApplicationRef: newAltinityClickHouseRef("ch-test"),
+ BackupClassName: "clickhouse-backup",
+ },
+ Status: backupsv1alpha1.BackupJobStatus{StartedAt: &now, Conditions: waitingForCredentials()},
+ }
+ r, _, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"),
+ clientfake.NewClientBuilder().WithObjects(backupJob, newAltinityStrategy("clickhouse-strategy"),
+ projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))).
+ WithStatusSubresource(&backupsv1alpha1.BackupJob{}))
+ r.CredentialsConfig.TargetSecretName = "cozy-backups-creds"
+ resolved := &ResolvedBackupConfig{
+ StrategyRef: corev1.TypedLocalObjectReference{
+ APIGroup: new(strategyv1alpha1.GroupVersion.Group),
+ Kind: strategyv1alpha1.AltinityStrategyKind,
+ Name: "clickhouse-strategy",
+ },
+ }
+ ctx := context.Background()
+ j := &backupsv1alpha1.BackupJob{}
+ if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), j); err != nil {
+ t.Fatalf("get BackupJob: %v", err)
+ }
+ if _, err := r.reconcileAltinity(ctx, j, resolved); err != nil {
+ t.Fatalf("reconcileAltinity: %v", err)
+ }
+ jobs := &batchv1.JobList{}
+ if err := r.List(ctx, jobs, client.InNamespace("tenant-test")); err != nil {
+ t.Fatalf("list Jobs: %v", err)
+ }
+ if len(jobs.Items) != 1 {
+ t.Fatalf("got %d strategy Jobs, want 1", len(jobs.Items))
+ }
+ stored := &backupsv1alpha1.BackupJob{}
+ if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), stored); err != nil {
+ t.Fatalf("get BackupJob: %v", err)
+ }
+ if ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForCredentials" {
+ t.Errorf("stored Ready = %+v while the Job runs", ready)
+ }
+}
+
+func TestReconcileAltinityRestore_ClearsTheCredentialsWaitOnceTheJobRuns(t *testing.T) {
+ now := metav1.Now()
+ backup := &backupsv1alpha1.Backup{
+ ObjectMeta: metav1.ObjectMeta{Name: "ch-backup", Namespace: "tenant-test"},
+ Spec: backupsv1alpha1.BackupSpec{
+ ApplicationRef: newAltinityClickHouseRef("ch-test"),
+ StrategyRef: corev1.TypedLocalObjectReference{
+ APIGroup: new(strategyv1alpha1.GroupVersion.Group),
+ Kind: strategyv1alpha1.AltinityStrategyKind,
+ Name: "clickhouse-strategy",
+ },
+ TakenAt: now,
+ },
+ }
+ restoreJob := &backupsv1alpha1.RestoreJob{
+ ObjectMeta: metav1.ObjectMeta{Name: "test-rj", Namespace: "tenant-test"},
+ Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}},
+ Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &now, Phase: backupsv1alpha1.RestoreJobPhaseRunning,
+ Conditions: waitingForCredentials()},
+ }
+ _, rr, _ := newAltinityTestEnv(t, newAltinityClickHouseApp("ch-test", "tenant-test"),
+ clientfake.NewClientBuilder().WithObjects(newAltinityStrategy("clickhouse-strategy"), backup, restoreJob,
+ projectedCredentials("tenant-test", time.Now().Add(-altinityCredentialsPropagation-time.Second))).
+ WithStatusSubresource(&backupsv1alpha1.RestoreJob{}))
+ rr.CredentialsConfig.TargetSecretName = "cozy-backups-creds"
+ ctx := context.Background()
+ rj := &backupsv1alpha1.RestoreJob{}
+ if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), rj); err != nil {
+ t.Fatalf("get RestoreJob: %v", err)
+ }
+ if _, err := rr.reconcileAltinityRestore(ctx, rj, backup); err != nil {
+ t.Fatalf("reconcileAltinityRestore: %v", err)
+ }
+ stored := &backupsv1alpha1.RestoreJob{}
+ if err := rr.Get(ctx, client.ObjectKeyFromObject(restoreJob), stored); err != nil {
+ t.Fatalf("get RestoreJob: %v", err)
+ }
+ if ready := meta.FindStatusCondition(stored.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForCredentials" {
+ t.Errorf("stored Ready = %+v while the Job runs", ready)
+ }
+}
+
// The stamp is only ever rewritten when the projected content changes, so a
// value the hold cannot use must not hold the job: an unreadable one would
// fail every reconcile, and one far in the future would wait again on each.
@@ -937,3 +1031,13 @@ func TestCredentialsPropagationWait_UnusableStampsDoNotHoldTheJob(t *testing.T)
})
}
}
+
+func TestClearCredentialsWait_KeepsAReadyConditionItDidNotSet(t *testing.T) {
+ conditions := []metav1.Condition{{Type: "Ready", Status: metav1.ConditionFalse, Reason: "StrategyNotReady"}}
+ if clearCredentialsWait(&conditions) {
+ t.Errorf("clearCredentialsWait reported clearing a Ready condition with reason StrategyNotReady")
+ }
+ if ready := meta.FindStatusCondition(conditions, "Ready"); ready == nil || ready.Reason != "StrategyNotReady" {
+ t.Errorf("Ready = %+v, want the StrategyNotReady condition kept", ready)
+ }
+}
From dbcc2af6791c320a60b80662e78074bcd23b87cf Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:43:55 +0300
Subject: [PATCH 075/346] test(clickhouse): require the system-bucket
round-trip to run on the sidecar that started without credentials
A Pod restart during the round-trip would hand the sidecar fresh
credentials at start, which is how the old environment-based sidecar
got them, and the Test would pass without the running sidecar ever
picking them up. The Pod uid and the sidecar's restart count must stay
the same across both jobs.
The probe deadline is now measured on the clock: counting only its
pauses let a hanging exec stretch it past the step budget, so the step
timeout would fire before the probe could say what stalled.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov
---
.../clickhouse/chainsaw-test.yaml | 9 ++++----
.../clickhouse/system-bucket-roundtrip.sh | 21 +++++++++++++++----
2 files changed, 22 insertions(+), 8 deletions(-)
diff --git a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml
index f5b444a84..d96013140 100644
--- a/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml
+++ b/hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml
@@ -521,13 +521,14 @@ spec:
- name: run
try:
# Inner budgets of system-bucket-roundtrip.sh:
- # 240s probe: CoreDNS picking up the rewrite and the Ingress serving
- # the Test certificate
+ # 280s probe: a 240s deadline for CoreDNS picking up the rewrite and
+ # the Ingress serving the Test certificate, plus one last 30s
+ # attempt and its 10s pause
# 480s BackupJob to Succeeded, including the driver's 2m hold
# 480s RestoreJob to Succeeded, same hold
# 90s the EXIT trap: Corefile replace plus a 60s Secret delete
- # 1290s (21m30s)
- # 25m leaves 3m30s for the SQL and kubectl calls between those waits, so
+ # 1330s (22m10s)
+ # 25m leaves 2m50s for the SQL and kubectl calls between those waits, so
# an inner wait is what fires and names what stalled.
- script:
timeout: 25m
diff --git a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh
index 4ca52024b..fe1361418 100755
--- a/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh
+++ b/hack/e2e-chainsaw/clickhouse/system-bucket-roundtrip.sh
@@ -111,11 +111,11 @@ log_step "Checking that ${S3_HOST} serves the Test certificate from inside ${NAM
kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.ca\.crt}' | base64 -d >"$workdir/ca.crt"
want=$(kubectl -n "$NAMESPACE" get secret "$CERT_SECRET" -o jsonpath='{.data.tls\.crt}' | base64 -d \
| openssl x509 -noout -fingerprint -sha256)
-elapsed=0
+deadline=$((SECONDS + PROBE_TIMEOUT))
while true; do
# --cacert makes curl verify both the chain and the host name; %{certs}
# prints the chain it was served, leaf first.
- if served=$(kubectl -n "$NAMESPACE" exec -i "$PROBE_POD" -- sh -c \
+ if served=$(timeout 30 kubectl -n "$NAMESPACE" exec -i "$PROBE_POD" -- sh -c \
"cat >/tmp/ca.crt && curl -sS -o /dev/null --max-time 10 --cacert /tmp/ca.crt -w '%{certs}' https://${S3_HOST}/" \
<"$workdir/ca.crt" 2>"$workdir/probe.err"); then
got=$(printf '%s\n' "$served" | sed -n '/BEGIN CERTIFICATE/,/END CERTIFICATE/p;/END CERTIFICATE/q' \
@@ -126,16 +126,26 @@ while true; do
fi
echo "served ${got}, want ${want}" >"$workdir/probe.err"
fi
- if [ "$elapsed" -ge "$PROBE_TIMEOUT" ]; then
+ if [ "$SECONDS" -ge "$deadline" ]; then
log_error "${S3_HOST} did not serve the Test certificate within ${PROBE_TIMEOUT}s: $(cat "$workdir/probe.err")"
kubectl -n kube-system get configmap coredns -o jsonpath='{.data.Corefile}' >&2 || true
kubectl -n "$NAMESPACE" describe ingress ingress-seaweedfs-s3 >&2 || true
exit 1
fi
sleep 10
- elapsed=$((elapsed + 10))
done
+# A restarted Pod reads its credentials afresh at start, which is what the
+# old environment-based sidecar needed; the round-trip proves the fix only if
+# the sidecar that started without them is the one that serves both jobs.
+sidecar_identity() {
+ kubectl -n "$NAMESPACE" get pod "chi-clickhouse-${CLICKHOUSE_NAME}-clickhouse-0-0-0" \
+ -o jsonpath='{.metadata.uid} {.status.containerStatuses[?(@.name=="clickhouse-backup")].restartCount}'
+}
+sidecar_before=$(sidecar_identity)
+[[ "$sidecar_before" =~ ^[0-9a-f-]+\ [0-9]+$ ]] \
+ || { log_error "could not read the ClickHouse Pod uid and clickhouse-backup restart count (got '${sidecar_before}')"; exit 1; }
+
log_step "Writing the sentinel row"
clickhouse_query "$CLICKHOUSE_NAME" "CREATE TABLE IF NOT EXISTS default.sentinel (id UInt32, name String) ENGINE = MergeTree ORDER BY id"
clickhouse_query "$CLICKHOUSE_NAME" "INSERT INTO default.sentinel VALUES (1, 'before-backup')"
@@ -182,4 +192,7 @@ wait_for_field restorejob "$RESTOREJOB_NAME" '{.status.phase}' Succeeded "$NAMES
count=$(clickhouse_query "$CLICKHOUSE_NAME" "SELECT count() FROM default.sentinel" | tr -d '[:space:]')
[[ "$count" =~ ^[0-9]+$ ]] || { log_error "non-numeric sentinel count: '${count}'"; exit 1; }
(( count >= 1 )) || { log_error "sentinel row count after restore is ${count}; expected >= 1"; exit 1; }
+sidecar_after=$(sidecar_identity)
+[ "$sidecar_after" = "$sidecar_before" ] \
+ || { log_error "the ClickHouse Pod or its clickhouse-backup container restarted during the round-trip (uid restarts: '${sidecar_before}' then '${sidecar_after}')"; exit 1; }
log_success "System-bucket round-trip verified: ${count} sentinel row(s) restored"
From 31b905a1ce938b84db90deaadde1cf91796f4387 Mon Sep 17 00:00:00 2001
From: Alexey Artamonov
Date: Wed, 4 Nov 2026 19:50:35 +0300
Subject: [PATCH 076/346] docs(clickhouse): note that the backup path stays in
the sidecar environment
The chart comment said every S3 setting of the system-bucket flow comes
from the mounted config file, but S3_PATH is still set in the
environment: the platform's file carries no path, and the prefix belongs
to the release rather than to the platform.
Assisted-by: LLM
Signed-off-by: Alexey Artamonov