From 1779df2fb3c743a93ddf779162e05ed1347c2c63 Mon Sep 17 00:00:00 2001
From: Roshan Ramani
Date: Sun, 18 Oct 2026 20:26:50 +0530
Subject: [PATCH 001/216] docs(vpc): generate README parameters from
values.yaml
The vpc Makefile never passed -r README.md to cozyvalues-gen, unlike
every other app, so its Parameters table was never regenerated. It
still described subnets as a map after the switch to a list, and the
README example, which also feeds the website reference page, fails
values.schema.json.
Assisted-by: LLM
Signed-off-by: Roshan Ramani
---
packages/apps/vpc/Makefile | 2 +-
packages/apps/vpc/README.md | 22 +++++++++++++++-------
2 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/packages/apps/vpc/Makefile b/packages/apps/vpc/Makefile
index fce015f3e..6cc55dbed 100644
--- a/packages/apps/vpc/Makefile
+++ b/packages/apps/vpc/Makefile
@@ -1,7 +1,7 @@
include ../../../hack/package.mk
generate:
- cozyvalues-gen -m 'vpc' -v values.yaml -s values.schema.json -g ../../../api/apps/v1alpha1/vpc/types.go
+ cozyvalues-gen -m 'vpc' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/vpc/types.go
../../../hack/update-crd.sh
update:
diff --git a/packages/apps/vpc/README.md b/packages/apps/vpc/README.md
index 890f12d89..f2dd2a5f9 100644
--- a/packages/apps/vpc/README.md
+++ b/packages/apps/vpc/README.md
@@ -27,10 +27,18 @@ A VM or a pod may be connected to multiple secondary Subnets at once. Each secon
### Common parameters
-| Name | Description | Type | Value |
-| -------------------- | -------------------------------- | ------------------- | ------- |
-| `subnets` | Subnets of a VPC | `map[string]object` | `{...}` |
-| `subnets[name].cidr` | Subnet CIDR, e.g. 192.168.0.0/24 | `cidr` | `{}` |
+| Name | Description | Type | Value |
+| -------------------------- | ----------------------------------------------------------------------------------------------------------------- | ---------- | ----- |
+| `subnets` | Subnets of a VPC | `[]object` | `[]` |
+| `subnets[i].name` | Subnet name | `string` | `""` |
+| `subnets[i].cidr` | IP address range | `string` | `""` |
+| `subnets[i].allowSubnets` | Additional source CIDRs admitted into this private subnet, such as remote networks routed in through a gateway VM | `[]string` | `[]` |
+| `peers` | VPC peering connections (bidirectional declaration required) | `[]object` | `[]` |
+| `peers[i].vpcName` | Logical name of the remote VPC (without "virtualprivatecloud-" prefix) | `string` | `""` |
+| `peers[i].tenantNamespace` | Namespace of the remote tenant | `string` | `""` |
+| `routes` | Static routes for the VPC | `[]object` | `[]` |
+| `routes[i].cidr` | Destination CIDR | `string` | `""` |
+| `routes[i].nextHopIP` | Next hop IP address | `string` | `""` |
## Examples
@@ -41,10 +49,10 @@ metadata:
name: vpc00
spec:
subnets:
- sub00:
+ - name: sub00
cidr: 172.16.0.0/24
- sub01:
+ - name: sub01
cidr: 172.16.1.0/24
- sub02:
+ - name: sub02
cidr: 172.16.2.0/24
```
From a2c9c07a1629eeb78e7cd454753e0e864cbf8a29 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 18 Oct 2026 22:56:06 +0300
Subject: [PATCH 002/216] fix(apps): keep Job and CronJob names within the
Kubernetes limits
The apps API admits release names of up to 53 characters, and several
charts name a Job -. Kubernetes copies a Job name into
the job-name label of its pods, and a label value takes at most 63
characters, so the apiserver refused these Jobs for the longest names. A
Kafka whose name has 42 to 44 characters could not be installed, a
VMDisk of 37 or more could not grow, a VMInstance of 40 or 41 could not
take an instance type, profile or external change, and a Bucket of 44 or
more or a Qdrant of 42 or more failed its post-delete hook. Those Job
names, and the pod label vm-instance copies from one, are now cut to 63
characters. A release name is never longer than 53, so only the fixed
suffix is cut and two releases still get two Jobs.
A CronJob name is refused over 52 characters, because the controller
names each Job it starts -. With the backup CronJob
on, a MariaDB whose name has 38 characters or more and a ClickHouse of
35 or more failed the whole release. A cut there reaches into the
release name and could give two releases one CronJob, so a name that
does not fit becomes the release name cut to 38 characters, a hash of
the full release name and -backup, the way the ClickHouse Keeper name
is already shortened.
The tenant cleanup Job is named after the tenant namespace, which a
nested tenant can fill to 63 characters by itself. The Job runs in
cozy-system next to that of every other tenant, where a plain cut could
give a parent and its child one name and let the child's
before-hook-creation delete the parent's Job while it runs, so it is
shortened with a hash of the namespace the same way.
Names that fit render exactly as before, and a name that did not fit
never produced an object, so nothing is renamed on upgrade. The hook
ServiceAccounts, Roles and RoleBindings have no such limit and keep
their names.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../bucket/templates/hooks/cleanup-acme.yaml | 2 +-
.../bucket/tests/job_name_budget_test.yaml | 80 +++++++++++
.../clickhouse/templates/backup-cronjob.yaml | 10 +-
.../tests/cronjob_name_budget_test.yaml | 94 +++++++++++++
.../apps/kafka/templates/migration-hook.yaml | 2 +-
.../kafka/tests/job_name_budget_test.yaml | 66 +++++++++
.../mariadb/templates/backup-cronjob.yaml | 10 +-
.../tests/cronjob_name_budget_test.yaml | 92 +++++++++++++
.../qdrant/templates/hooks/cleanup-pvc.yaml | 2 +-
.../qdrant/tests/job_name_budget_test.yaml | 64 +++++++++
.../apps/tenant/templates/cleanup-job.yaml | 15 ++-
.../tenant/tests/job_name_budget_test.yaml | 127 ++++++++++++++++++
.../vm-disk/templates/pvc-resize-hook.yaml | 2 +-
.../vm-disk/tests/job_name_budget_test.yaml | 127 ++++++++++++++++++
.../vm-instance/templates/vm-update-hook.yaml | 5 +-
.../tests/job_name_budget_test.yaml | 107 +++++++++++++++
16 files changed, 795 insertions(+), 10 deletions(-)
create mode 100644 packages/apps/bucket/tests/job_name_budget_test.yaml
create mode 100644 packages/apps/clickhouse/tests/cronjob_name_budget_test.yaml
create mode 100644 packages/apps/kafka/tests/job_name_budget_test.yaml
create mode 100644 packages/apps/mariadb/tests/cronjob_name_budget_test.yaml
create mode 100644 packages/apps/qdrant/tests/job_name_budget_test.yaml
create mode 100644 packages/apps/tenant/tests/job_name_budget_test.yaml
create mode 100644 packages/apps/vm-disk/tests/job_name_budget_test.yaml
create mode 100644 packages/apps/vm-instance/tests/job_name_budget_test.yaml
diff --git a/packages/apps/bucket/templates/hooks/cleanup-acme.yaml b/packages/apps/bucket/templates/hooks/cleanup-acme.yaml
index 94814a2c1..7680df52e 100644
--- a/packages/apps/bucket/templates/hooks/cleanup-acme.yaml
+++ b/packages/apps/bucket/templates/hooks/cleanup-acme.yaml
@@ -2,7 +2,7 @@
apiVersion: batch/v1
kind: Job
metadata:
- name: {{ .Release.Name }}-cleanup-acme
+ name: {{ printf "%s-cleanup-acme" .Release.Name | trunc 63 | trimSuffix "-" }}
labels:
app.kubernetes.io/instance: {{ .Release.Name }}
annotations:
diff --git a/packages/apps/bucket/tests/job_name_budget_test.yaml b/packages/apps/bucket/tests/job_name_budget_test.yaml
new file mode 100644
index 000000000..94a295b5a
--- /dev/null
+++ b/packages/apps/bucket/tests/job_name_budget_test.yaml
@@ -0,0 +1,80 @@
+suite: bucket cleanup Job name budget
+
+# The API server copies a Job's name into the job-name label of its pods, and a
+# label value takes 63 characters at most, while a release name may take 53.
+# The ServiceAccount, Role and RoleBinding have no such limit and keep the full
+# name, so a hook that ran before keeps the same objects.
+
+templates:
+ - templates/hooks/cleanup-acme.yaml
+
+set:
+ _namespace:
+ host: example.org
+
+tests:
+ - it: keeps -cleanup-acme for a short release name
+ release:
+ name: my-bucket
+ namespace: tenant-root
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: my-bucket-cleanup-acme
+
+ - it: keeps the name when it lands exactly on 63 characters
+ release:
+ name: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefg
+ namespace: tenant-root
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefg-cleanup-acme
+
+ - it: cuts the name to 63 characters for a 53-character release name
+ release:
+ name: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij
+ namespace: tenant-root
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-a
+ - matchRegex:
+ path: metadata.name
+ pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$
+ - equal:
+ path: spec.template.spec.serviceAccountName
+ value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-acme
+
+ - it: leaves the RBAC objects on the full name for a 53-character release name
+ release:
+ name: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij
+ namespace: tenant-root
+ asserts:
+ - equal:
+ path: metadata.name
+ value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-acme
+ documentSelector:
+ path: kind
+ value: ServiceAccount
+ - equal:
+ path: roleRef.name
+ value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-acme
+ documentSelector:
+ path: kind
+ value: RoleBinding
+ - equal:
+ path: subjects[0].name
+ value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-acme
+ documentSelector:
+ path: kind
+ value: RoleBinding
diff --git a/packages/apps/clickhouse/templates/backup-cronjob.yaml b/packages/apps/clickhouse/templates/backup-cronjob.yaml
index f792e8f9d..2a0bbd0b2 100644
--- a/packages/apps/clickhouse/templates/backup-cronjob.yaml
+++ b/packages/apps/clickhouse/templates/backup-cronjob.yaml
@@ -18,11 +18,19 @@
*/}}
{{- if and .Values.backup.enabled .Values.backup.schedule (not .Values.backup.useSystemBucket) }}
{{ $image := .Files.Get "images/backup.json" | fromJson }}
+{{- /* The CronJob controller starts each Job as -, so the API
+ refuses a CronJob name over 52 characters. Cutting the release name to
+ fit could give two releases one CronJob, so a name that does not fit
+ keeps a hash of the full release name instead. */}}
+{{- $cronJobName := printf "%s-backup" .Release.Name }}
+{{- if gt (len $cronJobName) 52 }}
+{{- $cronJobName = printf "%s-%s-backup" (.Release.Name | trunc 38 | trimSuffix "-") (.Release.Name | sha256sum | trunc 6) }}
+{{- end }}
apiVersion: batch/v1
kind: CronJob
metadata:
- name: {{ .Release.Name }}-backup
+ name: {{ $cronJobName }}
spec:
schedule: "{{ .Values.backup.schedule }}"
concurrencyPolicy: Forbid
diff --git a/packages/apps/clickhouse/tests/cronjob_name_budget_test.yaml b/packages/apps/clickhouse/tests/cronjob_name_budget_test.yaml
new file mode 100644
index 000000000..0ff3e2305
--- /dev/null
+++ b/packages/apps/clickhouse/tests/cronjob_name_budget_test.yaml
@@ -0,0 +1,94 @@
+suite: clickhouse legacy backup CronJob name budget
+
+# The API refuses a CronJob name over 52 characters, because the controller
+# names each Job it starts -, while a release name may take 53.
+# Cutting the release name alone would give two long names one CronJob, so a
+# name that does not fit is the release name cut to 38 characters, a hash of
+# the full release name and -backup.
+
+templates:
+ - templates/backup-cronjob.yaml
+ - templates/backup-script.yaml
+ - templates/backup-secret.yaml
+
+set:
+ _cluster: {}
+ backup:
+ enabled: true
+ schedule: "0 2 * * *"
+
+tests:
+ - it: keeps -backup for a short release name
+ release:
+ name: ch-test
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: CronJob
+ asserts:
+ - equal:
+ path: metadata.name
+ value: ch-test-backup
+
+ - it: keeps the name when it lands exactly on 52 characters
+ release:
+ name: clickhouse-abcdefghijklmnopqrstuvwxyz01234567
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: CronJob
+ asserts:
+ - equal:
+ path: metadata.name
+ value: clickhouse-abcdefghijklmnopqrstuvwxyz01234567-backup
+
+ - it: shortens the name one character past the budget
+ release:
+ name: clickhouse-abcdefghijklmnopqrstuvwxyz012345678
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: CronJob
+ asserts:
+ - equal:
+ path: metadata.name
+ value: clickhouse-abcdefghijklmnopqrstuvwxyz0-bec95e-backup
+
+ - it: fits a 53-character release name into 52 characters
+ release:
+ name: clickhouse-abcdefghijklmnopqrstuvwxyz0123456789abcdef
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: CronJob
+ asserts:
+ - equal:
+ path: metadata.name
+ value: clickhouse-abcdefghijklmnopqrstuvwxyz0-3e8cf9-backup
+ - matchRegex:
+ path: metadata.name
+ pattern: ^[a-z0-9]([-a-z0-9]{0,50}[a-z0-9])?$
+
+ - it: gives a release name differing only in its last character another CronJob
+ release:
+ name: clickhouse-abcdefghijklmnopqrstuvwxyz0123456789abcdex
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: CronJob
+ asserts:
+ - equal:
+ path: metadata.name
+ value: clickhouse-abcdefghijklmnopqrstuvwxyz0-bd3283-backup
+
+ - it: does not end the cut release name with a dash
+ release:
+ name: clickhouse-abcdefghijklmnopqrstuvwxyz-0123456789
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: CronJob
+ asserts:
+ - equal:
+ path: metadata.name
+ value: clickhouse-abcdefghijklmnopqrstuvwxyz-07ba9e-backup
diff --git a/packages/apps/kafka/templates/migration-hook.yaml b/packages/apps/kafka/templates/migration-hook.yaml
index 9877f699b..997b28ea1 100644
--- a/packages/apps/kafka/templates/migration-hook.yaml
+++ b/packages/apps/kafka/templates/migration-hook.yaml
@@ -79,7 +79,7 @@ subjects:
apiVersion: batch/v1
kind: Job
metadata:
- name: {{ .Release.Name }}-kafka-migration
+ name: {{ printf "%s-kafka-migration" .Release.Name | trunc 63 | trimSuffix "-" }}
annotations:
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-weight: "2"
diff --git a/packages/apps/kafka/tests/job_name_budget_test.yaml b/packages/apps/kafka/tests/job_name_budget_test.yaml
new file mode 100644
index 000000000..908ef75ed
--- /dev/null
+++ b/packages/apps/kafka/tests/job_name_budget_test.yaml
@@ -0,0 +1,66 @@
+suite: kafka migration Job name budget
+
+# The API server copies a Job's name into the job-name label of its pods, and a
+# label value takes 63 characters at most. The Kafka name cap leaves a release
+# name of up to 50 characters, and the migration hook runs on install, so an
+# overlong Job name would fail every install of the longest names. The
+# ServiceAccount, Role and RoleBinding have no such limit and keep the full
+# name, so the ones an earlier attempt left behind are replaced, not orphaned.
+
+templates:
+ - templates/migration-hook.yaml
+
+tests:
+ - it: keeps -kafka-migration for a short release name
+ release:
+ name: test-kafka
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: test-kafka-kafka-migration
+
+ - it: keeps the name when it lands exactly on 63 characters
+ release:
+ name: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcde
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcde-kafka-migration
+
+ - it: cuts the name to 63 characters for the longest release name the cap admits
+ release:
+ name: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh-kafka-migrat
+ - matchRegex:
+ path: metadata.name
+ pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$
+ - equal:
+ path: spec.template.spec.serviceAccountName
+ value: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh-kafka-migration
+
+ - it: leaves the ServiceAccount on the full name for the longest release name
+ release:
+ name: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: ServiceAccount
+ asserts:
+ - equal:
+ path: metadata.name
+ value: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh-kafka-migration
diff --git a/packages/apps/mariadb/templates/backup-cronjob.yaml b/packages/apps/mariadb/templates/backup-cronjob.yaml
index d39d5a8dd..1072a7c85 100644
--- a/packages/apps/mariadb/templates/backup-cronjob.yaml
+++ b/packages/apps/mariadb/templates/backup-cronjob.yaml
@@ -1,10 +1,18 @@
{{- if .Values.backup.enabled }}
{{ $image := .Files.Get "images/backup.json" | fromJson }}
+{{- /* The CronJob controller starts each Job as -, so the API
+ refuses a CronJob name over 52 characters. Cutting the release name to
+ fit could give two releases one CronJob, so a name that does not fit
+ keeps a hash of the full release name instead. */}}
+{{- $cronJobName := printf "%s-backup" .Release.Name }}
+{{- if gt (len $cronJobName) 52 }}
+{{- $cronJobName = printf "%s-%s-backup" (.Release.Name | trunc 38 | trimSuffix "-") (.Release.Name | sha256sum | trunc 6) }}
+{{- end }}
apiVersion: batch/v1
kind: CronJob
metadata:
- name: {{ .Release.Name }}-backup
+ name: {{ $cronJobName }}
spec:
schedule: "{{ .Values.backup.schedule }}"
concurrencyPolicy: Forbid
diff --git a/packages/apps/mariadb/tests/cronjob_name_budget_test.yaml b/packages/apps/mariadb/tests/cronjob_name_budget_test.yaml
new file mode 100644
index 000000000..18e5e40cf
--- /dev/null
+++ b/packages/apps/mariadb/tests/cronjob_name_budget_test.yaml
@@ -0,0 +1,92 @@
+suite: mariadb backup CronJob name budget
+
+# The API refuses a CronJob name over 52 characters, because the controller
+# names each Job it starts -, while a release name may take 53.
+# Cutting the release name alone would give two long names one CronJob, so a
+# name that does not fit is the release name cut to 38 characters, a hash of
+# the full release name and -backup.
+
+templates:
+ - templates/backup-cronjob.yaml
+ - templates/backup-script.yaml
+ - templates/backup-secret.yaml
+
+set:
+ backup:
+ enabled: true
+
+tests:
+ - it: keeps -backup for a short release name
+ release:
+ name: mariadb-test
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: CronJob
+ asserts:
+ - equal:
+ path: metadata.name
+ value: mariadb-test-backup
+
+ - it: keeps the name when it lands exactly on 52 characters
+ release:
+ name: mariadb-abcdefghijklmnopqrstuvwxyz0123456789a
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: CronJob
+ asserts:
+ - equal:
+ path: metadata.name
+ value: mariadb-abcdefghijklmnopqrstuvwxyz0123456789a-backup
+
+ - it: shortens the name one character past the budget
+ release:
+ name: mariadb-abcdefghijklmnopqrstuvwxyz0123456789ab
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: CronJob
+ asserts:
+ - equal:
+ path: metadata.name
+ value: mariadb-abcdefghijklmnopqrstuvwxyz0123-57a159-backup
+
+ - it: fits a 53-character release name into 52 characters
+ release:
+ name: mariadb-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: CronJob
+ asserts:
+ - equal:
+ path: metadata.name
+ value: mariadb-abcdefghijklmnopqrstuvwxyz0123-b5132c-backup
+ - matchRegex:
+ path: metadata.name
+ pattern: ^[a-z0-9]([-a-z0-9]{0,50}[a-z0-9])?$
+
+ - it: gives a release name differing only in its last character another CronJob
+ release:
+ name: mariadb-abcdefghijklmnopqrstuvwxyz0123456789abcdefghx
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: CronJob
+ asserts:
+ - equal:
+ path: metadata.name
+ value: mariadb-abcdefghijklmnopqrstuvwxyz0123-a49389-backup
+
+ - it: does not end the cut release name with a dash
+ release:
+ name: mariadb-abcdefghijklmnopqrstuvwxyz012-3456789abc
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: CronJob
+ asserts:
+ - equal:
+ path: metadata.name
+ value: mariadb-abcdefghijklmnopqrstuvwxyz012-2985e6-backup
diff --git a/packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml b/packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml
index bbc9547f9..3e2038e91 100644
--- a/packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml
+++ b/packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml
@@ -2,7 +2,7 @@
apiVersion: batch/v1
kind: Job
metadata:
- name: {{ .Release.Name }}-qdrant-cleanup
+ name: {{ printf "%s-qdrant-cleanup" .Release.Name | trunc 63 | trimSuffix "-" }}
labels:
app.kubernetes.io/instance: {{ .Release.Name }}
annotations:
diff --git a/packages/apps/qdrant/tests/job_name_budget_test.yaml b/packages/apps/qdrant/tests/job_name_budget_test.yaml
new file mode 100644
index 000000000..76416b387
--- /dev/null
+++ b/packages/apps/qdrant/tests/job_name_budget_test.yaml
@@ -0,0 +1,64 @@
+suite: qdrant cleanup Job name budget
+
+# The API server copies a Job's name into the job-name label of its pods, and a
+# label value takes 63 characters at most, while a release name may take 53.
+# The ServiceAccount, Role and RoleBinding have no such limit and keep the full
+# name.
+
+templates:
+ - templates/hooks/cleanup-pvc.yaml
+
+tests:
+ - it: keeps -qdrant-cleanup for a short release name
+ release:
+ name: qdrant-test
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: qdrant-test-qdrant-cleanup
+
+ - it: keeps the name when it lands exactly on 63 characters
+ release:
+ name: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcde
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcde-qdrant-cleanup
+
+ - it: cuts the name to 63 characters for a 53-character release name
+ release:
+ name: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-qdrant-cl
+ - matchRegex:
+ path: metadata.name
+ pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$
+ - equal:
+ path: spec.template.spec.serviceAccountName
+ value: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-qdrant-cleanup
+
+ - it: leaves the ServiceAccount on the full name for a 53-character release name
+ release:
+ name: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: ServiceAccount
+ asserts:
+ - equal:
+ path: metadata.name
+ value: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-qdrant-cleanup
diff --git a/packages/apps/tenant/templates/cleanup-job.yaml b/packages/apps/tenant/templates/cleanup-job.yaml
index ffa07d0fe..7285a4d79 100644
--- a/packages/apps/tenant/templates/cleanup-job.yaml
+++ b/packages/apps/tenant/templates/cleanup-job.yaml
@@ -40,11 +40,22 @@ subjects:
- kind: ServiceAccount
name: {{ include "tenant.name" . }}-cleanup
namespace: cozy-system
+{{- /* The Job name becomes the job-name label of its pod, which takes 63
+ characters at most, and a nested tenant namespace can use all 63 by
+ itself. Cutting the name to fit would let a parent and one of its
+ children land on the same Job in cozy-system, where the child's
+ before-hook-creation would delete the parent's running cleanup, so a
+ name that does not fit keeps a hash of the whole namespace instead. */}}
+{{- $tenantName := include "tenant.name" . }}
+{{- $jobName := printf "%s-cleanup" $tenantName }}
+{{- if gt (len $jobName) 63 }}
+{{- $jobName = printf "%s-%s-cleanup" ($tenantName | trunc 48 | trimSuffix "-") ($tenantName | sha256sum | trunc 6) }}
+{{- end }}
---
apiVersion: batch/v1
kind: Job
metadata:
- name: {{ include "tenant.name" . }}-cleanup
+ name: {{ $jobName }}
namespace: cozy-system
annotations:
helm.sh/hook: pre-delete
@@ -98,7 +109,7 @@ spec:
ttlSecondsAfterFinished: 300
template:
metadata:
- name: {{ include "tenant.name" . }}-cleanup
+ name: {{ $jobName }}
labels:
policy.cozystack.io/allow-to-apiserver: "true"
spec:
diff --git a/packages/apps/tenant/tests/job_name_budget_test.yaml b/packages/apps/tenant/tests/job_name_budget_test.yaml
new file mode 100644
index 000000000..5cf4b93f6
--- /dev/null
+++ b/packages/apps/tenant/tests/job_name_budget_test.yaml
@@ -0,0 +1,127 @@
+suite: tenant cleanup Job name budget
+
+# The API server copies a Job's name into the job-name label of its pods, and a
+# label value takes 63 characters at most, while a nested tenant namespace may
+# take all 63 by itself. Every tenant's cleanup Job runs in cozy-system, where
+# cutting the namespace alone could give a parent and its child one Job, and the
+# child's before-hook-creation would delete the parent's while it runs. So a
+# name that does not fit is the namespace cut to 48 characters, a hash of the
+# whole namespace and -cleanup. The ServiceAccount, Role and RoleBinding have no
+# such limit and keep the full name.
+
+templates:
+ - templates/cleanup-job.yaml
+
+tests:
+ - it: keeps -cleanup for a tenant under tenant-root
+ release:
+ name: tenant-test
+ namespace: tenant-root
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: tenant-test-cleanup
+
+ - it: keeps -cleanup for a short nested tenant
+ release:
+ name: tenant-child
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: tenant-test-child-cleanup
+
+ - it: keeps the name when it lands exactly on 63 characters
+ release:
+ name: tenant-0123456789abcdefghijk
+ namespace: tenant-abcdefghijklmnopqrstuvwxyz
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijk-cleanup
+
+ - it: shortens the name for a 63-character namespace
+ release:
+ name: tenant-0123456789abcdefghijklmnopqrs
+ namespace: tenant-abcdefghijklmnopqrstuvwxyz
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcd-7b3e3e-cleanup
+ - matchRegex:
+ path: metadata.name
+ pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$
+ - equal:
+ path: spec.template.metadata.name
+ value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcd-7b3e3e-cleanup
+ - equal:
+ path: spec.template.spec.serviceAccountName
+ value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmnopqrs-cleanup
+
+ # Cut to 63 characters, the parent below and its child named clea would both
+ # be tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmn-clea.
+ - it: gives a parent its own Job name
+ release:
+ name: tenant-0123456789abcdefghijklmn
+ namespace: tenant-abcdefghijklmnopqrstuvwxyz
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcd-571020-cleanup
+
+ - it: gives the child that would collide with that parent another Job name
+ release:
+ name: tenant-clea
+ namespace: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmn
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcd-ec5e9b-cleanup
+
+ - it: does not end the cut namespace with a dash
+ release:
+ name: tenant-abcdefghijklmno
+ namespace: tenant-abcdefghijklmnopqrstuvwxyz0123456789abcd
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: tenant-abcdefghijklmnopqrstuvwxyz0123456789abcd-477f49-cleanup
+
+ - it: leaves the RBAC objects on the full name for a 63-character namespace
+ release:
+ name: tenant-0123456789abcdefghijklmnopqrs
+ namespace: tenant-abcdefghijklmnopqrstuvwxyz
+ asserts:
+ - equal:
+ path: metadata.name
+ value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmnopqrs-cleanup
+ documentSelector:
+ path: kind
+ value: ServiceAccount
+ - equal:
+ path: subjects[0].name
+ value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmnopqrs-cleanup
+ documentSelector:
+ path: kind
+ value: RoleBinding
diff --git a/packages/apps/vm-disk/templates/pvc-resize-hook.yaml b/packages/apps/vm-disk/templates/pvc-resize-hook.yaml
index 5e327c6f5..224da33ba 100644
--- a/packages/apps/vm-disk/templates/pvc-resize-hook.yaml
+++ b/packages/apps/vm-disk/templates/pvc-resize-hook.yaml
@@ -15,7 +15,7 @@
apiVersion: batch/v1
kind: Job
metadata:
- name: "{{ .Release.Name }}-volume-resize-hook"
+ name: {{ printf "%s-volume-resize-hook" .Release.Name | trunc 63 | trimSuffix "-" | quote }}
annotations:
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-weight: "0"
diff --git a/packages/apps/vm-disk/tests/job_name_budget_test.yaml b/packages/apps/vm-disk/tests/job_name_budget_test.yaml
new file mode 100644
index 000000000..5922dcc7b
--- /dev/null
+++ b/packages/apps/vm-disk/tests/job_name_budget_test.yaml
@@ -0,0 +1,127 @@
+suite: vm-disk resize Job name budget
+
+# The API server copies a Job's name into the job-name label of its pods, and a
+# label value takes 63 characters at most, while a release name may take 53.
+# The hook renders only when an existing PVC grows, so each release below gets
+# a smaller PVC to look up. The ServiceAccount, Role and RoleBinding have no
+# such limit and keep the full name, so the ones an earlier resize left behind
+# are replaced, not orphaned.
+
+templates:
+ - templates/pvc-resize-hook.yaml
+
+set:
+ storage: 2Gi
+
+kubernetesProvider:
+ scheme:
+ "v1/PersistentVolumeClaim":
+ gvr:
+ group: ""
+ version: "v1"
+ resource: "persistentvolumeclaims"
+ namespaced: true
+ objects:
+ - kind: PersistentVolumeClaim
+ apiVersion: v1
+ metadata:
+ name: test-vm-disk
+ namespace: tenant-test
+ spec:
+ resources:
+ requests:
+ storage: 1Gi
+ - kind: PersistentVolumeClaim
+ apiVersion: v1
+ metadata:
+ name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789
+ namespace: tenant-test
+ spec:
+ resources:
+ requests:
+ storage: 1Gi
+ - kind: PersistentVolumeClaim
+ apiVersion: v1
+ metadata:
+ name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcd
+ namespace: tenant-test
+ spec:
+ resources:
+ requests:
+ storage: 1Gi
+ - kind: PersistentVolumeClaim
+ apiVersion: v1
+ metadata:
+ name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi
+ namespace: tenant-test
+ spec:
+ resources:
+ requests:
+ storage: 1Gi
+
+tests:
+ - it: keeps -volume-resize-hook for a short release name
+ release:
+ name: test-vm-disk
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: test-vm-disk-volume-resize-hook
+
+ - it: keeps the name when it lands exactly on 63 characters
+ release:
+ name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789-volume-resize-hook
+
+ - it: does not end the cut name with a dash
+ release:
+ name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcd
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcd-volume-resize
+
+ - it: cuts the name to 63 characters for a 53-character release name
+ release:
+ name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi-volume-re
+ - matchRegex:
+ path: metadata.name
+ pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$
+ - equal:
+ path: spec.template.spec.serviceAccountName
+ value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi-volume-resize-hook
+
+ - it: leaves the ServiceAccount on the full name for a 53-character release name
+ release:
+ name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: ServiceAccount
+ asserts:
+ - equal:
+ path: metadata.name
+ value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi-volume-resize-hook
diff --git a/packages/apps/vm-instance/templates/vm-update-hook.yaml b/packages/apps/vm-instance/templates/vm-update-hook.yaml
index 72bc617e2..ab77f21b1 100644
--- a/packages/apps/vm-instance/templates/vm-update-hook.yaml
+++ b/packages/apps/vm-instance/templates/vm-update-hook.yaml
@@ -1,5 +1,6 @@
{{- $vmName := include "virtual-machine.fullname" . -}}
{{- $namespace := .Release.Namespace -}}
+{{- $jobName := printf "%s-update-hook" .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- $existingVM := lookup "kubevirt.io/v1" "VirtualMachine" $namespace $vmName -}}
{{- $existingService := lookup "v1" "Service" $namespace $vmName -}}
@@ -48,7 +49,7 @@
apiVersion: batch/v1
kind: Job
metadata:
- name: "{{ $.Release.Name }}-update-hook"
+ name: "{{ $jobName }}"
annotations:
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-weight: "0"
@@ -57,7 +58,7 @@ spec:
template:
metadata:
labels:
- app: "{{ $.Release.Name }}-update-hook"
+ app: "{{ $jobName }}"
policy.cozystack.io/allow-to-apiserver: "true"
spec:
serviceAccountName: {{ $.Release.Name }}-update-hook
diff --git a/packages/apps/vm-instance/tests/job_name_budget_test.yaml b/packages/apps/vm-instance/tests/job_name_budget_test.yaml
new file mode 100644
index 000000000..fb4190930
--- /dev/null
+++ b/packages/apps/vm-instance/tests/job_name_budget_test.yaml
@@ -0,0 +1,107 @@
+suite: vm-instance update Job name budget
+
+# The API server copies a Job's name into the job-name label of its pods, and a
+# label value takes 63 characters at most, while a release name may take 53.
+# The pod's own app label carries the same name and is held to the same limit.
+# The hook renders only when an existing VM or Service needs reconciling, so
+# each release below gets a Service of the wrong type to look up. The
+# ServiceAccount, Role and RoleBinding have no such limit and keep the full
+# name, so the ones an earlier update left behind are replaced, not orphaned.
+
+templates:
+ - templates/vm-update-hook.yaml
+
+set:
+ external: false
+
+kubernetesProvider:
+ scheme:
+ "v1/Service":
+ gvr:
+ group: ""
+ version: "v1"
+ resource: "services"
+ namespaced: true
+ objects:
+ - kind: Service
+ apiVersion: v1
+ metadata:
+ name: vm-instance-test
+ namespace: tenant-test
+ spec:
+ type: LoadBalancer
+ - kind: Service
+ apiVersion: v1
+ metadata:
+ name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abc
+ namespace: tenant-test
+ spec:
+ type: LoadBalancer
+ - kind: Service
+ apiVersion: v1
+ metadata:
+ name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde
+ namespace: tenant-test
+ spec:
+ type: LoadBalancer
+
+tests:
+ - it: keeps -update-hook for a short release name
+ release:
+ name: vm-instance-test
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: vm-instance-test-update-hook
+ - equal:
+ path: spec.template.metadata.labels.app
+ value: vm-instance-test-update-hook
+
+ - it: keeps the name when it lands exactly on 63 characters
+ release:
+ name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abc
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abc-update-hook
+
+ - it: cuts the Job name and the app label to 63 characters for a 53-character release name
+ release:
+ name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: Job
+ asserts:
+ - equal:
+ path: metadata.name
+ value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde-update-ho
+ - matchRegex:
+ path: metadata.name
+ pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$
+ - equal:
+ path: spec.template.metadata.labels.app
+ value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde-update-ho
+ - equal:
+ path: spec.template.spec.serviceAccountName
+ value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde-update-hook
+
+ - it: leaves the ServiceAccount on the full name for a 53-character release name
+ release:
+ name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde
+ namespace: tenant-test
+ documentSelector:
+ path: kind
+ value: ServiceAccount
+ asserts:
+ - equal:
+ path: metadata.name
+ value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde-update-hook
From b74029cb390c79c1c7cf3a1a697e0f6b70d353d1 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 18 Oct 2026 23:11:02 +0300
Subject: [PATCH 003/216] fix(keycloak): apply keycloak-operator CRD updates on
upgrade
keycloak-operator ships its CRDs in the chart's crds/ directory and its
PackageSource sets no upgradeCRDs policy, so helm-controller never
replaces them. A cluster installed before v1.2.0 therefore runs the
v1.32.0 operator against the v1.25.0 CRDs, whose ClusterKeycloakRealm
schema has no spec.sessions, spec.login or spec.organizationsEnabled.
The apiserver drops those fields from the realm keycloak-configure
renders, so the operator never applies the session limits the platform
sets. On a v1.6.4 cluster in that state Keycloak also rejected the
dashboard client with HTTP 400, and replacing the CRDs from the chart
by hand, with an operator restart, cleared it.
The v1.32.0 CRDs are not purely additive. They drop the realm,
targetRealm and keycloakOwner fields that v1.25.0 marked deprecated and
require realmRef and keycloakRef instead. Every Keycloak object the
platform renders already uses the refs, and each CRD keeps its single
version, so replacing them loses nothing the platform relies on.
helm-controller applies the policy only when it upgrades the release,
and it runs an upgrade only when the release artifact or its values
change, which a PackageSource edit does not do. The package test added
here also changes the keycloak-operator artifact, so the replace runs
on the upgrade that ships this change even where nothing else in the
package changes. The test itself pins the realm fields
keycloak-configure sets against the CRD the chart ships.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../platform/sources/keycloak-operator.yaml | 7 ++++++
.../sources_keycloak_operator_crds_test.yaml | 22 +++++++++++++++++++
packages/system/keycloak-operator/Makefile | 3 +++
.../keycloak-operator/tests/crds_test.yaml | 19 ++++++++++++++++
4 files changed, 51 insertions(+)
create mode 100644 packages/core/platform/tests/sources_keycloak_operator_crds_test.yaml
create mode 100644 packages/system/keycloak-operator/tests/crds_test.yaml
diff --git a/packages/core/platform/sources/keycloak-operator.yaml b/packages/core/platform/sources/keycloak-operator.yaml
index ce3a5f7e0..5b116e5e6 100644
--- a/packages/core/platform/sources/keycloak-operator.yaml
+++ b/packages/core/platform/sources/keycloak-operator.yaml
@@ -20,3 +20,10 @@ spec:
install:
namespace: cozy-keycloak
releaseName: keycloak-operator
+ # helm-controller leaves existing CRDs alone on upgrade, so a cluster
+ # installed before v1.2.0 keeps the v1.25.0 schema and the apiserver
+ # prunes the realm session settings keycloak-configure sets. v1.32.0
+ # drops the realm, targetRealm and keycloakOwner fields deprecated in
+ # favour of realmRef and keycloakRef. The platform renders only the
+ # refs, so replace is safe.
+ upgradeCRDs: CreateReplace
diff --git a/packages/core/platform/tests/sources_keycloak_operator_crds_test.yaml b/packages/core/platform/tests/sources_keycloak_operator_crds_test.yaml
new file mode 100644
index 000000000..5cccfc757
--- /dev/null
+++ b/packages/core/platform/tests/sources_keycloak_operator_crds_test.yaml
@@ -0,0 +1,22 @@
+suite: keycloak-operator applies CRD updates on upgrade
+# keycloak-configure sets spec.sessions and spec.organizationsEnabled, and
+# spec.login when configured, on the cozy ClusterKeycloakRealm. The CRDs of
+# keycloak-operator v1.25.0 declare none of them. Helm never upgrades CRDs
+# shipped in crds/, so without CreateReplace a cluster installed before v1.2.0
+# keeps those CRDs, the apiserver drops the fields without an error, and the
+# operator never applies them to the realm. Pin the policy so an upgrade cannot
+# silently regress.
+templates:
+ - templates/sources.yaml
+release:
+ name: cozystack
+ namespace: cozy-system
+tests:
+ - it: keycloak-operator PackageSource sets upgradeCRDs to CreateReplace
+ documentSelector:
+ path: metadata.name
+ value: cozystack.keycloak-operator
+ asserts:
+ - equal:
+ path: spec.variants[0].components[0].install.upgradeCRDs
+ value: CreateReplace
diff --git a/packages/system/keycloak-operator/Makefile b/packages/system/keycloak-operator/Makefile
index e2f7f45d5..303f37eeb 100644
--- a/packages/system/keycloak-operator/Makefile
+++ b/packages/system/keycloak-operator/Makefile
@@ -4,6 +4,9 @@ export NAMESPACE=cozy-keycloak
include ../../../hack/common-envs.mk
include ../../../hack/package.mk
+test:
+ helm unittest .
+
image:
docker buildx build images/keycloak-operator \
$(call image-tags,keycloak-operator,v$(COZYSTACK_VERSION)) \
diff --git a/packages/system/keycloak-operator/tests/crds_test.yaml b/packages/system/keycloak-operator/tests/crds_test.yaml
new file mode 100644
index 000000000..1be3fe611
--- /dev/null
+++ b/packages/system/keycloak-operator/tests/crds_test.yaml
@@ -0,0 +1,19 @@
+suite: keycloak-operator CRDs declare the realm settings the platform sets
+# keycloak-configure renders the cozy ClusterKeycloakRealm with spec.sessions
+# and spec.organizationsEnabled, and spec.login when configured. The apiserver
+# drops any field the CRD does not declare without an error, so with a chart
+# that lacks them the realm never gets those settings.
+includeCrds: true
+templates:
+ - charts/keycloak-operator/crds/v1.edp.epam.com_clusterkeycloakrealms.yaml
+tests:
+ - it: declares the session, organization and login settings
+ asserts:
+ - exists:
+ path: spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.sessions.properties.ssoSessionSettings.properties.idleTimeout
+ - exists:
+ path: spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.sessions.properties.ssoSessionSettings.properties.maxLifespan
+ - exists:
+ path: spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.organizationsEnabled
+ - exists:
+ path: spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.login
From 8e96a3e9b73b127a116131327b73925e7c2ffa9a Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 18 Oct 2026 22:59:52 +0300
Subject: [PATCH 004/216] fix(foundationdb): apply foundationdb-operator CRD
updates on upgrade
The foundationdb-operator PackageSource sets no upgradeCRDs policy, so
a cluster installed before v1.6.0 runs the v2.30.0 operator on the
v2.13.0 CRDs it first installed. The apiserver prunes every field added
since, status.backupDetails.restorable on FoundationDBBackup among
them, so a backup never shows as restorable however long it has run.
The v2.30.0 CRDs only add fields. They keep the single v1beta2 version
and drop or retype nothing, and the one new constraint, a pattern on
perpetual_storage_wiggle_locality, sits on a field the platform does
not set, so CreateReplace is safe.
Nothing in the foundationdb-operator package changes here, so a cluster
picks the new CRDs up at the next upgrade of that release, which comes
with the next change to the package.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../sources/foundationdb-operator.yaml | 6 ++++++
...urces_foundationdb_operator_crds_test.yaml | 20 +++++++++++++++++++
2 files changed, 26 insertions(+)
create mode 100644 packages/core/platform/tests/sources_foundationdb_operator_crds_test.yaml
diff --git a/packages/core/platform/sources/foundationdb-operator.yaml b/packages/core/platform/sources/foundationdb-operator.yaml
index 0af881f60..28328c3d1 100644
--- a/packages/core/platform/sources/foundationdb-operator.yaml
+++ b/packages/core/platform/sources/foundationdb-operator.yaml
@@ -20,3 +20,9 @@ spec:
install:
namespace: cozy-foundationdb-operator
releaseName: foundationdb-operator
+ # helm-controller leaves existing CRDs alone on upgrade, so a cluster
+ # installed before v1.6.0 keeps the v2.13.0 schema under the v2.30.0
+ # operator, and the apiserver prunes the fields added since,
+ # status.backupDetails.restorable among them. The schema diff adds
+ # fields and drops none, so replace is safe.
+ upgradeCRDs: CreateReplace
diff --git a/packages/core/platform/tests/sources_foundationdb_operator_crds_test.yaml b/packages/core/platform/tests/sources_foundationdb_operator_crds_test.yaml
new file mode 100644
index 000000000..a0976ae57
--- /dev/null
+++ b/packages/core/platform/tests/sources_foundationdb_operator_crds_test.yaml
@@ -0,0 +1,20 @@
+suite: foundationdb-operator applies CRD updates on upgrade
+# The fdb-operator v2.30.0 CRDs add fields the v2.13.0 ones lack, among them
+# status.backupDetails.restorable on FoundationDBBackup. Helm never upgrades
+# CRDs shipped in crds/, so without CreateReplace a cluster installed before
+# v1.6.0 keeps the v2.13.0 CRDs and the apiserver prunes those fields on every
+# write. Pin the policy so an upgrade cannot silently regress.
+templates:
+ - templates/sources.yaml
+release:
+ name: cozystack
+ namespace: cozy-system
+tests:
+ - it: foundationdb-operator PackageSource sets upgradeCRDs to CreateReplace
+ documentSelector:
+ path: metadata.name
+ value: cozystack.foundationdb-operator
+ asserts:
+ - equal:
+ path: spec.variants[0].components[0].install.upgradeCRDs
+ value: CreateReplace
From 89028b455ef2fbf4772672c2fe6342150efc3200 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 18 Oct 2026 23:16:45 +0300
Subject: [PATCH 005/216] fix(monitoring): route InfoInhibitor to the blackhole
receiver
InfoInhibitor exists to be the source side of an Alertmanager inhibit
rule and, as its own description says, to go to a null receiver. The
platform Alertmanager config has no inhibit rule, and the rule has
carried severity major since it was imported alongside Alerta, so
Alerta shows it one level below critical. Alerta's Telegram and Slack
plugins forward it by default, and a notify_email label inherited from
the info alert gets it emailed too.
It fires whenever a rule labelled severity="info" is active with no
warning or critical alert in the same namespace and cluster. The
platform's own rules no longer use info, but rules it does not write
still do. CDI generates CDIStorageProfilesIncomplete, KubeVirt
generates KubeVirtVMGuestMemoryAvailableLow, and operators add their
own. Each of them turns into a major InfoInhibitor in Alerta.
none is the severity upstream kube-prometheus gives this rule, and the
platform config already sends severity="none" to the blackhole ahead
of every route that delivers. The severity contract refuses none
because such an alert is never delivered, so InfoInhibitor is named as
the one alert for which that is the point. Tests hold both ends, so
neither the label nor the route order can drift back to delivering it.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
hack/alert-severity-contract.bats | 89 +++++++++++++++++++
.../alerts/general.rules.yaml | 2 +-
.../tests/alertmanager_blackhole_test.yaml | 45 ++++++++++
3 files changed, 135 insertions(+), 1 deletion(-)
create mode 100644 packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
diff --git a/hack/alert-severity-contract.bats b/hack/alert-severity-contract.bats
index e1315eb96..44f197000 100644
--- a/hack/alert-severity-contract.bats
+++ b/hack/alert-severity-contract.bats
@@ -9,6 +9,10 @@ load test_helper
# scope: they are re-vendored by `make update`, so a bad severity there is
# fixed upstream or in patches/, not by editing the label.
ALERT_SEVERITY_ALLOWED=" security critical major minor warning indeterminate informational normal ok cleared debug trace unknown "
+# The alerts that exist only to be dropped, the one place `none` is right.
+# InfoInhibitor is the source side of an Alertmanager inhibit rule, which the
+# platform config does not have, so delivering it at any severity is noise.
+ALERT_SEVERITY_BLACKHOLED=" InfoInhibitor "
helm_actions_stripped() {
awk '
@@ -104,6 +108,14 @@ alert_severity_contract() {
break
fi
done
+ if [ "$severity" = "none" ]; then
+ for blackholed in $ALERT_SEVERITY_BLACKHOLED; do
+ if [ "$blackholed" = "$alert" ]; then
+ accepted=1
+ break
+ fi
+ done
+ fi
if [ "$accepted" -eq 0 ]; then
echo "$file: $alert -> '$severity'" >&2
bad=$((bad + 1))
@@ -128,6 +140,83 @@ alert_severity_contract() {
alert_severity_contract
}
+@test "InfoInhibitor is labelled for the blackhole route" {
+ rows=$(mktemp) || return 1
+ if ! alert_severity_rows > "$rows"; then
+ rm -f "$rows"
+ return 1
+ fi
+ tab=$(printf '\t')
+ found=$(grep -c "${tab}InfoInhibitor${tab}" "$rows" || true)
+ delivered=$(grep "${tab}InfoInhibitor${tab}" "$rows" | grep -vc "${tab}none\$" || true)
+ rm -f "$rows"
+ [ "$found" -gt 0 ]
+ [ "$delivered" -eq 0 ]
+}
+
+@test "severity none passes only on an alert that exists to be dropped" {
+ fixture=$(mktemp -d)
+ mkdir -p "$fixture/packages/system/example/alerts"
+ printf '%s\n' \
+ 'apiVersion: operator.victoriametrics.com/v1beta1' \
+ 'kind: VMRule' \
+ 'spec:' \
+ ' groups:' \
+ ' - name: general' \
+ ' rules:' \
+ ' - alert: InfoInhibitor' \
+ ' labels:' \
+ ' severity: none' \
+ > "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
+ git -C "$fixture" init -q
+ git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
+
+ inhibitor_succeeded=0
+ ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || inhibitor_succeeded=$?
+
+ printf '%s\n' \
+ 'apiVersion: operator.victoriametrics.com/v1beta1' \
+ 'kind: VMRule' \
+ 'spec:' \
+ ' groups:' \
+ ' - name: silent' \
+ ' rules:' \
+ ' - alert: SilentAlert' \
+ ' labels:' \
+ ' severity: none' \
+ > "$fixture/packages/system/example/alerts/01-silent.yaml"
+ git -C "$fixture" add packages/system/example/alerts/01-silent.yaml
+
+ silent_succeeded=0
+ ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || silent_succeeded=$?
+ rm -rf "$fixture"
+ [ "$inhibitor_succeeded" -eq 0 ]
+ [ "$silent_succeeded" -ne 0 ]
+}
+
+@test "an alert that exists to be dropped still fails on a refused severity" {
+ fixture=$(mktemp -d)
+ mkdir -p "$fixture/packages/system/example/alerts"
+ printf '%s\n' \
+ 'apiVersion: operator.victoriametrics.com/v1beta1' \
+ 'kind: VMRule' \
+ 'spec:' \
+ ' groups:' \
+ ' - name: general' \
+ ' rules:' \
+ ' - alert: InfoInhibitor' \
+ ' labels:' \
+ ' severity: info' \
+ > "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
+ git -C "$fixture" init -q
+ git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
+
+ contract_succeeded=0
+ ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$?
+ rm -rf "$fixture"
+ [ "$contract_succeeded" -ne 0 ]
+}
+
@test "a parser failure cannot leave the contract green" {
fixture=$(mktemp -d)
mkdir -p "$fixture/packages/system/example/alerts"
diff --git a/packages/system/monitoring-agents/alerts/general.rules.yaml b/packages/system/monitoring-agents/alerts/general.rules.yaml
index 924870c40..45dd6100f 100644
--- a/packages/system/monitoring-agents/alerts/general.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/general.rules.yaml
@@ -52,6 +52,6 @@ spec:
!= "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} ==
1
labels:
- severity: major
+ severity: none
exported_instance: global
service: general.rules
diff --git a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
new file mode 100644
index 000000000..e987bd405
--- /dev/null
+++ b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
@@ -0,0 +1,45 @@
+suite: Alertmanager blackhole route
+templates:
+ - templates/alerta/alerta.yaml
+
+release:
+ name: monitoring
+ namespace: tenant-root
+
+set:
+ _cluster:
+ root-host: example.org
+ _namespace:
+ host: tenant-root.example.org
+ ingress: tenant-root
+
+# Rules labelled severity="none", InfoInhibitor among them, exist to be
+# dropped, and Alerta answers that severity with a 500. Only the heartbeat
+# route may come ahead of the blackhole, so no delivering route claims them.
+tests:
+ - it: drops severity none ahead of every delivering route
+ asserts:
+ - documentSelector:
+ path: metadata.labels.app
+ value: alertmanager
+ matchRegex:
+ path: stringData["alertmanager.yaml"]
+ pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'
+
+ - it: keeps severity none out of email even when its alert name is listed
+ set:
+ alerta:
+ alerts:
+ email:
+ to: "oncall@example.org"
+ from: "alerts@example.org"
+ smarthost: "smtp.example.org:465"
+ alertnames:
+ - InfoInhibitor
+ asserts:
+ - documentSelector:
+ path: metadata.labels.app
+ value: alertmanager
+ matchRegex:
+ path: stringData["alertmanager.yaml"]
+ pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'
From 8d82348e85fb9588ad213d763ece81f7b88e84a8 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Mon, 19 Oct 2026 16:04:58 +0300
Subject: [PATCH 006/216] fix(vm-disk): refuse a disk smaller than the golden
image it clones
CDI checks a clone against its source before it creates the claim. When
the golden image's PVC is in Block mode and requests more than the disk,
it records CloneValidationFailed and leaves the DataVolume pending for
good. The DataVolume spec is immutable, so raising storage on the VMDisk
afterwards changes nothing, and the resize hook finds no claim to patch.
Every image in vm-default-images is 20Gi on replicated, where CDI picks
Block mode for the LINSTOR driver, while the chart defaults storage to
5Gi, so a disk created from an image at the default size never comes up.
The chart now refuses such a disk at render time and names both sizes,
so the user sets storage to the golden's size before any DataVolume
exists. It compares what validateClone compares in CDI v1.66.1, the
golden PVC's request against the disk's request raised to the target
StorageProfile's minimumSupportedPvcSize. A Filesystem golden is not
compared, because CDI sizes that clone from the image inside it or with
a filesystem overhead, and neither is a golden whose PVC does not exist
yet. An existing disk is exempt, as for the other golden checks.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
packages/apps/vm-disk/templates/dv.yaml | 20 ++
.../vm-disk/tests/dv_golden_size_test.yaml | 252 ++++++++++++++++++
2 files changed, 272 insertions(+)
create mode 100644 packages/apps/vm-disk/tests/dv_golden_size_test.yaml
diff --git a/packages/apps/vm-disk/templates/dv.yaml b/packages/apps/vm-disk/templates/dv.yaml
index 09831fc79..9bec73f5f 100644
--- a/packages/apps/vm-disk/templates/dv.yaml
+++ b/packages/apps/vm-disk/templates/dv.yaml
@@ -146,6 +146,26 @@
{{- fail (printf "source.image %q: golden image %q lives in StorageClass %q, and a clone into StorageClass %q of the same provisioner gets no host-assisted fallback from CDI. LINSTOR creates the clone next to the source data, and StorageClass %q does not allow remote volume access from every node, so the clone may never bind. Set storageClass to %q, or ask the platform administrator to set cloneStrategyOverride to copy in the kubevirt-cdi package." .Values.source.image.name $desiredSource.pvc.name $goldenSC $targetSC $targetSC $goldenSC) }}
{{- end }}
{{- end }}
+ {{- /* CDI v1.66.1 refuses to clone a Block source PVC into a smaller
+ request before it creates the claim (validateClone in
+ pkg/controller/datavolume/pvc-clone-controller.go), and the
+ DataVolume then waits forever. It raises the request to the target
+ StorageProfile's minimumSupportedPvcSize before comparing. A
+ Filesystem source is not compared, as CDI sizes that clone from
+ the image inside it or with a filesystem overhead the chart does
+ not reproduce. */}}
+ {{- $goldenSize := dig "spec" "resources" "requests" "storage" "" ($goldenPVC | default dict) | toString }}
+ {{- if and .Values.storage $goldenSize (eq (dig "spec" "volumeMode" "" ($goldenPVC | default dict) | toString) "Block") }}
+ {{- $goldenBytes := include "cozy-lib.resources.toFloat" $goldenSize | float64 }}
+ {{- $diskBytes := include "cozy-lib.resources.toFloat" (.Values.storage | toString) | float64 }}
+ {{- if and (lt $diskBytes $goldenBytes) $targetSC }}
+ {{- $minimum := dig "metadata" "annotations" "cdi.kubevirt.io/minimumSupportedPvcSize" "" (lookup "cdi.kubevirt.io/v1beta1" "StorageProfile" "" $targetSC | default dict) | toString }}
+ {{- $diskBytes = maxf $diskBytes (include "cozy-lib.resources.toFloat" $minimum | float64) }}
+ {{- end }}
+ {{- if lt $diskBytes $goldenBytes }}
+ {{- fail (printf "storage %s is smaller than golden image %q (%s), so CDI refuses to clone it and the disk would stay pending forever. Set storage to %s or more." (.Values.storage | toString) $desiredSource.pvc.name $goldenSize $goldenSize) }}
+ {{- end }}
+ {{- end }}
{{- end }}
{{- else if hasKey .Values.source "http" }}
{{- $desiredSource = dict "http" (dict "url" (required "A valid .Values.source.http.url entry required!" .Values.source.http.url)) }}
diff --git a/packages/apps/vm-disk/tests/dv_golden_size_test.yaml b/packages/apps/vm-disk/tests/dv_golden_size_test.yaml
new file mode 100644
index 000000000..6a68874c7
--- /dev/null
+++ b/packages/apps/vm-disk/tests/dv_golden_size_test.yaml
@@ -0,0 +1,252 @@
+suite: vm-disk refuses a disk smaller than the golden image it clones
+release:
+ name: test-vm-disk
+ namespace: tenant-test
+templates:
+ - templates/dv.yaml
+tests:
+ - it: refuses a new disk at the chart default size from a larger Block golden
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: &scheme
+ "cdi.kubevirt.io/v1beta1/DataVolume":
+ gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: datavolumes}
+ namespaced: true
+ "cdi.kubevirt.io/v1beta1/StorageProfile":
+ gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: storageprofiles}
+ namespaced: false
+ "v1/Namespace":
+ gvr: {group: "", version: v1, resource: namespaces}
+ namespaced: false
+ "v1/PersistentVolumeClaim":
+ gvr: {group: "", version: v1, resource: persistentvolumeclaims}
+ namespaced: true
+ "storage.k8s.io/v1/StorageClass":
+ gvr: {group: storage.k8s.io, version: v1, resource: storageclasses}
+ namespaced: false
+ objects:
+ - &namespace
+ kind: Namespace
+ apiVersion: v1
+ metadata: {name: tenant-test}
+ - &goldenPVC
+ kind: PersistentVolumeClaim
+ apiVersion: v1
+ metadata: {name: vm-default-images-ubuntu, namespace: cozy-public}
+ spec:
+ storageClassName: replicated
+ volumeMode: Block
+ resources: {requests: {storage: 20Gi}}
+ asserts:
+ - failedTemplate:
+ errorPattern: 'storage 5Gi is smaller than golden image "vm-default-images-ubuntu" \(20Gi\), so CDI refuses to clone it and the disk would stay pending forever\. Set storage to 20Gi or more\.'
+
+ - it: clones into a disk the size of the golden
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 20Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - equal:
+ path: spec.storage.resources.requests.storage
+ value: 20Gi
+
+ - it: clones into a disk larger than the golden
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 25Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - equal:
+ path: spec.storage.resources.requests.storage
+ value: 25Gi
+
+ - it: compares bytes, so 21G is smaller than a 20Gi golden
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 21G
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - failedTemplate:
+ errorPattern: 'storage 21G is smaller than golden image "vm-default-images-ubuntu" \(20Gi\)'
+
+ - it: compares bytes, so 20480Mi is the size of a 20Gi golden
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 20480Mi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - equal:
+ path: spec.storage.resources.requests.storage
+ value: 20480Mi
+
+ - it: clones when the target StorageProfile's minimum PVC size covers the golden
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ - kind: StorageProfile
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata:
+ name: replicated
+ annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 20Gi}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: refuses when the target StorageProfile's minimum PVC size is still below the golden
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ - kind: StorageProfile
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata:
+ name: replicated
+ annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 10Gi}
+ asserts:
+ - failedTemplate:
+ errorPattern: 'storage 5Gi is smaller than golden image "vm-default-images-ubuntu" \(20Gi\)'
+
+ - it: reads the minimum PVC size of the default class for an empty storageClass
+ set:
+ source: {image: {name: ubuntu}}
+ storageClass: ""
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ - kind: StorageClass
+ apiVersion: storage.k8s.io/v1
+ metadata:
+ name: replicated
+ annotations: {storageclass.kubernetes.io/is-default-class: "true"}
+ provisioner: linstor.csi.linbit.com
+ - kind: StorageProfile
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata:
+ name: replicated
+ annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 20Gi}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: does not compare sizes for a Filesystem golden
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - kind: PersistentVolumeClaim
+ apiVersion: v1
+ metadata: {name: vm-default-images-ubuntu, namespace: cozy-public}
+ spec:
+ storageClassName: replicated
+ volumeMode: Filesystem
+ resources: {requests: {storage: 20Gi}}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: renders without a cluster to look the golden up in
+ set:
+ source: {image: {name: ubuntu}}
+ asserts:
+ - equal:
+ path: spec.storage.resources.requests.storage
+ value: 5Gi
+
+ - it: clones a golden whose PVC does not exist yet
+ set:
+ source: {image: {name: ubuntu}}
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - kind: DataVolume
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata: {name: vm-default-images-ubuntu, namespace: cozy-public}
+ status: {phase: ImportScheduled}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: leaves a disk without a size to CDI, which takes the golden's
+ set:
+ source: {image: {name: ubuntu}}
+ storage: null
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: leaves an existing disk alone
+ set:
+ source: {image: {name: ubuntu}}
+ storage: 25Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ - kind: DataVolume
+ apiVersion: cdi.kubevirt.io/v1beta1
+ metadata:
+ name: test-vm-disk
+ namespace: tenant-test
+ annotations:
+ vm-disk.cozystack.io/source: '{"image":{"name":"ubuntu"}}'
+ vm-disk.cozystack.io/storage-class: replicated
+ spec:
+ source: {pvc: {name: vm-default-images-ubuntu, namespace: cozy-public}}
+ storage: {resources: {requests: {storage: 10Gi}}, storageClassName: replicated}
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: does not compare sizes for an http source
+ set:
+ source: {http: {url: "https://example.com/disk.img"}}
+ storage: 1Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - notFailedTemplate: {}
+
+ - it: does not compare sizes for an upload source
+ set:
+ source: {upload: {}}
+ storage: 1Gi
+ kubernetesProvider:
+ scheme: *scheme
+ objects:
+ - *namespace
+ - *goldenPVC
+ asserts:
+ - notFailedTemplate: {}
From 85026299958edba4ccdcfaafef5b17fad7f055fc Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Mon, 19 Oct 2026 19:33:25 +0300
Subject: [PATCH 007/216] fix(piraeus-operator): keep the LINSTOR error-report
alerts quiet when the controller pod moves
linstor_error_reports_count comes from the LINSTOR controller, so its
series carry the controller pod's instance and controller_node labels.
When the pod moves to another node, or an external label of the scrape
changes, the existing report counts come back as new series.
VictoriaMetrics' increase() counts the first samples of a series from
zero when they are small, and report counts are small, so
linstorSatelliteErrorRate fires for every node with an old report and
keeps firing for 15 minutes. The up guard matches on instance and lets
the new series through.
The controller also leaves a satellite out of the metric while it is
not connected, so a satellite that comes back after more than five
minutes has its old count counted again in the same way.
Keyed on hostname, a moved controller or a changed label continues the
same series, and holding the last value for an hour bridges a
disconnect or a slow controller restart. The up guard matches on()
because the result no longer has an instance label. The controller
rule takes the same form. Its hostname is the pod name and a new pod
starts with no reports, but a label change hit it the same way.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
hack/linstor-error-report-alerts.bats | 32 +++
.../linstor-error-report-alerts/unittest.yaml | 198 ++++++++++++++++++
.../alerts/piraeus-datastore.yaml | 6 +-
3 files changed, 233 insertions(+), 3 deletions(-)
create mode 100644 hack/linstor-error-report-alerts.bats
create mode 100644 hack/testdata/linstor-error-report-alerts/unittest.yaml
diff --git a/hack/linstor-error-report-alerts.bats b/hack/linstor-error-report-alerts.bats
new file mode 100644
index 000000000..4c6124aad
--- /dev/null
+++ b/hack/linstor-error-report-alerts.bats
@@ -0,0 +1,32 @@
+#!/usr/bin/env bats
+# Replays the LINSTOR error-report alerts from
+# packages/system/piraeus-operator/alerts/piraeus-datastore.yaml through
+# vmalert-tool, pinned to the VictoriaMetrics release that the
+# victoria-metrics-operator deploys by default. Its increase() counts the first
+# samples of a series as new when they are small, and the report counts are
+# small, so the cases pin that a series which starts over after a controller
+# move, a label change or a disconnect does not report old error reports again.
+# Only pulling the image touches the network.
+
+load test_helper
+
+REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
+VMALERT_TOOL_IMAGE="victoriametrics/vmalert-tool:v1.136.0@sha256:417346d7b6fdb66d8e3984887c2ad114535ec7d983eeef53381b56c776f333f1"
+
+@test "LINSTOR error-report alerts pass their vmalert-tool cases" {
+ docker info >/dev/null 2>&1 || {
+ echo "docker is required to run the cases in $VMALERT_TOOL_IMAGE" >&2
+ return 1
+ }
+ work=$(mktemp -d)
+ yq '.spec' "$REPO_ROOT/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml" > "$work/rules.yaml"
+ cp "$REPO_ROOT/hack/testdata/linstor-error-report-alerts/unittest.yaml" "$work/unittest.yaml"
+ rc=0
+ docker run --rm --network none -v "$work:/w:ro" "$VMALERT_TOOL_IMAGE" \
+ unittest --files /w/unittest.yaml > "$work/out" 2>&1 || rc=$?
+ cat "$work/out"
+ passed=$(grep -cx 'SUCCESS' "$work/out" || true)
+ rm -rf "$work"
+ [ "$rc" -eq 0 ]
+ [ "$passed" -eq 1 ]
+}
diff --git a/hack/testdata/linstor-error-report-alerts/unittest.yaml b/hack/testdata/linstor-error-report-alerts/unittest.yaml
new file mode 100644
index 000000000..41e332169
--- /dev/null
+++ b/hack/testdata/linstor-error-report-alerts/unittest.yaml
@@ -0,0 +1,198 @@
+rule_files:
+ - rules.yaml
+
+evaluation_interval: 15s
+
+tests:
+ - name: controller pod moves to another node
+ interval: 30s
+ input_series:
+ - series: 'up{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a"}'
+ values: '1x38 0 stale'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a", hostname="node-a", module="SATELLITE"}'
+ values: '1x38 stale'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a", hostname="node-b", module="SATELLITE"}'
+ values: '2x38 stale'
+ - series: 'up{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b"}'
+ values: '_x41 1x79'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b", hostname="node-a", module="SATELLITE"}'
+ values: '_x41 1x58 2x20'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b", hostname="node-b", module="SATELLITE"}'
+ values: '_x41 2x79'
+ alert_rule_test:
+ - eval_time: 21m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ - eval_time: 28m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ - eval_time: 35m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ - eval_time: 45m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ - eval_time: 51m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ exp_alerts:
+ - exp_labels:
+ hostname: node-a
+ severity: warning
+ exp_annotations:
+ description: |
+ LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes.
+ Use "linstor error-reports list --nodes node-a" to inspect the reports.
+
+ - name: an external label of the scrape changes
+ interval: 30s
+ input_series:
+ - series: 'up{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack"}'
+ values: '1x39 stale'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack", hostname="node-a", module="SATELLITE"}'
+ values: '1x39 stale'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack", hostname="linstor-controller-abc", module="CONTROLLER"}'
+ values: '1x39 stale'
+ - series: 'up{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a"}'
+ values: '_x41 1x79'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a", hostname="node-a", module="SATELLITE"}'
+ values: '_x41 1x79'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a", hostname="linstor-controller-abc", module="CONTROLLER"}'
+ values: '_x41 1x79'
+ alert_rule_test:
+ - eval_time: 21m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ - eval_time: 30m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ - eval_time: 21m
+ groupname: linstor.rules
+ alertname: linstorControllerErrorRate
+ - eval_time: 30m
+ groupname: linstor.rules
+ alertname: linstorControllerErrorRate
+
+ - name: a satellite drops out of the metric while it is disconnected
+ interval: 30s
+ input_series:
+ - series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}'
+ values: '1x220'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}'
+ values: '1x38 stale _x80 1x100'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-b", module="SATELLITE"}'
+ values: '2x220'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-c", module="SATELLITE"}'
+ values: '1x38 stale _x80 2x100'
+ alert_rule_test:
+ - eval_time: 61m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ exp_alerts:
+ - exp_labels:
+ hostname: node-c
+ severity: warning
+ exp_annotations:
+ description: |
+ LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes.
+ Use "linstor error-reports list --nodes node-c" to inspect the reports.
+ - eval_time: 70m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ exp_alerts:
+ - exp_labels:
+ hostname: node-c
+ severity: warning
+ exp_annotations:
+ description: |
+ LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes.
+ Use "linstor error-reports list --nodes node-c" to inspect the reports.
+ - eval_time: 76m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+
+ - name: a new report fires for the 15 minute window
+ interval: 30s
+ input_series:
+ - series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}'
+ values: '1x120'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}'
+ values: '1x39 2x80'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-b", module="SATELLITE"}'
+ values: '2x120'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-c", module="SATELLITE"}'
+ values: '0x39 1x80'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="linstor-controller-abc", module="CONTROLLER"}'
+ values: '0x39 1x80'
+ alert_rule_test:
+ - eval_time: 21m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ exp_alerts:
+ - exp_labels:
+ hostname: node-a
+ severity: warning
+ exp_annotations:
+ description: |
+ LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes.
+ Use "linstor error-reports list --nodes node-a" to inspect the reports.
+ - exp_labels:
+ hostname: node-c
+ severity: warning
+ exp_annotations:
+ description: |
+ LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes.
+ Use "linstor error-reports list --nodes node-c" to inspect the reports.
+ - eval_time: 34m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ exp_alerts:
+ - exp_labels:
+ hostname: node-a
+ severity: warning
+ exp_annotations:
+ description: |
+ LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes.
+ Use "linstor error-reports list --nodes node-a" to inspect the reports.
+ - exp_labels:
+ hostname: node-c
+ severity: warning
+ exp_annotations:
+ description: |
+ LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes.
+ Use "linstor error-reports list --nodes node-c" to inspect the reports.
+ - eval_time: 36m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ - eval_time: 21m
+ groupname: linstor.rules
+ alertname: linstorControllerErrorRate
+ exp_alerts:
+ - exp_labels:
+ hostname: linstor-controller-abc
+ severity: warning
+ exp_annotations:
+ description: |
+ LINSTOR Controller reports 1 errors in the last 15 minutes.
+ Use "linstor error-reports list --since 15minutes" to see them.
+ - eval_time: 36m
+ groupname: linstor.rules
+ alertname: linstorControllerErrorRate
+
+ - name: a failed controller scrape still holds the alert back
+ interval: 30s
+ input_series:
+ - series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}'
+ values: '1x19 0x140 1x60'
+ - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}'
+ values: '1x19 stale _x140 1x60'
+ alert_rule_test:
+ - eval_time: 85m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ - eval_time: 93m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
+ - eval_time: 100m
+ groupname: linstor.rules
+ alertname: linstorSatelliteErrorRate
diff --git a/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml b/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml
index ab679d663..8be364a72 100644
--- a/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml
+++ b/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml
@@ -29,8 +29,8 @@ spec:
LINSTOR Satellite "{{ $labels.hostname }}" reports {{ $value }} errors in the last 15 minutes.
Use "linstor error-reports list --nodes {{ $labels.hostname }}" to inspect the reports.
expr: |
- increase(linstor_error_reports_count{module="SATELLITE"}[15m]) > 0
- and on(instance, job)
+ increase((max by (hostname) (last_over_time(linstor_error_reports_count{module="SATELLITE"}[1h])))[15m:1m]) > 0
+ and on()
min_over_time(up{job="linstor-controller"}[15m]) == 1
labels:
severity: warning
@@ -39,7 +39,7 @@ spec:
description: |
LINSTOR Controller reports {{ $value }} errors in the last 15 minutes.
Use "linstor error-reports list --since 15minutes" to see them.
- expr: increase(linstor_error_reports_count{module="CONTROLLER"}[15m]) > 0
+ expr: increase((max by (hostname) (last_over_time(linstor_error_reports_count{module="CONTROLLER"}[1h])))[15m:1m]) > 0
labels:
severity: warning
- alert: linstorSatelliteNotOnline
From 78cadf53f9f88a2b5569024031c399a5aba6e075 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Mon, 19 Oct 2026 19:55:08 +0300
Subject: [PATCH 008/216] fix(tcp-balancer): render the kubernetes and talos
Service ports only with kubernetesAndTalos
The Service rendered ports 6443 and 50000 under httpAndHttps, while the
Deployment declares the kubernetes and talos container ports and HAProxy
binds those frontends only under kubernetesAndTalos. So with the default
values every Service carried two ports whose targetPort names no
container declares, with no endpoints behind them, and with
external: true the LoadBalancer Service carried them as well. A release
with kubernetesAndTalos and without httpAndHttps got no Service port for
those frontends at all.
kubernetesAndTalos is not documented in values.yaml, but the apps API
keeps the top level of spec open and hands it to the chart as values,
so the key reaches the chart and the Deployment and the ConfigMap
already follow it. Gating the Service on the same key keeps the three
templates in step, while dropping the block would leave HAProxy
listening on ports that no Service exposes.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../apps/tcp-balancer/templates/service.yaml | 2 +-
.../apps/tcp-balancer/tests/service_test.yaml | 97 +++++++++++++++++++
2 files changed, 98 insertions(+), 1 deletion(-)
create mode 100644 packages/apps/tcp-balancer/tests/service_test.yaml
diff --git a/packages/apps/tcp-balancer/templates/service.yaml b/packages/apps/tcp-balancer/templates/service.yaml
index 8384cc85b..689db7e16 100644
--- a/packages/apps/tcp-balancer/templates/service.yaml
+++ b/packages/apps/tcp-balancer/templates/service.yaml
@@ -27,7 +27,7 @@ spec:
port: 443
targetPort: https
{{- end }}
- {{- with .Values.httpAndHttps }}
+ {{- with .Values.kubernetesAndTalos }}
- name: kubernetes
protocol: TCP
port: 6443
diff --git a/packages/apps/tcp-balancer/tests/service_test.yaml b/packages/apps/tcp-balancer/tests/service_test.yaml
new file mode 100644
index 000000000..57191115c
--- /dev/null
+++ b/packages/apps/tcp-balancer/tests/service_test.yaml
@@ -0,0 +1,97 @@
+suite: haproxy service
+
+templates:
+ - templates/service.yaml
+ - templates/deployment.yaml
+ - templates/configmap.yaml
+
+tests:
+ - it: publishes only http and https by default, both on container ports the Deployment declares
+ asserts:
+ - equal:
+ path: spec.ports
+ value:
+ - name: http
+ protocol: TCP
+ port: 80
+ targetPort: http
+ - name: https
+ protocol: TCP
+ port: 443
+ targetPort: https
+ template: templates/service.yaml
+ - equal:
+ path: spec.template.spec.containers[0].ports
+ value:
+ - containerPort: 8080
+ name: http
+ - containerPort: 8443
+ name: https
+ template: templates/deployment.yaml
+
+ - it: publishes kubernetes and talos for kubernetesAndTalos alone, both on container ports the Deployment declares
+ set:
+ httpAndHttps: null
+ kubernetesAndTalos:
+ endpoints:
+ - 10.1.1.1
+ asserts:
+ - equal:
+ path: spec.ports
+ value:
+ - name: kubernetes
+ protocol: TCP
+ port: 6443
+ targetPort: kubernetes
+ - name: talos
+ protocol: TCP
+ port: 50000
+ targetPort: talos
+ template: templates/service.yaml
+ - equal:
+ path: spec.template.spec.containers[0].ports
+ value:
+ - containerPort: 6443
+ name: kubernetes
+ - containerPort: 50000
+ name: talos
+ template: templates/deployment.yaml
+
+ - it: publishes all four ports when both are set, each on a container port the Deployment declares
+ set:
+ kubernetesAndTalos:
+ endpoints:
+ - 10.1.1.1
+ asserts:
+ - equal:
+ path: spec.ports
+ value:
+ - name: http
+ protocol: TCP
+ port: 80
+ targetPort: http
+ - name: https
+ protocol: TCP
+ port: 443
+ targetPort: https
+ - name: kubernetes
+ protocol: TCP
+ port: 6443
+ targetPort: kubernetes
+ - name: talos
+ protocol: TCP
+ port: 50000
+ targetPort: talos
+ template: templates/service.yaml
+ - equal:
+ path: spec.template.spec.containers[0].ports
+ value:
+ - containerPort: 8080
+ name: http
+ - containerPort: 8443
+ name: https
+ - containerPort: 6443
+ name: kubernetes
+ - containerPort: 50000
+ name: talos
+ template: templates/deployment.yaml
From f1f30b3071254e3640292b4eb483b04f5d40d77a Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Tue, 20 Oct 2026 12:05:51 +0300
Subject: [PATCH 009/216] fix(etcd): require an even spread of members over
nodes by default
The default spread of the etcd members was ScheduleAnyway, which only
adds to the score of a node, and other scores can outweigh it. A new
etcd on a three-node cluster got its first two members on one node.
On a node-local StorageClass a member cannot leave its node, so losing
that node takes two of the three members and the quorum.
DoNotSchedule with maxSkew 1 still lets an etcd with more members than
nodes start, because the skew is counted between the nodes that exist.
One node takes all three members and two nodes take two and one.
nodeTaintsPolicy is Ignore, so a cordoned node still counts and a
member waits for it instead of landing next to another member.
The operator copies the constraints into an EtcdMember when it creates
it and builds the Pod from that copy, so members that exist keep the
old setting and only members created later get the new one.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../extra/etcd/templates/etcd-cluster.yaml | 3 +-
.../etcd/tests/topology-spread_test.yaml | 46 ++++++++++++++-----
2 files changed, 37 insertions(+), 12 deletions(-)
diff --git a/packages/extra/etcd/templates/etcd-cluster.yaml b/packages/extra/etcd/templates/etcd-cluster.yaml
index 2824ce371..67dd34bd9 100644
--- a/packages/extra/etcd/templates/etcd-cluster.yaml
+++ b/packages/extra/etcd/templates/etcd-cluster.yaml
@@ -69,7 +69,8 @@ spec:
topologySpreadConstraints:
- maxSkew: 1
topologyKey: "kubernetes.io/hostname"
- whenUnsatisfiable: ScheduleAnyway
+ whenUnsatisfiable: DoNotSchedule
+ nodeTaintsPolicy: Ignore
labelSelector:
matchLabels:
app.kubernetes.io/instance: etcd
diff --git a/packages/extra/etcd/tests/topology-spread_test.yaml b/packages/extra/etcd/tests/topology-spread_test.yaml
index 4b96fc310..11bb92243 100644
--- a/packages/extra/etcd/tests/topology-spread_test.yaml
+++ b/packages/extra/etcd/tests/topology-spread_test.yaml
@@ -8,7 +8,7 @@ templates:
- templates/etcd-cluster.yaml
tests:
- - it: uses a soft per-host spread by default
+ - it: uses a strict per-host spread by default
set:
_cluster: {}
documentSelector:
@@ -16,17 +16,41 @@ tests:
value: EtcdCluster
asserts:
- equal:
- path: spec.topologySpreadConstraints[0].maxSkew
- value: 1
+ path: spec.topologySpreadConstraints[0]
+ value:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: DoNotSchedule
+ nodeTaintsPolicy: Ignore
+ labelSelector:
+ matchLabels:
+ app.kubernetes.io/instance: etcd
+ - lengthEqual:
+ path: spec.topologySpreadConstraints
+ count: 1
+
+ - it: renders a soft cluster-wide spread policy as written
+ set:
+ _cluster:
+ scheduling:
+ globalAppTopologySpreadConstraints: |
+ topologySpreadConstraints:
+ - maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+ documentSelector:
+ path: kind
+ value: EtcdCluster
+ asserts:
- equal:
- path: spec.topologySpreadConstraints[0].topologyKey
- value: kubernetes.io/hostname
- - equal:
- path: spec.topologySpreadConstraints[0].whenUnsatisfiable
- value: ScheduleAnyway
- - equal:
- path: spec.topologySpreadConstraints[0].labelSelector.matchLabels["app.kubernetes.io/instance"]
- value: etcd
+ path: spec.topologySpreadConstraints[0]
+ value:
+ maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+ labelSelector:
+ matchLabels:
+ app.kubernetes.io/instance: etcd
- lengthEqual:
path: spec.topologySpreadConstraints
count: 1
From 5632596fe84fa358191c62f5f316df62f32c865c Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Tue, 20 Oct 2026 13:38:55 +0300
Subject: [PATCH 010/216] feat(monitoring): alert when pods of one workload
share a node
Two etcd members, two Redis Sentinels or both replicas of a storage
tier can end up on one node through a soft spread rule, a drain, or a
placement made before a spread rule existed. Nothing reported it, and
the scheduler does not move running pods, so it stays like that until
a pod is recreated. One node failure then costs the quorum or every
copy at once.
etcd-operator owns each member pod by its own EtcdMember, so members
are grouped by the EtcdCluster name their names start with. Owner and
node are read from kube_pod_info because kube_pod_owner and
kube_pod_status_phase get the node of the kube-state-metrics pod as a
target label. Nodes with a NoSchedule or NoExecute taint do not count,
which covers cordoned, not ready and unreachable nodes. The alert waits
an hour, longer than a replacement pod runs next to its terminating
predecessor or a node takes to drain and come back.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../workload-placement-alerts/unittest.yaml | 572 ++++++++++++++++++
hack/workload-placement-alerts.bats | 26 +
.../alerts/workload-placement.yaml | 81 +++
3 files changed, 679 insertions(+)
create mode 100644 hack/testdata/workload-placement-alerts/unittest.yaml
create mode 100644 hack/workload-placement-alerts.bats
create mode 100644 packages/system/monitoring-agents/alerts/workload-placement.yaml
diff --git a/hack/testdata/workload-placement-alerts/unittest.yaml b/hack/testdata/workload-placement-alerts/unittest.yaml
new file mode 100644
index 000000000..2565a8241
--- /dev/null
+++ b/hack/testdata/workload-placement-alerts/unittest.yaml
@@ -0,0 +1,572 @@
+rule_files:
+ - rules.yaml
+
+evaluation_interval: 15s
+
+tests:
+ - name: two of three StatefulSet pods on one node
+ interval: 1m
+ input_series:
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", node="node-b", created_by_kind="StatefulSet", created_by_name="db"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", phase="Running", node="node-c"}'
+ values: '1x150'
+ alert_rule_test:
+ - eval_time: 59m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ - eval_time: 61m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ exp_alerts:
+ - exp_labels:
+ cluster: cozystack
+ namespace: tenant-a
+ owner_kind: StatefulSet
+ owner_name: db
+ node: node-a
+ severity: warning
+ exported_instance: tenant-a/db
+ service: workload-placement
+ exp_annotations:
+ summary: Pods of one workload share a node.
+ description: >-
+ 2 running pods of StatefulSet tenant-a/db are on node node-a in cluster cozystack,
+ although the cluster has a schedulable node for each of its pods. Losing this node
+ stops these pods together, which can cost a quorum-based workload its quorum. The
+ scheduler does not move running pods, so they stay together until they are recreated.
+
+ - name: StatefulSet pods spread over three nodes
+ interval: 1m
+ input_series:
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", node="node-b", created_by_kind="StatefulSet", created_by_name="db"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", node="node-c", created_by_kind="StatefulSet", created_by_name="db"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", phase="Running", node="node-c"}'
+ values: '1x150'
+ alert_rule_test:
+ - eval_time: 61m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ - eval_time: 120m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+
+ - name: five ReplicaSet pods on three nodes
+ interval: 1m
+ input_series:
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-bbbbb", node="node-a", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ddddd", node="node-b", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-eeeee", node="node-c", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-aaaaa", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-bbbbb", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ccccc", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ddddd", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-eeeee", phase="Running", node="node-c"}'
+ values: '1x150'
+ alert_rule_test:
+ - eval_time: 61m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ - eval_time: 120m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+
+ - name: DaemonSet surge pod next to the old one
+ interval: 1m
+ input_series:
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-x7k2p", node="node-a", created_by_kind="DaemonSet", created_by_name="node-agent"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-q9z4m", node="node-a", created_by_kind="DaemonSet", created_by_name="node-agent"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-b5n8w", node="node-b", created_by_kind="DaemonSet", created_by_name="node-agent"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-x7k2p", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-q9z4m", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-b5n8w", phase="Running", node="node-c"}'
+ values: '1x150'
+ alert_rule_test:
+ - eval_time: 61m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ - eval_time: 120m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+
+ - name: two pods of one Job on one node
+ interval: 1m
+ input_series:
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-h2k4m", node="node-a", created_by_kind="Job", created_by_name="backup-29345678"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-p8r6t", node="node-a", created_by_kind="Job", created_by_name="backup-29345678"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-h2k4m", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-p8r6t", phase="Running", node="node-c"}'
+ values: '1x150'
+ alert_rule_test:
+ - eval_time: 61m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+
+ - name: etcd members of one EtcdCluster on one node
+ interval: 1m
+ input_series:
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-x7k2p", node="node-a", created_by_kind="EtcdMember", created_by_name="etcd-x7k2p"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-q9z4m", node="node-a", created_by_kind="EtcdMember", created_by_name="etcd-q9z4m"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-b5n8w", node="node-b", created_by_kind="EtcdMember", created_by_name="etcd-b5n8w"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-0", node="node-b", created_by_kind="EtcdMember", created_by_name="etcd-0"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-1", node="node-c", created_by_kind="EtcdMember", created_by_name="etcd-1"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-2", node="node-c", created_by_kind="EtcdMember", created_by_name="etcd-2"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-x7k2p", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-q9z4m", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-b5n8w", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-0", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-1", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-2", phase="Running", node="node-c"}'
+ values: '1x150'
+ alert_rule_test:
+ - eval_time: 61m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ exp_alerts:
+ - exp_labels:
+ cluster: cozystack
+ namespace: tenant-a
+ owner_kind: EtcdCluster
+ owner_name: etcd
+ node: node-a
+ severity: warning
+ exported_instance: tenant-a/etcd
+ service: workload-placement
+ exp_annotations:
+ summary: Pods of one workload share a node.
+ description: >-
+ 2 running pods of EtcdCluster tenant-a/etcd are on node node-a in cluster cozystack,
+ although the cluster has a schedulable node for each of its pods. Losing this node
+ stops these pods together, which can cost a quorum-based workload its quorum. The
+ scheduler does not move running pods, so they stay together until they are recreated.
+ - exp_labels:
+ cluster: cozystack
+ namespace: tenant-b
+ owner_kind: EtcdCluster
+ owner_name: etcd
+ node: node-c
+ severity: warning
+ exported_instance: tenant-b/etcd
+ service: workload-placement
+ exp_annotations:
+ summary: Pods of one workload share a node.
+ description: >-
+ 2 running pods of EtcdCluster tenant-b/etcd are on node node-c in cluster cozystack,
+ although the cluster has a schedulable node for each of its pods. Losing this node
+ stops these pods together, which can cost a quorum-based workload its quorum. The
+ scheduler does not move running pods, so they stay together until they are recreated.
+
+ - name: replacement pod next to its terminating predecessor for twenty minutes
+ interval: 1m
+ input_series:
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}'
+ values: '1x30 stale'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}'
+ values: '_x10 1x140'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", phase="Running", node="node-c"}'
+ values: '1x30 stale'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", phase="Running", node="node-c"}'
+ values: '_x10 1x140'
+ alert_rule_test:
+ - eval_time: 29m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ - eval_time: 75m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ - eval_time: 120m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+
+ - name: the same overlap held past the hour
+ interval: 1m
+ input_series:
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}'
+ values: '1x100 stale'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}'
+ values: '_x10 1x140'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", phase="Running", node="node-c"}'
+ values: '1x100 stale'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", phase="Running", node="node-c"}'
+ values: '_x10 1x140'
+ alert_rule_test:
+ - eval_time: 69m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ - eval_time: 71m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ exp_alerts:
+ - exp_labels:
+ cluster: cozystack
+ namespace: tenant-a
+ owner_kind: ReplicaSet
+ owner_name: web-6b8f4
+ node: node-b
+ severity: warning
+ exported_instance: tenant-a/web-6b8f4
+ service: workload-placement
+ exp_annotations:
+ summary: Pods of one workload share a node.
+ description: >-
+ 2 running pods of ReplicaSet tenant-a/web-6b8f4 are on node node-b in cluster cozystack,
+ although the cluster has a schedulable node for each of its pods. Losing this node
+ stops these pods together, which can cost a quorum-based workload its quorum. The
+ scheduler does not move running pods, so they stay together until they are recreated.
+ - eval_time: 105m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+
+ - name: a cordoned node does not count until it is uncordoned
+ interval: 1m
+ input_series:
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '0x150'
+ - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '0x150'
+ - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x30 0x120'
+ - series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unschedulable", value="", effect="NoSchedule"}'
+ values: '1x30 stale'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", node="node-b", created_by_kind="StatefulSet", created_by_name="kv"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", phase="Running", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", phase="Running", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", phase="Running", node="node-a"}'
+ values: '1x150'
+ alert_rule_test:
+ - eval_time: 61m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ - eval_time: 90m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ - eval_time: 92m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ exp_alerts:
+ - exp_labels:
+ cluster: cozystack
+ namespace: tenant-a
+ owner_kind: StatefulSet
+ owner_name: kv
+ node: node-a
+ severity: warning
+ exported_instance: tenant-a/kv
+ service: workload-placement
+ exp_annotations:
+ summary: Pods of one workload share a node.
+ description: >-
+ 2 running pods of StatefulSet tenant-a/kv are on node node-a in cluster cozystack,
+ although the cluster has a schedulable node for each of its pods. Losing this node
+ stops these pods together, which can cost a quorum-based workload its quorum. The
+ scheduler does not move running pods, so they stay together until they are recreated.
+
+ - name: a node that is down does not count
+ interval: 1m
+ input_series:
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '0x150'
+ - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '0x150'
+ - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '0x150'
+ - series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unreachable", value="", effect="NoSchedule"}'
+ values: '1x150'
+ - series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unreachable", value="", effect="NoExecute"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", node="node-b", created_by_kind="StatefulSet", created_by_name="kv"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", phase="Running", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", phase="Running", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", phase="Running", node="node-a"}'
+ values: '1x150'
+ alert_rule_test:
+ - eval_time: 61m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ - eval_time: 120m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+
+ - name: pods that are not running do not count
+ interval: 1m
+ input_series:
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", node="node-a", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-aaaaa", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", phase="Running", node="node-c"}'
+ values: '0x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", phase="Failed", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-ccccc", phase="Running", node="node-c"}'
+ values: '1x150'
+ alert_rule_test:
+ - eval_time: 61m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+
+ - name: one pod each in two clusters that share workload and node names
+ interval: 1m
+ input_series:
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="kube-system", pod="coredns-5d78c9869d-x7k2p", node="node-a", created_by_kind="ReplicaSet", created_by_name="coredns-5d78c9869d"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="kubernetes-example", namespace="kube-system", pod="coredns-5d78c9869d-q9z4m", node="node-a", created_by_kind="ReplicaSet", created_by_name="coredns-5d78c9869d"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="kube-system", pod="coredns-5d78c9869d-x7k2p", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="kubernetes-example", namespace="kube-system", pod="coredns-5d78c9869d-q9z4m", phase="Running", node="node-b"}'
+ values: '1x150'
+ alert_rule_test:
+ - eval_time: 61m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+
+ - name: ReplicaSet, CNPG Cluster and StrimziPodSet pods on one node
+ interval: 1m
+ input_series:
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
+ values: '1x150'
+ - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-h2k4m", node="node-a", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-p8r6t", node="node-a", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-w3x5z", node="node-b", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-1", node="node-a", created_by_kind="Cluster", created_by_name="postgres-db"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-2", node="node-b", created_by_kind="Cluster", created_by_name="postgres-db"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-3", node="node-b", created_by_kind="Cluster", created_by_name="postgres-db"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-0", node="node-c", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-1", node="node-c", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}'
+ values: '1x150'
+ - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-2", node="node-a", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-h2k4m", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-p8r6t", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-w3x5z", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-1", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-2", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-3", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-0", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-1", phase="Running", node="node-c"}'
+ values: '1x150'
+ - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-2", phase="Running", node="node-c"}'
+ values: '1x150'
+ alert_rule_test:
+ - eval_time: 61m
+ groupname: workload-placement
+ alertname: PodReplicasColocated
+ exp_alerts:
+ - exp_labels:
+ cluster: cozystack
+ namespace: tenant-a
+ owner_kind: ReplicaSet
+ owner_name: rfs-cache-6f7d8b9c5d
+ node: node-a
+ severity: warning
+ exported_instance: tenant-a/rfs-cache-6f7d8b9c5d
+ service: workload-placement
+ exp_annotations:
+ summary: Pods of one workload share a node.
+ description: >-
+ 2 running pods of ReplicaSet tenant-a/rfs-cache-6f7d8b9c5d are on node node-a in cluster
+ cozystack, although the cluster has a schedulable node for each of its pods. Losing this
+ node stops these pods together, which can cost a quorum-based workload its quorum. The
+ scheduler does not move running pods, so they stay together until they are recreated.
+ - exp_labels:
+ cluster: cozystack
+ namespace: tenant-a
+ owner_kind: Cluster
+ owner_name: postgres-db
+ node: node-b
+ severity: warning
+ exported_instance: tenant-a/postgres-db
+ service: workload-placement
+ exp_annotations:
+ summary: Pods of one workload share a node.
+ description: >-
+ 2 running pods of Cluster tenant-a/postgres-db are on node node-b in cluster cozystack,
+ although the cluster has a schedulable node for each of its pods. Losing this node
+ stops these pods together, which can cost a quorum-based workload its quorum. The
+ scheduler does not move running pods, so they stay together until they are recreated.
+ - exp_labels:
+ cluster: cozystack
+ namespace: tenant-a
+ owner_kind: StrimziPodSet
+ owner_name: kafka-kafka
+ node: node-c
+ severity: warning
+ exported_instance: tenant-a/kafka-kafka
+ service: workload-placement
+ exp_annotations:
+ summary: Pods of one workload share a node.
+ description: >-
+ 2 running pods of StrimziPodSet tenant-a/kafka-kafka are on node node-c in cluster
+ cozystack, although the cluster has a schedulable node for each of its pods. Losing
+ this node stops these pods together, which can cost a quorum-based workload its quorum.
+ The scheduler does not move running pods, so they stay together until they are
+ recreated.
diff --git a/hack/workload-placement-alerts.bats b/hack/workload-placement-alerts.bats
new file mode 100644
index 000000000..3f7b9b961
--- /dev/null
+++ b/hack/workload-placement-alerts.bats
@@ -0,0 +1,26 @@
+#!/usr/bin/env bats
+
+load test_helper
+
+REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
+VMALERT_TOOL_IMAGE="victoriametrics/vmalert-tool:v1.136.0@sha256:417346d7b6fdb66d8e3984887c2ad114535ec7d983eeef53381b56c776f333f1"
+
+@test "workload placement alerts pass their vmalert-tool cases" {
+ docker info >/dev/null 2>&1 || {
+ echo "docker is required to run the cases in $VMALERT_TOOL_IMAGE" >&2
+ return 1
+ }
+ work=$(mktemp -d)
+ rc=0
+ yq '.spec' "$REPO_ROOT/packages/system/monitoring-agents/alerts/workload-placement.yaml" > "$work/rules.yaml" || rc=$?
+ cp "$REPO_ROOT/hack/testdata/workload-placement-alerts/unittest.yaml" "$work/unittest.yaml" || rc=$?
+ if [ "$rc" -eq 0 ]; then
+ docker run --rm --network none -v "$work:/w:ro" "$VMALERT_TOOL_IMAGE" \
+ unittest --files /w/unittest.yaml > "$work/out" 2>&1 || rc=$?
+ cat "$work/out"
+ fi
+ passed=$(grep -cx 'SUCCESS' "$work/out" 2>/dev/null || true)
+ rm -rf "$work"
+ [ "$rc" -eq 0 ]
+ [ "$passed" -eq 1 ]
+}
diff --git a/packages/system/monitoring-agents/alerts/workload-placement.yaml b/packages/system/monitoring-agents/alerts/workload-placement.yaml
new file mode 100644
index 000000000..3ae9c8f50
--- /dev/null
+++ b/packages/system/monitoring-agents/alerts/workload-placement.yaml
@@ -0,0 +1,81 @@
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMRule
+metadata:
+ name: alerts-workload-placement
+spec:
+ groups:
+ - name: workload-placement
+ params: {}
+ rules:
+ - alert: PodReplicasColocated
+ expr: |-
+ count by (cluster, namespace, owner_kind, owner_name, node) (
+ label_replace(
+ label_replace(
+ (
+ max by (cluster, namespace, pod, node, created_by_kind, created_by_name) (
+ kube_pod_info{job="kube-state-metrics", created_by_kind=~"StatefulSet|ReplicaSet|Cluster|StrimziPodSet"}
+ )
+ or
+ label_replace(
+ label_replace(
+ max by (cluster, namespace, pod, node, created_by_kind, created_by_name) (
+ kube_pod_info{job="kube-state-metrics", created_by_kind="EtcdMember"}
+ ),
+ "created_by_name", "$1", "created_by_name", "(.+)-[^-]+"
+ ),
+ "created_by_kind", "EtcdCluster", "", ""
+ )
+ )
+ and on (cluster, namespace, pod)
+ kube_pod_status_phase{job="kube-state-metrics", phase="Running"} == 1,
+ "owner_kind", "$1", "created_by_kind", "(.+)"
+ ),
+ "owner_name", "$1", "created_by_name", "(.+)"
+ )
+ ) > 1
+ and on (cluster, namespace, owner_kind, owner_name)
+ count by (cluster, namespace, owner_kind, owner_name) (
+ label_replace(
+ label_replace(
+ (
+ max by (cluster, namespace, pod, node, created_by_kind, created_by_name) (
+ kube_pod_info{job="kube-state-metrics", created_by_kind=~"StatefulSet|ReplicaSet|Cluster|StrimziPodSet"}
+ )
+ or
+ label_replace(
+ label_replace(
+ max by (cluster, namespace, pod, node, created_by_kind, created_by_name) (
+ kube_pod_info{job="kube-state-metrics", created_by_kind="EtcdMember"}
+ ),
+ "created_by_name", "$1", "created_by_name", "(.+)-[^-]+"
+ ),
+ "created_by_kind", "EtcdCluster", "", ""
+ )
+ )
+ and on (cluster, namespace, pod)
+ kube_pod_status_phase{job="kube-state-metrics", phase="Running"} == 1,
+ "owner_kind", "$1", "created_by_kind", "(.+)"
+ ),
+ "owner_name", "$1", "created_by_name", "(.+)"
+ )
+ )
+ <= on (cluster) group_left ()
+ count by (cluster) (
+ max by (cluster, node) (kube_node_info{job="kube-state-metrics"})
+ unless on (cluster, node)
+ kube_node_spec_taint{job="kube-state-metrics", effect=~"NoSchedule|NoExecute"}
+ )
+ for: 1h
+ labels:
+ severity: warning
+ exported_instance: '{{ $labels.namespace }}/{{ $labels.owner_name }}'
+ service: workload-placement
+ annotations:
+ summary: Pods of one workload share a node.
+ description: >-
+ {{ $value }} running pods of {{ $labels.owner_kind }} {{ $labels.namespace }}/{{ $labels.owner_name }}
+ are on node {{ $labels.node }} in cluster {{ $labels.cluster }}, although the cluster has a
+ schedulable node for each of its pods. Losing this node stops these pods together, which can cost
+ a quorum-based workload its quorum. The scheduler does not move running pods, so they stay
+ together until they are recreated.
From 4a706901b501aa0a5d620de73c352659a9768027 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Wed, 21 Oct 2026 04:14:26 +0300
Subject: [PATCH 011/216] fix(cozy-lib)!: refuse memory below 1Mi instead of
rendering a few bytes
A quantity without a unit counts bytes. The app schemas take a bare
number for memory and the dashboard sends it as typed, so 8 meant as
8Gi rendered a memory limit and request of 8 bytes and the app did not
work. Charts size their containers through sanitize, directly or from
defaultingSanitize, so the floor goes there and the render stops with a
message that names the value and asks for a unit. 1Mi is far below the
smallest preset, 128Mi, and above any bare number typed for Mi or Gi.
0 is refused as well, since kubelet sets no cgroup memory limit for a
zero limit. flatten renders tenant quotas from the same conversion
without the floor, because a quota of 0 memory is a valid way to allow
none.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../library/cozy-lib/templates/_resources.tpl | 29 ++-
.../templates/tests/resources-defaulting.yaml | 11 +
.../tests/resources_memory_floor_test.yaml | 203 ++++++++++++++++++
3 files changed, 242 insertions(+), 1 deletion(-)
create mode 100644 packages/tests/cozy-lib-tests/templates/tests/resources-defaulting.yaml
create mode 100644 packages/tests/cozy-lib-tests/tests/resources_memory_floor_test.yaml
diff --git a/packages/library/cozy-lib/templates/_resources.tpl b/packages/library/cozy-lib/templates/_resources.tpl
index 48b0718fd..6ee63af0e 100644
--- a/packages/library/cozy-lib/templates/_resources.tpl
+++ b/packages/library/cozy-lib/templates/_resources.tpl
@@ -68,6 +68,8 @@
number of milli-CPUs (a CPU quantity finer than 1m is invalid for the VPA
admission webhook and meaningless for the scheduler).
+ A memory below 1Mi fails the render, see cozy-lib.resources.checkMemory.
+
The template now expects **one flat map** as input (no nested `requests:` /
`limits:` sections). Each value in that map is taken as the *limit* for the
corresponding resource. Usage example:
@@ -92,6 +94,31 @@
devices.com/nvidia: "1" # = limit
*/}}
{{- define "cozy-lib.resources.sanitize" }}
+{{- include "cozy-lib.resources.checkMemory" (index . 0) }}
+{{- include "cozy-lib.resources.requestsAndLimits" . }}
+{{- end }}
+
+{{- /*
+ A quantity without a unit counts bytes, so a memory of `8` typed for 8Gi
+ sizes the container at 8 bytes and nothing else refuses it. Every preset is
+ 128Mi or more, so a floor of 1Mi only stops that kind of slip.
+*/}}
+{{- define "cozy-lib.resources.checkMemory" }}
+{{- if hasKey . "memory" }}
+{{- $memory := index . "memory" }}
+{{- if not (kindIs "invalid" $memory) }}
+{{- if lt (include "cozy-lib.resources.toFloat" $memory | float64) (include "cozy-lib.resources.toFloat" "1Mi" | float64) }}
+{{- fail (printf "memory (%v) must be at least 1Mi. A value without a unit is read as bytes, so add the unit, such as 512Mi or 8Gi." $memory) }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{- /*
+ sanitize without the memory floor. flatten renders a ResourceQuota with it,
+ where a memory of 0 is a valid quota that allows none.
+*/}}
+{{- define "cozy-lib.resources.requestsAndLimits" }}
{{- $cpuAllocationRatio := include "cozy-lib.resources.cpuAllocationRatio" . | float64 }}
{{- $memoryAllocationRatio := include "cozy-lib.resources.memoryAllocationRatio" . | float64 }}
{{- $ephemeralStorageAllocationRatio := include "cozy-lib.resources.ephemeralStorageAllocationRatio" . | float64 }}
@@ -204,7 +231,7 @@
{{- define "cozy-lib.resources.flatten" -}}
{{- $out := dict -}}
-{{- $res := include "cozy-lib.resources.sanitize" . | fromYaml -}}
+{{- $res := include "cozy-lib.resources.requestsAndLimits" . | fromYaml -}}
{{- range $section, $values := $res }}
{{- range $k, $v := $values }}
{{- with include "cozy-lib.resources.flattenResource" (list $section $k) }}
diff --git a/packages/tests/cozy-lib-tests/templates/tests/resources-defaulting.yaml b/packages/tests/cozy-lib-tests/templates/tests/resources-defaulting.yaml
new file mode 100644
index 000000000..3e6d9a9eb
--- /dev/null
+++ b/packages/tests/cozy-lib-tests/templates/tests/resources-defaulting.yaml
@@ -0,0 +1,11 @@
+{{- with .Values.defaulting }}
+apiVersion: v1
+kind: Pod
+metadata:
+ name: {{ $.Release.Name }}-defaulting
+spec:
+ containers:
+ - name: main
+ image: busybox
+ resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .preset .resources $) | nindent 6 }}
+{{- end }}
diff --git a/packages/tests/cozy-lib-tests/tests/resources_memory_floor_test.yaml b/packages/tests/cozy-lib-tests/tests/resources_memory_floor_test.yaml
new file mode 100644
index 000000000..485829054
--- /dev/null
+++ b/packages/tests/cozy-lib-tests/tests/resources_memory_floor_test.yaml
@@ -0,0 +1,203 @@
+# ./tests/resources_memory_floor_test.yaml
+suite: resources memory floor
+
+# A quantity without a unit counts bytes, so a memory of 8 typed for 8Gi would
+# size the container at 8 bytes. sanitize refuses anything below 1Mi, which
+# every preset clears by far. flatten renders tenant quotas without the floor,
+# because a quota of 0 memory is a valid way to allow none.
+
+templates:
+ - templates/tests/resources.yaml
+ - templates/tests/resources-defaulting.yaml
+ - templates/tests/quota.yaml
+
+release:
+ name: myrelease
+ namespace: default
+
+tests:
+ - it: refuses a quoted number without a unit
+ template: templates/tests/resources.yaml
+ set:
+ resources:
+ cpu: "1"
+ memory: "8"
+ asserts:
+ - failedTemplate:
+ errorMessage: memory (8) must be at least 1Mi. A value without a unit is read as bytes, so add the unit, such as 512Mi or 8Gi.
+
+ - it: refuses an unquoted number without a unit
+ template: templates/tests/resources.yaml
+ set:
+ resources:
+ cpu: "1"
+ memory: 8
+ asserts:
+ - failedTemplate:
+ errorPattern: '^memory \(8\) must be at least 1Mi\.'
+
+ - it: refuses a zero memory
+ template: templates/tests/resources.yaml
+ set:
+ resources:
+ cpu: "1"
+ memory: 0
+ asserts:
+ - failedTemplate:
+ errorPattern: '^memory \(0\) must be at least 1Mi\.'
+
+ - it: refuses a byte count one below 1Mi
+ template: templates/tests/resources.yaml
+ set:
+ resources:
+ cpu: "1"
+ memory: "1048575"
+ asserts:
+ - failedTemplate:
+ errorPattern: '^memory \(1048575\) must be at least 1Mi\.'
+
+ - it: refuses a suffixed value below 1Mi
+ template: templates/tests/resources.yaml
+ set:
+ resources:
+ cpu: "1"
+ memory: 1023Ki
+ asserts:
+ - failedTemplate:
+ errorPattern: '^memory \(1023Ki\) must be at least 1Mi\.'
+
+ - it: refuses a negative memory
+ template: templates/tests/resources.yaml
+ set:
+ resources:
+ cpu: "1"
+ memory: -1Gi
+ asserts:
+ - failedTemplate:
+ errorPattern: '^memory \(-1Gi\) must be at least 1Mi\.'
+
+ - it: renders exactly 1Mi
+ template: templates/tests/resources.yaml
+ set:
+ resources:
+ cpu: "1"
+ memory: 1Mi
+ asserts:
+ - equal:
+ path: spec.containers[0].resources.limits.memory
+ value: 1Mi
+ - equal:
+ path: spec.containers[0].resources.requests.memory
+ value: "1048576"
+
+ - it: renders an unquoted byte count of exactly 1Mi
+ template: templates/tests/resources.yaml
+ set:
+ resources:
+ cpu: "1"
+ memory: 1048576
+ asserts:
+ - notFailedTemplate: {}
+ - equal:
+ path: spec.containers[0].resources.requests.memory
+ value: "1048576"
+
+ - it: renders 512Mi
+ template: templates/tests/resources.yaml
+ set:
+ resources:
+ cpu: "1"
+ memory: 512Mi
+ asserts:
+ - equal:
+ path: spec.containers[0].resources.limits.memory
+ value: 512Mi
+ - equal:
+ path: spec.containers[0].resources.requests.memory
+ value: "536870912"
+
+ - it: renders 8Gi
+ template: templates/tests/resources.yaml
+ set:
+ resources:
+ cpu: "1"
+ memory: 8Gi
+ asserts:
+ - equal:
+ path: spec.containers[0].resources.limits.memory
+ value: 8Gi
+ - equal:
+ path: spec.containers[0].resources.requests.memory
+ value: "8589934592"
+
+ - it: leaves a map without memory alone
+ template: templates/tests/resources.yaml
+ set:
+ resources:
+ ephemeral-storage: 2Gi
+ asserts:
+ - notFailedTemplate: {}
+ - notExists:
+ path: spec.containers[0].resources.limits.memory
+
+ - it: refuses a number without a unit set over a preset
+ template: templates/tests/resources-defaulting.yaml
+ set:
+ defaulting:
+ preset: t1.nano
+ resources:
+ memory: "8"
+ asserts:
+ - failedTemplate:
+ errorPattern: '^memory \(8\) must be at least 1Mi\.'
+
+ - it: renders the smallest preset unchanged
+ template: templates/tests/resources-defaulting.yaml
+ set:
+ defaulting:
+ preset: t1.nano
+ asserts:
+ - equal:
+ path: spec.containers[0].resources.limits.memory
+ value: 128Mi
+ - equal:
+ path: spec.containers[0].resources.requests.memory
+ value: "134217728"
+
+ - it: renders the smallest legacy preset unchanged
+ template: templates/tests/resources-defaulting.yaml
+ set:
+ defaulting:
+ preset: nano
+ asserts:
+ - equal:
+ path: spec.containers[0].resources.limits.memory
+ value: 128Mi
+
+ - it: renders an explicit memory with a unit over a preset
+ template: templates/tests/resources-defaulting.yaml
+ set:
+ defaulting:
+ preset: t1.nano
+ resources:
+ memory: 8Gi
+ asserts:
+ - equal:
+ path: spec.containers[0].resources.limits.memory
+ value: 8Gi
+ - equal:
+ path: spec.containers[0].resources.limits.cpu
+ value: 250m
+
+ - it: still renders a tenant quota of 0 memory
+ template: templates/tests/quota.yaml
+ set:
+ quota:
+ memory: 0
+ asserts:
+ - equal:
+ path: spec.hard["limits.memory"]
+ value: "0"
+ - equal:
+ path: spec.hard["requests.memory"]
+ value: "0"
From 6bb0fda706e9ccaec5f433f6cd81eb2cdb7e6ef0 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Wed, 21 Oct 2026 04:30:33 +0300
Subject: [PATCH 012/216] fix(cozystack-basics): refuse a direct delete of a
tenant namespace
A tenant namespace is meant to go away with its Tenant. The tenant
chart's pre-delete hook deletes the applications first and the tenant
modules such as etcd after them, and helm-controller deletes the
namespace at the end. A direct delete hands the namespace to the
namespace controller instead. It deletes the pods first and then
everything else together, so the tenant etcd and its etcd-ca-tls Secret
can be gone before the TenantControlPlanes that use them. Kamaji then
cannot reach the datastore to clean up, the TenantControlPlane finalizer
never clears and the namespace stays Terminating.
The policy refuses DELETE of a namespace labelled
tenant.cozystack.io/tenant-root, which the tenant chart puts on every
tenant namespace below tenant-root. tenant-root carries no such label
and stays with the no-delete guardrail. Two callers are let through.
helm-controller and its shards run as the flux ServiceAccount in
cozy-fluxcd and delete the namespace during a Tenant uninstall. The
garbage collector runs in kube-system and removes a child namespace
through its ownerReference once the parent namespace is gone. The
namespace controller only finalizes a namespace and never sends a
DELETE for it, so it is not affected. Everyone else is refused,
system:masters included, as with the no-delete guardrail. The message
names the Tenant to delete and the label to remove when the Tenant is
already gone.
The policy test helpers take the template path now, so the new test
compiles the shipped template with the apiserver CEL compiler like the
external TLS policy test does.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
hack/e2e-install-cozystack.bats | 37 ++++
.../tenant-namespace-delete-policy.yaml | 59 +++++++
...ostname-policies-capability-gate_test.yaml | 4 +
.../tenant-namespace-delete-policy_test.yaml | 51 ++++++
.../tests/tenant-root_test.yaml | 6 +
.../application/external_tls_policy_test.go | 22 +--
.../tenant_namespace_delete_policy_test.go | 163 ++++++++++++++++++
7 files changed, 331 insertions(+), 11 deletions(-)
create mode 100644 packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml
create mode 100644 packages/system/cozystack-basics/tests/tenant-namespace-delete-policy_test.yaml
create mode 100644 pkg/registry/apps/application/tenant_namespace_delete_policy_test.go
diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats
index 47315ac85..4f0884269 100644
--- a/hack/e2e-install-cozystack.bats
+++ b/hack/e2e-install-cozystack.bats
@@ -561,3 +561,40 @@ data:
version: "${version}"
EOF
}
+
+@test "Tenant namespace delete policy refuses a direct delete and keeps the Tenant uninstall" {
+ if ! kubectl api-resources --api-group=admissionregistration.k8s.io \
+ 2>/dev/null | grep -qw validatingadmissionpolicies; then
+ skip "ValidatingAdmissionPolicy API not available on this cluster"
+ fi
+ kubectl get validatingadmissionpolicy cozystack-tenant-namespace-delete-policy
+ kubectl get validatingadmissionpolicybinding cozystack-tenant-namespace-delete-policy
+
+ kubectl apply -f - </dev/null 2>&1; do sleep 2; done'
+ kubectl wait hr/tenant-deleteprobe -n tenant-root --timeout=1m --for=condition=ready
+
+ local selected output rc still_there
+ selected=$(kubectl get namespaces --selector tenant.cozystack.io/tenant-root --field-selector metadata.name=tenant-deleteprobe -o name)
+ [ "$selected" = "namespace/tenant-deleteprobe" ]
+
+ output=$(kubectl delete namespace tenant-deleteprobe --dry-run=server 2>&1) && rc=0 || rc=$?
+ echo "kubectl delete exit=$rc, output=$output"
+ [ "$rc" -ne 0 ]
+ echo "$output" | grep -qF "ValidatingAdmissionPolicy 'cozystack-tenant-namespace-delete-policy'"
+ echo "$output" | grep -qF 'kubectl delete tenants.apps.cozystack.io deleteprobe --namespace tenant-root.'
+
+ kubectl delete tenants.apps.cozystack.io deleteprobe -n tenant-root --timeout=5m
+ kubectl wait namespace/tenant-deleteprobe --for=delete --timeout=5m
+ if ! still_there=$(kubectl get namespace tenant-deleteprobe --ignore-not-found -o name); then
+ echo "FAIL: reading the tenant-deleteprobe namespace did not answer"; false
+ fi
+ if [ -n "$still_there" ]; then echo "FAIL: deleting the Tenant must remove its namespace"; false; fi
+}
diff --git a/packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml b/packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml
new file mode 100644
index 000000000..cbc928dc4
--- /dev/null
+++ b/packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml
@@ -0,0 +1,59 @@
+{{- if .Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" }}
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicy
+metadata:
+ name: cozystack-tenant-namespace-delete-policy
+ labels:
+ internal.cozystack.io/managed-by-cozystack: ""
+spec:
+ failurePolicy: Fail
+ matchConstraints:
+ resourceRules:
+ - apiGroups: [""]
+ apiVersions: ["v1"]
+ operations: ["DELETE"]
+ resources: ["namespaces"]
+ variables:
+ - name: platformCaller
+ expression: >-
+ has(request.userInfo.groups) && request.userInfo.groups.exists(g,
+ g == "system:serviceaccounts:cozy-fluxcd" ||
+ g == "system:serviceaccounts:kube-system")
+ - name: name
+ expression: oldObject.metadata.name
+ - name: cut
+ expression: variables.name.lastIndexOf("-")
+ - name: tenant
+ expression: variables.name.substring(variables.cut + 1)
+ - name: parentNamespace
+ expression: >-
+ variables.cut > 0 && variables.name.substring(0, variables.cut) != "tenant"
+ ? variables.name.substring(0, variables.cut)
+ : "tenant-root"
+ validations:
+ - expression: variables.platformCaller
+ messageExpression: >-
+ "Namespace " + variables.name + " belongs to the Tenant " + variables.tenant +
+ " in " + variables.parentNamespace + ". Delete the Tenant instead with kubectl delete tenants.apps.cozystack.io " +
+ variables.tenant + " --namespace " + variables.parentNamespace +
+ ". Deleting the namespace directly can remove the tenant etcd before the Kubernetes clusters that use it and leave the namespace stuck in Terminating." +
+ " If the Tenant is already gone, remove the label first with kubectl label namespace " + variables.name +
+ " tenant.cozystack.io/tenant-root-"
+ reason: Forbidden
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicyBinding
+metadata:
+ name: cozystack-tenant-namespace-delete-policy
+ labels:
+ internal.cozystack.io/managed-by-cozystack: ""
+spec:
+ policyName: cozystack-tenant-namespace-delete-policy
+ validationActions: [Deny]
+ matchResources:
+ objectSelector:
+ matchExpressions:
+ - key: tenant.cozystack.io/tenant-root
+ operator: Exists
+{{- end }}
diff --git a/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml
index 61b8be507..96cfdfaa3 100644
--- a/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml
+++ b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml
@@ -1,5 +1,6 @@
suite: cozystack-basics VAP policies are gated on the ValidatingAdmissionPolicy API
templates:
+ - templates/tenant-namespace-delete-policy.yaml
- templates/route-hostname-policy.yaml
- templates/gateway-hostname-policy.yaml
- templates/ingress-hostname-policy.yaml
@@ -20,6 +21,9 @@ capabilities:
tests:
- it: none of the VAP policies render when the API is unavailable
asserts:
+ - hasDocuments:
+ count: 0
+ template: templates/tenant-namespace-delete-policy.yaml
- hasDocuments:
count: 0
template: templates/route-hostname-policy.yaml
diff --git a/packages/system/cozystack-basics/tests/tenant-namespace-delete-policy_test.yaml b/packages/system/cozystack-basics/tests/tenant-namespace-delete-policy_test.yaml
new file mode 100644
index 000000000..a0e53c77f
--- /dev/null
+++ b/packages/system/cozystack-basics/tests/tenant-namespace-delete-policy_test.yaml
@@ -0,0 +1,51 @@
+suite: tenant namespace delete policy renders with its binding
+templates:
+ - templates/tenant-namespace-delete-policy.yaml
+release:
+ name: cozystack-basics
+ namespace: cozy-system
+capabilities:
+ apiVersions:
+ - admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy
+tests:
+ - it: renders the policy and a Deny binding to it
+ asserts:
+ - hasDocuments:
+ count: 2
+ - documentIndex: 0
+ equal:
+ path: kind
+ value: ValidatingAdmissionPolicy
+ - documentIndex: 0
+ equal:
+ path: metadata.name
+ value: cozystack-tenant-namespace-delete-policy
+ - documentIndex: 1
+ equal:
+ path: spec.policyName
+ value: cozystack-tenant-namespace-delete-policy
+ - documentIndex: 1
+ equal:
+ path: spec.validationActions
+ value: [Deny]
+
+ - it: matches only DELETE of core namespaces
+ asserts:
+ - documentIndex: 0
+ equal:
+ path: spec.matchConstraints.resourceRules
+ value:
+ - apiGroups: [""]
+ apiVersions: ["v1"]
+ operations: ["DELETE"]
+ resources: ["namespaces"]
+
+ - it: binds to the namespaces the tenant chart labels as descendants of tenant-root
+ asserts:
+ - documentIndex: 1
+ equal:
+ path: spec.matchResources.objectSelector
+ value:
+ matchExpressions:
+ - key: tenant.cozystack.io/tenant-root
+ operator: Exists
diff --git a/packages/system/cozystack-basics/tests/tenant-root_test.yaml b/packages/system/cozystack-basics/tests/tenant-root_test.yaml
index e22dbb649..54210174c 100644
--- a/packages/system/cozystack-basics/tests/tenant-root_test.yaml
+++ b/packages/system/cozystack-basics/tests/tenant-root_test.yaml
@@ -79,3 +79,9 @@ tests:
equal:
path: spec.chartRef.name
value: cozystack-tenant-application-default-tenant
+
+ - it: tenant-root namespace carries no tenant.cozystack.io/tenant-root label, which the tenant namespace delete policy selects on
+ asserts:
+ - documentIndex: 0
+ notExists:
+ path: metadata.labels["tenant.cozystack.io/tenant-root"]
diff --git a/pkg/registry/apps/application/external_tls_policy_test.go b/pkg/registry/apps/application/external_tls_policy_test.go
index 1cbc808fc..ff5704a23 100644
--- a/pkg/registry/apps/application/external_tls_policy_test.go
+++ b/pkg/registry/apps/application/external_tls_policy_test.go
@@ -41,14 +41,14 @@ import (
// expressions a cluster actually receives rather than a copy of them.
const externalTLSPolicyPath = "../../../../packages/system/cozystack-basics/templates/database-external-tls-policy.yaml"
-// loadExternalTLSPolicy returns the policy and its binding from the chart
-// template. The template's only directives are the capability guard lines
-// around the manifests; any other directive would mean the test is no longer
-// looking at what renders, so it fails instead of guessing.
-func loadExternalTLSPolicy(t *testing.T) (*admissionregistrationv1.ValidatingAdmissionPolicy, *admissionregistrationv1.ValidatingAdmissionPolicyBinding) {
+// loadPolicy returns the policy and its binding from a chart template. The
+// template's only directives are the capability guard lines around the
+// manifests; any other directive would mean the test is no longer looking at
+// what renders, so it fails instead of guessing.
+func loadPolicy(t *testing.T, path string) (*admissionregistrationv1.ValidatingAdmissionPolicy, *admissionregistrationv1.ValidatingAdmissionPolicyBinding) {
t.Helper()
- raw, err := os.ReadFile(externalTLSPolicyPath)
+ raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read policy template: %v", err)
}
@@ -92,11 +92,11 @@ func loadExternalTLSPolicy(t *testing.T) (*admissionregistrationv1.ValidatingAdm
return policy, binding
}
-// compileExternalTLSPolicy mirrors compilePolicy in
+// compilePolicy mirrors compilePolicy in
// k8s.io/apiserver/pkg/admission/plugin/policy/validating, which is
// unexported: the same composition environment, the same variable, match
// condition, validation and message compilers.
-func compileExternalTLSPolicy(t *testing.T, policy *admissionregistrationv1.ValidatingAdmissionPolicy) validating.Validator {
+func compilePolicy(t *testing.T, policy *admissionregistrationv1.ValidatingAdmissionPolicy) validating.Validator {
t.Helper()
compiler, err := plugincel.NewCompositedCompiler(environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion()))
@@ -139,7 +139,7 @@ func compileExternalTLSPolicy(t *testing.T, policy *admissionregistrationv1.Vali
}
func TestDatabaseExternalTLSPolicy_Binding(t *testing.T) {
- policy, binding := loadExternalTLSPolicy(t)
+ policy, binding := loadPolicy(t, externalTLSPolicyPath)
if binding.Spec.PolicyName != policy.Name {
t.Errorf("binding names policy %q, want %q", binding.Spec.PolicyName, policy.Name)
@@ -171,8 +171,8 @@ func TestDatabaseExternalTLSPolicy_Binding(t *testing.T) {
}
func TestDatabaseExternalTLSPolicy_Decisions(t *testing.T) {
- policy, _ := loadExternalTLSPolicy(t)
- validator := compileExternalTLSPolicy(t, policy)
+ policy, _ := loadPolicy(t, externalTLSPolicyPath)
+ validator := compilePolicy(t, policy)
tlsOn := map[string]any{"enabled": true}
tlsOff := map[string]any{"enabled": false}
diff --git a/pkg/registry/apps/application/tenant_namespace_delete_policy_test.go b/pkg/registry/apps/application/tenant_namespace_delete_policy_test.go
new file mode 100644
index 000000000..6c8870ea2
--- /dev/null
+++ b/pkg/registry/apps/application/tenant_namespace_delete_policy_test.go
@@ -0,0 +1,163 @@
+/*
+Copyright 2026 The Cozystack Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package application
+
+import (
+ "context"
+ "slices"
+ "strings"
+ "testing"
+
+ admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+ "k8s.io/apimachinery/pkg/labels"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "k8s.io/apiserver/pkg/admission"
+ "k8s.io/apiserver/pkg/admission/plugin/policy/validating"
+ celconfig "k8s.io/apiserver/pkg/apis/cel"
+ "k8s.io/apiserver/pkg/authentication/user"
+)
+
+const tenantNamespaceDeletePolicyPath = "../../../../packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml"
+
+func TestTenantNamespaceDeletePolicy_Binding(t *testing.T) {
+ policy, binding := loadPolicy(t, tenantNamespaceDeletePolicyPath)
+
+ if binding.Spec.PolicyName != policy.Name {
+ t.Errorf("binding names policy %q, want %q", binding.Spec.PolicyName, policy.Name)
+ }
+ if !slices.Equal(binding.Spec.ValidationActions, []admissionregistrationv1.ValidationAction{admissionregistrationv1.Deny}) {
+ t.Errorf("binding validationActions = %v, want [Deny]", binding.Spec.ValidationActions)
+ }
+ if policy.Spec.FailurePolicy == nil || *policy.Spec.FailurePolicy != admissionregistrationv1.Fail {
+ t.Errorf("failurePolicy must be Fail")
+ }
+
+ rules := policy.Spec.MatchConstraints.ResourceRules
+ if len(rules) != 1 {
+ t.Fatalf("want one resource rule, got %d", len(rules))
+ }
+ rule := rules[0]
+ if !slices.Equal(rule.APIGroups, []string{""}) || !slices.Equal(rule.APIVersions, []string{"v1"}) {
+ t.Errorf("rule must match core v1 only, got apiGroups %v apiVersions %v", rule.APIGroups, rule.APIVersions)
+ }
+ if !slices.Equal(rule.Operations, []admissionregistrationv1.OperationType{admissionregistrationv1.Delete}) {
+ t.Errorf("operations = %v, want [DELETE]", rule.Operations)
+ }
+ if !slices.Equal(rule.Resources, []string{"namespaces"}) {
+ t.Errorf("resources = %v, want [namespaces]", rule.Resources)
+ }
+
+ if binding.Spec.MatchResources == nil || binding.Spec.MatchResources.ObjectSelector == nil {
+ t.Fatal("binding must select namespaces by label")
+ }
+ selector, err := metav1.LabelSelectorAsSelector(binding.Spec.MatchResources.ObjectSelector)
+ if err != nil {
+ t.Fatalf("binding objectSelector: %v", err)
+ }
+ selected := []struct {
+ name string
+ labels labels.Set
+ want bool
+ }{
+ {name: "tenant namespace under tenant-root", labels: labels.Set{"tenant.cozystack.io/tenant-root": "", "tenant.cozystack.io/tenant-foo": "", "namespace.cozystack.io/host": "foo.example.org"}, want: true},
+ {name: "nested tenant namespace", labels: labels.Set{"tenant.cozystack.io/tenant-root": "", "tenant.cozystack.io/tenant-foo": "", "tenant.cozystack.io/tenant-foo-bar": ""}, want: true},
+ {name: "tenant-root namespace", labels: labels.Set{"platform.cozystack.io/no-delete": "true", "namespace.cozystack.io/host": "example.org", "namespace.cozystack.io/gateway": "tenant-root"}},
+ {name: "namespace named tenant-* without tenant labels", labels: labels.Set{"namespace.cozystack.io/host": "case-probe.example.org"}},
+ {name: "unlabelled namespace", labels: labels.Set{}},
+ }
+ for _, tc := range selected {
+ if got := selector.Matches(tc.labels); got != tc.want {
+ t.Errorf("%s: selected = %v, want %v", tc.name, got, tc.want)
+ }
+ }
+}
+
+func TestTenantNamespaceDeletePolicy_Decisions(t *testing.T) {
+ policy, _ := loadPolicy(t, tenantNamespaceDeletePolicyPath)
+ validator := compilePolicy(t, policy)
+
+ fluxGroups := []string{"system:serviceaccounts", "system:serviceaccounts:cozy-fluxcd", "system:authenticated"}
+ kubeSystemGroups := []string{"system:serviceaccounts", "system:serviceaccounts:kube-system", "system:authenticated"}
+ adminGroups := []string{"system:masters", "system:authenticated"}
+
+ cases := []struct {
+ name string
+ namespace string
+ user string
+ groups []string
+ denied bool
+ tenant string
+ parent string
+ }{
+ {name: "helm-controller uninstalling a Tenant", namespace: "tenant-foo", user: "system:serviceaccount:cozy-fluxcd:flux", groups: fluxGroups},
+ {name: "helm-controller uninstalling a nested Tenant", namespace: "tenant-foo-bar", user: "system:serviceaccount:cozy-fluxcd:flux", groups: fluxGroups},
+ {name: "garbage collector following a deleted parent namespace", namespace: "tenant-foo-bar", user: "system:serviceaccount:kube-system:generic-garbage-collector", groups: kubeSystemGroups},
+ {name: "cluster admin deleting a tenant namespace", namespace: "tenant-foo", user: "admin", groups: adminGroups, denied: true, tenant: "foo", parent: "tenant-root"},
+ {name: "cluster admin deleting a nested tenant namespace", namespace: "tenant-foo-bar", user: "admin", groups: adminGroups, denied: true, tenant: "bar", parent: "tenant-foo"},
+ {name: "cluster admin deleting a third-level tenant namespace", namespace: "tenant-foo-bar-baz", user: "admin", groups: adminGroups, denied: true, tenant: "baz", parent: "tenant-foo-bar"},
+ {name: "cluster admin deleting a labelled namespace without a dash", namespace: "standalone", user: "admin", groups: adminGroups, denied: true, tenant: "standalone", parent: "tenant-root"},
+ {name: "OIDC user bound to cluster-admin", namespace: "tenant-foo", user: "cluster-admin-user", groups: []string{"cozystack-cluster-admin", "system:authenticated"}, denied: true, tenant: "foo", parent: "tenant-root"},
+ {name: "cozy-system service account", namespace: "tenant-foo", user: "system:serviceaccount:cozy-system:cozystack-api", groups: []string{"system:serviceaccounts", "system:serviceaccounts:cozy-system", "system:authenticated"}, denied: true, tenant: "foo", parent: "tenant-root"},
+ {name: "tenant service account", namespace: "tenant-foo-bar", user: "system:serviceaccount:tenant-foo:default", groups: []string{"system:serviceaccounts", "system:serviceaccounts:tenant-foo", "system:authenticated"}, denied: true, tenant: "bar", parent: "tenant-foo"},
+ {name: "user without groups", namespace: "tenant-foo", user: "no-groups", denied: true, tenant: "foo", parent: "tenant-root"},
+ }
+
+ gvk := schema.GroupVersionKind{Version: "v1", Kind: "Namespace"}
+ gvr := schema.GroupVersionResource{Version: "v1", Resource: "namespaces"}
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ oldObj := &unstructured.Unstructured{Object: map[string]any{
+ "apiVersion": "v1",
+ "kind": "Namespace",
+ "metadata": map[string]any{
+ "name": tc.namespace,
+ "labels": map[string]any{"tenant.cozystack.io/tenant-root": ""},
+ },
+ }}
+ var newObj runtime.Object
+ attrs := admission.NewAttributesRecord(newObj, oldObj, gvk, tc.namespace, tc.namespace, gvr, "", admission.Delete, &metav1.DeleteOptions{}, false, &user.DefaultInfo{Name: tc.user, Groups: tc.groups})
+ versioned := &admission.VersionedAttributes{Attributes: attrs, VersionedObject: admission.NewLazyObject(newObj), VersionedOldObject: admission.NewLazyObject(oldObj), VersionedKind: gvk}
+
+ result := validator.Validate(context.Background(), gvr, versioned, nil, nil, celconfig.RuntimeCELCostBudget, nil)
+ var denials []validating.PolicyDecision
+ for _, d := range result.Decisions {
+ if d.Evaluation == validating.EvalError {
+ t.Fatalf("evaluation error: %s", d.Message)
+ }
+ if d.Action == validating.ActionDeny {
+ denials = append(denials, d)
+ }
+ }
+ if got := len(denials) > 0; got != tc.denied {
+ t.Fatalf("denied = %v, want %v (decisions %+v)", got, tc.denied, result.Decisions)
+ }
+ for _, d := range denials {
+ for _, want := range []string{
+ "kubectl delete tenants.apps.cozystack.io " + tc.tenant + " --namespace " + tc.parent + ".",
+ "kubectl label namespace " + tc.namespace + " tenant.cozystack.io/tenant-root-",
+ } {
+ if !strings.Contains(d.Message, want) {
+ t.Errorf("denial message must carry %q, got %q", want, d.Message)
+ }
+ }
+ }
+ })
+ }
+}
From 9f6fdd77c6ca964e0adcc387e50993487f2e594e Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sat, 24 Oct 2026 15:52:37 +0300
Subject: [PATCH 013/216] docs(etcd): update the affinity notes for the hard
member spread
The affinity description and the comment above the affinity block
explained the soft default by a member that would stay Pending on its
node-local volume. Now that the spread is DoNotSchedule by default, the
spread keeps such a member Pending as well, once its node holds another
member and some other node holds none. Both texts now explain the soft
default by what is left, that a required rule forbids two members on
one node even when every node has one, and state the node-local case
for both rules.
The description also says that the operator copies affinity into each
EtcdMember when it creates it, so a recreated Pod keeps the rule its
member was created with. README, values.schema.json and the
ApplicationDefinition are regenerated from it.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
packages/extra/etcd/README.md | 20 +++++++++----------
.../extra/etcd/templates/etcd-cluster.yaml | 14 ++++++++-----
packages/extra/etcd/values.schema.json | 2 +-
packages/extra/etcd/values.yaml | 2 +-
packages/system/etcd-rd/cozyrds/etcd.yaml | 2 +-
5 files changed, 22 insertions(+), 18 deletions(-)
diff --git a/packages/extra/etcd/README.md b/packages/extra/etcd/README.md
index e96c63832..91341ff32 100644
--- a/packages/extra/etcd/README.md
+++ b/packages/extra/etcd/README.md
@@ -34,14 +34,14 @@ coordinates).
### Common parameters
-| Name | Description | Type | Value |
-| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------- |
-| `version` | etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image. | `string` | `3.6.11` |
-| `size` | Persistent Volume size. | `quantity` | `4Gi` |
-| `storageClass` | StorageClass used to store the data. | `string` | `""` |
-| `replicas` | Number of etcd replicas. | `int` | `3` |
-| `resources` | Resource configuration for etcd. | `object` | `{}` |
-| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` |
-| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` |
-| `affinity` | Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively. | `object` | `{}` |
+| Name | Description | Type | Value |
+| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------- |
+| `version` | etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image. | `string` | `3.6.11` |
+| `size` | Persistent Volume size. | `quantity` | `4Gi` |
+| `storageClass` | StorageClass used to store the data. | `string` | `""` |
+| `replicas` | Number of etcd replicas. | `int` | `3` |
+| `resources` | Resource configuration for etcd. | `object` | `{}` |
+| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` |
+| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` |
+| `affinity` | Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones. | `object` | `{}` |
diff --git a/packages/extra/etcd/templates/etcd-cluster.yaml b/packages/extra/etcd/templates/etcd-cluster.yaml
index 67dd34bd9..3676f53cb 100644
--- a/packages/extra/etcd/templates/etcd-cluster.yaml
+++ b/packages/extra/etcd/templates/etcd-cluster.yaml
@@ -79,11 +79,15 @@ spec:
affinity:
{{- toYaml . | nindent 4 }}
{{- else }}
- # Soft anti-affinity: spread etcd members across nodes on a best-effort
- # basis so a single node failure is unlikely to take out more than one
- # member. Preferred (not required) because a hard rule would leave a member
- # Pending when replicas exceed schedulable nodes, or when node-local storage
- # pins a member to a node. Override via .Values.affinity for a hard rule.
+ # Soft anti-affinity, a preference on top of the spread constraint above,
+ # which by default is the hard rule that keeps two members off one node
+ # while another node has none. Preferred, not required, because a required
+ # rule also forbids two members on one node when every node already has
+ # one, so an etcd with more members than schedulable nodes could not start
+ # all of them. With node-local storage a recreated member Pod can only
+ # return to its own node, and the default spread keeps it Pending there
+ # while that node holds another member and some other node holds none.
+ # Override via .Values.affinity for a hard rule.
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
diff --git a/packages/extra/etcd/values.schema.json b/packages/extra/etcd/values.schema.json
index b438ca9d7..c33219f4f 100644
--- a/packages/extra/etcd/values.schema.json
+++ b/packages/extra/etcd/values.schema.json
@@ -67,7 +67,7 @@
}
},
"affinity": {
- "description": "Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.",
+ "description": "Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.",
"type": "object",
"default": {},
"x-kubernetes-preserve-unknown-fields": true
diff --git a/packages/extra/etcd/values.yaml b/packages/extra/etcd/values.yaml
index 61b994d7c..c5be94909 100644
--- a/packages/extra/etcd/values.yaml
+++ b/packages/extra/etcd/values.yaml
@@ -23,5 +23,5 @@ resources:
cpu: 1000m
memory: 512Mi
-## @param {object} [affinity] - Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.
+## @param {object} [affinity] - Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.
affinity: {}
diff --git a/packages/system/etcd-rd/cozyrds/etcd.yaml b/packages/system/etcd-rd/cozyrds/etcd.yaml
index 90ed8a80a..1ce88e8ae 100644
--- a/packages/system/etcd-rd/cozyrds/etcd.yaml
+++ b/packages/system/etcd-rd/cozyrds/etcd.yaml
@@ -8,7 +8,7 @@ spec:
plural: etcds
singular: etcd
openAPISchema: |-
- {"title":"Chart Values","type":"object","properties":{"version":{"description":"etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image.","type":"string","default":"3.6.11"},"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"affinity":{"description":"Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}
+ {"title":"Chart Values","type":"object","properties":{"version":{"description":"etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image.","type":"string","default":"3.6.11"},"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"affinity":{"description":"Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}
release:
prefix: ""
labels:
From 00c11c7a745db9b57d419631c3c88b7ac8c274cb Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:29:50 +0300
Subject: [PATCH 014/216] test(monitoring): check the listed alert still
reaches email after the blackhole
The case lists InfoInhibitor in email.alertnames but only asserted the
route order up to the blackhole, so it would still pass if the email
route for a listed name stopped being rendered.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../monitoring/tests/alertmanager_blackhole_test.yaml | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
index e987bd405..d4e366180 100644
--- a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
+++ b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml
@@ -43,3 +43,9 @@ tests:
matchRegex:
path: stringData["alertmanager.yaml"]
pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n'
+ - documentSelector:
+ path: metadata.labels.app
+ value: alertmanager
+ matchRegex:
+ path: stringData["alertmanager.yaml"]
+ pattern: '(?s)\n - receiver: blackhole\n matchers:\n - .severity="none".\n.*\n - receiver: email\n group_wait: 1m\n group_interval: 5m\n repeat_interval: .12h.\n matchers:\n - .alertname=~"\^\(InfoInhibitor\)\$".\n'
From 7bda2bdbc07dd941724e5ea12e51e8c808313b1b Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:30:14 +0300
Subject: [PATCH 015/216] fix(monitoring): select the vmalert Service the
operator creates
The VMServiceScrape selected app.kubernetes.io/instance=vmalert, while the
chart names the VMAlert vmalert- and the
operator puts that name into the instance label of its Service. The scrape
matched nothing, so the metrics of vmalert itself were never collected.
The instance stays in the selector so the scrape keeps pointing at the one
VMAlert this release renders.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../templates/vm/vmalert-scrape.yaml | 4 +-
.../monitoring/tests/vmalert_scrape_test.yaml | 38 +++++++++++++++++++
2 files changed, 41 insertions(+), 1 deletion(-)
create mode 100644 packages/system/monitoring/tests/vmalert_scrape_test.yaml
diff --git a/packages/system/monitoring/templates/vm/vmalert-scrape.yaml b/packages/system/monitoring/templates/vm/vmalert-scrape.yaml
index ee406a275..b467d5564 100644
--- a/packages/system/monitoring/templates/vm/vmalert-scrape.yaml
+++ b/packages/system/monitoring/templates/vm/vmalert-scrape.yaml
@@ -1,3 +1,4 @@
+{{- with first .Values.metricsStorages }}
---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMServiceScrape
@@ -13,5 +14,6 @@ spec:
selector:
matchLabels:
app.kubernetes.io/component: monitoring
- app.kubernetes.io/instance: vmalert
+ app.kubernetes.io/instance: vmalert-{{ .name }}
app.kubernetes.io/name: vmalert
+{{- end }}
diff --git a/packages/system/monitoring/tests/vmalert_scrape_test.yaml b/packages/system/monitoring/tests/vmalert_scrape_test.yaml
new file mode 100644
index 000000000..1dfb9ec3e
--- /dev/null
+++ b/packages/system/monitoring/tests/vmalert_scrape_test.yaml
@@ -0,0 +1,38 @@
+suite: vmalert metrics are scraped
+templates:
+ - templates/vm/vmalert-scrape.yaml
+
+release:
+ name: monitoring
+ namespace: tenant-root
+
+set:
+ _cluster:
+ root-host: example.org
+ _namespace:
+ host: tenant-root.example.org
+ ingress: tenant-root
+ metricsStorages:
+ - name: shortterm
+ retentionPeriod: "3d"
+ deduplicationInterval: "15s"
+ - name: longterm
+ retentionPeriod: "14d"
+ deduplicationInterval: "5m"
+
+tests:
+ - it: selects the Service of the VMAlert the chart renders
+ asserts:
+ - equal:
+ path: spec.selector.matchLabels
+ value:
+ app.kubernetes.io/component: monitoring
+ app.kubernetes.io/instance: vmalert-shortterm
+ app.kubernetes.io/name: vmalert
+
+ - it: renders nothing without metrics storages, like the VMAlert itself
+ set:
+ metricsStorages: []
+ asserts:
+ - hasDocuments:
+ count: 0
From 34766ff0b50af0801f2c94c1435dd744211bbc23 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 17:06:52 +0300
Subject: [PATCH 016/216] fix(vertical-pod-autoscaler): run two admission
controllers spread over nodes
With one admission controller and failurePolicy Ignore, a pod created
while that controller is being rescheduled is admitted without its VPA
recommendation, and with updateMode Initial it keeps the defaults for
the rest of its life. On a v1.6.5 cluster a vmstorage pod created during
a node drain got the operator defaults, a 1500Mi limit against a 3.4Gi
recommendation, and was OOMKilled weeks later.
A second replica on another node and a PodDisruptionBudget keep one
controller serving through a drain. The controller still deletes and
recreates its webhook configuration on start, so a restart leaves only
that short gap instead of the whole reschedule.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../tests/vpa_test.yaml | 29 +++++++++++++++++++
.../vertical-pod-autoscaler/values.yaml | 12 ++++++++
2 files changed, 41 insertions(+)
diff --git a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
index 6b8c82675..1d2011330 100644
--- a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
+++ b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
@@ -242,3 +242,32 @@ tests:
count: 0
+
+ - it: runs two admission controllers spread over nodes
+ template: charts/vertical-pod-autoscaler/templates/admission-controller/deployment.yaml
+ asserts:
+ - equal:
+ path: spec.replicas
+ value: 2
+ - equal:
+ path: spec.template.spec.topologySpreadConstraints[0].topologyKey
+ value: kubernetes.io/hostname
+ - equal:
+ path: spec.template.spec.topologySpreadConstraints[0].labelSelector.matchLabels
+ value:
+ app.kubernetes.io/name: vertical-pod-autoscaler
+ app.kubernetes.io/instance: vpa
+ app.kubernetes.io/component: admission-controller
+
+ - it: keeps one admission controller up through a voluntary disruption
+ template: charts/vertical-pod-autoscaler/templates/admission-controller/pdb.yaml
+ asserts:
+ - equal:
+ path: spec.minAvailable
+ value: 1
+ - equal:
+ path: spec.selector.matchLabels
+ value:
+ app.kubernetes.io/name: vertical-pod-autoscaler
+ app.kubernetes.io/instance: vpa
+ app.kubernetes.io/component: admission-controller
diff --git a/packages/system/vertical-pod-autoscaler/values.yaml b/packages/system/vertical-pod-autoscaler/values.yaml
index 46678903e..456b99912 100644
--- a/packages/system/vertical-pod-autoscaler/values.yaml
+++ b/packages/system/vertical-pod-autoscaler/values.yaml
@@ -37,6 +37,18 @@ vertical-pod-autoscaler:
prometheus-cadvisor-job-name: cadvisor
admissionController:
+ replicaCount: 2
+ pdb:
+ create: true
+ topologySpreadConstraints:
+ - maxSkew: 1
+ topologyKey: kubernetes.io/hostname
+ whenUnsatisfiable: ScheduleAnyway
+ labelSelector:
+ matchLabels:
+ app.kubernetes.io/name: vertical-pod-autoscaler
+ app.kubernetes.io/instance: "{{ .Release.Name }}"
+ app.kubernetes.io/component: admission-controller
resources:
limits:
memory: 256Mi
From 61289c721011af0ca4a61eb278ae2664ae7cd8e7 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 17:09:16 +0300
Subject: [PATCH 017/216] fix(etcd): size the defrag job for the client it runs
The hourly defrag pod runs the etcd-defrag client, which only sends
defragment calls to the members, yet it requested 200m CPU and 256Mi
from the tenant quota. On a sandbox tenant whose quota was mostly taken
by its own etcd and SeaweedFS, the job could not create a pod and the
defrag never ran. The client peaked at 5.1 MiB working set over a day
on a v1.6.5 cluster.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
packages/extra/etcd/templates/etcd-defrag.yaml | 8 ++++----
packages/extra/etcd/tests/defrag_test.yaml | 13 +++++++++++++
2 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/packages/extra/etcd/templates/etcd-defrag.yaml b/packages/extra/etcd/templates/etcd-defrag.yaml
index 52cc63518..c1eefa0db 100644
--- a/packages/extra/etcd/templates/etcd-defrag.yaml
+++ b/packages/extra/etcd/templates/etcd-defrag.yaml
@@ -29,11 +29,11 @@ spec:
drop: ["ALL"]
resources:
requests:
- cpu: 200m
- memory: 256Mi
+ cpu: 10m
+ memory: 32Mi
limits:
- cpu: 500m
- memory: 512Mi
+ cpu: 100m
+ memory: 64Mi
args:
# The v1alpha2 operator names the headless Service after the
# cluster ({{ $.Release.Name }}) and does not use StatefulSet
diff --git a/packages/extra/etcd/tests/defrag_test.yaml b/packages/extra/etcd/tests/defrag_test.yaml
index 8d74e1a6b..b3acc3401 100644
--- a/packages/extra/etcd/tests/defrag_test.yaml
+++ b/packages/extra/etcd/tests/defrag_test.yaml
@@ -41,3 +41,16 @@ tests:
documentSelector:
path: kind
value: EtcdCluster
+
+ - it: asks the tenant quota only for what the defrag client uses
+ template: templates/etcd-defrag.yaml
+ asserts:
+ - equal:
+ path: spec.jobTemplate.spec.template.spec.containers[0].resources
+ value:
+ requests:
+ cpu: 10m
+ memory: 32Mi
+ limits:
+ cpu: 100m
+ memory: 64Mi
From a7ec05f9eb759e0c1025cf4bf95631df338a85c0 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 17:15:04 +0300
Subject: [PATCH 018/216] fix(monitoring-agents): make a silent Watchdog an
alarm in Alerta
Alerta turns the Watchdog into a heartbeat and stores its severity in the
heartbeat attributes. The alerta heartbeats --alert job of the alerta-web
image takes the severity of a stale heartbeat from those attributes ahead
of its HEARTBEAT_SEVERITY default, so with severity ok the HeartbeatFail
it raises is a normal alert and nothing says the alerting pipeline
stopped. Without a timeout label the heartbeat also took a day to go
stale, while Alertmanager refreshes it every minute.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
.../alerts/general.rules.yaml | 3 +-
.../tests/watchdog_heartbeat_test.yaml | 29 +++++++++++++++++++
2 files changed, 31 insertions(+), 1 deletion(-)
create mode 100644 packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml
diff --git a/packages/system/monitoring-agents/alerts/general.rules.yaml b/packages/system/monitoring-agents/alerts/general.rules.yaml
index 924870c40..0ac83fe72 100644
--- a/packages/system/monitoring-agents/alerts/general.rules.yaml
+++ b/packages/system/monitoring-agents/alerts/general.rules.yaml
@@ -33,7 +33,8 @@ spec:
is working properly.
expr: vector(1)
labels:
- severity: ok
+ severity: major
+ timeout: "300"
exported_instance: global
service: general.rules
event: Heartbeat
diff --git a/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml b/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml
new file mode 100644
index 000000000..fe630a80c
--- /dev/null
+++ b/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml
@@ -0,0 +1,29 @@
+suite: Watchdog heartbeat raises an alarm in Alerta when it stops
+templates:
+ - templates/alerts.yaml
+
+release:
+ name: monitoring-agents
+ namespace: cozy-monitoring
+
+tests:
+ - it: gives the heartbeat a severity Alerta reports as an alarm
+ documentSelector:
+ path: metadata.name
+ value: alerts-general.rules
+ asserts:
+ - equal:
+ path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.severity
+ value: major
+
+ - it: lets the heartbeat go stale within minutes, not a day
+ documentSelector:
+ path: metadata.name
+ value: alerts-general.rules
+ asserts:
+ - equal:
+ path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.timeout
+ value: "300"
+ - equal:
+ path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.event
+ value: Heartbeat
From 70e1e8b1454db65222a0b3c0952650fa69239fdc Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 20:33:46 +0300
Subject: [PATCH 019/216] test(vertical-pod-autoscaler): check maxSkew and
whenUnsatisfiable of the spread
The test only looked at the topology key and the label selector, so a
change to maxSkew or to whenUnsatisfiable went unnoticed.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
index 1d2011330..0ff338b48 100644
--- a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
+++ b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml
@@ -252,6 +252,12 @@ tests:
- equal:
path: spec.template.spec.topologySpreadConstraints[0].topologyKey
value: kubernetes.io/hostname
+ - equal:
+ path: spec.template.spec.topologySpreadConstraints[0].maxSkew
+ value: 1
+ - equal:
+ path: spec.template.spec.topologySpreadConstraints[0].whenUnsatisfiable
+ value: ScheduleAnyway
- equal:
path: spec.template.spec.topologySpreadConstraints[0].labelSelector.matchLabels
value:
From bcc5f3216a7ea4ec93bf2597ad4821ff97e855ac Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Mon, 26 Oct 2026 18:21:32 +0300
Subject: [PATCH 020/216] fix(kubevirt): let tenant admins upload a VMDisk
image with virtctl
A VMDisk created with source upload waits in UploadReady for an image,
but no tenant role can fill it. virtctl image-upload reads the disk PVC
and CDI's PVC Prime and requests an upload token, and the tenant roles
grant neither get on persistentvolumeclaims nor create on
uploadtokenrequests, so the command stops on a forbidden PVC read.
cozy:tenant:admin:base now grants both, so tenant admins and
super-admins get them in their own namespaces through the existing
RoleBindings, while the use and view tiers do not. Reading the
DataVolume is already granted for the disk's own name by the vm-disk
chart's dashboard role. The vm-disk notes print the command with
--no-create, because the chart has already created the DataVolume, and
the security self-assessment and threat model list the new grants.
The rules, their tests, the notes change and the docs lines come from
#3847, which also carries a console upload panel and an upload proxy
certificate that are still in review there.
Fixes #3759
Co-authored-by: Myasnikov Daniil
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
docs/security/self-assessment.md | 2 +-
docs/security/threat-model.md | 4 +-
packages/apps/vm-disk/templates/NOTES.txt | 2 +-
.../apps/vm-disk/tests/upload_notes_test.yaml | 18 ++++
.../templates/clusterroles.yaml | 11 +++
.../tests/cdi_upload_rbac_test.yaml | 97 +++++++++++++++++++
6 files changed, 130 insertions(+), 4 deletions(-)
create mode 100644 packages/apps/vm-disk/tests/upload_notes_test.yaml
create mode 100644 packages/system/cozystack-basics/tests/cdi_upload_rbac_test.yaml
diff --git a/docs/security/self-assessment.md b/docs/security/self-assessment.md
index cb8360904..b59cbda4a 100644
--- a/docs/security/self-assessment.md
+++ b/docs/security/self-assessment.md
@@ -55,7 +55,7 @@ Actors are decomposed by **trust boundary**: two components are distinct actors
| GitOps / Flux delivery layer | T1 — platform control plane | Reconciles HelmReleases/Kustomizations into arbitrary privileged manifests; effectively cluster-admin. | Tenants cannot create HelmReleases (RBAC); not tenant-network-reachable. |
| `cozystack-api` aggregated API server | T1 — platform control plane | Converts tenant CRs into HelmReleases; can CRUD HelmReleases, Secrets, and Cilium policies cluster-wide. Acts as the policy-enforcing gateway. | Constrains tenants via fixed chart reference, name/quota/host validation, and admission. |
| System operators / controllers | T1 — platform control plane | Domain-scoped cluster privileges (provision control planes, mint certs/secrets, program CNI). | Distinct ServiceAccounts; isolated from tenants by RBAC and network policy. |
-| Tenant administrator (OIDC `-admin` / `-super-admin`) | T2 — tenant boundary | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` in its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) `super-admin` holds the `apps.cozystack.io/*` wildcard (and can create child `Tenant`s); `admin` has an enumerated allowlist excluding some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`). | Namespace-scoped RBAC + admission policies + quota; no HelmRelease or cross-tenant access. |
+| Tenant administrator (OIDC `-admin` / `-super-admin`) | T2 — tenant boundary | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` in its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); `get` on persistentvolumeclaims and `create` on `upload.cdi.kubevirt.io/uploadtokenrequests`, the pair `virtctl image-upload` needs to upload into an existing VMDisk; no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) `super-admin` holds the `apps.cozystack.io/*` wildcard (and can create child `Tenant`s); `admin` has an enumerated allowlist excluding some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`). | Namespace-scoped RBAC + admission policies + quota; no HelmRelease or cross-tenant access. |
| Tenant user (OIDC `-view` / `-use`) | T2 — tenant boundary | Read-only view plus operational verbs (VM console/vnc/portforward, start/stop/restart). Cannot create Application CRs. | Strictly dominated by tenant-admin; cannot escalate. |
| Parent vs child (nested) tenant | T2 — tenant boundary | Parent ServiceAccounts bind into descendant namespaces; children do not bind upward. Quota hierarchically capped. | Ancestor→descendant trust is one-way. |
| Tenant workloads / managed services | T3 — workload boundary | Run tenant-controlled code; no cluster-API rights beyond their ServiceAccount. | Namespace-scoped RBAC/identity isolate API objects; on the network, Cilium egress confines each tenant pod to its own subtree + platform system services + `world`, so unrelated tenants cannot reach each other over the pod network (enforced on egress; ingress is open to `cluster` — see Non-goals). |
diff --git a/docs/security/threat-model.md b/docs/security/threat-model.md
index 9790d9591..0a4ea0bb3 100644
--- a/docs/security/threat-model.md
+++ b/docs/security/threat-model.md
@@ -39,7 +39,7 @@ The actors below are decomposed **by trust boundary**, not by component: two com
| GitOps / Flux delivery layer | T1 | Reconciles HelmReleases/Kustomizations into arbitrary privileged manifests; effectively cluster-admin. | Tenants cannot create HelmReleases/Kustomizations (RBAC); not tenant-network-reachable. |
| `cozystack-api` aggregated API server (ServiceAccount) | T1 | CRUD HelmReleases, Secrets, and CiliumNetworkPolicies cluster-wide; reads Namespaces/RBAC/quotas/nodes. Acts as the policy-enforcing gateway that converts tenant CRs to HelmReleases. | A distinct, network-exposed attack surface from Flux; constrains tenants via fixed chart ref + name/quota/host validation + admission. |
| System operators / controllers | T1 | Domain-scoped cluster privileges (provision control planes, mint certs/secrets, program CNI). | Distinct ServiceAccounts, but all cluster-admin-adjacent; isolated from tenants only by RBAC and network policy. |
-| Tenant administrator (OIDC groups `-admin`, `-super-admin`) | T2 | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` within its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) The `-super-admin` group holds the `apps.cozystack.io/*` wildcard (and so can create child `Tenant`s); the `-admin` group has an enumerated allowlist that excludes some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`, `external-dns`, `bootbox`). | Namespace-scoped RoleBindings + admission policies + quota; no access to HelmReleases, arbitrary core writes, or other tenants. |
+| Tenant administrator (OIDC groups `-admin`, `-super-admin`) | T2 | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` within its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); `get` on persistentvolumeclaims and `create` on `upload.cdi.kubevirt.io/uploadtokenrequests`, the pair `virtctl image-upload` needs to upload into an existing VMDisk; no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) The `-super-admin` group holds the `apps.cozystack.io/*` wildcard (and so can create child `Tenant`s); the `-admin` group has an enumerated allowlist that excludes some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`, `external-dns`, `bootbox`). | Namespace-scoped RoleBindings + admission policies + quota; no access to HelmReleases, arbitrary core writes, or other tenants. |
| Tenant user (OIDC groups `-view`, `-use`) | T2 | Read-only view of Application CRs, pods, and services, plus operational verbs only (VM console/vnc/portforward, start/stop/restart). Cannot create Application CRs. | Strictly dominated by tenant-admin; cannot escalate to create/delete, so it is a distinct actor. |
| Parent tenant vs child (nested) tenant | T2 | Parent ServiceAccounts are bound into descendant namespaces; child ServiceAccounts are not bound upward. Quota is hierarchically capped. | Ancestor→descendant only; compromising a parent compromises its children, not the reverse. |
| Tenant workloads / managed services | T3 | Run tenant-controlled code; no cluster-API rights beyond their ServiceAccount. | Namespace-scoped RBAC/identity isolate their API objects. On the network, Cilium egress confines each tenant pod to its own tenant subtree, platform system services, and `world` (out-of-cluster), so unrelated tenants cannot reach each other over the pod network — enforced on the egress side (tenant-pod ingress is open to `cluster`; see Non-goals for residual caveats). |
@@ -92,7 +92,7 @@ The following are explicitly out of scope for the platform's security guarantees
The single most load-bearing isolation control is the combination of two facts:
-1. The tenant ClusterRoles (`packages/system/cozystack-basics/templates/clusterroles.yaml`) grant **no** `helm.toolkit.fluxcd.io` access and **no** create/update of arbitrary core resources — only CRUD on `apps.cozystack.io` application kinds (super-admin holds the `*` wildcard; admin an enumerated allowlist) and `sdn.cozystack.io/securitygroups`, read access to pods/services/ingresses, and (for tenant-admins) `delete` on a fixed set of core objects (pods/services/endpoints/events; the `persistentvolumes` entry in the ClusterRole is inert under the namespace-scoped RoleBinding). Tenant isolation is enforced by the *absence* of privileged grants in RBAC, not by an admission policy that whitelists `apps.cozystack.io/*` writes.
+1. The tenant ClusterRoles (`packages/system/cozystack-basics/templates/clusterroles.yaml`) grant **no** `helm.toolkit.fluxcd.io` access and **no** create/update of arbitrary core resources — only CRUD on `apps.cozystack.io` application kinds (super-admin holds the `*` wildcard; admin an enumerated allowlist) and `sdn.cozystack.io/securitygroups`, read access to pods/services/ingresses, and (for tenant-admins) `delete` on a fixed set of core objects (pods/services/endpoints/events; the `persistentvolumes` entry in the ClusterRole is inert under the namespace-scoped RoleBinding), `get` on persistentvolumeclaims and `create` on `upload.cdi.kubevirt.io/uploadtokenrequests` (for uploading an image into an existing VMDisk with `virtctl image-upload`). Tenant isolation is enforced by the *absence* of privileged grants in RBAC, not by an admission policy that whitelists `apps.cozystack.io/*` writes.
2. The Application-to-HelmRelease conversion (`pkg/registry/apps/application/rest.go`) takes the chart reference, release-name prefix, and `valuesFrom` from server-side configuration; the tenant controls only the Helm values (`app.Spec`).
If a future `ApplicationDefinition`, RBAC change, or API change ever let a tenant influence the chart reference, `valuesFrom`, or write HelmReleases directly, the T2→T1 isolation would collapse. Changes touching tenant RBAC or the Application REST layer should be reviewed against this invariant.
diff --git a/packages/apps/vm-disk/templates/NOTES.txt b/packages/apps/vm-disk/templates/NOTES.txt
index 73d07cc59..0940e75ba 100644
--- a/packages/apps/vm-disk/templates/NOTES.txt
+++ b/packages/apps/vm-disk/templates/NOTES.txt
@@ -1,5 +1,5 @@
{{- if and .Values.source (hasKey .Values.source "upload") }}
To upload your disk run:
-virtctl image-upload dv -n {{ .Release.Namespace }} {{ .Release.Name }} --image-path
+virtctl image-upload dv {{ .Release.Name }} --no-create --namespace {{ .Release.Namespace }} --image-path
{{- end }}
diff --git a/packages/apps/vm-disk/tests/upload_notes_test.yaml b/packages/apps/vm-disk/tests/upload_notes_test.yaml
new file mode 100644
index 000000000..16a6432a3
--- /dev/null
+++ b/packages/apps/vm-disk/tests/upload_notes_test.yaml
@@ -0,0 +1,18 @@
+suite: VM disk upload notes
+templates:
+ - templates/NOTES.txt
+
+release:
+ name: vm-disk-demo
+ namespace: tenant-root
+
+tests:
+ - it: reuses the chart-created DataVolume for local image upload
+ set:
+ source:
+ upload: {}
+ asserts:
+ - matchRegexRaw:
+ pattern: 'virtctl image-upload dv vm-disk-demo --no-create --namespace tenant-root --image-path '
+ - notMatchRegexRaw:
+ pattern: 'virtctl image-upload dv -n'
diff --git a/packages/system/cozystack-basics/templates/clusterroles.yaml b/packages/system/cozystack-basics/templates/clusterroles.yaml
index 9807d32f1..5df762886 100644
--- a/packages/system/cozystack-basics/templates/clusterroles.yaml
+++ b/packages/system/cozystack-basics/templates/clusterroles.yaml
@@ -235,6 +235,17 @@ metadata:
labels:
rbac.cozystack.io/aggregate-to-tenant-admin: "true"
rules:
+# virtctl reads both the disk PVC and CDI's dynamically named PVC Prime.
+- apiGroups: [""]
+ resources:
+ - persistentvolumeclaims
+ verbs:
+ - get
+- apiGroups: ["upload.cdi.kubevirt.io"]
+ resources:
+ - uploadtokenrequests
+ verbs:
+ - create
- apiGroups: [""]
resources:
- pods
diff --git a/packages/system/cozystack-basics/tests/cdi_upload_rbac_test.yaml b/packages/system/cozystack-basics/tests/cdi_upload_rbac_test.yaml
new file mode 100644
index 000000000..b99df44bf
--- /dev/null
+++ b/packages/system/cozystack-basics/tests/cdi_upload_rbac_test.yaml
@@ -0,0 +1,97 @@
+suite: tenant admin CDI upload RBAC
+
+templates:
+ - templates/clusterroles.yaml
+
+release:
+ name: cozystack-basics
+ namespace: cozy-system
+
+tests:
+ - it: lets tenant admins inspect upload PVCs and create CDI upload tokens
+ documentSelector:
+ path: metadata.name
+ value: cozy:tenant:admin:base
+ asserts:
+ - equal:
+ path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant-admin"]
+ value: "true"
+ - contains:
+ path: rules
+ content:
+ apiGroups: [""]
+ resources: ["persistentvolumeclaims"]
+ verbs: ["get"]
+ - contains:
+ path: rules
+ content:
+ apiGroups: ["upload.cdi.kubevirt.io"]
+ resources: ["uploadtokenrequests"]
+ verbs: ["create"]
+
+ - it: keeps the admin grant out of the use, view and service-account aggregation
+ documentSelector:
+ path: metadata.name
+ value: cozy:tenant:admin:base
+ asserts:
+ - notExists:
+ path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant-use"]
+ - notExists:
+ path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant-view"]
+ - notExists:
+ path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant"]
+
+ - it: does not grant CDI upload permissions to the use tier
+ documentSelector:
+ path: metadata.name
+ value: cozy:tenant:use:base
+ asserts:
+ - notContains:
+ path: rules
+ content:
+ apiGroups: [""]
+ resources: ["persistentvolumeclaims"]
+ verbs: ["get"]
+ - notContains:
+ path: rules
+ content:
+ apiGroups: ["upload.cdi.kubevirt.io"]
+ resources: ["uploadtokenrequests"]
+ verbs: ["create"]
+
+ - it: aggregates upload permissions through admin into super-admin
+ documentSelector:
+ path: metadata.name
+ value: cozy:tenant:admin
+ asserts:
+ - equal:
+ path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant-super-admin"]
+ value: "true"
+ - contains:
+ path: aggregationRule.clusterRoleSelectors
+ content:
+ matchLabels:
+ rbac.cozystack.io/aggregate-to-tenant-admin: "true"
+
+ - it: selects admin permissions for super-admin
+ documentSelector:
+ path: metadata.name
+ value: cozy:tenant:super-admin
+ asserts:
+ - contains:
+ path: aggregationRule.clusterRoleSelectors
+ content:
+ matchLabels:
+ rbac.cozystack.io/aggregate-to-tenant-super-admin: "true"
+
+ - it: does not grant CDI upload token creation to the view tier
+ documentSelector:
+ path: metadata.name
+ value: cozy:tenant:view:base
+ asserts:
+ - notContains:
+ path: rules
+ content:
+ apiGroups: ["upload.cdi.kubevirt.io"]
+ resources: ["uploadtokenrequests"]
+ verbs: ["create"]
From 982db83dd22d6a0257beda788cf75839c738ab42 Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Mon, 26 Oct 2026 20:10:59 +0300
Subject: [PATCH 021/216] fix(postgres): accept a standard five-field backup
schedule
The mongodb app and backup Plans take a standard five-field cron, but
the postgres chart hands backup.schedule to a CloudNativePG
ScheduledBackup, which parses it with robfig/cron v1 Parse. That parser
reads the first field as seconds and makes day-of-week optional, so a
five-field value passes admission with only a warning and runs one
field off. 0 2 * * * runs hourly and 5 * * * * every minute.
A five-field schedule now gets a leading 0 seconds field. Six fields
and descriptors pass through, and any other field count fails the
render. A release that already sets five fields gets the standard
meaning on upgrade, so 0 2 * * * goes from hourly to daily.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
api/apps/v1alpha1/postgresql/types.go | 2 +-
packages/apps/postgres/README.md | 15 ++--
packages/apps/postgres/templates/backup.yaml | 19 ++++-
.../postgres/tests/backup_schedule_test.yaml | 80 +++++++++++++++++++
packages/apps/postgres/values.schema.json | 2 +-
packages/apps/postgres/values.yaml | 2 +-
.../system/postgres-rd/cozyrds/postgres.yaml | 2 +-
7 files changed, 107 insertions(+), 15 deletions(-)
create mode 100644 packages/apps/postgres/tests/backup_schedule_test.yaml
diff --git a/api/apps/v1alpha1/postgresql/types.go b/api/apps/v1alpha1/postgresql/types.go
index f0415f398..a1e32ed26 100644
--- a/api/apps/v1alpha1/postgresql/types.go
+++ b/api/apps/v1alpha1/postgresql/types.go
@@ -124,7 +124,7 @@ type Backup struct {
// DEPRECATED. See `s3AccessKey`.
// +kubebuilder:default:=""
S3SecretKey string `json:"s3SecretKey,omitempty"`
- // Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.
+ // Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.
// +kubebuilder:default:=""
Schedule string `json:"schedule,omitempty"`
// Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`.
diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md
index bdc13ea9c..49f08d441 100644
--- a/packages/apps/postgres/README.md
+++ b/packages/apps/postgres/README.md
@@ -20,7 +20,7 @@ together rather than picking one:
| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- |
| **Archive plumbing** (chart, legacy) | Renders `spec.plugins` (referencing a barman-cloud `ObjectStore`) on the cnpg.io Cluster from helm-install onwards. CNPG runs the barman-cloud plugin as a WAL-archiver sidecar, every WAL switch ships to object storage, and any backup driver gets a complete WAL chain to start replay from. Renders only when `backup.enabled=true`, `useSystemBucket=false`, `destinationPath` is non-empty, AND inline-or-external creds are supplied. **Skipped in the platform `useSystemBucket=true` mode** — the CNPG backup driver SSA-applies the `ObjectStore` and patches `spec.plugins` onto the live Cluster at first BackupJob time; until that first BackupJob fires, plugin WAL archiving is not active and WAL accumulates on the PVC. Fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. | Legacy: `backup.enabled=true` plus `backup.destinationPath`, `backup.endpointURL`, and either `backup.s3AccessKey`+`backup.s3SecretKey` or `backup.s3CredentialsSecret`. Platform: `backup.enabled=true` plus `backup.useSystemBucket=true` — no S3 fields. |
| **Backup orchestration** (recommended) | Drives ad-hoc and scheduled backups, retention, and restores from `backups.cozystack.io` resources that can span multiple Postgres apps in a tenant. The driver SSA-applies its own barman-cloud `ObjectStore` and patches `spec.plugins` with `ForceOwnership` (so compression / target settings flow from the strategy template, while the live Cluster's `serverName` — its WAL-archive S3 prefix — is always preserved); chart-managed values cover the same destination so there is no tug-of-war on the live Cluster. | `strategy.backups.cozystack.io/CNPG` + `BackupClass` + `Plan` (recurring) or `BackupJob` (ad-hoc), restores via `RestoreJob` |
-| Legacy chart-emitted scheduled backup | The chart can also emit a `cnpg.io/ScheduledBackup` directly. Superseded by the BackupClass + Plan path, kept around for clusters that did not migrate. **Off by default** - rendered only when both `backup.enabled` and `backup.schedule` are non-empty. With `backup.schedule=""` plugin WAL archiving still runs; only the chart-emitted scheduled backup is silent. | `backup.enabled=true` plus `backup.schedule` (CNPG 6-field cron, e.g. `"0 2 * * * *"`) |
+| Legacy chart-emitted scheduled backup | The chart can also emit a `cnpg.io/ScheduledBackup` directly. Superseded by the BackupClass + Plan path, kept around for clusters that did not migrate. **Off by default** - rendered only when both `backup.enabled` and `backup.schedule` are non-empty. With `backup.schedule=""` plugin WAL archiving still runs; only the chart-emitted scheduled backup is silent. | `backup.enabled=true` plus `backup.schedule` (five-field cron such as `"0 2 * * *"`, or six fields with leading seconds) |
The **canonical setup** depends on whether the cluster ships the platform `cozy-default` BackupClass.
@@ -87,16 +87,11 @@ field.
### How to enable chart-managed scheduled backups (legacy)
-The chart can also emit a `cnpg.io/ScheduledBackup` directly, without a
-BackupClass. Superseded by the BackupClass + Plan path above and kept
-around for clusters that did not migrate. It does not run by default -
-`backup.schedule` defaults to an empty string, which gates the chart's
-ScheduledBackup template off. To turn it on, fill in a CNPG 6-field cron
-expression:
+The chart can also emit a `cnpg.io/ScheduledBackup` directly, without a BackupClass. Superseded by the BackupClass + Plan path above and kept around for clusters that did not migrate. It does not run by default - `backup.schedule` defaults to an empty string, which gates the chart's ScheduledBackup template off. To turn it on, set a standard five-field cron such as `0 2 * * *` for every day at 02:00, or the CloudNativePG six-field form with a leading seconds field such as `0 0 2 * * *` for the same time. The chart prepends a `0` seconds field to a five-field value, because CloudNativePG reads the first field as seconds and would otherwise run `0 2 * * *` every hour.
```yaml
## @param backup.enabled Enable plugin WAL archiving + render the chart-managed ScheduledBackup
-## @param backup.schedule Cron schedule (CNPG 6-field). Empty means no chart-managed ScheduledBackup
+## @param backup.schedule Cron schedule, five fields or six with leading seconds. Empty means no chart-managed ScheduledBackup
## @param backup.retentionPolicy Retention policy
## @param backup.destinationPath Path to store the backup (i.e. s3://bucket/path/to/folder)
## @param backup.endpointURL S3 Endpoint used to upload data to the cloud
@@ -107,7 +102,7 @@ backup:
retentionPolicy: 30d
destinationPath: s3://bucket/path/to/folder/
endpointURL: http://minio-gateway-service:9000
- schedule: "0 2 * * * *" # opt in - empty (the default) means no chart-managed schedule
+ schedule: "0 2 * * *" # opt in - empty (the default) means no chart-managed schedule
s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu
s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog
```
@@ -295,7 +290,7 @@ For `sslmode=verify-full` to work, the CA bundle retrieved above must be saved t
| `backup` | Backup configuration. | `object` | `{}` |
| `backup.enabled` | Enable regular backups. | `bool` | `false` |
| `backup.useSystemBucket` | Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`. | `bool` | `false` |
-| `backup.schedule` | Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. | `string` | `""` |
+| `backup.schedule` | Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. | `string` | `""` |
| `backup.retentionPolicy` | Retention policy (e.g. "30d"). | `string` | `30d` |
| `backup.destinationPath` | DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility. | `string` | `s3://bucket/path/to/folder/` |
| `backup.endpointURL` | DEPRECATED. See `destinationPath`. | `string` | `http://minio-gateway-service:9000` |
diff --git a/packages/apps/postgres/templates/backup.yaml b/packages/apps/postgres/templates/backup.yaml
index aa1d888e3..4025ca571 100644
--- a/packages/apps/postgres/templates/backup.yaml
+++ b/packages/apps/postgres/templates/backup.yaml
@@ -23,15 +23,32 @@
against a Cluster with no backup configuration and fail. This mirrors
the ClickHouse `useSystemBucket` gates on the chart-managed backup
resources.
+
+ CNPG parses `schedule` with robfig/cron v1 `Parse`, which takes the first
+ field as seconds and makes day-of-week optional. A standard five-field
+ cron passes its webhook with only a warning and then runs one field off,
+ so `5 * * * *` fires every minute instead of hourly. A five-field value
+ therefore gets a leading `0` seconds field. Six fields and descriptors
+ such as `@daily` pass through as they are, and any other field count
+ fails the render, which CNPG would reject at admission anyway.
*/}}
{{- if and .Values.backup.enabled .Values.backup.schedule (not .Values.backup.useSystemBucket) }}
+{{- $schedule := .Values.backup.schedule }}
+{{- $fields := regexFindAll `\S+` $schedule -1 }}
+{{- if not (hasPrefix "@" (trim $schedule)) }}
+{{- if eq (len $fields) 5 }}
+{{- $schedule = printf "0 %s" (join " " $fields) }}
+{{- else if ne (len $fields) 6 }}
+{{- fail (printf "postgres: backup.schedule %q is not a cron schedule with 5 or 6 fields (it has %d). Use a standard five-field cron such as \"0 2 * * *\", a six-field cron with leading seconds such as \"0 0 2 * * *\", or a descriptor such as \"@daily\"." $schedule (len $fields)) }}
+{{- end }}
+{{- end }}
---
apiVersion: postgresql.cnpg.io/v1
kind: ScheduledBackup
metadata:
name: {{ .Release.Name }}
spec:
- schedule: {{ .Values.backup.schedule | quote }}
+ schedule: {{ $schedule | quote }}
backupOwnerReference: self
method: plugin
pluginConfiguration:
diff --git a/packages/apps/postgres/tests/backup_schedule_test.yaml b/packages/apps/postgres/tests/backup_schedule_test.yaml
new file mode 100644
index 000000000..e78526848
--- /dev/null
+++ b/packages/apps/postgres/tests/backup_schedule_test.yaml
@@ -0,0 +1,80 @@
+suite: chart-managed scheduled backup takes five-field and six-field cron
+
+# CNPG parses the ScheduledBackup schedule with robfig/cron v1 Parse, which
+# takes the first field as seconds and accepts five fields with only a
+# warning, so an unconverted "5 * * * *" would run every minute.
+
+templates:
+ - templates/backup.yaml
+
+release:
+ name: pg-test
+ namespace: tenant-test
+
+tests:
+ - it: prepends a zero seconds field to a standard five-field schedule
+ set:
+ backup:
+ enabled: true
+ schedule: "5 * * * *"
+ asserts:
+ - equal:
+ path: spec.schedule
+ value: "0 5 * * * *"
+
+ - it: joins the fields of a five-field schedule with single spaces
+ set:
+ backup:
+ enabled: true
+ schedule: " 0 2 * *\t* "
+ asserts:
+ - equal:
+ path: spec.schedule
+ value: "0 0 2 * * *"
+
+ - it: passes a six-field schedule through unchanged
+ set:
+ backup:
+ enabled: true
+ schedule: "0 5 * * * *"
+ asserts:
+ - equal:
+ path: spec.schedule
+ value: "0 5 * * * *"
+
+ - it: passes a descriptor through unchanged
+ set:
+ backup:
+ enabled: true
+ schedule: "@every 6h"
+ asserts:
+ - equal:
+ path: spec.schedule
+ value: "@every 6h"
+
+ - it: refuses a schedule with four fields
+ set:
+ backup:
+ enabled: true
+ schedule: "5 * * *"
+ asserts:
+ - failedTemplate:
+ errorMessage: 'postgres: backup.schedule "5 * * *" is not a cron schedule with 5 or 6 fields (it has 4). Use a standard five-field cron such as "0 2 * * *", a six-field cron with leading seconds such as "0 0 2 * * *", or a descriptor such as "@daily".'
+
+ - it: refuses a schedule with seven fields
+ set:
+ backup:
+ enabled: true
+ schedule: "0 0 2 * * * 2026"
+ asserts:
+ - failedTemplate:
+ errorMessage: 'postgres: backup.schedule "0 0 2 * * * 2026" is not a cron schedule with 5 or 6 fields (it has 7). Use a standard five-field cron such as "0 2 * * *", a six-field cron with leading seconds such as "0 0 2 * * *", or a descriptor such as "@daily".'
+
+ - it: refuses a schedule of only spaces
+ set:
+ backup:
+ enabled: true
+ schedule: " "
+ asserts:
+ - failedTemplate:
+ errorMessage: 'postgres: backup.schedule " " is not a cron schedule with 5 or 6 fields (it has 0). Use a standard five-field cron such as "0 2 * * *", a six-field cron with leading seconds such as "0 0 2 * * *", or a descriptor such as "@daily".'
diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json
index 91747ba24..799b05776 100644
--- a/packages/apps/postgres/values.schema.json
+++ b/packages/apps/postgres/values.schema.json
@@ -427,7 +427,7 @@
"default": ""
},
"schedule": {
- "description": "Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.",
+ "description": "Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.",
"type": "string",
"default": ""
},
diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml
index 83b658a64..a938497d2 100644
--- a/packages/apps/postgres/values.yaml
+++ b/packages/apps/postgres/values.yaml
@@ -321,7 +321,7 @@ databases: {}
## @typedef {struct} Backup - Backup configuration.
## @field {bool} enabled - Enable regular backups.
## @field {bool} [useSystemBucket] - Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`.
-## @field {string} [schedule] - Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.
+## @field {string} [schedule] - Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.
## @field {string} [retentionPolicy] - Retention policy (e.g. "30d").
## @field {string} [destinationPath] - DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility.
## @field {string} [endpointURL] - DEPRECATED. See `destinationPath`.
diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml
index 39adc0f58..aeea68e12 100644
--- a/packages/system/postgres-rd/cozyrds/postgres.yaml
+++ b/packages/system/postgres-rd/cozyrds/postgres.yaml
@@ -30,7 +30,7 @@ spec:
singular: postgres
plural: postgreses
openAPISchema: |-
- {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas. Under active autoscaling this is the resting/seed count, not the live count — KEDA drives the live count via `/scale`. Disabling autoscaling does NOT automatically bring the live count back to `replicas`: the client-side merge writes `instances` only when the rendered value changes, so a cluster KEDA grew past `replicas` stays there with no autoscaler left to reduce it — lower `replicas` to the count you want and it rebases down (shedding the surplus replicas + PVCs). Lowering it below the live count, or disabling/dry-running while it sits below the live count, sheds in one step past the scaleDown pacing; stage it to the live count first (suspend the HelmRelease and `kubectl scale` if needed) to avoid a transient. Must stay greater than `quorum.maxSyncReplicas`.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"flavor":{"description":"PostgreSQL image flavor, fixed when the cluster is created. `postgresql` uses the CloudNativePG PostgreSQL image. `postgis` uses the CloudNativePG PostGIS image, which adds postgis, postgis_raster, postgis_topology, postgis_sfcgal, postgis_tiger_geocoder and address_standardizer; enable them per database in `databases..extensions`. Changing the flavor of an existing cluster is refused, and a backup restores only into a cluster of the flavor it was taken from: to change flavor, create a new cluster and move the data with a logical dump (pg_dump / pg_restore).","type":"string","default":"postgresql","enum":["postgresql","postgis"]},"tls":{"description":"TLS configuration for server connections.","type":"object","default":{},"properties":{"enabled":{"description":"Tri-state switch controlling whether the chart injects the external hostnames into the operator-managed CNPG cert via spec.certificates.serverAltDNSNames. When omitted, the chart injects them if `external: true` and skips them otherwise. Set explicitly to `true` to inject regardless of `external` (no-op when `external: false` since there is no external hostname to add). Set to `false` to skip injection. Note that CNPG keeps its built-in TLS on the wire regardless of this flag — this toggle only controls the chart-side SAN injection; to disable PostgreSQL TLS entirely set `postgresql.parameters.ssl = \"off\"` at the CNPG layer.","type":"boolean"},"required":{"description":"Tri-state switch controlling whether the server refuses sessions that are not encrypted. When omitted, it is on if `external: true` and off otherwise. When on, the chart renders `spec.postgresql.pg_hba` with `hostssl all all all scram-sha-256` (`md5` on PostgreSQL 13, matching CloudNativePG's default rule) followed by `host all all all reject`, so a client that connects with `sslmode=disable` is refused instead of getting a plaintext session; in-cluster clients are held to the same rule, since the pod network cannot be told apart from an external client. Set to `false` to keep accepting plaintext sessions on an external instance, or to `true` to require TLS on an in-cluster one. Rendering fails if `postgresql.parameters.ssl` is off while this is in effect.","type":"boolean"}}},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters. Values may be strings or integers; integers are coerced to strings by the template (e.g. both `max_connections: 100` and `max_connections: \"100\"` are accepted). BLOCKED (enable arbitrary code execution): archive_command, restore_command, ssl_passphrase_command, archive_cleanup_command, recovery_end_command, dynamic_library_path, local_preload_libraries, session_preload_libraries, shared_preload_libraries. Do NOT override CloudNativePG-managed parameters: archive_mode, primary_conninfo, wal_level, max_replication_slots.","type":"object","default":{"max_connections":"100"},"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}},"sharedPreloadLibraries":{"description":"Libraries loaded at server start, restricted to an allowlist of modules already shipped in the PostgreSQL image. CloudNativePG also preloads `pg_stat_statements`, `auto_explain` and `pgaudit` by itself whenever `parameters` sets a key under their prefix (e.g. `pg_stat_statements.track`), so removing one of them from this list does not unload it while such a key is set; `pg_prewarm` is preloaded only through this list. Changing the list restarts PostgreSQL on every instance, the primary included, so writes pause briefly.","type":"array","items":{"type":"string","enum":["pg_stat_statements","auto_explain","pgaudit","pg_prewarm"]}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas). Under autoscaling it also sets the scale-down floor (the effective minimum is `max(minReplicas, maxSyncReplicas+1, 2)`); the chart fails to render if that floor exceeds `autoscaling.maxReplicas`.","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"autoscaling":{"description":"Read-replica autoscaling configuration.","type":"object","default":{},"required":["dryRun","enabled","maxReplicas","maxReplicationLagSeconds","metric","minReplicas","serverAddress","target","transition"],"properties":{"dryRun":{"description":"Recommendation mode: keep the static count and render the ScaledObject with both scaling directions paused, so KEDA keeps the HPA and keeps serving its trigger metric but actuates neither way. Note both scaling policies are disabled, so the HPA's desiredReplicas is frozen to the current count - read the recommendation from the matching read-load dashboard panel for your metric (connections or CPU): desired = 1 + ceil(read load / target), clamped to [effectiveMin..maxReplicas] where effectiveMin = max(minReplicas, maxSyncReplicas+1, 2), not from the desired-replicas line. Like disabling, flipping a LIVE autoscaled cluster into dryRun re-renders `instances: replicas` (the raw static value, which can be below the autoscaling floor); if `replicas` is below the live count, stage it to the live count first (see the enablement note) or the cluster sheds down to `replicas`. No shed occurs when the rendered value is unchanged (replicas already equals the active seed) — client-side merge is then a no-op and the live count is preserved. Permanent, unlike `transition`.","type":"boolean","default":false},"enabled":{"description":"Enable horizontal autoscaling of read replicas. Requires the platform `keda` package to be enabled by an administrator first; the chart fails to render (rather than break the release) if KEDA is not installed. Prerequisite: the queried vmselect must hold BOTH the CNPG metrics (cnpg_backends_total / container CPU) AND kube_pod_labels (kube-state-metrics). This holds for a tenant on the shared root monitoring stack (`monitoring: false`, both land in tenant-root). A tenant running its OWN Monitoring app (`monitoring: true`) has the CNPG metrics but not kube_pod_labels (KSM is root-only), so it must set `serverAddress` to a vmselect carrying both series or autoscaling never actuates.","type":"boolean","default":false},"maxReplicas":{"description":"Maximum total instances. Must be >= minReplicas and >= the synchronous-quorum floor (maxSyncReplicas+1); the chart fails to render (rather than silently run an unbounded instance count) if the effective floor exceeds it - raise maxReplicas. HAZARD: lowering maxReplicas below the live count makes the HPA cut straight to the new maximum in one step (the current>max branch bypasses the scale-down pacing), shedding the surplus replicas + PVCs at once; to lower it deliberately, suspend the HelmRelease and pin the live count with `kubectl scale` first.","type":"integer","default":6},"maxReplicationLagSeconds":{"description":"Freeze scaling while replication lag exceeds this (seconds) and the primary is writing. 0 disables the brake. Default 0: the freeze branch has only been validated on a live cluster in its non-braking (pass-through) form, so the lag brake is opt-in until the braking path is exercised under load; set a positive value (e.g. 30) to enable it.","type":"integer","default":0},"metric":{"description":"Read-load signal to scale on.","type":"string","default":"ReadConnections","enum":["ReadConnections","ReadCPUUtilization"]},"minReplicas":{"description":"Minimum total instances; raised to the synchronous-quorum floor (`maxSyncReplicas + 1`) and to 2 when either is higher.","type":"integer","default":2},"serverAddress":{"description":"Override the Prometheus/vmselect URL the ScaledObject queries. Empty (default) derives `http://vmselect-shortterm..svc:8481/select/0/prometheus`. Set it when the derived vmselect does not hold both the CNPG metrics and kube_pod_labels (e.g. `monitoring: true` tenants, or a non-default `metricsStorages` name). SSRF constraint: the host must be an in-cluster Service DNS name (`..svc[.]`) whose namespace is within the tenant's own subtree (its namespace, its parents, or `tenant-root`) — the cluster-privileged KEDA operator fetches this URL every poll, so it is refused a metadata endpoint, an external host, or a sibling/system namespace. Residual surface (accepted trade-off): `tenant-root` is always allowed because the derived default points there, so a tenant can still aim the operator at any Service in `tenant-root` on any port/path (a blind in-cluster reachability probe — the operator reads no response body back to the tenant); the egress NetworkPolicy does not narrow that.","type":"string","default":""},"target":{"description":"Target read load per read-serving replica (unit depends on `metric`: active connections, or CPU millicores). Dimensionless integer - a resource.Quantity here would accept `150m`, which PromQL reads as 150 minutes and KEDA cannot use as a threshold.","type":"integer","default":150},"transition":{"description":"Migration phase 1: render `.spec.instances` from `replicas` (which the operator stages to the current live count) AND pause the ScaledObject in both directions, so KEDA stands up with its HPA present but not actuating. Flip to false (phase 2) to hand the runtime count to KEDA. Because the both-direction pause keeps the HPA (not the HPA-deleting full pause), KEDA is genuinely warm at phase 2 — no HPA recreate — and because the active seed is max(replicas, effectiveMin), phase 2 re-renders the same instance count phase 1 did (when `replicas` is staged at/above the floor), so the flip is a merge no-op with no dip. The HPA's scaleUp stabilization window still smooths the first actuation.","type":"boolean","default":false}}},"users":{"description":"Users configuration map. Passwords are always auto-generated and stored in the `-credentials` Secret; they cannot be set from values — read a user's password from that Secret. A `password` left over in values from before the field was removed is ignored by the render and draws an admission warning, but it is not inert on an upgraded release: the chart preserves whatever password is already in the Secret, which on the first upgrade is the value that was set before removal, so that value stays the live credential until it is rotated (dedicated rotation is tracked in cozystack/community#72). The live password lives only in the Secret.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"bypassRls":{"description":"Whether the user can bypass row-level security policies (BYPASSRLS). A role can only create other roles carrying BYPASSRLS if it holds the attribute itself, so this is required alongside `createRole` to provision a privileged service role.","type":"boolean"},"createRole":{"description":"Whether the user can create and manage roles (CREATEROLE). Requires PostgreSQL 16 or newer. Setting it back to `false` withdraws the attribute only: the roles the user created, and its ADMIN and SET memberships in them, stay. Once any user has held it, the chart no longer adopts an existing role it did not create, so a user or database declared later under the name of such a role fails the init job until a platform administrator drops that role.","type":"boolean"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility.","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointCA":{"description":"DEPRECATED. Pre-existing Secret with the CA bundle the barman-cloud plugin should trust when reaching a self-signed S3 endpoint. Used for both backup and bootstrap recovery in the legacy chart-managed flow.","type":"object","default":{},"properties":{"key":{"description":"Key within the Secret containing the CA bundle. Defaults to `ca.crt`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means no endpointCA is emitted (the plugin uses the system trust store).","type":"string","default":""}}},"endpointURL":{"description":"DEPRECATED. See `destinationPath`.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"DEPRECATED. Tenants no longer supply S3 keys; the system Bucket Secret is projected into the tenant namespace by the backup controller. Ignored when `s3CredentialsSecret.name` is set or `useSystemBucket` is true. The chart skips materialising `-s3-creds` whenever this field is empty so a default install does not leak placeholder credentials into the tenant namespace.","type":"string","default":""},"s3CredentialsSecret":{"description":"DEPRECATED. Pre-existing Secret with S3 credentials. Use the platform-managed `cozy-default` BackupClass instead. When set, the chart references this Secret directly (legacy chart-managed flow). The CNPG backup driver writes this field on restore so credentials never land in the CR `.spec`.","type":"object","default":{},"properties":{"accessKeyIDKey":{"description":"Key in the Secret holding the access key ID. Defaults to `AWS_ACCESS_KEY_ID`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means the chart materialises `-s3-creds` from `s3AccessKey`/`s3SecretKey`.","type":"string","default":""},"secretAccessKeyKey":{"description":"Key in the Secret holding the secret access key. Defaults to `AWS_SECRET_ACCESS_KEY`.","type":"string","default":""}}},"s3SecretKey":{"description":"DEPRECATED. See `s3AccessKey`.","type":"string","default":""},"schedule":{"description":"Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.","type":"string","default":""},"useSystemBucket":{"description":"Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`.","type":"boolean","default":false}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"backupID":{"description":"Barman ID of the base backup to recover from, passed as `recoveryTarget.backupID`. Empty lets the barman-cloud plugin pick one: the newest backup when `recoveryTime` is empty, otherwise the newest one ending at or before it, on any timeline. The CNPG backup driver sets this on restore.","type":"string","default":""},"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"newServerName":{"description":"WAL-archive server name (S3 path prefix) the RESTORED cluster writes to. Must differ from `serverName` (the recovery source) so a restored cluster archives to a fresh, empty prefix and barman-cloud-check-wal-archive passes while it recovers from the source's prefix. The CNPG backup driver sets this on restore; empty means the cluster archives under its own name.","type":"string","default":""},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Server name (S3 path prefix) used by the original cluster when writing backups; passed to the barman-cloud plugin via `externalClusters[].plugin.parameters.serverName`. Defaults to `bootstrap.oldName`. Set this only when the original cluster wrote backups under an explicit server name that differed from its Kubernetes resource name.","type":"string","default":""}}}}}
+ {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas. Under active autoscaling this is the resting/seed count, not the live count — KEDA drives the live count via `/scale`. Disabling autoscaling does NOT automatically bring the live count back to `replicas`: the client-side merge writes `instances` only when the rendered value changes, so a cluster KEDA grew past `replicas` stays there with no autoscaler left to reduce it — lower `replicas` to the count you want and it rebases down (shedding the surplus replicas + PVCs). Lowering it below the live count, or disabling/dry-running while it sits below the live count, sheds in one step past the scaleDown pacing; stage it to the live count first (suspend the HelmRelease and `kubectl scale` if needed) to avoid a transient. Must stay greater than `quorum.maxSyncReplicas`.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"flavor":{"description":"PostgreSQL image flavor, fixed when the cluster is created. `postgresql` uses the CloudNativePG PostgreSQL image. `postgis` uses the CloudNativePG PostGIS image, which adds postgis, postgis_raster, postgis_topology, postgis_sfcgal, postgis_tiger_geocoder and address_standardizer; enable them per database in `databases..extensions`. Changing the flavor of an existing cluster is refused, and a backup restores only into a cluster of the flavor it was taken from: to change flavor, create a new cluster and move the data with a logical dump (pg_dump / pg_restore).","type":"string","default":"postgresql","enum":["postgresql","postgis"]},"tls":{"description":"TLS configuration for server connections.","type":"object","default":{},"properties":{"enabled":{"description":"Tri-state switch controlling whether the chart injects the external hostnames into the operator-managed CNPG cert via spec.certificates.serverAltDNSNames. When omitted, the chart injects them if `external: true` and skips them otherwise. Set explicitly to `true` to inject regardless of `external` (no-op when `external: false` since there is no external hostname to add). Set to `false` to skip injection. Note that CNPG keeps its built-in TLS on the wire regardless of this flag — this toggle only controls the chart-side SAN injection; to disable PostgreSQL TLS entirely set `postgresql.parameters.ssl = \"off\"` at the CNPG layer.","type":"boolean"},"required":{"description":"Tri-state switch controlling whether the server refuses sessions that are not encrypted. When omitted, it is on if `external: true` and off otherwise. When on, the chart renders `spec.postgresql.pg_hba` with `hostssl all all all scram-sha-256` (`md5` on PostgreSQL 13, matching CloudNativePG's default rule) followed by `host all all all reject`, so a client that connects with `sslmode=disable` is refused instead of getting a plaintext session; in-cluster clients are held to the same rule, since the pod network cannot be told apart from an external client. Set to `false` to keep accepting plaintext sessions on an external instance, or to `true` to require TLS on an in-cluster one. Rendering fails if `postgresql.parameters.ssl` is off while this is in effect.","type":"boolean"}}},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters. Values may be strings or integers; integers are coerced to strings by the template (e.g. both `max_connections: 100` and `max_connections: \"100\"` are accepted). BLOCKED (enable arbitrary code execution): archive_command, restore_command, ssl_passphrase_command, archive_cleanup_command, recovery_end_command, dynamic_library_path, local_preload_libraries, session_preload_libraries, shared_preload_libraries. Do NOT override CloudNativePG-managed parameters: archive_mode, primary_conninfo, wal_level, max_replication_slots.","type":"object","default":{"max_connections":"100"},"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}},"sharedPreloadLibraries":{"description":"Libraries loaded at server start, restricted to an allowlist of modules already shipped in the PostgreSQL image. CloudNativePG also preloads `pg_stat_statements`, `auto_explain` and `pgaudit` by itself whenever `parameters` sets a key under their prefix (e.g. `pg_stat_statements.track`), so removing one of them from this list does not unload it while such a key is set; `pg_prewarm` is preloaded only through this list. Changing the list restarts PostgreSQL on every instance, the primary included, so writes pause briefly.","type":"array","items":{"type":"string","enum":["pg_stat_statements","auto_explain","pgaudit","pg_prewarm"]}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas). Under autoscaling it also sets the scale-down floor (the effective minimum is `max(minReplicas, maxSyncReplicas+1, 2)`); the chart fails to render if that floor exceeds `autoscaling.maxReplicas`.","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"autoscaling":{"description":"Read-replica autoscaling configuration.","type":"object","default":{},"required":["dryRun","enabled","maxReplicas","maxReplicationLagSeconds","metric","minReplicas","serverAddress","target","transition"],"properties":{"dryRun":{"description":"Recommendation mode: keep the static count and render the ScaledObject with both scaling directions paused, so KEDA keeps the HPA and keeps serving its trigger metric but actuates neither way. Note both scaling policies are disabled, so the HPA's desiredReplicas is frozen to the current count - read the recommendation from the matching read-load dashboard panel for your metric (connections or CPU): desired = 1 + ceil(read load / target), clamped to [effectiveMin..maxReplicas] where effectiveMin = max(minReplicas, maxSyncReplicas+1, 2), not from the desired-replicas line. Like disabling, flipping a LIVE autoscaled cluster into dryRun re-renders `instances: replicas` (the raw static value, which can be below the autoscaling floor); if `replicas` is below the live count, stage it to the live count first (see the enablement note) or the cluster sheds down to `replicas`. No shed occurs when the rendered value is unchanged (replicas already equals the active seed) — client-side merge is then a no-op and the live count is preserved. Permanent, unlike `transition`.","type":"boolean","default":false},"enabled":{"description":"Enable horizontal autoscaling of read replicas. Requires the platform `keda` package to be enabled by an administrator first; the chart fails to render (rather than break the release) if KEDA is not installed. Prerequisite: the queried vmselect must hold BOTH the CNPG metrics (cnpg_backends_total / container CPU) AND kube_pod_labels (kube-state-metrics). This holds for a tenant on the shared root monitoring stack (`monitoring: false`, both land in tenant-root). A tenant running its OWN Monitoring app (`monitoring: true`) has the CNPG metrics but not kube_pod_labels (KSM is root-only), so it must set `serverAddress` to a vmselect carrying both series or autoscaling never actuates.","type":"boolean","default":false},"maxReplicas":{"description":"Maximum total instances. Must be >= minReplicas and >= the synchronous-quorum floor (maxSyncReplicas+1); the chart fails to render (rather than silently run an unbounded instance count) if the effective floor exceeds it - raise maxReplicas. HAZARD: lowering maxReplicas below the live count makes the HPA cut straight to the new maximum in one step (the current>max branch bypasses the scale-down pacing), shedding the surplus replicas + PVCs at once; to lower it deliberately, suspend the HelmRelease and pin the live count with `kubectl scale` first.","type":"integer","default":6},"maxReplicationLagSeconds":{"description":"Freeze scaling while replication lag exceeds this (seconds) and the primary is writing. 0 disables the brake. Default 0: the freeze branch has only been validated on a live cluster in its non-braking (pass-through) form, so the lag brake is opt-in until the braking path is exercised under load; set a positive value (e.g. 30) to enable it.","type":"integer","default":0},"metric":{"description":"Read-load signal to scale on.","type":"string","default":"ReadConnections","enum":["ReadConnections","ReadCPUUtilization"]},"minReplicas":{"description":"Minimum total instances; raised to the synchronous-quorum floor (`maxSyncReplicas + 1`) and to 2 when either is higher.","type":"integer","default":2},"serverAddress":{"description":"Override the Prometheus/vmselect URL the ScaledObject queries. Empty (default) derives `http://vmselect-shortterm..svc:8481/select/0/prometheus`. Set it when the derived vmselect does not hold both the CNPG metrics and kube_pod_labels (e.g. `monitoring: true` tenants, or a non-default `metricsStorages` name). SSRF constraint: the host must be an in-cluster Service DNS name (`..svc[.]`) whose namespace is within the tenant's own subtree (its namespace, its parents, or `tenant-root`) — the cluster-privileged KEDA operator fetches this URL every poll, so it is refused a metadata endpoint, an external host, or a sibling/system namespace. Residual surface (accepted trade-off): `tenant-root` is always allowed because the derived default points there, so a tenant can still aim the operator at any Service in `tenant-root` on any port/path (a blind in-cluster reachability probe — the operator reads no response body back to the tenant); the egress NetworkPolicy does not narrow that.","type":"string","default":""},"target":{"description":"Target read load per read-serving replica (unit depends on `metric`: active connections, or CPU millicores). Dimensionless integer - a resource.Quantity here would accept `150m`, which PromQL reads as 150 minutes and KEDA cannot use as a threshold.","type":"integer","default":150},"transition":{"description":"Migration phase 1: render `.spec.instances` from `replicas` (which the operator stages to the current live count) AND pause the ScaledObject in both directions, so KEDA stands up with its HPA present but not actuating. Flip to false (phase 2) to hand the runtime count to KEDA. Because the both-direction pause keeps the HPA (not the HPA-deleting full pause), KEDA is genuinely warm at phase 2 — no HPA recreate — and because the active seed is max(replicas, effectiveMin), phase 2 re-renders the same instance count phase 1 did (when `replicas` is staged at/above the floor), so the flip is a merge no-op with no dip. The HPA's scaleUp stabilization window still smooths the first actuation.","type":"boolean","default":false}}},"users":{"description":"Users configuration map. Passwords are always auto-generated and stored in the `-credentials` Secret; they cannot be set from values — read a user's password from that Secret. A `password` left over in values from before the field was removed is ignored by the render and draws an admission warning, but it is not inert on an upgraded release: the chart preserves whatever password is already in the Secret, which on the first upgrade is the value that was set before removal, so that value stays the live credential until it is rotated (dedicated rotation is tracked in cozystack/community#72). The live password lives only in the Secret.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"bypassRls":{"description":"Whether the user can bypass row-level security policies (BYPASSRLS). A role can only create other roles carrying BYPASSRLS if it holds the attribute itself, so this is required alongside `createRole` to provision a privileged service role.","type":"boolean"},"createRole":{"description":"Whether the user can create and manage roles (CREATEROLE). Requires PostgreSQL 16 or newer. Setting it back to `false` withdraws the attribute only: the roles the user created, and its ADMIN and SET memberships in them, stay. Once any user has held it, the chart no longer adopts an existing role it did not create, so a user or database declared later under the name of such a role fails the init job until a platform administrator drops that role.","type":"boolean"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility.","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointCA":{"description":"DEPRECATED. Pre-existing Secret with the CA bundle the barman-cloud plugin should trust when reaching a self-signed S3 endpoint. Used for both backup and bootstrap recovery in the legacy chart-managed flow.","type":"object","default":{},"properties":{"key":{"description":"Key within the Secret containing the CA bundle. Defaults to `ca.crt`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means no endpointCA is emitted (the plugin uses the system trust store).","type":"string","default":""}}},"endpointURL":{"description":"DEPRECATED. See `destinationPath`.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"DEPRECATED. Tenants no longer supply S3 keys; the system Bucket Secret is projected into the tenant namespace by the backup controller. Ignored when `s3CredentialsSecret.name` is set or `useSystemBucket` is true. The chart skips materialising `-s3-creds` whenever this field is empty so a default install does not leak placeholder credentials into the tenant namespace.","type":"string","default":""},"s3CredentialsSecret":{"description":"DEPRECATED. Pre-existing Secret with S3 credentials. Use the platform-managed `cozy-default` BackupClass instead. When set, the chart references this Secret directly (legacy chart-managed flow). The CNPG backup driver writes this field on restore so credentials never land in the CR `.spec`.","type":"object","default":{},"properties":{"accessKeyIDKey":{"description":"Key in the Secret holding the access key ID. Defaults to `AWS_ACCESS_KEY_ID`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means the chart materialises `-s3-creds` from `s3AccessKey`/`s3SecretKey`.","type":"string","default":""},"secretAccessKeyKey":{"description":"Key in the Secret holding the secret access key. Defaults to `AWS_SECRET_ACCESS_KEY`.","type":"string","default":""}}},"s3SecretKey":{"description":"DEPRECATED. See `s3AccessKey`.","type":"string","default":""},"schedule":{"description":"Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.","type":"string","default":""},"useSystemBucket":{"description":"Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`.","type":"boolean","default":false}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"backupID":{"description":"Barman ID of the base backup to recover from, passed as `recoveryTarget.backupID`. Empty lets the barman-cloud plugin pick one: the newest backup when `recoveryTime` is empty, otherwise the newest one ending at or before it, on any timeline. The CNPG backup driver sets this on restore.","type":"string","default":""},"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"newServerName":{"description":"WAL-archive server name (S3 path prefix) the RESTORED cluster writes to. Must differ from `serverName` (the recovery source) so a restored cluster archives to a fresh, empty prefix and barman-cloud-check-wal-archive passes while it recovers from the source's prefix. The CNPG backup driver sets this on restore; empty means the cluster archives under its own name.","type":"string","default":""},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Server name (S3 path prefix) used by the original cluster when writing backups; passed to the barman-cloud plugin via `externalClusters[].plugin.parameters.serverName`. Defaults to `bootstrap.oldName`. Set this only when the original cluster wrote backups under an explicit server name that differed from its Kubernetes resource name.","type":"string","default":""}}}}}
release:
prefix: postgres-
labels:
From c0c5f2580fb3d878efd34b91c9e088c9aef4db8d Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Tue, 27 Oct 2026 12:20:45 +0300
Subject: [PATCH 022/216] docs(postgres): note how an existing backup schedule
changes on upgrade
A release that already sets backup.schedule to "0 2 * * *" runs hourly
at minute 2 today and daily at 02:00 after this change. The README now
says so and gives "0 2 * * * *" for keeping the hourly run.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
packages/apps/postgres/README.md | 2 ++
1 file changed, 2 insertions(+)
diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md
index 49f08d441..9185b35d5 100644
--- a/packages/apps/postgres/README.md
+++ b/packages/apps/postgres/README.md
@@ -89,6 +89,8 @@ field.
The chart can also emit a `cnpg.io/ScheduledBackup` directly, without a BackupClass. Superseded by the BackupClass + Plan path above and kept around for clusters that did not migrate. It does not run by default - `backup.schedule` defaults to an empty string, which gates the chart's ScheduledBackup template off. To turn it on, set a standard five-field cron such as `0 2 * * *` for every day at 02:00, or the CloudNativePG six-field form with a leading seconds field such as `0 0 2 * * *` for the same time. The chart prepends a `0` seconds field to a five-field value, because CloudNativePG reads the first field as seconds and would otherwise run `0 2 * * *` every hour.
+On upgrade, an existing `backup.schedule: "0 2 * * *"` changes from hourly at minute 2 to daily at 02:00. To keep the previous hourly schedule, set `backup.schedule: "0 2 * * * *"` before upgrading.
+
```yaml
## @param backup.enabled Enable plugin WAL archiving + render the chart-managed ScheduledBackup
## @param backup.schedule Cron schedule, five fields or six with leading seconds. Empty means no chart-managed ScheduledBackup
From 3990268195c1b0133d365bb1c40d8c777cd7e1a1 Mon Sep 17 00:00:00 2001
From: Fedor Batonogov
Date: Wed, 28 Oct 2026 11:42:37 +0300
Subject: [PATCH 023/216] feat(platform): expose Keycloak truststores through
platform values
Assisted-by: LLM
Signed-off-by: Fedor Batonogov
---
.../platform/templates/bundles/system.yaml | 10 +
.../bundles_keycloak_truststores_test.yaml | 203 ++++++++++++++++++
packages/core/platform/values.yaml | 13 ++
3 files changed, 226 insertions(+)
create mode 100644 packages/core/platform/tests/bundles_keycloak_truststores_test.yaml
diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml
index b175714a2..2d0fa3506 100644
--- a/packages/core/platform/templates/bundles/system.yaml
+++ b/packages/core/platform/templates/bundles/system.yaml
@@ -178,9 +178,19 @@
{{- $controllerValues := dict "cozystackController" (dict "disableTelemetry" true) -}}
{{- $_ := set $cozystackEngineComponents "cozystack-controller" (dict "values" $controllerValues) -}}
{{- end -}}
+{{- $keycloakTruststores := .Values.authentication.oidc.keycloakTruststores -}}
+{{- $hasKeycloakTruststores := and (not (kindIs "invalid" $keycloakTruststores)) (or (not (kindIs "slice" $keycloakTruststores)) (not (empty $keycloakTruststores))) -}}
+{{- $keycloakComponents := dict -}}
+{{- if $hasKeycloakTruststores }}
+{{- $_ := set $keycloakComponents "keycloak" (dict "values" (dict "truststores" $keycloakTruststores)) -}}
+{{- end }}
{{- if .Values.authentication.oidc.enabled }}
{{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }}
+{{- if $hasKeycloakTruststores }}
+{{include "cozystack.platform.package" (list "cozystack.keycloak" "default" $ $keycloakComponents) }}
+{{- else }}
{{include "cozystack.platform.package.default" (list "cozystack.keycloak" $) }}
+{{- end }}
{{include "cozystack.platform.package.default" (list "cozystack.keycloak-operator" $) }}
{{- else }}
{{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "default" $ $cozystackEngineComponents) }}
diff --git a/packages/core/platform/tests/bundles_keycloak_truststores_test.yaml b/packages/core/platform/tests/bundles_keycloak_truststores_test.yaml
new file mode 100644
index 000000000..6319c6dd9
--- /dev/null
+++ b/packages/core/platform/tests/bundles_keycloak_truststores_test.yaml
@@ -0,0 +1,203 @@
+suite: bundles.system Keycloak truststores wiring
+templates:
+ - templates/bundles/system.yaml
+release:
+ name: cozystack
+ namespace: cozy-system
+tests:
+ - it: emits cozystack.keycloak without a components block when keycloakTruststores is unset
+ set:
+ bundles:
+ system:
+ enabled: true
+ variant: isp-full
+ authentication:
+ oidc:
+ enabled: true
+ documentSelector:
+ path: metadata.name
+ value: cozystack.keycloak
+ asserts:
+ - equal:
+ path: kind
+ value: Package
+ - equal:
+ path: spec.variant
+ value: default
+ - notExists:
+ path: spec.components
+
+ - it: omits the components block when keycloakTruststores is explicitly empty
+ set:
+ bundles:
+ system:
+ enabled: true
+ variant: isp-full
+ authentication:
+ oidc:
+ enabled: true
+ keycloakTruststores: []
+ documentSelector:
+ path: metadata.name
+ value: cozystack.keycloak
+ asserts:
+ - notExists:
+ path: spec.components
+
+ - it: omits the components block when keycloakTruststores is explicitly null
+ set:
+ bundles:
+ system:
+ enabled: true
+ variant: isp-full
+ authentication:
+ oidc:
+ enabled: true
+ keycloakTruststores: null
+ documentSelector:
+ path: metadata.name
+ value: cozystack.keycloak
+ asserts:
+ - notExists:
+ path: spec.components
+
+ - it: forwards false so the Keycloak chart can reject it instead of silently removing trust
+ set:
+ bundles:
+ system:
+ enabled: true
+ variant: isp-full
+ authentication:
+ oidc:
+ enabled: true
+ keycloakTruststores: false
+ documentSelector:
+ path: metadata.name
+ value: cozystack.keycloak
+ asserts:
+ - equal:
+ path: spec.components.keycloak.values.truststores
+ value: false
+
+ - it: forwards zero so the Keycloak chart can reject it instead of silently removing trust
+ set:
+ bundles:
+ system:
+ enabled: true
+ variant: isp-full
+ authentication:
+ oidc:
+ enabled: true
+ keycloakTruststores: 0
+ documentSelector:
+ path: metadata.name
+ value: cozystack.keycloak
+ asserts:
+ - equal:
+ path: spec.components.keycloak.values.truststores
+ value: 0
+
+ - it: forwards an empty string so the Keycloak chart can reject it instead of silently removing trust
+ set:
+ bundles:
+ system:
+ enabled: true
+ variant: isp-full
+ authentication:
+ oidc:
+ enabled: true
+ keycloakTruststores: ""
+ documentSelector:
+ path: metadata.name
+ value: cozystack.keycloak
+ asserts:
+ - equal:
+ path: spec.components.keycloak.values.truststores
+ value: ""
+
+ - it: forwards an empty map so the Keycloak chart can reject it instead of silently removing trust
+ set:
+ bundles:
+ system:
+ enabled: true
+ variant: isp-full
+ authentication:
+ oidc:
+ enabled: true
+ keycloakTruststores: {}
+ documentSelector:
+ path: metadata.name
+ value: cozystack.keycloak
+ asserts:
+ - equal:
+ path: spec.components.keycloak.values.truststores
+ value: {}
+
+ - it: forwards the complete truststores list without taking over chart defaults
+ set:
+ bundles:
+ system:
+ enabled: true
+ variant: isp-full
+ authentication:
+ oidc:
+ enabled: true
+ keycloakTruststores:
+ - configMapName: organization-ca
+ - secretName: partner-ca
+ key: certificates.pem
+ - configMapName: secondary-ca
+ key: chain.pem
+ documentSelector:
+ path: metadata.name
+ value: cozystack.keycloak
+ asserts:
+ - equal:
+ path: spec.components.keycloak.values
+ value:
+ truststores:
+ - configMapName: organization-ca
+ - secretName: partner-ca
+ key: certificates.pem
+ - configMapName: secondary-ca
+ key: chain.pem
+
+ - it: forwards keycloakTruststores on the isp-slim variant too
+ set:
+ bundles:
+ system:
+ enabled: true
+ variant: isp-slim
+ authentication:
+ oidc:
+ enabled: true
+ keycloakTruststores:
+ - configMapName: organization-ca
+ key: ca.crt
+ documentSelector:
+ path: metadata.name
+ value: cozystack.keycloak
+ asserts:
+ - equal:
+ path: spec.components.keycloak.values.truststores
+ value:
+ - configMapName: organization-ca
+ key: ca.crt
+
+ - it: does not emit cozystack.keycloak for a malformed false-y value when OIDC is disabled
+ set:
+ bundles:
+ system:
+ enabled: true
+ variant: isp-full
+ authentication:
+ oidc:
+ enabled: false
+ keycloakTruststores: false
+ asserts:
+ - containsDocument:
+ apiVersion: cozystack.io/v1alpha1
+ kind: Package
+ name: cozystack.keycloak
+ any: true
+ not: true
diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml
index d60effbc8..c02ad7b23 100644
--- a/packages/core/platform/values.yaml
+++ b/packages/core/platform/values.yaml
@@ -549,6 +549,19 @@ authentication:
# OIDC discovery; any other value -> used verbatim. An explicit value is not rewritten, so a
# custom realm paired with an explicit URL must name the same realm.
keycloakInternalUrl: null
+ # Additional CA trust anchors for the bundled Keycloak server (LDAPS,
+ # external IdPs, SMTP). Each entry selects one key from an existing Secret
+ # or ConfigMap in cozy-keycloak; key defaults to ca.crt. Trust applies to
+ # the entire server across all realms. See packages/system/keycloak/values.yaml
+ # for supported formats, incoming-client-mTLS scope and rotation behavior.
+ # Ignored when oidc.enabled is false: no Keycloak Package is emitted. An
+ # existing Package and its rendered truststores remain on the cluster
+ # (helm.sh/resource-policy: keep).
+ keycloakTruststores: []
+ # - configMapName: organization-ca
+ # key: ca.crt
+ # - secretName: partner-ca
+ # key: certificates.pem
# Gateway API configuration
gateway:
# Enable Gateway API support across the platform. When true:
From 9d33037566a35979ecdb95ae256f6f209afb078d Mon Sep 17 00:00:00 2001
From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
Date: Sun, 25 Oct 2026 16:40:54 +0300
Subject: [PATCH 024/216] fix(workloadmonitor): keep a bucket not ready until
its claim is provisioned
A bucket WorkloadMonitor has no pods and minReplicas 0, so it stayed
operational, and the Bucket app stayed Ready, while its BucketClaim was
never provisioned. A tenant whose nearest S3 ancestor runs no SeaweedFS
gets a claim for a bucket class that does not exist, and the dashboard
showed a ready bucket that was never created.
Assisted-by: LLM
Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com>
---
api/v1alpha1/workloadmonitor_types.go | 4 +
.../controller/workloadmonitor_controller.go | 35 ++++-
.../workloadmonitor_controller_test.go | 132 ++++++++++++++++++
3 files changed, 168 insertions(+), 3 deletions(-)
diff --git a/api/v1alpha1/workloadmonitor_types.go b/api/v1alpha1/workloadmonitor_types.go
index 0a4aa04cc..b5089652a 100644
--- a/api/v1alpha1/workloadmonitor_types.go
+++ b/api/v1alpha1/workloadmonitor_types.go
@@ -39,6 +39,10 @@ type WorkloadMonitorSpec struct {
// that is not operational because a DataVolume it selects is not ready.
const WorkloadMonitorReasonDataVolumeNotReady = "DataVolumeNotReady"
+// WorkloadMonitorReasonBucketClaimNotReady is the status Reason of a monitor
+// that is not operational because a BucketClaim it selects is not provisioned.
+const WorkloadMonitorReasonBucketClaimNotReady = "BucketClaimNotReady"
+
// WorkloadMonitorStatus defines the observed state of WorkloadMonitor
type WorkloadMonitorStatus struct {
// Operational indicates if the workload meets all operational requirements
diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go
index 984ab7b21..d5d30e4e4 100644
--- a/internal/controller/workloadmonitor_controller.go
+++ b/internal/controller/workloadmonitor_controller.go
@@ -374,6 +374,25 @@ func (r *WorkloadMonitorReconciler) isBucketClaimReady(bc *cosiv1alpha1.BucketCl
return bc.Status.BucketReady
}
+// bucketClaimsMessage names every BucketClaim that has not been provisioned,
+// with its bucket class, and is empty when there is none.
+func (r *WorkloadMonitorReconciler) bucketClaimsMessage(claims []cosiv1alpha1.BucketClaim) string {
+ var pending []string
+ for i := range claims {
+ bc := &claims[i]
+ if r.isBucketClaimReady(bc) {
+ continue
+ }
+ entry := fmt.Sprintf("BucketClaim %s is not provisioned", bc.Name)
+ if bc.Spec.BucketClassName != "" {
+ entry = fmt.Sprintf("BucketClaim %s of bucket class %s is not provisioned", bc.Name, bc.Spec.BucketClassName)
+ }
+ pending = append(pending, entry)
+ }
+ sort.Strings(pending)
+ return strings.Join(pending, "; ")
+}
+
// isServiceReady checks if the service has an external IP bound
func (r *WorkloadMonitorReconciler) isServiceReady(svc *corev1.Service) bool {
return len(svc.Status.LoadBalancer.Ingress) > 0
@@ -1157,6 +1176,8 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
}
}
+ bucketsMessage := r.bucketClaimsMessage(bucketClaimList.Items)
+
// Update WorkloadMonitor status based on observed pods
monitor.Status.ObservedReplicas = observedReplicas
monitor.Status.AvailableReplicas = availableReplicas
@@ -1178,15 +1199,23 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
if fresh.Spec.MinReplicas != nil && availableReplicas < *fresh.Spec.MinReplicas {
fresh.Status.Operational = pointer.Bool(false)
}
- // Only the DataVolume check writes Message and Reason, so the stored ones
- // are the last DataVolume verdict: kept while no reader is installed yet, so a
- // controller restart does not report a stuck disk as ready.
+ // A stored DataVolume verdict is the last one: kept while no reader is
+ // installed yet, so a controller restart does not report a stuck disk as
+ // ready. BucketClaims are listed on every pass, so their verdict is
+ // recomputed and only fills fields no DataVolume verdict holds.
if dataVolumesRead {
fresh.Status.Message = dataVolumesMessage
fresh.Status.Reason = ""
if dataVolumesMessage != "" {
fresh.Status.Reason = cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady
}
+ } else if fresh.Status.Reason == cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady {
+ fresh.Status.Message = ""
+ fresh.Status.Reason = ""
+ }
+ if fresh.Status.Message == "" && bucketsMessage != "" {
+ fresh.Status.Message = bucketsMessage
+ fresh.Status.Reason = cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady
}
if fresh.Status.Message != "" {
fresh.Status.Operational = pointer.Bool(false)
diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go
index 353066c4b..f8c761da5 100644
--- a/internal/controller/workloadmonitor_controller_test.go
+++ b/internal/controller/workloadmonitor_controller_test.go
@@ -2975,3 +2975,135 @@ func TestReconcileBucketApp_LegacySizeWinsOverOwnerlessNewWorkload(t *testing.T)
t.Errorf("size = %s, want the legacy 689Gi over the stale 10Gi", got.String())
}
}
+
+func newBucketClaim(name string, labels map[string]string, class string, ready bool) *cosiv1alpha1.BucketClaim {
+ return &cosiv1alpha1.BucketClaim{
+ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Labels: labels},
+ Spec: cosiv1alpha1.BucketClaimSpec{
+ BucketClassName: class,
+ Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3},
+ },
+ Status: cosiv1alpha1.BucketClaimStatus{BucketReady: ready},
+ }
+}
+
+func reconcileBucketMonitor(t *testing.T, c client.Client, s *runtime.Scheme, withReader bool) *cozyv1alpha1.WorkloadMonitor {
+ t.Helper()
+ reconciler := &WorkloadMonitorReconciler{Client: c, Scheme: s}
+ if withReader {
+ reconciler.DataVolumeReader = c
+ }
+ req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "my-bucket", Namespace: "default"}}
+ if _, err := reconciler.Reconcile(context.TODO(), req); err != nil {
+ t.Fatalf("Reconcile returned error: %v", err)
+ }
+ updated := &cozyv1alpha1.WorkloadMonitor{}
+ if err := c.Get(context.TODO(), req.NamespacedName, updated); err != nil {
+ t.Fatalf("Failed to get updated WorkloadMonitor: %v", err)
+ }
+ if updated.Status.Operational == nil {
+ t.Fatal("Expected Operational to be set, got nil")
+ }
+ return updated
+}
+
+func newBucketMonitorClient(s *runtime.Scheme, objs ...client.Object) client.Client {
+ monitor := &cozyv1alpha1.WorkloadMonitor{
+ ObjectMeta: metav1.ObjectMeta{Name: "my-bucket", Namespace: "default"},
+ Spec: cozyv1alpha1.WorkloadMonitorSpec{
+ Kind: "bucket",
+ Type: "s3",
+ Selector: map[string]string{"app.kubernetes.io/instance": "my-bucket"},
+ Replicas: ptr.To[int32](0),
+ MinReplicas: ptr.To[int32](0),
+ },
+ }
+ return fake.NewClientBuilder().
+ WithScheme(s).
+ WithObjects(append([]client.Object{monitor}, objs...)...).
+ WithStatusSubresource(monitor).
+ Build()
+}
+
+func TestReconcile_BucketClaimDecidesOperational(t *testing.T) {
+ selected := map[string]string{"app.kubernetes.io/instance": "my-bucket"}
+ cases := []struct {
+ name string
+ claim *cosiv1alpha1.BucketClaim
+ operational bool
+ message string
+ }{
+ {"provisioned", newBucketClaim("my-bucket", selected, "tenant-root", true), true, ""},
+ {"not provisioned", newBucketClaim("my-bucket", selected, "tenant-root", false), false, "BucketClaim my-bucket of bucket class tenant-root is not provisioned"},
+ {"not provisioned without a class", newBucketClaim("my-bucket", selected, "", false), false, "BucketClaim my-bucket is not provisioned"},
+ {"outside the selector", newBucketClaim("other", map[string]string{"app.kubernetes.io/instance": "other"}, "tenant-root", false), true, ""},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ s := newTestScheme()
+ got := reconcileBucketMonitor(t, newBucketMonitorClient(s, tc.claim), s, true)
+ if *got.Status.Operational != tc.operational {
+ t.Errorf("Operational = %v, want %v", *got.Status.Operational, tc.operational)
+ }
+ if got.Status.Message != tc.message {
+ t.Errorf("Message = %q, want %q", got.Status.Message, tc.message)
+ }
+ wantReason := ""
+ if tc.message != "" {
+ wantReason = cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady
+ }
+ if got.Status.Reason != wantReason {
+ t.Errorf("Reason = %q, want %q", got.Status.Reason, wantReason)
+ }
+ })
+ }
+}
+
+func TestReconcile_ProvisionedBucketClaimClearsTheVerdict(t *testing.T) {
+ for _, withReader := range []bool{true, false} {
+ t.Run(fmt.Sprintf("dataVolumeReader=%v", withReader), func(t *testing.T) {
+ s := newTestScheme()
+ claim := newBucketClaim("my-bucket", map[string]string{"app.kubernetes.io/instance": "my-bucket"}, "tenant-root", false)
+ c := newBucketMonitorClient(s, claim)
+
+ got := reconcileBucketMonitor(t, c, s, withReader)
+ if *got.Status.Operational || got.Status.Reason != cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady {
+ t.Fatalf("before provisioning: Operational = %v, Reason = %q", *got.Status.Operational, got.Status.Reason)
+ }
+
+ stored := &cosiv1alpha1.BucketClaim{}
+ if err := c.Get(context.TODO(), types.NamespacedName{Name: "my-bucket", Namespace: "default"}, stored); err != nil {
+ t.Fatalf("Failed to get BucketClaim: %v", err)
+ }
+ stored.Status.BucketReady = true
+ if err := c.Update(context.TODO(), stored); err != nil {
+ t.Fatalf("Failed to update BucketClaim: %v", err)
+ }
+
+ got = reconcileBucketMonitor(t, c, s, withReader)
+ if !*got.Status.Operational || got.Status.Message != "" || got.Status.Reason != "" {
+ t.Errorf("after provisioning: Operational = %v, Message = %q, Reason = %q", *got.Status.Operational, got.Status.Message, got.Status.Reason)
+ }
+ })
+ }
+}
+
+func TestReconcile_DataVolumeVerdictOutranksBucketClaim(t *testing.T) {
+ s := newTestScheme()
+ selected := map[string]string{"app.kubernetes.io/instance": "my-bucket"}
+ phase := "ImportInProgress"
+ c := newBucketMonitorClient(s,
+ newBucketClaim("my-bucket", selected, "tenant-root", false),
+ newDataVolume("my-bucket", selected, &phase),
+ )
+ got := reconcileBucketMonitor(t, c, s, true)
+ if *got.Status.Operational {
+ t.Error("Operational = true, want false")
+ }
+ if got.Status.Reason != cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady {
+ t.Errorf("Reason = %q, want %q", got.Status.Reason, cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady)
+ }
+ if got.Status.Message != "DataVolume my-bucket is ImportInProgress" {
+ t.Errorf("Message = %q", got.Status.Message)
+ }
+}
From e96ceedd7ebd6a8e138e8b363ef36a2ffbf7e20f Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:25:58 +0700
Subject: [PATCH 025/216] feat(dashboard): add a console client for the
observability proxy
The observability proxy serves metrics and logs under Kubernetes-shaped
paths but answers in native Prometheus JSON and VictoriaLogs NDJSON, so
the typed K8sClient methods cannot read it. A raw fetch on the client
keeps the same credentials and 401 handling the rest of the console
uses, and the proxy module maps each documented status to its own
error kind so a cluster without the proxy (404) can be told apart from
a denied request or a store that is down.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../__tests__/k8s-client/client.raw.test.ts | 47 ++++
.../src/lib/observability/proxy.test.ts | 179 +++++++++++++++
.../console/src/lib/observability/proxy.ts | 203 ++++++++++++++++++
.../console/packages/k8s-client/src/client.ts | 18 ++
4 files changed, 447 insertions(+)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
diff --git a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
new file mode 100644
index 000000000..ab6c056f5
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts
@@ -0,0 +1,47 @@
+import { describe, it, expect, vi, afterEach } from "vitest"
+import { K8sClient } from "@cozystack/k8s-client"
+
+afterEach(() => {
+ vi.unstubAllGlobals()
+ vi.restoreAllMocks()
+})
+
+describe("K8sClient.raw", () => {
+ it("returns the response untouched, including a non-2xx one", async () => {
+ const res = new Response("not found", { status: 404 })
+ vi.stubGlobal("fetch", vi.fn(async () => res))
+ const client = new K8sClient()
+
+ await expect(client.raw("/x")).resolves.toBe(res)
+ })
+
+ it("prefixes the base URL and attaches the bearer token", async () => {
+ const fetchMock = vi.fn(async () => new Response("{}"))
+ vi.stubGlobal("fetch", fetchMock)
+ const client = new K8sClient({ baseUrl: "/base", getToken: async () => "t0k" })
+
+ await client.raw("/apis/x", { headers: { Accept: "application/json" } })
+
+ const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit]
+ expect(url).toBe("/base/apis/x")
+ expect(init.headers).toEqual({ Accept: "application/json", Authorization: "Bearer t0k" })
+ })
+
+ it("routes a 401 through the client's unauthorized handler", async () => {
+ const onUnauthorized = vi.fn()
+ vi.stubGlobal("fetch", vi.fn(async () => new Response("", { status: 401 })))
+ const client = new K8sClient({ onUnauthorized })
+
+ const res = await client.raw("/x")
+
+ expect(res.status).toBe(401)
+ expect(onUnauthorized).toHaveBeenCalledTimes(1)
+ })
+
+ it("does not swallow a network failure", async () => {
+ vi.stubGlobal("fetch", vi.fn(async () => { throw new TypeError("offline") }))
+ const client = new K8sClient()
+
+ await expect(client.raw("/x")).rejects.toThrow("offline")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
new file mode 100644
index 000000000..f06835ff7
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts
@@ -0,0 +1,179 @@
+import { describe, it, expect, vi } from "vitest"
+import { K8sClient } from "@cozystack/k8s-client"
+import {
+ ProxyError,
+ errorKindForStatus,
+ fieldValues,
+ logsRoot,
+ metricsRoot,
+ queryLogs,
+ queryRange,
+ tailLogs,
+} from "./proxy.ts"
+
+const app = { namespace: "tenant-a", plural: "postgreses", name: "db" }
+const ns = { namespace: "tenant-a" }
+
+function clientReturning(...responses: Response[]) {
+ const client = new K8sClient({ onUnauthorized: () => {} })
+ const raw = vi.spyOn(client, "raw")
+ for (const r of responses) raw.mockResolvedValueOnce(r)
+ return { client, raw }
+}
+
+function calledUrl(raw: ReturnType["raw"], call = 0): URL {
+ return new URL(raw.mock.calls[call][0], "http://x")
+}
+
+describe("roots", () => {
+ it("builds the application roots under apps.cozystack.io", () => {
+ expect(metricsRoot(app)).toBe("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/")
+ expect(logsRoot(app)).toBe("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/")
+ })
+
+ it("builds the namespace roots under cozystack.io", () => {
+ expect(metricsRoot(ns)).toBe("/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/")
+ expect(logsRoot(ns)).toBe("/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/")
+ })
+})
+
+describe("errorKindForStatus", () => {
+ it.each([
+ [400, "bad-request"],
+ [401, "unauthorized"],
+ [403, "forbidden"],
+ [404, "not-available"],
+ [422, "limit"],
+ [429, "rate-limited"],
+ [502, "store-unreachable"],
+ [503, "unavailable"],
+ [504, "timeout"],
+ [500, "unknown"],
+ ])("maps %i to %s", (status, kind) => {
+ expect(errorKindForStatus(status)).toBe(kind)
+ })
+})
+
+describe("queryRange", () => {
+ it("GETs query_range under the root and returns the matrix", async () => {
+ const body = { status: "success", data: { resultType: "matrix", result: [{ metric: { pod: "p" }, values: [[1, "2"]] }] } }
+ const { client, raw } = clientReturning(Response.json(body))
+
+ const result = await queryRange(client, metricsRoot(app), { query: "up", start: 10, end: 20, step: 5 })
+
+ const url = calledUrl(raw)
+ expect(url.pathname).toBe(`${metricsRoot(app)}api/v1/query_range`)
+ expect(Object.fromEntries(url.searchParams)).toEqual({ query: "up", start: "10", end: "20", step: "5" })
+ expect(result).toEqual(body.data.result)
+ })
+
+ it("throws a ProxyError carrying the Prometheus error text", async () => {
+ const { client } = clientReturning(
+ Response.json({ status: "error", error: "range too long" }, { status: 422 }),
+ )
+
+ const err = await queryRange(client, metricsRoot(app), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
+
+ expect(err).toBeInstanceOf(ProxyError)
+ expect(err).toMatchObject({ kind: "limit", status: 422, message: "range too long" })
+ })
+
+ it("reports a fetch failure as a network error", async () => {
+ const client = new K8sClient()
+ vi.spyOn(client, "raw").mockRejectedValue(new TypeError("offline"))
+
+ const err = await queryRange(client, metricsRoot(app), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e)
+
+ expect(err).toMatchObject({ kind: "network" })
+ })
+})
+
+describe("queryLogs", () => {
+ it("parses NDJSON lines and sorts them by _time ascending", async () => {
+ const ndjson = [
+ JSON.stringify({ _time: "2026-11-04T10:00:02Z", _msg: "b" }),
+ "",
+ JSON.stringify({ _time: "2026-11-04T10:00:01Z", _msg: "a" }),
+ ].join("\n")
+ const { client, raw } = clientReturning(new Response(ndjson))
+
+ const entries = await queryLogs(client, logsRoot(ns), { query: "*", start: "s", end: "e", limit: 100 })
+
+ expect(calledUrl(raw).pathname).toBe(`${logsRoot(ns)}select/logsql/query`)
+ expect(Object.fromEntries(calledUrl(raw).searchParams)).toEqual({ query: "*", start: "s", end: "e", limit: "100" })
+ expect(entries.map((e) => e._msg)).toEqual(["a", "b"])
+ })
+
+ it("keeps a plain-text error body as the message", async () => {
+ const { client } = clientReturning(new Response("cannot parse query", { status: 400 }))
+
+ const err = await queryLogs(client, logsRoot(ns), { query: "(", start: "s", end: "e", limit: 1 }).catch((e) => e)
+
+ expect(err).toMatchObject({ kind: "bad-request", message: "cannot parse query" })
+ })
+})
+
+describe("fieldValues", () => {
+ it("uses stream_field_values for stream fields and returns the values", async () => {
+ const { client, raw } = clientReturning(
+ Response.json({ values: [{ value: "pod-b", hits: 1 }, { value: "pod-a", hits: 9 }] }),
+ )
+
+ const values = await fieldValues(client, logsRoot(app), {
+ field: "kubernetes_pod_name",
+ stream: true,
+ query: "*",
+ start: "s",
+ end: "e",
+ })
+
+ expect(calledUrl(raw).pathname).toBe(`${logsRoot(app)}select/logsql/stream_field_values`)
+ expect(calledUrl(raw).searchParams.get("field")).toBe("kubernetes_pod_name")
+ expect(values).toEqual(["pod-a", "pod-b"])
+ })
+
+ it("uses field_values for other fields", async () => {
+ const { client, raw } = clientReturning(Response.json({ values: [] }))
+
+ await fieldValues(client, logsRoot(app), { field: "level", stream: false, query: "*", start: "s", end: "e" })
+
+ expect(calledUrl(raw).pathname).toBe(`${logsRoot(app)}select/logsql/field_values`)
+ })
+})
+
+describe("tailLogs", () => {
+ it("delivers every line as it arrives, including one split across chunks", async () => {
+ const enc = new TextEncoder()
+ const chunks = ['{"_time":"t1","_msg":"a"}\n{"_ti', 'me":"t2","_msg":"b"}\n']
+ const body = new ReadableStream({
+ start(c) {
+ for (const ch of chunks) c.enqueue(enc.encode(ch))
+ c.close()
+ },
+ })
+ const { client, raw } = clientReturning(new Response(body))
+ const seen: string[] = []
+
+ await tailLogs(client, logsRoot(ns), { query: "*", since: "t0" }, (e) => seen.push(e._msg))
+
+ expect(calledUrl(raw).pathname).toBe(`${logsRoot(ns)}select/logsql/tail`)
+ expect(calledUrl(raw).searchParams.get("since")).toBe("t0")
+ expect(seen).toEqual(["a", "b"])
+ })
+
+ it("omits since on the first connect", async () => {
+ const { client, raw } = clientReturning(new Response(""))
+
+ await tailLogs(client, logsRoot(ns), { query: "*" }, () => {})
+
+ expect(calledUrl(raw).searchParams.has("since")).toBe(false)
+ })
+
+ it("throws a ProxyError when the tail is refused", async () => {
+ const { client } = clientReturning(new Response("", { status: 403 }))
+
+ const err = await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}).catch((e) => e)
+
+ expect(err).toMatchObject({ kind: "forbidden" })
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
new file mode 100644
index 000000000..a257825c9
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts
@@ -0,0 +1,203 @@
+import type { K8sClient } from "@cozystack/k8s-client"
+
+export type ObservabilityScope =
+ | { namespace: string }
+ | { namespace: string; plural: string; name: string }
+
+function appBase(scope: ObservabilityScope): string | null {
+ if (!("plural" in scope)) return null
+ return `/apis/apps.cozystack.io/v1alpha1/namespaces/${scope.namespace}/${scope.plural}/${scope.name}`
+}
+
+export function metricsRoot(scope: ObservabilityScope): string {
+ const app = appBase(scope)
+ return app ? `${app}/metrics/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/metrics/`
+}
+
+export function logsRoot(scope: ObservabilityScope): string {
+ const app = appBase(scope)
+ return app ? `${app}/log/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/logs/`
+}
+
+export type ProxyErrorKind =
+ | "bad-request"
+ | "unauthorized"
+ | "forbidden"
+ | "not-available"
+ | "limit"
+ | "rate-limited"
+ | "store-unreachable"
+ | "unavailable"
+ | "timeout"
+ | "network"
+ | "unknown"
+
+const KIND_BY_STATUS: Record = {
+ 400: "bad-request",
+ 401: "unauthorized",
+ 403: "forbidden",
+ 404: "not-available",
+ 422: "limit",
+ 429: "rate-limited",
+ 502: "store-unreachable",
+ 503: "unavailable",
+ 504: "timeout",
+}
+
+export function errorKindForStatus(status: number): ProxyErrorKind {
+ return KIND_BY_STATUS[status] ?? "unknown"
+}
+
+export class ProxyError extends Error {
+ kind: ProxyErrorKind
+ status: number
+
+ constructor(kind: ProxyErrorKind, status: number, message: string) {
+ super(message)
+ this.name = "ProxyError"
+ this.kind = kind
+ this.status = status
+ }
+}
+
+// Prometheus answers {"error": "..."}, VictoriaLogs and nginx answer plain text.
+function errorMessage(text: string, status: number): string {
+ try {
+ const parsed: unknown = JSON.parse(text)
+ if (parsed && typeof parsed === "object" && "error" in parsed && typeof parsed.error === "string") {
+ return parsed.error
+ }
+ } catch {
+ // not JSON
+ }
+ const trimmed = text.trim()
+ return trimmed && trimmed.length < 300 && !trimmed.startsWith("<") ? trimmed : `Server returned ${status}`
+}
+
+async function proxyGet(
+ client: K8sClient,
+ path: string,
+ params: Record,
+ signal?: AbortSignal,
+): Promise {
+ const search = new URLSearchParams()
+ for (const [k, v] of Object.entries(params)) if (v !== undefined) search.set(k, String(v))
+ let res: Response
+ try {
+ res = await client.raw(`${path}?${search}`, { signal })
+ } catch (err) {
+ if (signal?.aborted) throw err
+ throw new ProxyError("network", 0, "Unable to connect to the server")
+ }
+ if (!res.ok) {
+ const text = await res.text().catch(() => "")
+ throw new ProxyError(errorKindForStatus(res.status), res.status, errorMessage(text, res.status))
+ }
+ return res
+}
+
+export interface PromSeries {
+ metric: Record
+ values: [number, string][]
+}
+
+export interface RangeParams {
+ query: string
+ start: number
+ end: number
+ step: number
+}
+
+export async function queryRange(
+ client: K8sClient,
+ root: string,
+ params: RangeParams,
+ signal?: AbortSignal,
+): Promise {
+ const res = await proxyGet(client, `${root}api/v1/query_range`, { ...params }, signal)
+ const body = (await res.json()) as { data?: { result?: PromSeries[] } }
+ return body.data?.result ?? []
+}
+
+export interface LogEntry {
+ _time: string
+ _msg: string
+ _stream_id?: string
+ [field: string]: string | undefined
+}
+
+function parseLine(line: string): LogEntry | null {
+ if (!line.trim()) return null
+ try {
+ const value: unknown = JSON.parse(line)
+ if (value && typeof value === "object" && "_time" in value) return value as LogEntry
+ } catch {
+ // a torn or non-JSON line carries nothing to show
+ }
+ return null
+}
+
+export interface LogsQueryParams {
+ query: string
+ start: string
+ end: string
+ limit: number
+}
+
+export async function queryLogs(
+ client: K8sClient,
+ root: string,
+ params: LogsQueryParams,
+ signal?: AbortSignal,
+): Promise {
+ const res = await proxyGet(client, `${root}select/logsql/query`, { ...params }, signal)
+ const entries = (await res.text()).split("\n").map(parseLine).filter((e) => e !== null)
+ return entries.sort((a, b) => (a._time < b._time ? -1 : a._time > b._time ? 1 : 0))
+}
+
+export interface FieldValuesParams {
+ field: string
+ stream: boolean
+ query: string
+ start: string
+ end: string
+}
+
+export async function fieldValues(
+ client: K8sClient,
+ root: string,
+ { stream, ...params }: FieldValuesParams,
+ signal?: AbortSignal,
+): Promise {
+ const endpoint = stream ? "stream_field_values" : "field_values"
+ const res = await proxyGet(client, `${root}select/logsql/${endpoint}`, { ...params }, signal)
+ const body = (await res.json()) as { values?: { value: string }[] }
+ return (body.values ?? []).map((v) => v.value).sort()
+}
+
+export async function tailLogs(
+ client: K8sClient,
+ root: string,
+ params: { query: string; since?: string },
+ onEntry: (entry: LogEntry) => void,
+ signal?: AbortSignal,
+): Promise {
+ const res = await proxyGet(client, `${root}select/logsql/tail`, params, signal)
+ if (!res.body) return
+ const reader = res.body.getReader()
+ const decoder = new TextDecoder()
+ let buffer = ""
+ for (;;) {
+ const { done, value } = await reader.read()
+ if (done) break
+ buffer += decoder.decode(value, { stream: true })
+ const lines = buffer.split("\n")
+ buffer = lines.pop() ?? ""
+ for (const line of lines) {
+ const entry = parseLine(line)
+ if (entry) onEntry(entry)
+ }
+ }
+ const last = parseLine(buffer)
+ if (last) onEntry(last)
+}
diff --git a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
index 12f50e0fd..04c528760 100644
--- a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
+++ b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts
@@ -241,6 +241,24 @@ export class K8sClient {
})
}
+ /**
+ * Fetch a same-origin path with the client's credentials and 401 handling,
+ * leaving status and body to the caller. For endpoints that do not speak
+ * Kubernetes JSON, such as the observability proxy's streamed NDJSON.
+ */
+ async raw(path: string, init?: RequestInit): Promise {
+ const headers: Record = {
+ ...(init?.headers as Record),
+ }
+ if (this.getToken) {
+ headers["Authorization"] = `Bearer ${await this.getToken()}`
+ }
+ const res = await fetch(`${this.baseUrl}${path}`, { ...init, headers })
+ if (res.status === 401) this.handleUnauthorized()
+ else if (res.ok) this.unauthorizedHandled = false
+ return res
+ }
+
getApiGroups(): Promise {
return this.request("/apis")
}
From 063e56ede57a481b2db223a37641e601a2d51a05 Mon Sep 17 00:00:00 2001
From: ZverGuy
Date: Wed, 4 Nov 2026 05:30:03 +0700
Subject: [PATCH 026/216] feat(dashboard): show resource consumption on
application pages
Tenants had no way to see what their applications consume. A Metrics
tab on every application detail page (except VM disks, which run no
containers) charts CPU, memory and network for a selectable range
through the observability proxy's application root, which injects the
scope server-side, so the queries stay kind-agnostic and no per-kind
UI is needed.
Charts use uPlot: about 25 KB gzipped with no dependencies, built for
time series, and loaded only with the Metrics tab so other pages do
not pay for it. A cluster without the proxy answers 404 and the tab
says metrics are not available there instead of showing an error.
Assisted-by: LLM
Signed-off-by: ZverGuy
---
.../images/console/apps/console/package.json | 3 +-
.../observability/MetricsView.test.tsx | 88 +++++++++++++
.../components/observability/MetricsView.tsx | 72 +++++++++++
.../observability/ProxyNotice.test.tsx | 43 +++++++
.../components/observability/ProxyNotice.tsx | 49 +++++++
.../components/observability/RangePicker.tsx | 30 +++++
.../observability/TimeSeriesChart.test.tsx | 46 +++++++
.../observability/TimeSeriesChart.tsx | 51 ++++++++
.../src/lib/observability/metrics.test.ts | 107 ++++++++++++++++
.../console/src/lib/observability/metrics.ts | 121 ++++++++++++++++++
.../src/lib/observability/proxy.test.ts | 10 ++
.../console/src/lib/observability/proxy.ts | 8 ++
.../detail/ApplicationDetailPage.test.tsx | 28 ++++
.../routes/detail/ApplicationDetailPage.tsx | 12 ++
.../console/src/routes/detail/MetricsTab.tsx | 15 +++
.../dashboard/images/console/pnpm-lock.yaml | 8 ++
16 files changed, 690 insertions(+), 1 deletion(-)
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts
create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx
diff --git a/packages/system/dashboard/images/console/apps/console/package.json b/packages/system/dashboard/images/console/apps/console/package.json
index fcddb7023..254195f28 100644
--- a/packages/system/dashboard/images/console/apps/console/package.json
+++ b/packages/system/dashboard/images/console/apps/console/package.json
@@ -31,7 +31,8 @@
"react": "^19.2.4",
"react-dom": "^19.2.4",
"react-router": "^7.6.1",
- "simple-icons": "^16.22.0"
+ "simple-icons": "^16.22.0",
+ "uplot": "^1.6.32"
},
"devDependencies": {
"@tailwindcss/vite": "^4.2.2",
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
new file mode 100644
index 000000000..f09e15201
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx
@@ -0,0 +1,88 @@
+import { describe, it, expect, vi } from "vitest"
+import { screen, waitFor } from "@testing-library/react"
+import userEvent from "@testing-library/user-event"
+import { K8sClient } from "@cozystack/k8s-client"
+import { renderWithK8sProvider } from "../../test-utils/render.tsx"
+import type { ChartData } from "../../lib/observability/metrics.ts"
+
+vi.mock("./TimeSeriesChart.tsx", () => ({
+ TimeSeriesChart: ({ data }: { data: ChartData }) => (
+
{data.series.map((s) => s.label).join(",")}
+ ),
+}))
+
+const { MetricsView } = await import("./MetricsView.tsx")
+
+const scope = { namespace: "tenant-a", plural: "postgreses", name: "db" }
+
+function matrix(pod: string) {
+ return Response.json({
+ status: "success",
+ data: { resultType: "matrix", result: [{ metric: { pod }, values: [[1, "1"]] }] },
+ })
+}
+
+function clientWith(handler: (url: URL) => Response) {
+ const client = new K8sClient()
+ const raw = vi.spyOn(client, "raw").mockImplementation(async (path) => handler(new URL(path, "http://x")))
+ return { client, raw }
+}
+
+describe("MetricsView", () => {
+ it("draws CPU, memory and network from the application root", async () => {
+ const { client, raw } = clientWith(() => matrix("db-0"))
+
+ renderWithK8sProvider(, { client })
+
+ await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3))
+ expect(screen.getByText("CPU (cores)")).toBeInTheDocument()
+ expect(screen.getByText("Memory (working set)")).toBeInTheDocument()
+ expect(screen.getByText("Network")).toBeInTheDocument()
+ for (const [path] of raw.mock.calls) {
+ expect(path.startsWith("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query_range?")).toBe(true)
+ }
+ })
+
+ it("requeries with a coarser step when a longer range is picked", async () => {
+ const { client, raw } = clientWith(() => matrix("db-0"))
+ renderWithK8sProvider(, { client })
+ await waitFor(() => expect(raw).toHaveBeenCalled())
+
+ await userEvent.click(screen.getByRole("button", { name: "7d" }))
+
+ await waitFor(() =>
+ expect(raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("step") === "2520")).toBe(true),
+ )
+ })
+
+ it("shows one quiet notice instead of three errors when the proxy is absent", async () => {
+ const { client } = clientWith(() => new Response("404 page not found", { status: 404 }))
+
+ renderWithK8sProvider(, { client })
+
+ expect(await screen.findByText("Metrics are not available on this cluster.")).toBeInTheDocument()
+ expect(screen.getAllByRole("status")).toHaveLength(1)
+ expect(screen.queryByText("CPU (cores)")).toBeNull()
+ })
+
+ it("keeps the other panels when one query hits a limit", async () => {
+ const { client } = clientWith((url) =>
+ url.searchParams.get("query")?.includes("memory")
+ ? Response.json({ status: "error", error: "too many series" }, { status: 422 })
+ : matrix("db-0"),
+ )
+
+ renderWithK8sProvider(, { client })
+
+ expect(await screen.findByText("too many series. Choose a shorter range.")).toBeInTheDocument()
+ await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(2))
+ })
+
+ it("says so when a panel has no data", async () => {
+ const { client } = clientWith(() => Response.json({ status: "success", data: { resultType: "matrix", result: [] } }))
+
+ renderWithK8sProvider(, { client })
+
+ await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(3))
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
new file mode 100644
index 000000000..35c8e8417
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx
@@ -0,0 +1,72 @@
+import { useState } from "react"
+import { useQueries } from "@tanstack/react-query"
+import { useK8sClient } from "@cozystack/k8s-client"
+import { Section, Spinner } from "@cozystack/ui"
+import {
+ METRIC_PANELS,
+ RANGES,
+ rangeWindow,
+ toChartData,
+ type RangeKey,
+} from "../../lib/observability/metrics.ts"
+import { isScopeWide, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts"
+import { ProxyNotice } from "./ProxyNotice.tsx"
+import { TimeSeriesChart } from "./TimeSeriesChart.tsx"
+import { RangePicker } from "./RangePicker.tsx"
+
+const REFRESH_MS = 60_000
+
+export function MetricsView({ scope }: { scope: ObservabilityScope }) {
+ const client = useK8sClient()
+ const root = metricsRoot(scope)
+ const [range, setRange] = useState("1h")
+
+ const results = useQueries({
+ queries: METRIC_PANELS.map((panel) => ({
+ queryKey: ["observability", "metrics", root, panel.id, range],
+ queryFn: async ({ signal }: { signal: AbortSignal }) => {
+ const window = rangeWindow(range, Math.floor(Date.now() / 1000))
+ const series = await Promise.all(
+ panel.queries(window).map(async (q) => ({
+ label: q.label,
+ series: await queryRange(client, root, { query: q.expr, ...window }, signal),
+ })),
+ )
+ return toChartData(series)
+ },
+ retry: false,
+ refetchInterval: REFRESH_MS,
+ })),
+ })
+
+ const wide = results.find((r) => isScopeWide(r.error))?.error
+
+ return (
+
+ )
+}
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
new file mode 100644
index 000000000..fd001bcae
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx
@@ -0,0 +1,43 @@
+import { describe, it, expect } from "vitest"
+import { render, screen } from "@testing-library/react"
+import { ProxyError } from "../../lib/observability/proxy.ts"
+import { ProxyNotice } from "./ProxyNotice.tsx"
+
+describe("ProxyNotice", () => {
+ it("renders a 404 as not available on this cluster, not as an error", () => {
+ render()
+
+ expect(screen.getByRole("status")).toHaveTextContent("Metrics are not available on this cluster.")
+ expect(screen.queryByRole("alert")).toBeNull()
+ })
+
+ it("renders a 502 as a quiet notice about the store", () => {
+ render()
+
+ expect(screen.getByRole("status")).toHaveTextContent("The logs store is unreachable right now.")
+ })
+
+ it.each([
+ ["forbidden", 403, "You do not have permission to read metrics here."],
+ ["limit", 422, "too many points. Choose a shorter range."],
+ ["rate-limited", 429, "Too many requests. Try again in a moment."],
+ ["timeout", 504, "The store did not answer in time. Choose a shorter range or a narrower filter."],
+ ["unavailable", 503, "The service is temporarily unavailable. Try again in a moment."],
+ ] as const)("explains %s", (kind, status, text) => {
+ render()
+
+ expect(screen.getByRole("alert")).toHaveTextContent(text)
+ })
+
+ it("shows the server message for a rejected query", () => {
+ render()
+
+ expect(screen.getByRole("alert")).toHaveTextContent("cannot parse query")
+ })
+
+ it("falls back to the message of an unexpected error", () => {
+ render()
+
+ expect(screen.getByRole("alert")).toHaveTextContent("boom")
+ })
+})
diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
new file mode 100644
index 000000000..a60b831b0
--- /dev/null
+++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx
@@ -0,0 +1,49 @@
+import { ProxyError } from "../../lib/observability/proxy.ts"
+
+interface ProxyNoticeProps {
+ what: "Metrics" | "Logs"
+ error: unknown
+}
+
+function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: boolean; text: string } {
+ const lower = what.toLowerCase()
+ if (!(error instanceof ProxyError)) {
+ return { quiet: false, text: error instanceof Error ? error.message : String(error) }
+ }
+ switch (error.kind) {
+ case "not-available":
+ return { quiet: true, text: `${what} are not available on this cluster.` }
+ case "store-unreachable":
+ return { quiet: true, text: `The ${lower} store is unreachable right now.` }
+ case "unauthorized":
+ return { quiet: false, text: "Your session has expired. Sign in again." }
+ case "forbidden":
+ return { quiet: false, text: `You do not have permission to read ${lower} here.` }
+ case "limit":
+ return { quiet: false, text: `${error.message}. Choose a shorter range.` }
+ case "rate-limited":
+ return { quiet: false, text: "Too many requests. Try again in a moment." }
+ case "timeout":
+ return { quiet: false, text: "The store did not answer in time. Choose a shorter range or a narrower filter." }
+ case "unavailable":
+ return { quiet: false, text: "The service is temporarily unavailable. Try again in a moment." }
+ default:
+ return { quiet: false, text: error.message }
+ }
+}
+
+export function ProxyNotice({ what, error }: ProxyNoticeProps) {
+ const { quiet, text } = describe(what, error)
+ if (quiet) {
+ return (
+