From a2c9c07a1629eeb78e7cd454753e0e864cbf8a29 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:56:06 +0300 Subject: [PATCH 001/174] fix(apps): keep Job and CronJob names within the Kubernetes limits The apps API admits release names of up to 53 characters, and several charts name a Job -. Kubernetes copies a Job name into the job-name label of its pods, and a label value takes at most 63 characters, so the apiserver refused these Jobs for the longest names. A Kafka whose name has 42 to 44 characters could not be installed, a VMDisk of 37 or more could not grow, a VMInstance of 40 or 41 could not take an instance type, profile or external change, and a Bucket of 44 or more or a Qdrant of 42 or more failed its post-delete hook. Those Job names, and the pod label vm-instance copies from one, are now cut to 63 characters. A release name is never longer than 53, so only the fixed suffix is cut and two releases still get two Jobs. A CronJob name is refused over 52 characters, because the controller names each Job it starts -. With the backup CronJob on, a MariaDB whose name has 38 characters or more and a ClickHouse of 35 or more failed the whole release. A cut there reaches into the release name and could give two releases one CronJob, so a name that does not fit becomes the release name cut to 38 characters, a hash of the full release name and -backup, the way the ClickHouse Keeper name is already shortened. The tenant cleanup Job is named after the tenant namespace, which a nested tenant can fill to 63 characters by itself. The Job runs in cozy-system next to that of every other tenant, where a plain cut could give a parent and its child one name and let the child's before-hook-creation delete the parent's Job while it runs, so it is shortened with a hash of the namespace the same way. Names that fit render exactly as before, and a name that did not fit never produced an object, so nothing is renamed on upgrade. The hook ServiceAccounts, Roles and RoleBindings have no such limit and keep their names. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- .../bucket/templates/hooks/cleanup-acme.yaml | 2 +- .../bucket/tests/job_name_budget_test.yaml | 80 +++++++++++ .../clickhouse/templates/backup-cronjob.yaml | 10 +- .../tests/cronjob_name_budget_test.yaml | 94 +++++++++++++ .../apps/kafka/templates/migration-hook.yaml | 2 +- .../kafka/tests/job_name_budget_test.yaml | 66 +++++++++ .../mariadb/templates/backup-cronjob.yaml | 10 +- .../tests/cronjob_name_budget_test.yaml | 92 +++++++++++++ .../qdrant/templates/hooks/cleanup-pvc.yaml | 2 +- .../qdrant/tests/job_name_budget_test.yaml | 64 +++++++++ .../apps/tenant/templates/cleanup-job.yaml | 15 ++- .../tenant/tests/job_name_budget_test.yaml | 127 ++++++++++++++++++ .../vm-disk/templates/pvc-resize-hook.yaml | 2 +- .../vm-disk/tests/job_name_budget_test.yaml | 127 ++++++++++++++++++ .../vm-instance/templates/vm-update-hook.yaml | 5 +- .../tests/job_name_budget_test.yaml | 107 +++++++++++++++ 16 files changed, 795 insertions(+), 10 deletions(-) create mode 100644 packages/apps/bucket/tests/job_name_budget_test.yaml create mode 100644 packages/apps/clickhouse/tests/cronjob_name_budget_test.yaml create mode 100644 packages/apps/kafka/tests/job_name_budget_test.yaml create mode 100644 packages/apps/mariadb/tests/cronjob_name_budget_test.yaml create mode 100644 packages/apps/qdrant/tests/job_name_budget_test.yaml create mode 100644 packages/apps/tenant/tests/job_name_budget_test.yaml create mode 100644 packages/apps/vm-disk/tests/job_name_budget_test.yaml create mode 100644 packages/apps/vm-instance/tests/job_name_budget_test.yaml diff --git a/packages/apps/bucket/templates/hooks/cleanup-acme.yaml b/packages/apps/bucket/templates/hooks/cleanup-acme.yaml index 94814a2c1..7680df52e 100644 --- a/packages/apps/bucket/templates/hooks/cleanup-acme.yaml +++ b/packages/apps/bucket/templates/hooks/cleanup-acme.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: {{ .Release.Name }}-cleanup-acme + name: {{ printf "%s-cleanup-acme" .Release.Name | trunc 63 | trimSuffix "-" }} labels: app.kubernetes.io/instance: {{ .Release.Name }} annotations: diff --git a/packages/apps/bucket/tests/job_name_budget_test.yaml b/packages/apps/bucket/tests/job_name_budget_test.yaml new file mode 100644 index 000000000..94a295b5a --- /dev/null +++ b/packages/apps/bucket/tests/job_name_budget_test.yaml @@ -0,0 +1,80 @@ +suite: bucket cleanup Job name budget + +# The API server copies a Job's name into the job-name label of its pods, and a +# label value takes 63 characters at most, while a release name may take 53. +# The ServiceAccount, Role and RoleBinding have no such limit and keep the full +# name, so a hook that ran before keeps the same objects. + +templates: + - templates/hooks/cleanup-acme.yaml + +set: + _namespace: + host: example.org + +tests: + - it: keeps -cleanup-acme for a short release name + release: + name: my-bucket + namespace: tenant-root + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: my-bucket-cleanup-acme + + - it: keeps the name when it lands exactly on 63 characters + release: + name: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefg + namespace: tenant-root + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefg-cleanup-acme + + - it: cuts the name to 63 characters for a 53-character release name + release: + name: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij + namespace: tenant-root + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-a + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ + - equal: + path: spec.template.spec.serviceAccountName + value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-acme + + - it: leaves the RBAC objects on the full name for a 53-character release name + release: + name: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij + namespace: tenant-root + asserts: + - equal: + path: metadata.name + value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-acme + documentSelector: + path: kind + value: ServiceAccount + - equal: + path: roleRef.name + value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-acme + documentSelector: + path: kind + value: RoleBinding + - equal: + path: subjects[0].name + value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-acme + documentSelector: + path: kind + value: RoleBinding diff --git a/packages/apps/clickhouse/templates/backup-cronjob.yaml b/packages/apps/clickhouse/templates/backup-cronjob.yaml index f792e8f9d..2a0bbd0b2 100644 --- a/packages/apps/clickhouse/templates/backup-cronjob.yaml +++ b/packages/apps/clickhouse/templates/backup-cronjob.yaml @@ -18,11 +18,19 @@ */}} {{- if and .Values.backup.enabled .Values.backup.schedule (not .Values.backup.useSystemBucket) }} {{ $image := .Files.Get "images/backup.json" | fromJson }} +{{- /* The CronJob controller starts each Job as -, so the API + refuses a CronJob name over 52 characters. Cutting the release name to + fit could give two releases one CronJob, so a name that does not fit + keeps a hash of the full release name instead. */}} +{{- $cronJobName := printf "%s-backup" .Release.Name }} +{{- if gt (len $cronJobName) 52 }} +{{- $cronJobName = printf "%s-%s-backup" (.Release.Name | trunc 38 | trimSuffix "-") (.Release.Name | sha256sum | trunc 6) }} +{{- end }} apiVersion: batch/v1 kind: CronJob metadata: - name: {{ .Release.Name }}-backup + name: {{ $cronJobName }} spec: schedule: "{{ .Values.backup.schedule }}" concurrencyPolicy: Forbid diff --git a/packages/apps/clickhouse/tests/cronjob_name_budget_test.yaml b/packages/apps/clickhouse/tests/cronjob_name_budget_test.yaml new file mode 100644 index 000000000..0ff3e2305 --- /dev/null +++ b/packages/apps/clickhouse/tests/cronjob_name_budget_test.yaml @@ -0,0 +1,94 @@ +suite: clickhouse legacy backup CronJob name budget + +# The API refuses a CronJob name over 52 characters, because the controller +# names each Job it starts -, while a release name may take 53. +# Cutting the release name alone would give two long names one CronJob, so a +# name that does not fit is the release name cut to 38 characters, a hash of +# the full release name and -backup. + +templates: + - templates/backup-cronjob.yaml + - templates/backup-script.yaml + - templates/backup-secret.yaml + +set: + _cluster: {} + backup: + enabled: true + schedule: "0 2 * * *" + +tests: + - it: keeps -backup for a short release name + release: + name: ch-test + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: ch-test-backup + + - it: keeps the name when it lands exactly on 52 characters + release: + name: clickhouse-abcdefghijklmnopqrstuvwxyz01234567 + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: clickhouse-abcdefghijklmnopqrstuvwxyz01234567-backup + + - it: shortens the name one character past the budget + release: + name: clickhouse-abcdefghijklmnopqrstuvwxyz012345678 + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: clickhouse-abcdefghijklmnopqrstuvwxyz0-bec95e-backup + + - it: fits a 53-character release name into 52 characters + release: + name: clickhouse-abcdefghijklmnopqrstuvwxyz0123456789abcdef + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: clickhouse-abcdefghijklmnopqrstuvwxyz0-3e8cf9-backup + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,50}[a-z0-9])?$ + + - it: gives a release name differing only in its last character another CronJob + release: + name: clickhouse-abcdefghijklmnopqrstuvwxyz0123456789abcdex + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: clickhouse-abcdefghijklmnopqrstuvwxyz0-bd3283-backup + + - it: does not end the cut release name with a dash + release: + name: clickhouse-abcdefghijklmnopqrstuvwxyz-0123456789 + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: clickhouse-abcdefghijklmnopqrstuvwxyz-07ba9e-backup diff --git a/packages/apps/kafka/templates/migration-hook.yaml b/packages/apps/kafka/templates/migration-hook.yaml index 9877f699b..997b28ea1 100644 --- a/packages/apps/kafka/templates/migration-hook.yaml +++ b/packages/apps/kafka/templates/migration-hook.yaml @@ -79,7 +79,7 @@ subjects: apiVersion: batch/v1 kind: Job metadata: - name: {{ .Release.Name }}-kafka-migration + name: {{ printf "%s-kafka-migration" .Release.Name | trunc 63 | trimSuffix "-" }} annotations: helm.sh/hook: pre-install,pre-upgrade helm.sh/hook-weight: "2" diff --git a/packages/apps/kafka/tests/job_name_budget_test.yaml b/packages/apps/kafka/tests/job_name_budget_test.yaml new file mode 100644 index 000000000..908ef75ed --- /dev/null +++ b/packages/apps/kafka/tests/job_name_budget_test.yaml @@ -0,0 +1,66 @@ +suite: kafka migration Job name budget + +# The API server copies a Job's name into the job-name label of its pods, and a +# label value takes 63 characters at most. The Kafka name cap leaves a release +# name of up to 50 characters, and the migration hook runs on install, so an +# overlong Job name would fail every install of the longest names. The +# ServiceAccount, Role and RoleBinding have no such limit and keep the full +# name, so the ones an earlier attempt left behind are replaced, not orphaned. + +templates: + - templates/migration-hook.yaml + +tests: + - it: keeps -kafka-migration for a short release name + release: + name: test-kafka + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: test-kafka-kafka-migration + + - it: keeps the name when it lands exactly on 63 characters + release: + name: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcde + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcde-kafka-migration + + - it: cuts the name to 63 characters for the longest release name the cap admits + release: + name: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh-kafka-migrat + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ + - equal: + path: spec.template.spec.serviceAccountName + value: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh-kafka-migration + + - it: leaves the ServiceAccount on the full name for the longest release name + release: + name: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh + namespace: tenant-test + documentSelector: + path: kind + value: ServiceAccount + asserts: + - equal: + path: metadata.name + value: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh-kafka-migration diff --git a/packages/apps/mariadb/templates/backup-cronjob.yaml b/packages/apps/mariadb/templates/backup-cronjob.yaml index d39d5a8dd..1072a7c85 100644 --- a/packages/apps/mariadb/templates/backup-cronjob.yaml +++ b/packages/apps/mariadb/templates/backup-cronjob.yaml @@ -1,10 +1,18 @@ {{- if .Values.backup.enabled }} {{ $image := .Files.Get "images/backup.json" | fromJson }} +{{- /* The CronJob controller starts each Job as -, so the API + refuses a CronJob name over 52 characters. Cutting the release name to + fit could give two releases one CronJob, so a name that does not fit + keeps a hash of the full release name instead. */}} +{{- $cronJobName := printf "%s-backup" .Release.Name }} +{{- if gt (len $cronJobName) 52 }} +{{- $cronJobName = printf "%s-%s-backup" (.Release.Name | trunc 38 | trimSuffix "-") (.Release.Name | sha256sum | trunc 6) }} +{{- end }} apiVersion: batch/v1 kind: CronJob metadata: - name: {{ .Release.Name }}-backup + name: {{ $cronJobName }} spec: schedule: "{{ .Values.backup.schedule }}" concurrencyPolicy: Forbid diff --git a/packages/apps/mariadb/tests/cronjob_name_budget_test.yaml b/packages/apps/mariadb/tests/cronjob_name_budget_test.yaml new file mode 100644 index 000000000..18e5e40cf --- /dev/null +++ b/packages/apps/mariadb/tests/cronjob_name_budget_test.yaml @@ -0,0 +1,92 @@ +suite: mariadb backup CronJob name budget + +# The API refuses a CronJob name over 52 characters, because the controller +# names each Job it starts -, while a release name may take 53. +# Cutting the release name alone would give two long names one CronJob, so a +# name that does not fit is the release name cut to 38 characters, a hash of +# the full release name and -backup. + +templates: + - templates/backup-cronjob.yaml + - templates/backup-script.yaml + - templates/backup-secret.yaml + +set: + backup: + enabled: true + +tests: + - it: keeps -backup for a short release name + release: + name: mariadb-test + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: mariadb-test-backup + + - it: keeps the name when it lands exactly on 52 characters + release: + name: mariadb-abcdefghijklmnopqrstuvwxyz0123456789a + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: mariadb-abcdefghijklmnopqrstuvwxyz0123456789a-backup + + - it: shortens the name one character past the budget + release: + name: mariadb-abcdefghijklmnopqrstuvwxyz0123456789ab + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: mariadb-abcdefghijklmnopqrstuvwxyz0123-57a159-backup + + - it: fits a 53-character release name into 52 characters + release: + name: mariadb-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: mariadb-abcdefghijklmnopqrstuvwxyz0123-b5132c-backup + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,50}[a-z0-9])?$ + + - it: gives a release name differing only in its last character another CronJob + release: + name: mariadb-abcdefghijklmnopqrstuvwxyz0123456789abcdefghx + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: mariadb-abcdefghijklmnopqrstuvwxyz0123-a49389-backup + + - it: does not end the cut release name with a dash + release: + name: mariadb-abcdefghijklmnopqrstuvwxyz012-3456789abc + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: mariadb-abcdefghijklmnopqrstuvwxyz012-2985e6-backup diff --git a/packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml b/packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml index bbc9547f9..3e2038e91 100644 --- a/packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml +++ b/packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: {{ .Release.Name }}-qdrant-cleanup + name: {{ printf "%s-qdrant-cleanup" .Release.Name | trunc 63 | trimSuffix "-" }} labels: app.kubernetes.io/instance: {{ .Release.Name }} annotations: diff --git a/packages/apps/qdrant/tests/job_name_budget_test.yaml b/packages/apps/qdrant/tests/job_name_budget_test.yaml new file mode 100644 index 000000000..76416b387 --- /dev/null +++ b/packages/apps/qdrant/tests/job_name_budget_test.yaml @@ -0,0 +1,64 @@ +suite: qdrant cleanup Job name budget + +# The API server copies a Job's name into the job-name label of its pods, and a +# label value takes 63 characters at most, while a release name may take 53. +# The ServiceAccount, Role and RoleBinding have no such limit and keep the full +# name. + +templates: + - templates/hooks/cleanup-pvc.yaml + +tests: + - it: keeps -qdrant-cleanup for a short release name + release: + name: qdrant-test + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: qdrant-test-qdrant-cleanup + + - it: keeps the name when it lands exactly on 63 characters + release: + name: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcde + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcde-qdrant-cleanup + + - it: cuts the name to 63 characters for a 53-character release name + release: + name: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-qdrant-cl + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ + - equal: + path: spec.template.spec.serviceAccountName + value: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-qdrant-cleanup + + - it: leaves the ServiceAccount on the full name for a 53-character release name + release: + name: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij + namespace: tenant-test + documentSelector: + path: kind + value: ServiceAccount + asserts: + - equal: + path: metadata.name + value: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-qdrant-cleanup diff --git a/packages/apps/tenant/templates/cleanup-job.yaml b/packages/apps/tenant/templates/cleanup-job.yaml index ffa07d0fe..7285a4d79 100644 --- a/packages/apps/tenant/templates/cleanup-job.yaml +++ b/packages/apps/tenant/templates/cleanup-job.yaml @@ -40,11 +40,22 @@ subjects: - kind: ServiceAccount name: {{ include "tenant.name" . }}-cleanup namespace: cozy-system +{{- /* The Job name becomes the job-name label of its pod, which takes 63 + characters at most, and a nested tenant namespace can use all 63 by + itself. Cutting the name to fit would let a parent and one of its + children land on the same Job in cozy-system, where the child's + before-hook-creation would delete the parent's running cleanup, so a + name that does not fit keeps a hash of the whole namespace instead. */}} +{{- $tenantName := include "tenant.name" . }} +{{- $jobName := printf "%s-cleanup" $tenantName }} +{{- if gt (len $jobName) 63 }} +{{- $jobName = printf "%s-%s-cleanup" ($tenantName | trunc 48 | trimSuffix "-") ($tenantName | sha256sum | trunc 6) }} +{{- end }} --- apiVersion: batch/v1 kind: Job metadata: - name: {{ include "tenant.name" . }}-cleanup + name: {{ $jobName }} namespace: cozy-system annotations: helm.sh/hook: pre-delete @@ -98,7 +109,7 @@ spec: ttlSecondsAfterFinished: 300 template: metadata: - name: {{ include "tenant.name" . }}-cleanup + name: {{ $jobName }} labels: policy.cozystack.io/allow-to-apiserver: "true" spec: diff --git a/packages/apps/tenant/tests/job_name_budget_test.yaml b/packages/apps/tenant/tests/job_name_budget_test.yaml new file mode 100644 index 000000000..5cf4b93f6 --- /dev/null +++ b/packages/apps/tenant/tests/job_name_budget_test.yaml @@ -0,0 +1,127 @@ +suite: tenant cleanup Job name budget + +# The API server copies a Job's name into the job-name label of its pods, and a +# label value takes 63 characters at most, while a nested tenant namespace may +# take all 63 by itself. Every tenant's cleanup Job runs in cozy-system, where +# cutting the namespace alone could give a parent and its child one Job, and the +# child's before-hook-creation would delete the parent's while it runs. So a +# name that does not fit is the namespace cut to 48 characters, a hash of the +# whole namespace and -cleanup. The ServiceAccount, Role and RoleBinding have no +# such limit and keep the full name. + +templates: + - templates/cleanup-job.yaml + +tests: + - it: keeps -cleanup for a tenant under tenant-root + release: + name: tenant-test + namespace: tenant-root + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-test-cleanup + + - it: keeps -cleanup for a short nested tenant + release: + name: tenant-child + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-test-child-cleanup + + - it: keeps the name when it lands exactly on 63 characters + release: + name: tenant-0123456789abcdefghijk + namespace: tenant-abcdefghijklmnopqrstuvwxyz + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijk-cleanup + + - it: shortens the name for a 63-character namespace + release: + name: tenant-0123456789abcdefghijklmnopqrs + namespace: tenant-abcdefghijklmnopqrstuvwxyz + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcd-7b3e3e-cleanup + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ + - equal: + path: spec.template.metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcd-7b3e3e-cleanup + - equal: + path: spec.template.spec.serviceAccountName + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmnopqrs-cleanup + + # Cut to 63 characters, the parent below and its child named clea would both + # be tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmn-clea. + - it: gives a parent its own Job name + release: + name: tenant-0123456789abcdefghijklmn + namespace: tenant-abcdefghijklmnopqrstuvwxyz + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcd-571020-cleanup + + - it: gives the child that would collide with that parent another Job name + release: + name: tenant-clea + namespace: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmn + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcd-ec5e9b-cleanup + + - it: does not end the cut namespace with a dash + release: + name: tenant-abcdefghijklmno + namespace: tenant-abcdefghijklmnopqrstuvwxyz0123456789abcd + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz0123456789abcd-477f49-cleanup + + - it: leaves the RBAC objects on the full name for a 63-character namespace + release: + name: tenant-0123456789abcdefghijklmnopqrs + namespace: tenant-abcdefghijklmnopqrstuvwxyz + asserts: + - equal: + path: metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmnopqrs-cleanup + documentSelector: + path: kind + value: ServiceAccount + - equal: + path: subjects[0].name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmnopqrs-cleanup + documentSelector: + path: kind + value: RoleBinding diff --git a/packages/apps/vm-disk/templates/pvc-resize-hook.yaml b/packages/apps/vm-disk/templates/pvc-resize-hook.yaml index 5e327c6f5..224da33ba 100644 --- a/packages/apps/vm-disk/templates/pvc-resize-hook.yaml +++ b/packages/apps/vm-disk/templates/pvc-resize-hook.yaml @@ -15,7 +15,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: "{{ .Release.Name }}-volume-resize-hook" + name: {{ printf "%s-volume-resize-hook" .Release.Name | trunc 63 | trimSuffix "-" | quote }} annotations: helm.sh/hook: pre-install,pre-upgrade helm.sh/hook-weight: "0" diff --git a/packages/apps/vm-disk/tests/job_name_budget_test.yaml b/packages/apps/vm-disk/tests/job_name_budget_test.yaml new file mode 100644 index 000000000..5922dcc7b --- /dev/null +++ b/packages/apps/vm-disk/tests/job_name_budget_test.yaml @@ -0,0 +1,127 @@ +suite: vm-disk resize Job name budget + +# The API server copies a Job's name into the job-name label of its pods, and a +# label value takes 63 characters at most, while a release name may take 53. +# The hook renders only when an existing PVC grows, so each release below gets +# a smaller PVC to look up. The ServiceAccount, Role and RoleBinding have no +# such limit and keep the full name, so the ones an earlier resize left behind +# are replaced, not orphaned. + +templates: + - templates/pvc-resize-hook.yaml + +set: + storage: 2Gi + +kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + objects: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: test-vm-disk + namespace: tenant-test + spec: + resources: + requests: + storage: 1Gi + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789 + namespace: tenant-test + spec: + resources: + requests: + storage: 1Gi + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcd + namespace: tenant-test + spec: + resources: + requests: + storage: 1Gi + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi + namespace: tenant-test + spec: + resources: + requests: + storage: 1Gi + +tests: + - it: keeps -volume-resize-hook for a short release name + release: + name: test-vm-disk + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: test-vm-disk-volume-resize-hook + + - it: keeps the name when it lands exactly on 63 characters + release: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789 + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789-volume-resize-hook + + - it: does not end the cut name with a dash + release: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcd + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcd-volume-resize + + - it: cuts the name to 63 characters for a 53-character release name + release: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi-volume-re + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ + - equal: + path: spec.template.spec.serviceAccountName + value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi-volume-resize-hook + + - it: leaves the ServiceAccount on the full name for a 53-character release name + release: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi + namespace: tenant-test + documentSelector: + path: kind + value: ServiceAccount + asserts: + - equal: + path: metadata.name + value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi-volume-resize-hook diff --git a/packages/apps/vm-instance/templates/vm-update-hook.yaml b/packages/apps/vm-instance/templates/vm-update-hook.yaml index 72bc617e2..ab77f21b1 100644 --- a/packages/apps/vm-instance/templates/vm-update-hook.yaml +++ b/packages/apps/vm-instance/templates/vm-update-hook.yaml @@ -1,5 +1,6 @@ {{- $vmName := include "virtual-machine.fullname" . -}} {{- $namespace := .Release.Namespace -}} +{{- $jobName := printf "%s-update-hook" .Release.Name | trunc 63 | trimSuffix "-" -}} {{- $existingVM := lookup "kubevirt.io/v1" "VirtualMachine" $namespace $vmName -}} {{- $existingService := lookup "v1" "Service" $namespace $vmName -}} @@ -48,7 +49,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: "{{ $.Release.Name }}-update-hook" + name: "{{ $jobName }}" annotations: helm.sh/hook: pre-install,pre-upgrade helm.sh/hook-weight: "0" @@ -57,7 +58,7 @@ spec: template: metadata: labels: - app: "{{ $.Release.Name }}-update-hook" + app: "{{ $jobName }}" policy.cozystack.io/allow-to-apiserver: "true" spec: serviceAccountName: {{ $.Release.Name }}-update-hook diff --git a/packages/apps/vm-instance/tests/job_name_budget_test.yaml b/packages/apps/vm-instance/tests/job_name_budget_test.yaml new file mode 100644 index 000000000..fb4190930 --- /dev/null +++ b/packages/apps/vm-instance/tests/job_name_budget_test.yaml @@ -0,0 +1,107 @@ +suite: vm-instance update Job name budget + +# The API server copies a Job's name into the job-name label of its pods, and a +# label value takes 63 characters at most, while a release name may take 53. +# The pod's own app label carries the same name and is held to the same limit. +# The hook renders only when an existing VM or Service needs reconciling, so +# each release below gets a Service of the wrong type to look up. The +# ServiceAccount, Role and RoleBinding have no such limit and keep the full +# name, so the ones an earlier update left behind are replaced, not orphaned. + +templates: + - templates/vm-update-hook.yaml + +set: + external: false + +kubernetesProvider: + scheme: + "v1/Service": + gvr: + group: "" + version: "v1" + resource: "services" + namespaced: true + objects: + - kind: Service + apiVersion: v1 + metadata: + name: vm-instance-test + namespace: tenant-test + spec: + type: LoadBalancer + - kind: Service + apiVersion: v1 + metadata: + name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abc + namespace: tenant-test + spec: + type: LoadBalancer + - kind: Service + apiVersion: v1 + metadata: + name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde + namespace: tenant-test + spec: + type: LoadBalancer + +tests: + - it: keeps -update-hook for a short release name + release: + name: vm-instance-test + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: vm-instance-test-update-hook + - equal: + path: spec.template.metadata.labels.app + value: vm-instance-test-update-hook + + - it: keeps the name when it lands exactly on 63 characters + release: + name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abc + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abc-update-hook + + - it: cuts the Job name and the app label to 63 characters for a 53-character release name + release: + name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde-update-ho + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ + - equal: + path: spec.template.metadata.labels.app + value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde-update-ho + - equal: + path: spec.template.spec.serviceAccountName + value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde-update-hook + + - it: leaves the ServiceAccount on the full name for a 53-character release name + release: + name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde + namespace: tenant-test + documentSelector: + path: kind + value: ServiceAccount + asserts: + - equal: + path: metadata.name + value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde-update-hook From b74029cb390c79c1c7cf3a1a697e0f6b70d353d1 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Sun, 18 Oct 2026 23:11:02 +0300 Subject: [PATCH 002/174] fix(keycloak): apply keycloak-operator CRD updates on upgrade keycloak-operator ships its CRDs in the chart's crds/ directory and its PackageSource sets no upgradeCRDs policy, so helm-controller never replaces them. A cluster installed before v1.2.0 therefore runs the v1.32.0 operator against the v1.25.0 CRDs, whose ClusterKeycloakRealm schema has no spec.sessions, spec.login or spec.organizationsEnabled. The apiserver drops those fields from the realm keycloak-configure renders, so the operator never applies the session limits the platform sets. On a v1.6.4 cluster in that state Keycloak also rejected the dashboard client with HTTP 400, and replacing the CRDs from the chart by hand, with an operator restart, cleared it. The v1.32.0 CRDs are not purely additive. They drop the realm, targetRealm and keycloakOwner fields that v1.25.0 marked deprecated and require realmRef and keycloakRef instead. Every Keycloak object the platform renders already uses the refs, and each CRD keeps its single version, so replacing them loses nothing the platform relies on. helm-controller applies the policy only when it upgrades the release, and it runs an upgrade only when the release artifact or its values change, which a PackageSource edit does not do. The package test added here also changes the keycloak-operator artifact, so the replace runs on the upgrade that ships this change even where nothing else in the package changes. The test itself pins the realm fields keycloak-configure sets against the CRD the chart ships. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- .../platform/sources/keycloak-operator.yaml | 7 ++++++ .../sources_keycloak_operator_crds_test.yaml | 22 +++++++++++++++++++ packages/system/keycloak-operator/Makefile | 3 +++ .../keycloak-operator/tests/crds_test.yaml | 19 ++++++++++++++++ 4 files changed, 51 insertions(+) create mode 100644 packages/core/platform/tests/sources_keycloak_operator_crds_test.yaml create mode 100644 packages/system/keycloak-operator/tests/crds_test.yaml diff --git a/packages/core/platform/sources/keycloak-operator.yaml b/packages/core/platform/sources/keycloak-operator.yaml index ce3a5f7e0..5b116e5e6 100644 --- a/packages/core/platform/sources/keycloak-operator.yaml +++ b/packages/core/platform/sources/keycloak-operator.yaml @@ -20,3 +20,10 @@ spec: install: namespace: cozy-keycloak releaseName: keycloak-operator + # helm-controller leaves existing CRDs alone on upgrade, so a cluster + # installed before v1.2.0 keeps the v1.25.0 schema and the apiserver + # prunes the realm session settings keycloak-configure sets. v1.32.0 + # drops the realm, targetRealm and keycloakOwner fields deprecated in + # favour of realmRef and keycloakRef. The platform renders only the + # refs, so replace is safe. + upgradeCRDs: CreateReplace diff --git a/packages/core/platform/tests/sources_keycloak_operator_crds_test.yaml b/packages/core/platform/tests/sources_keycloak_operator_crds_test.yaml new file mode 100644 index 000000000..5cccfc757 --- /dev/null +++ b/packages/core/platform/tests/sources_keycloak_operator_crds_test.yaml @@ -0,0 +1,22 @@ +suite: keycloak-operator applies CRD updates on upgrade +# keycloak-configure sets spec.sessions and spec.organizationsEnabled, and +# spec.login when configured, on the cozy ClusterKeycloakRealm. The CRDs of +# keycloak-operator v1.25.0 declare none of them. Helm never upgrades CRDs +# shipped in crds/, so without CreateReplace a cluster installed before v1.2.0 +# keeps those CRDs, the apiserver drops the fields without an error, and the +# operator never applies them to the realm. Pin the policy so an upgrade cannot +# silently regress. +templates: + - templates/sources.yaml +release: + name: cozystack + namespace: cozy-system +tests: + - it: keycloak-operator PackageSource sets upgradeCRDs to CreateReplace + documentSelector: + path: metadata.name + value: cozystack.keycloak-operator + asserts: + - equal: + path: spec.variants[0].components[0].install.upgradeCRDs + value: CreateReplace diff --git a/packages/system/keycloak-operator/Makefile b/packages/system/keycloak-operator/Makefile index e2f7f45d5..303f37eeb 100644 --- a/packages/system/keycloak-operator/Makefile +++ b/packages/system/keycloak-operator/Makefile @@ -4,6 +4,9 @@ export NAMESPACE=cozy-keycloak include ../../../hack/common-envs.mk include ../../../hack/package.mk +test: + helm unittest . + image: docker buildx build images/keycloak-operator \ $(call image-tags,keycloak-operator,v$(COZYSTACK_VERSION)) \ diff --git a/packages/system/keycloak-operator/tests/crds_test.yaml b/packages/system/keycloak-operator/tests/crds_test.yaml new file mode 100644 index 000000000..1be3fe611 --- /dev/null +++ b/packages/system/keycloak-operator/tests/crds_test.yaml @@ -0,0 +1,19 @@ +suite: keycloak-operator CRDs declare the realm settings the platform sets +# keycloak-configure renders the cozy ClusterKeycloakRealm with spec.sessions +# and spec.organizationsEnabled, and spec.login when configured. The apiserver +# drops any field the CRD does not declare without an error, so with a chart +# that lacks them the realm never gets those settings. +includeCrds: true +templates: + - charts/keycloak-operator/crds/v1.edp.epam.com_clusterkeycloakrealms.yaml +tests: + - it: declares the session, organization and login settings + asserts: + - exists: + path: spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.sessions.properties.ssoSessionSettings.properties.idleTimeout + - exists: + path: spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.sessions.properties.ssoSessionSettings.properties.maxLifespan + - exists: + path: spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.organizationsEnabled + - exists: + path: spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.login From 8e96a3e9b73b127a116131327b73925e7c2ffa9a Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:59:52 +0300 Subject: [PATCH 003/174] fix(foundationdb): apply foundationdb-operator CRD updates on upgrade The foundationdb-operator PackageSource sets no upgradeCRDs policy, so a cluster installed before v1.6.0 runs the v2.30.0 operator on the v2.13.0 CRDs it first installed. The apiserver prunes every field added since, status.backupDetails.restorable on FoundationDBBackup among them, so a backup never shows as restorable however long it has run. The v2.30.0 CRDs only add fields. They keep the single v1beta2 version and drop or retype nothing, and the one new constraint, a pattern on perpetual_storage_wiggle_locality, sits on a field the platform does not set, so CreateReplace is safe. Nothing in the foundationdb-operator package changes here, so a cluster picks the new CRDs up at the next upgrade of that release, which comes with the next change to the package. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- .../sources/foundationdb-operator.yaml | 6 ++++++ ...urces_foundationdb_operator_crds_test.yaml | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 packages/core/platform/tests/sources_foundationdb_operator_crds_test.yaml diff --git a/packages/core/platform/sources/foundationdb-operator.yaml b/packages/core/platform/sources/foundationdb-operator.yaml index 0af881f60..28328c3d1 100644 --- a/packages/core/platform/sources/foundationdb-operator.yaml +++ b/packages/core/platform/sources/foundationdb-operator.yaml @@ -20,3 +20,9 @@ spec: install: namespace: cozy-foundationdb-operator releaseName: foundationdb-operator + # helm-controller leaves existing CRDs alone on upgrade, so a cluster + # installed before v1.6.0 keeps the v2.13.0 schema under the v2.30.0 + # operator, and the apiserver prunes the fields added since, + # status.backupDetails.restorable among them. The schema diff adds + # fields and drops none, so replace is safe. + upgradeCRDs: CreateReplace diff --git a/packages/core/platform/tests/sources_foundationdb_operator_crds_test.yaml b/packages/core/platform/tests/sources_foundationdb_operator_crds_test.yaml new file mode 100644 index 000000000..a0976ae57 --- /dev/null +++ b/packages/core/platform/tests/sources_foundationdb_operator_crds_test.yaml @@ -0,0 +1,20 @@ +suite: foundationdb-operator applies CRD updates on upgrade +# The fdb-operator v2.30.0 CRDs add fields the v2.13.0 ones lack, among them +# status.backupDetails.restorable on FoundationDBBackup. Helm never upgrades +# CRDs shipped in crds/, so without CreateReplace a cluster installed before +# v1.6.0 keeps the v2.13.0 CRDs and the apiserver prunes those fields on every +# write. Pin the policy so an upgrade cannot silently regress. +templates: + - templates/sources.yaml +release: + name: cozystack + namespace: cozy-system +tests: + - it: foundationdb-operator PackageSource sets upgradeCRDs to CreateReplace + documentSelector: + path: metadata.name + value: cozystack.foundationdb-operator + asserts: + - equal: + path: spec.variants[0].components[0].install.upgradeCRDs + value: CreateReplace From 89028b455ef2fbf4772672c2fe6342150efc3200 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Sun, 18 Oct 2026 23:16:45 +0300 Subject: [PATCH 004/174] fix(monitoring): route InfoInhibitor to the blackhole receiver InfoInhibitor exists to be the source side of an Alertmanager inhibit rule and, as its own description says, to go to a null receiver. The platform Alertmanager config has no inhibit rule, and the rule has carried severity major since it was imported alongside Alerta, so Alerta shows it one level below critical. Alerta's Telegram and Slack plugins forward it by default, and a notify_email label inherited from the info alert gets it emailed too. It fires whenever a rule labelled severity="info" is active with no warning or critical alert in the same namespace and cluster. The platform's own rules no longer use info, but rules it does not write still do. CDI generates CDIStorageProfilesIncomplete, KubeVirt generates KubeVirtVMGuestMemoryAvailableLow, and operators add their own. Each of them turns into a major InfoInhibitor in Alerta. none is the severity upstream kube-prometheus gives this rule, and the platform config already sends severity="none" to the blackhole ahead of every route that delivers. The severity contract refuses none because such an alert is never delivered, so InfoInhibitor is named as the one alert for which that is the point. Tests hold both ends, so neither the label nor the route order can drift back to delivering it. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- hack/alert-severity-contract.bats | 89 +++++++++++++++++++ .../alerts/general.rules.yaml | 2 +- .../tests/alertmanager_blackhole_test.yaml | 45 ++++++++++ 3 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 packages/system/monitoring/tests/alertmanager_blackhole_test.yaml diff --git a/hack/alert-severity-contract.bats b/hack/alert-severity-contract.bats index e1315eb96..44f197000 100644 --- a/hack/alert-severity-contract.bats +++ b/hack/alert-severity-contract.bats @@ -9,6 +9,10 @@ load test_helper # scope: they are re-vendored by `make update`, so a bad severity there is # fixed upstream or in patches/, not by editing the label. ALERT_SEVERITY_ALLOWED=" security critical major minor warning indeterminate informational normal ok cleared debug trace unknown " +# The alerts that exist only to be dropped, the one place `none` is right. +# InfoInhibitor is the source side of an Alertmanager inhibit rule, which the +# platform config does not have, so delivering it at any severity is noise. +ALERT_SEVERITY_BLACKHOLED=" InfoInhibitor " helm_actions_stripped() { awk ' @@ -104,6 +108,14 @@ alert_severity_contract() { break fi done + if [ "$severity" = "none" ]; then + for blackholed in $ALERT_SEVERITY_BLACKHOLED; do + if [ "$blackholed" = "$alert" ]; then + accepted=1 + break + fi + done + fi if [ "$accepted" -eq 0 ]; then echo "$file: $alert -> '$severity'" >&2 bad=$((bad + 1)) @@ -128,6 +140,83 @@ alert_severity_contract() { alert_severity_contract } +@test "InfoInhibitor is labelled for the blackhole route" { + rows=$(mktemp) || return 1 + if ! alert_severity_rows > "$rows"; then + rm -f "$rows" + return 1 + fi + tab=$(printf '\t') + found=$(grep -c "${tab}InfoInhibitor${tab}" "$rows" || true) + delivered=$(grep "${tab}InfoInhibitor${tab}" "$rows" | grep -vc "${tab}none\$" || true) + rm -f "$rows" + [ "$found" -gt 0 ] + [ "$delivered" -eq 0 ] +} + +@test "severity none passes only on an alert that exists to be dropped" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/alerts" + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: general' \ + ' rules:' \ + ' - alert: InfoInhibitor' \ + ' labels:' \ + ' severity: none' \ + > "$fixture/packages/system/example/alerts/00-inhibitor.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml + + inhibitor_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || inhibitor_succeeded=$? + + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: silent' \ + ' rules:' \ + ' - alert: SilentAlert' \ + ' labels:' \ + ' severity: none' \ + > "$fixture/packages/system/example/alerts/01-silent.yaml" + git -C "$fixture" add packages/system/example/alerts/01-silent.yaml + + silent_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || silent_succeeded=$? + rm -rf "$fixture" + [ "$inhibitor_succeeded" -eq 0 ] + [ "$silent_succeeded" -ne 0 ] +} + +@test "an alert that exists to be dropped still fails on a refused severity" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/alerts" + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: general' \ + ' rules:' \ + ' - alert: InfoInhibitor' \ + ' labels:' \ + ' severity: info' \ + > "$fixture/packages/system/example/alerts/00-inhibitor.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml + + contract_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$? + rm -rf "$fixture" + [ "$contract_succeeded" -ne 0 ] +} + @test "a parser failure cannot leave the contract green" { fixture=$(mktemp -d) mkdir -p "$fixture/packages/system/example/alerts" diff --git a/packages/system/monitoring-agents/alerts/general.rules.yaml b/packages/system/monitoring-agents/alerts/general.rules.yaml index 924870c40..45dd6100f 100644 --- a/packages/system/monitoring-agents/alerts/general.rules.yaml +++ b/packages/system/monitoring-agents/alerts/general.rules.yaml @@ -52,6 +52,6 @@ spec: != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 labels: - severity: major + severity: none exported_instance: global service: general.rules diff --git a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml new file mode 100644 index 000000000..e987bd405 --- /dev/null +++ b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml @@ -0,0 +1,45 @@ +suite: Alertmanager blackhole route +templates: + - templates/alerta/alerta.yaml + +release: + name: monitoring + namespace: tenant-root + +set: + _cluster: + root-host: example.org + _namespace: + host: tenant-root.example.org + ingress: tenant-root + +# Rules labelled severity="none", InfoInhibitor among them, exist to be +# dropped, and Alerta answers that severity with a 500. Only the heartbeat +# route may come ahead of the blackhole, so no delivering route claims them. +tests: + - it: drops severity none ahead of every delivering route + asserts: + - documentSelector: + path: metadata.labels.app + value: alertmanager + matchRegex: + path: stringData["alertmanager.yaml"] + pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n' + + - it: keeps severity none out of email even when its alert name is listed + set: + alerta: + alerts: + email: + to: "oncall@example.org" + from: "alerts@example.org" + smarthost: "smtp.example.org:465" + alertnames: + - InfoInhibitor + asserts: + - documentSelector: + path: metadata.labels.app + value: alertmanager + matchRegex: + path: stringData["alertmanager.yaml"] + pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n' From 8d82348e85fb9588ad213d763ece81f7b88e84a8 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:04:58 +0300 Subject: [PATCH 005/174] fix(vm-disk): refuse a disk smaller than the golden image it clones CDI checks a clone against its source before it creates the claim. When the golden image's PVC is in Block mode and requests more than the disk, it records CloneValidationFailed and leaves the DataVolume pending for good. The DataVolume spec is immutable, so raising storage on the VMDisk afterwards changes nothing, and the resize hook finds no claim to patch. Every image in vm-default-images is 20Gi on replicated, where CDI picks Block mode for the LINSTOR driver, while the chart defaults storage to 5Gi, so a disk created from an image at the default size never comes up. The chart now refuses such a disk at render time and names both sizes, so the user sets storage to the golden's size before any DataVolume exists. It compares what validateClone compares in CDI v1.66.1, the golden PVC's request against the disk's request raised to the target StorageProfile's minimumSupportedPvcSize. A Filesystem golden is not compared, because CDI sizes that clone from the image inside it or with a filesystem overhead, and neither is a golden whose PVC does not exist yet. An existing disk is exempt, as for the other golden checks. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- packages/apps/vm-disk/templates/dv.yaml | 20 ++ .../vm-disk/tests/dv_golden_size_test.yaml | 252 ++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 packages/apps/vm-disk/tests/dv_golden_size_test.yaml diff --git a/packages/apps/vm-disk/templates/dv.yaml b/packages/apps/vm-disk/templates/dv.yaml index 09831fc79..9bec73f5f 100644 --- a/packages/apps/vm-disk/templates/dv.yaml +++ b/packages/apps/vm-disk/templates/dv.yaml @@ -146,6 +146,26 @@ {{- fail (printf "source.image %q: golden image %q lives in StorageClass %q, and a clone into StorageClass %q of the same provisioner gets no host-assisted fallback from CDI. LINSTOR creates the clone next to the source data, and StorageClass %q does not allow remote volume access from every node, so the clone may never bind. Set storageClass to %q, or ask the platform administrator to set cloneStrategyOverride to copy in the kubevirt-cdi package." .Values.source.image.name $desiredSource.pvc.name $goldenSC $targetSC $targetSC $goldenSC) }} {{- end }} {{- end }} + {{- /* CDI v1.66.1 refuses to clone a Block source PVC into a smaller + request before it creates the claim (validateClone in + pkg/controller/datavolume/pvc-clone-controller.go), and the + DataVolume then waits forever. It raises the request to the target + StorageProfile's minimumSupportedPvcSize before comparing. A + Filesystem source is not compared, as CDI sizes that clone from + the image inside it or with a filesystem overhead the chart does + not reproduce. */}} + {{- $goldenSize := dig "spec" "resources" "requests" "storage" "" ($goldenPVC | default dict) | toString }} + {{- if and .Values.storage $goldenSize (eq (dig "spec" "volumeMode" "" ($goldenPVC | default dict) | toString) "Block") }} + {{- $goldenBytes := include "cozy-lib.resources.toFloat" $goldenSize | float64 }} + {{- $diskBytes := include "cozy-lib.resources.toFloat" (.Values.storage | toString) | float64 }} + {{- if and (lt $diskBytes $goldenBytes) $targetSC }} + {{- $minimum := dig "metadata" "annotations" "cdi.kubevirt.io/minimumSupportedPvcSize" "" (lookup "cdi.kubevirt.io/v1beta1" "StorageProfile" "" $targetSC | default dict) | toString }} + {{- $diskBytes = maxf $diskBytes (include "cozy-lib.resources.toFloat" $minimum | float64) }} + {{- end }} + {{- if lt $diskBytes $goldenBytes }} + {{- fail (printf "storage %s is smaller than golden image %q (%s), so CDI refuses to clone it and the disk would stay pending forever. Set storage to %s or more." (.Values.storage | toString) $desiredSource.pvc.name $goldenSize $goldenSize) }} + {{- end }} + {{- end }} {{- end }} {{- else if hasKey .Values.source "http" }} {{- $desiredSource = dict "http" (dict "url" (required "A valid .Values.source.http.url entry required!" .Values.source.http.url)) }} diff --git a/packages/apps/vm-disk/tests/dv_golden_size_test.yaml b/packages/apps/vm-disk/tests/dv_golden_size_test.yaml new file mode 100644 index 000000000..6a68874c7 --- /dev/null +++ b/packages/apps/vm-disk/tests/dv_golden_size_test.yaml @@ -0,0 +1,252 @@ +suite: vm-disk refuses a disk smaller than the golden image it clones +release: + name: test-vm-disk + namespace: tenant-test +templates: + - templates/dv.yaml +tests: + - it: refuses a new disk at the chart default size from a larger Block golden + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: &scheme + "cdi.kubevirt.io/v1beta1/DataVolume": + gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: datavolumes} + namespaced: true + "cdi.kubevirt.io/v1beta1/StorageProfile": + gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: storageprofiles} + namespaced: false + "v1/Namespace": + gvr: {group: "", version: v1, resource: namespaces} + namespaced: false + "v1/PersistentVolumeClaim": + gvr: {group: "", version: v1, resource: persistentvolumeclaims} + namespaced: true + "storage.k8s.io/v1/StorageClass": + gvr: {group: storage.k8s.io, version: v1, resource: storageclasses} + namespaced: false + objects: + - &namespace + kind: Namespace + apiVersion: v1 + metadata: {name: tenant-test} + - &goldenPVC + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + spec: + storageClassName: replicated + volumeMode: Block + resources: {requests: {storage: 20Gi}} + asserts: + - failedTemplate: + errorPattern: 'storage 5Gi is smaller than golden image "vm-default-images-ubuntu" \(20Gi\), so CDI refuses to clone it and the disk would stay pending forever\. Set storage to 20Gi or more\.' + + - it: clones into a disk the size of the golden + set: + source: {image: {name: ubuntu}} + storage: 20Gi + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - equal: + path: spec.storage.resources.requests.storage + value: 20Gi + + - it: clones into a disk larger than the golden + set: + source: {image: {name: ubuntu}} + storage: 25Gi + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - equal: + path: spec.storage.resources.requests.storage + value: 25Gi + + - it: compares bytes, so 21G is smaller than a 20Gi golden + set: + source: {image: {name: ubuntu}} + storage: 21G + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - failedTemplate: + errorPattern: 'storage 21G is smaller than golden image "vm-default-images-ubuntu" \(20Gi\)' + + - it: compares bytes, so 20480Mi is the size of a 20Gi golden + set: + source: {image: {name: ubuntu}} + storage: 20480Mi + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - equal: + path: spec.storage.resources.requests.storage + value: 20480Mi + + - it: clones when the target StorageProfile's minimum PVC size covers the golden + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + - kind: StorageProfile + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: replicated + annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 20Gi} + asserts: + - notFailedTemplate: {} + + - it: refuses when the target StorageProfile's minimum PVC size is still below the golden + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + - kind: StorageProfile + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: replicated + annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 10Gi} + asserts: + - failedTemplate: + errorPattern: 'storage 5Gi is smaller than golden image "vm-default-images-ubuntu" \(20Gi\)' + + - it: reads the minimum PVC size of the default class for an empty storageClass + set: + source: {image: {name: ubuntu}} + storageClass: "" + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: replicated + annotations: {storageclass.kubernetes.io/is-default-class: "true"} + provisioner: linstor.csi.linbit.com + - kind: StorageProfile + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: replicated + annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 20Gi} + asserts: + - notFailedTemplate: {} + + - it: does not compare sizes for a Filesystem golden + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + spec: + storageClassName: replicated + volumeMode: Filesystem + resources: {requests: {storage: 20Gi}} + asserts: + - notFailedTemplate: {} + + - it: renders without a cluster to look the golden up in + set: + source: {image: {name: ubuntu}} + asserts: + - equal: + path: spec.storage.resources.requests.storage + value: 5Gi + + - it: clones a golden whose PVC does not exist yet + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + status: {phase: ImportScheduled} + asserts: + - notFailedTemplate: {} + + - it: leaves a disk without a size to CDI, which takes the golden's + set: + source: {image: {name: ubuntu}} + storage: null + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - notFailedTemplate: {} + + - it: leaves an existing disk alone + set: + source: {image: {name: ubuntu}} + storage: 25Gi + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: test-vm-disk + namespace: tenant-test + annotations: + vm-disk.cozystack.io/source: '{"image":{"name":"ubuntu"}}' + vm-disk.cozystack.io/storage-class: replicated + spec: + source: {pvc: {name: vm-default-images-ubuntu, namespace: cozy-public}} + storage: {resources: {requests: {storage: 10Gi}}, storageClassName: replicated} + asserts: + - notFailedTemplate: {} + + - it: does not compare sizes for an http source + set: + source: {http: {url: "https://example.com/disk.img"}} + storage: 1Gi + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - notFailedTemplate: {} + + - it: does not compare sizes for an upload source + set: + source: {upload: {}} + storage: 1Gi + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - notFailedTemplate: {} From 85026299958edba4ccdcfaafef5b17fad7f055fc Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:33:25 +0300 Subject: [PATCH 006/174] fix(piraeus-operator): keep the LINSTOR error-report alerts quiet when the controller pod moves linstor_error_reports_count comes from the LINSTOR controller, so its series carry the controller pod's instance and controller_node labels. When the pod moves to another node, or an external label of the scrape changes, the existing report counts come back as new series. VictoriaMetrics' increase() counts the first samples of a series from zero when they are small, and report counts are small, so linstorSatelliteErrorRate fires for every node with an old report and keeps firing for 15 minutes. The up guard matches on instance and lets the new series through. The controller also leaves a satellite out of the metric while it is not connected, so a satellite that comes back after more than five minutes has its old count counted again in the same way. Keyed on hostname, a moved controller or a changed label continues the same series, and holding the last value for an hour bridges a disconnect or a slow controller restart. The up guard matches on() because the result no longer has an instance label. The controller rule takes the same form. Its hostname is the pod name and a new pod starts with no reports, but a label change hit it the same way. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- hack/linstor-error-report-alerts.bats | 32 +++ .../linstor-error-report-alerts/unittest.yaml | 198 ++++++++++++++++++ .../alerts/piraeus-datastore.yaml | 6 +- 3 files changed, 233 insertions(+), 3 deletions(-) create mode 100644 hack/linstor-error-report-alerts.bats create mode 100644 hack/testdata/linstor-error-report-alerts/unittest.yaml diff --git a/hack/linstor-error-report-alerts.bats b/hack/linstor-error-report-alerts.bats new file mode 100644 index 000000000..4c6124aad --- /dev/null +++ b/hack/linstor-error-report-alerts.bats @@ -0,0 +1,32 @@ +#!/usr/bin/env bats +# Replays the LINSTOR error-report alerts from +# packages/system/piraeus-operator/alerts/piraeus-datastore.yaml through +# vmalert-tool, pinned to the VictoriaMetrics release that the +# victoria-metrics-operator deploys by default. Its increase() counts the first +# samples of a series as new when they are small, and the report counts are +# small, so the cases pin that a series which starts over after a controller +# move, a label change or a disconnect does not report old error reports again. +# Only pulling the image touches the network. + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +VMALERT_TOOL_IMAGE="victoriametrics/vmalert-tool:v1.136.0@sha256:417346d7b6fdb66d8e3984887c2ad114535ec7d983eeef53381b56c776f333f1" + +@test "LINSTOR error-report alerts pass their vmalert-tool cases" { + docker info >/dev/null 2>&1 || { + echo "docker is required to run the cases in $VMALERT_TOOL_IMAGE" >&2 + return 1 + } + work=$(mktemp -d) + yq '.spec' "$REPO_ROOT/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml" > "$work/rules.yaml" + cp "$REPO_ROOT/hack/testdata/linstor-error-report-alerts/unittest.yaml" "$work/unittest.yaml" + rc=0 + docker run --rm --network none -v "$work:/w:ro" "$VMALERT_TOOL_IMAGE" \ + unittest --files /w/unittest.yaml > "$work/out" 2>&1 || rc=$? + cat "$work/out" + passed=$(grep -cx 'SUCCESS' "$work/out" || true) + rm -rf "$work" + [ "$rc" -eq 0 ] + [ "$passed" -eq 1 ] +} diff --git a/hack/testdata/linstor-error-report-alerts/unittest.yaml b/hack/testdata/linstor-error-report-alerts/unittest.yaml new file mode 100644 index 000000000..41e332169 --- /dev/null +++ b/hack/testdata/linstor-error-report-alerts/unittest.yaml @@ -0,0 +1,198 @@ +rule_files: + - rules.yaml + +evaluation_interval: 15s + +tests: + - name: controller pod moves to another node + interval: 30s + input_series: + - series: 'up{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a"}' + values: '1x38 0 stale' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a", hostname="node-a", module="SATELLITE"}' + values: '1x38 stale' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a", hostname="node-b", module="SATELLITE"}' + values: '2x38 stale' + - series: 'up{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b"}' + values: '_x41 1x79' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b", hostname="node-a", module="SATELLITE"}' + values: '_x41 1x58 2x20' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b", hostname="node-b", module="SATELLITE"}' + values: '_x41 2x79' + alert_rule_test: + - eval_time: 21m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 28m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 35m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 45m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 51m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + exp_alerts: + - exp_labels: + hostname: node-a + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-a" to inspect the reports. + + - name: an external label of the scrape changes + interval: 30s + input_series: + - series: 'up{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack"}' + values: '1x39 stale' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack", hostname="node-a", module="SATELLITE"}' + values: '1x39 stale' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack", hostname="linstor-controller-abc", module="CONTROLLER"}' + values: '1x39 stale' + - series: 'up{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a"}' + values: '_x41 1x79' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a", hostname="node-a", module="SATELLITE"}' + values: '_x41 1x79' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a", hostname="linstor-controller-abc", module="CONTROLLER"}' + values: '_x41 1x79' + alert_rule_test: + - eval_time: 21m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 30m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 21m + groupname: linstor.rules + alertname: linstorControllerErrorRate + - eval_time: 30m + groupname: linstor.rules + alertname: linstorControllerErrorRate + + - name: a satellite drops out of the metric while it is disconnected + interval: 30s + input_series: + - series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}' + values: '1x220' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}' + values: '1x38 stale _x80 1x100' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-b", module="SATELLITE"}' + values: '2x220' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-c", module="SATELLITE"}' + values: '1x38 stale _x80 2x100' + alert_rule_test: + - eval_time: 61m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + exp_alerts: + - exp_labels: + hostname: node-c + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-c" to inspect the reports. + - eval_time: 70m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + exp_alerts: + - exp_labels: + hostname: node-c + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-c" to inspect the reports. + - eval_time: 76m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + + - name: a new report fires for the 15 minute window + interval: 30s + input_series: + - series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}' + values: '1x120' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}' + values: '1x39 2x80' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-b", module="SATELLITE"}' + values: '2x120' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-c", module="SATELLITE"}' + values: '0x39 1x80' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="linstor-controller-abc", module="CONTROLLER"}' + values: '0x39 1x80' + alert_rule_test: + - eval_time: 21m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + exp_alerts: + - exp_labels: + hostname: node-a + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-a" to inspect the reports. + - exp_labels: + hostname: node-c + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-c" to inspect the reports. + - eval_time: 34m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + exp_alerts: + - exp_labels: + hostname: node-a + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-a" to inspect the reports. + - exp_labels: + hostname: node-c + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-c" to inspect the reports. + - eval_time: 36m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 21m + groupname: linstor.rules + alertname: linstorControllerErrorRate + exp_alerts: + - exp_labels: + hostname: linstor-controller-abc + severity: warning + exp_annotations: + description: | + LINSTOR Controller reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --since 15minutes" to see them. + - eval_time: 36m + groupname: linstor.rules + alertname: linstorControllerErrorRate + + - name: a failed controller scrape still holds the alert back + interval: 30s + input_series: + - series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}' + values: '1x19 0x140 1x60' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}' + values: '1x19 stale _x140 1x60' + alert_rule_test: + - eval_time: 85m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 93m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 100m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate diff --git a/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml b/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml index ab679d663..8be364a72 100644 --- a/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml +++ b/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml @@ -29,8 +29,8 @@ spec: LINSTOR Satellite "{{ $labels.hostname }}" reports {{ $value }} errors in the last 15 minutes. Use "linstor error-reports list --nodes {{ $labels.hostname }}" to inspect the reports. expr: | - increase(linstor_error_reports_count{module="SATELLITE"}[15m]) > 0 - and on(instance, job) + increase((max by (hostname) (last_over_time(linstor_error_reports_count{module="SATELLITE"}[1h])))[15m:1m]) > 0 + and on() min_over_time(up{job="linstor-controller"}[15m]) == 1 labels: severity: warning @@ -39,7 +39,7 @@ spec: description: | LINSTOR Controller reports {{ $value }} errors in the last 15 minutes. Use "linstor error-reports list --since 15minutes" to see them. - expr: increase(linstor_error_reports_count{module="CONTROLLER"}[15m]) > 0 + expr: increase((max by (hostname) (last_over_time(linstor_error_reports_count{module="CONTROLLER"}[1h])))[15m:1m]) > 0 labels: severity: warning - alert: linstorSatelliteNotOnline From 78cadf53f9f88a2b5569024031c399a5aba6e075 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:55:08 +0300 Subject: [PATCH 007/174] fix(tcp-balancer): render the kubernetes and talos Service ports only with kubernetesAndTalos The Service rendered ports 6443 and 50000 under httpAndHttps, while the Deployment declares the kubernetes and talos container ports and HAProxy binds those frontends only under kubernetesAndTalos. So with the default values every Service carried two ports whose targetPort names no container declares, with no endpoints behind them, and with external: true the LoadBalancer Service carried them as well. A release with kubernetesAndTalos and without httpAndHttps got no Service port for those frontends at all. kubernetesAndTalos is not documented in values.yaml, but the apps API keeps the top level of spec open and hands it to the chart as values, so the key reaches the chart and the Deployment and the ConfigMap already follow it. Gating the Service on the same key keeps the three templates in step, while dropping the block would leave HAProxy listening on ports that no Service exposes. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- .../apps/tcp-balancer/templates/service.yaml | 2 +- .../apps/tcp-balancer/tests/service_test.yaml | 97 +++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 packages/apps/tcp-balancer/tests/service_test.yaml diff --git a/packages/apps/tcp-balancer/templates/service.yaml b/packages/apps/tcp-balancer/templates/service.yaml index 8384cc85b..689db7e16 100644 --- a/packages/apps/tcp-balancer/templates/service.yaml +++ b/packages/apps/tcp-balancer/templates/service.yaml @@ -27,7 +27,7 @@ spec: port: 443 targetPort: https {{- end }} - {{- with .Values.httpAndHttps }} + {{- with .Values.kubernetesAndTalos }} - name: kubernetes protocol: TCP port: 6443 diff --git a/packages/apps/tcp-balancer/tests/service_test.yaml b/packages/apps/tcp-balancer/tests/service_test.yaml new file mode 100644 index 000000000..57191115c --- /dev/null +++ b/packages/apps/tcp-balancer/tests/service_test.yaml @@ -0,0 +1,97 @@ +suite: haproxy service + +templates: + - templates/service.yaml + - templates/deployment.yaml + - templates/configmap.yaml + +tests: + - it: publishes only http and https by default, both on container ports the Deployment declares + asserts: + - equal: + path: spec.ports + value: + - name: http + protocol: TCP + port: 80 + targetPort: http + - name: https + protocol: TCP + port: 443 + targetPort: https + template: templates/service.yaml + - equal: + path: spec.template.spec.containers[0].ports + value: + - containerPort: 8080 + name: http + - containerPort: 8443 + name: https + template: templates/deployment.yaml + + - it: publishes kubernetes and talos for kubernetesAndTalos alone, both on container ports the Deployment declares + set: + httpAndHttps: null + kubernetesAndTalos: + endpoints: + - 10.1.1.1 + asserts: + - equal: + path: spec.ports + value: + - name: kubernetes + protocol: TCP + port: 6443 + targetPort: kubernetes + - name: talos + protocol: TCP + port: 50000 + targetPort: talos + template: templates/service.yaml + - equal: + path: spec.template.spec.containers[0].ports + value: + - containerPort: 6443 + name: kubernetes + - containerPort: 50000 + name: talos + template: templates/deployment.yaml + + - it: publishes all four ports when both are set, each on a container port the Deployment declares + set: + kubernetesAndTalos: + endpoints: + - 10.1.1.1 + asserts: + - equal: + path: spec.ports + value: + - name: http + protocol: TCP + port: 80 + targetPort: http + - name: https + protocol: TCP + port: 443 + targetPort: https + - name: kubernetes + protocol: TCP + port: 6443 + targetPort: kubernetes + - name: talos + protocol: TCP + port: 50000 + targetPort: talos + template: templates/service.yaml + - equal: + path: spec.template.spec.containers[0].ports + value: + - containerPort: 8080 + name: http + - containerPort: 8443 + name: https + - containerPort: 6443 + name: kubernetes + - containerPort: 50000 + name: talos + template: templates/deployment.yaml From f1f30b3071254e3640292b4eb483b04f5d40d77a Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:05:51 +0300 Subject: [PATCH 008/174] fix(etcd): require an even spread of members over nodes by default The default spread of the etcd members was ScheduleAnyway, which only adds to the score of a node, and other scores can outweigh it. A new etcd on a three-node cluster got its first two members on one node. On a node-local StorageClass a member cannot leave its node, so losing that node takes two of the three members and the quorum. DoNotSchedule with maxSkew 1 still lets an etcd with more members than nodes start, because the skew is counted between the nodes that exist. One node takes all three members and two nodes take two and one. nodeTaintsPolicy is Ignore, so a cordoned node still counts and a member waits for it instead of landing next to another member. The operator copies the constraints into an EtcdMember when it creates it and builds the Pod from that copy, so members that exist keep the old setting and only members created later get the new one. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- .../extra/etcd/templates/etcd-cluster.yaml | 3 +- .../etcd/tests/topology-spread_test.yaml | 46 ++++++++++++++----- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/packages/extra/etcd/templates/etcd-cluster.yaml b/packages/extra/etcd/templates/etcd-cluster.yaml index 2824ce371..67dd34bd9 100644 --- a/packages/extra/etcd/templates/etcd-cluster.yaml +++ b/packages/extra/etcd/templates/etcd-cluster.yaml @@ -69,7 +69,8 @@ spec: topologySpreadConstraints: - maxSkew: 1 topologyKey: "kubernetes.io/hostname" - whenUnsatisfiable: ScheduleAnyway + whenUnsatisfiable: DoNotSchedule + nodeTaintsPolicy: Ignore labelSelector: matchLabels: app.kubernetes.io/instance: etcd diff --git a/packages/extra/etcd/tests/topology-spread_test.yaml b/packages/extra/etcd/tests/topology-spread_test.yaml index 4b96fc310..11bb92243 100644 --- a/packages/extra/etcd/tests/topology-spread_test.yaml +++ b/packages/extra/etcd/tests/topology-spread_test.yaml @@ -8,7 +8,7 @@ templates: - templates/etcd-cluster.yaml tests: - - it: uses a soft per-host spread by default + - it: uses a strict per-host spread by default set: _cluster: {} documentSelector: @@ -16,17 +16,41 @@ tests: value: EtcdCluster asserts: - equal: - path: spec.topologySpreadConstraints[0].maxSkew - value: 1 + path: spec.topologySpreadConstraints[0] + value: + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + nodeTaintsPolicy: Ignore + labelSelector: + matchLabels: + app.kubernetes.io/instance: etcd + - lengthEqual: + path: spec.topologySpreadConstraints + count: 1 + + - it: renders a soft cluster-wide spread policy as written + set: + _cluster: + scheduling: + globalAppTopologySpreadConstraints: | + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + documentSelector: + path: kind + value: EtcdCluster + asserts: - equal: - path: spec.topologySpreadConstraints[0].topologyKey - value: kubernetes.io/hostname - - equal: - path: spec.topologySpreadConstraints[0].whenUnsatisfiable - value: ScheduleAnyway - - equal: - path: spec.topologySpreadConstraints[0].labelSelector.matchLabels["app.kubernetes.io/instance"] - value: etcd + path: spec.topologySpreadConstraints[0] + value: + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/instance: etcd - lengthEqual: path: spec.topologySpreadConstraints count: 1 From 5632596fe84fa358191c62f5f316df62f32c865c Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:38:55 +0300 Subject: [PATCH 009/174] feat(monitoring): alert when pods of one workload share a node Two etcd members, two Redis Sentinels or both replicas of a storage tier can end up on one node through a soft spread rule, a drain, or a placement made before a spread rule existed. Nothing reported it, and the scheduler does not move running pods, so it stays like that until a pod is recreated. One node failure then costs the quorum or every copy at once. etcd-operator owns each member pod by its own EtcdMember, so members are grouped by the EtcdCluster name their names start with. Owner and node are read from kube_pod_info because kube_pod_owner and kube_pod_status_phase get the node of the kube-state-metrics pod as a target label. Nodes with a NoSchedule or NoExecute taint do not count, which covers cordoned, not ready and unreachable nodes. The alert waits an hour, longer than a replacement pod runs next to its terminating predecessor or a node takes to drain and come back. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- .../workload-placement-alerts/unittest.yaml | 572 ++++++++++++++++++ hack/workload-placement-alerts.bats | 26 + .../alerts/workload-placement.yaml | 81 +++ 3 files changed, 679 insertions(+) create mode 100644 hack/testdata/workload-placement-alerts/unittest.yaml create mode 100644 hack/workload-placement-alerts.bats create mode 100644 packages/system/monitoring-agents/alerts/workload-placement.yaml diff --git a/hack/testdata/workload-placement-alerts/unittest.yaml b/hack/testdata/workload-placement-alerts/unittest.yaml new file mode 100644 index 000000000..2565a8241 --- /dev/null +++ b/hack/testdata/workload-placement-alerts/unittest.yaml @@ -0,0 +1,572 @@ +rule_files: + - rules.yaml + +evaluation_interval: 15s + +tests: + - name: two of three StatefulSet pods on one node + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", node="node-b", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 59m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: StatefulSet + owner_name: db + node: node-a + severity: warning + exported_instance: tenant-a/db + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of StatefulSet tenant-a/db are on node node-a in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + + - name: StatefulSet pods spread over three nodes + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", node="node-b", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", node="node-c", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: five ReplicaSet pods on three nodes + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-bbbbb", node="node-a", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ddddd", node="node-b", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-eeeee", node="node-c", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-aaaaa", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-bbbbb", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ccccc", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ddddd", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-eeeee", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: DaemonSet surge pod next to the old one + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-x7k2p", node="node-a", created_by_kind="DaemonSet", created_by_name="node-agent"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-q9z4m", node="node-a", created_by_kind="DaemonSet", created_by_name="node-agent"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-b5n8w", node="node-b", created_by_kind="DaemonSet", created_by_name="node-agent"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-x7k2p", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-q9z4m", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-b5n8w", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: two pods of one Job on one node + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-h2k4m", node="node-a", created_by_kind="Job", created_by_name="backup-29345678"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-p8r6t", node="node-a", created_by_kind="Job", created_by_name="backup-29345678"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-h2k4m", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-p8r6t", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: etcd members of one EtcdCluster on one node + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-x7k2p", node="node-a", created_by_kind="EtcdMember", created_by_name="etcd-x7k2p"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-q9z4m", node="node-a", created_by_kind="EtcdMember", created_by_name="etcd-q9z4m"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-b5n8w", node="node-b", created_by_kind="EtcdMember", created_by_name="etcd-b5n8w"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-0", node="node-b", created_by_kind="EtcdMember", created_by_name="etcd-0"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-1", node="node-c", created_by_kind="EtcdMember", created_by_name="etcd-1"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-2", node="node-c", created_by_kind="EtcdMember", created_by_name="etcd-2"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-x7k2p", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-q9z4m", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-b5n8w", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-0", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-2", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: EtcdCluster + owner_name: etcd + node: node-a + severity: warning + exported_instance: tenant-a/etcd + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of EtcdCluster tenant-a/etcd are on node node-a in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + - exp_labels: + cluster: cozystack + namespace: tenant-b + owner_kind: EtcdCluster + owner_name: etcd + node: node-c + severity: warning + exported_instance: tenant-b/etcd + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of EtcdCluster tenant-b/etcd are on node node-c in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + + - name: replacement pod next to its terminating predecessor for twenty minutes + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '1x30 stale' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '_x10 1x140' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", phase="Running", node="node-c"}' + values: '1x30 stale' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", phase="Running", node="node-c"}' + values: '_x10 1x140' + alert_rule_test: + - eval_time: 29m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 75m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: the same overlap held past the hour + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '1x100 stale' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '_x10 1x140' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", phase="Running", node="node-c"}' + values: '1x100 stale' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", phase="Running", node="node-c"}' + values: '_x10 1x140' + alert_rule_test: + - eval_time: 69m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 71m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: ReplicaSet + owner_name: web-6b8f4 + node: node-b + severity: warning + exported_instance: tenant-a/web-6b8f4 + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of ReplicaSet tenant-a/web-6b8f4 are on node node-b in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + - eval_time: 105m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: a cordoned node does not count until it is uncordoned + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '0x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '0x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x30 0x120' + - series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unschedulable", value="", effect="NoSchedule"}' + values: '1x30 stale' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", node="node-b", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", phase="Running", node="node-a"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", phase="Running", node="node-a"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", phase="Running", node="node-a"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 90m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 92m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: StatefulSet + owner_name: kv + node: node-a + severity: warning + exported_instance: tenant-a/kv + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of StatefulSet tenant-a/kv are on node node-a in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + + - name: a node that is down does not count + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '0x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '0x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '0x150' + - series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unreachable", value="", effect="NoSchedule"}' + values: '1x150' + - series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unreachable", value="", effect="NoExecute"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", node="node-b", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", phase="Running", node="node-a"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", phase="Running", node="node-a"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", phase="Running", node="node-a"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: pods that are not running do not count + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", node="node-a", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-aaaaa", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", phase="Running", node="node-c"}' + values: '0x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", phase="Failed", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-ccccc", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: one pod each in two clusters that share workload and node names + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="kube-system", pod="coredns-5d78c9869d-x7k2p", node="node-a", created_by_kind="ReplicaSet", created_by_name="coredns-5d78c9869d"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="kubernetes-example", namespace="kube-system", pod="coredns-5d78c9869d-q9z4m", node="node-a", created_by_kind="ReplicaSet", created_by_name="coredns-5d78c9869d"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="kube-system", pod="coredns-5d78c9869d-x7k2p", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="kubernetes-example", namespace="kube-system", pod="coredns-5d78c9869d-q9z4m", phase="Running", node="node-b"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: ReplicaSet, CNPG Cluster and StrimziPodSet pods on one node + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-h2k4m", node="node-a", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-p8r6t", node="node-a", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-w3x5z", node="node-b", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-1", node="node-a", created_by_kind="Cluster", created_by_name="postgres-db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-2", node="node-b", created_by_kind="Cluster", created_by_name="postgres-db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-3", node="node-b", created_by_kind="Cluster", created_by_name="postgres-db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-0", node="node-c", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-1", node="node-c", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-2", node="node-a", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-h2k4m", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-p8r6t", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-w3x5z", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-2", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-3", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-0", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-2", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: ReplicaSet + owner_name: rfs-cache-6f7d8b9c5d + node: node-a + severity: warning + exported_instance: tenant-a/rfs-cache-6f7d8b9c5d + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of ReplicaSet tenant-a/rfs-cache-6f7d8b9c5d are on node node-a in cluster + cozystack, although the cluster has a schedulable node for each of its pods. Losing this + node stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: Cluster + owner_name: postgres-db + node: node-b + severity: warning + exported_instance: tenant-a/postgres-db + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of Cluster tenant-a/postgres-db are on node node-b in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: StrimziPodSet + owner_name: kafka-kafka + node: node-c + severity: warning + exported_instance: tenant-a/kafka-kafka + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of StrimziPodSet tenant-a/kafka-kafka are on node node-c in cluster + cozystack, although the cluster has a schedulable node for each of its pods. Losing + this node stops these pods together, which can cost a quorum-based workload its quorum. + The scheduler does not move running pods, so they stay together until they are + recreated. diff --git a/hack/workload-placement-alerts.bats b/hack/workload-placement-alerts.bats new file mode 100644 index 000000000..3f7b9b961 --- /dev/null +++ b/hack/workload-placement-alerts.bats @@ -0,0 +1,26 @@ +#!/usr/bin/env bats + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +VMALERT_TOOL_IMAGE="victoriametrics/vmalert-tool:v1.136.0@sha256:417346d7b6fdb66d8e3984887c2ad114535ec7d983eeef53381b56c776f333f1" + +@test "workload placement alerts pass their vmalert-tool cases" { + docker info >/dev/null 2>&1 || { + echo "docker is required to run the cases in $VMALERT_TOOL_IMAGE" >&2 + return 1 + } + work=$(mktemp -d) + rc=0 + yq '.spec' "$REPO_ROOT/packages/system/monitoring-agents/alerts/workload-placement.yaml" > "$work/rules.yaml" || rc=$? + cp "$REPO_ROOT/hack/testdata/workload-placement-alerts/unittest.yaml" "$work/unittest.yaml" || rc=$? + if [ "$rc" -eq 0 ]; then + docker run --rm --network none -v "$work:/w:ro" "$VMALERT_TOOL_IMAGE" \ + unittest --files /w/unittest.yaml > "$work/out" 2>&1 || rc=$? + cat "$work/out" + fi + passed=$(grep -cx 'SUCCESS' "$work/out" 2>/dev/null || true) + rm -rf "$work" + [ "$rc" -eq 0 ] + [ "$passed" -eq 1 ] +} diff --git a/packages/system/monitoring-agents/alerts/workload-placement.yaml b/packages/system/monitoring-agents/alerts/workload-placement.yaml new file mode 100644 index 000000000..3ae9c8f50 --- /dev/null +++ b/packages/system/monitoring-agents/alerts/workload-placement.yaml @@ -0,0 +1,81 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: alerts-workload-placement +spec: + groups: + - name: workload-placement + params: {} + rules: + - alert: PodReplicasColocated + expr: |- + count by (cluster, namespace, owner_kind, owner_name, node) ( + label_replace( + label_replace( + ( + max by (cluster, namespace, pod, node, created_by_kind, created_by_name) ( + kube_pod_info{job="kube-state-metrics", created_by_kind=~"StatefulSet|ReplicaSet|Cluster|StrimziPodSet"} + ) + or + label_replace( + label_replace( + max by (cluster, namespace, pod, node, created_by_kind, created_by_name) ( + kube_pod_info{job="kube-state-metrics", created_by_kind="EtcdMember"} + ), + "created_by_name", "$1", "created_by_name", "(.+)-[^-]+" + ), + "created_by_kind", "EtcdCluster", "", "" + ) + ) + and on (cluster, namespace, pod) + kube_pod_status_phase{job="kube-state-metrics", phase="Running"} == 1, + "owner_kind", "$1", "created_by_kind", "(.+)" + ), + "owner_name", "$1", "created_by_name", "(.+)" + ) + ) > 1 + and on (cluster, namespace, owner_kind, owner_name) + count by (cluster, namespace, owner_kind, owner_name) ( + label_replace( + label_replace( + ( + max by (cluster, namespace, pod, node, created_by_kind, created_by_name) ( + kube_pod_info{job="kube-state-metrics", created_by_kind=~"StatefulSet|ReplicaSet|Cluster|StrimziPodSet"} + ) + or + label_replace( + label_replace( + max by (cluster, namespace, pod, node, created_by_kind, created_by_name) ( + kube_pod_info{job="kube-state-metrics", created_by_kind="EtcdMember"} + ), + "created_by_name", "$1", "created_by_name", "(.+)-[^-]+" + ), + "created_by_kind", "EtcdCluster", "", "" + ) + ) + and on (cluster, namespace, pod) + kube_pod_status_phase{job="kube-state-metrics", phase="Running"} == 1, + "owner_kind", "$1", "created_by_kind", "(.+)" + ), + "owner_name", "$1", "created_by_name", "(.+)" + ) + ) + <= on (cluster) group_left () + count by (cluster) ( + max by (cluster, node) (kube_node_info{job="kube-state-metrics"}) + unless on (cluster, node) + kube_node_spec_taint{job="kube-state-metrics", effect=~"NoSchedule|NoExecute"} + ) + for: 1h + labels: + severity: warning + exported_instance: '{{ $labels.namespace }}/{{ $labels.owner_name }}' + service: workload-placement + annotations: + summary: Pods of one workload share a node. + description: >- + {{ $value }} running pods of {{ $labels.owner_kind }} {{ $labels.namespace }}/{{ $labels.owner_name }} + are on node {{ $labels.node }} in cluster {{ $labels.cluster }}, although the cluster has a + schedulable node for each of its pods. Losing this node stops these pods together, which can cost + a quorum-based workload its quorum. The scheduler does not move running pods, so they stay + together until they are recreated. From 4a706901b501aa0a5d620de73c352659a9768027 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:14:26 +0300 Subject: [PATCH 010/174] fix(cozy-lib)!: refuse memory below 1Mi instead of rendering a few bytes A quantity without a unit counts bytes. The app schemas take a bare number for memory and the dashboard sends it as typed, so 8 meant as 8Gi rendered a memory limit and request of 8 bytes and the app did not work. Charts size their containers through sanitize, directly or from defaultingSanitize, so the floor goes there and the render stops with a message that names the value and asks for a unit. 1Mi is far below the smallest preset, 128Mi, and above any bare number typed for Mi or Gi. 0 is refused as well, since kubelet sets no cgroup memory limit for a zero limit. flatten renders tenant quotas from the same conversion without the floor, because a quota of 0 memory is a valid way to allow none. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- .../library/cozy-lib/templates/_resources.tpl | 29 ++- .../templates/tests/resources-defaulting.yaml | 11 + .../tests/resources_memory_floor_test.yaml | 203 ++++++++++++++++++ 3 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 packages/tests/cozy-lib-tests/templates/tests/resources-defaulting.yaml create mode 100644 packages/tests/cozy-lib-tests/tests/resources_memory_floor_test.yaml diff --git a/packages/library/cozy-lib/templates/_resources.tpl b/packages/library/cozy-lib/templates/_resources.tpl index 48b0718fd..6ee63af0e 100644 --- a/packages/library/cozy-lib/templates/_resources.tpl +++ b/packages/library/cozy-lib/templates/_resources.tpl @@ -68,6 +68,8 @@ number of milli-CPUs (a CPU quantity finer than 1m is invalid for the VPA admission webhook and meaningless for the scheduler). + A memory below 1Mi fails the render, see cozy-lib.resources.checkMemory. + The template now expects **one flat map** as input (no nested `requests:` / `limits:` sections). Each value in that map is taken as the *limit* for the corresponding resource. Usage example: @@ -92,6 +94,31 @@ devices.com/nvidia: "1" # = limit */}} {{- define "cozy-lib.resources.sanitize" }} +{{- include "cozy-lib.resources.checkMemory" (index . 0) }} +{{- include "cozy-lib.resources.requestsAndLimits" . }} +{{- end }} + +{{- /* + A quantity without a unit counts bytes, so a memory of `8` typed for 8Gi + sizes the container at 8 bytes and nothing else refuses it. Every preset is + 128Mi or more, so a floor of 1Mi only stops that kind of slip. +*/}} +{{- define "cozy-lib.resources.checkMemory" }} +{{- if hasKey . "memory" }} +{{- $memory := index . "memory" }} +{{- if not (kindIs "invalid" $memory) }} +{{- if lt (include "cozy-lib.resources.toFloat" $memory | float64) (include "cozy-lib.resources.toFloat" "1Mi" | float64) }} +{{- fail (printf "memory (%v) must be at least 1Mi. A value without a unit is read as bytes, so add the unit, such as 512Mi or 8Gi." $memory) }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} + +{{- /* + sanitize without the memory floor. flatten renders a ResourceQuota with it, + where a memory of 0 is a valid quota that allows none. +*/}} +{{- define "cozy-lib.resources.requestsAndLimits" }} {{- $cpuAllocationRatio := include "cozy-lib.resources.cpuAllocationRatio" . | float64 }} {{- $memoryAllocationRatio := include "cozy-lib.resources.memoryAllocationRatio" . | float64 }} {{- $ephemeralStorageAllocationRatio := include "cozy-lib.resources.ephemeralStorageAllocationRatio" . | float64 }} @@ -204,7 +231,7 @@ {{- define "cozy-lib.resources.flatten" -}} {{- $out := dict -}} -{{- $res := include "cozy-lib.resources.sanitize" . | fromYaml -}} +{{- $res := include "cozy-lib.resources.requestsAndLimits" . | fromYaml -}} {{- range $section, $values := $res }} {{- range $k, $v := $values }} {{- with include "cozy-lib.resources.flattenResource" (list $section $k) }} diff --git a/packages/tests/cozy-lib-tests/templates/tests/resources-defaulting.yaml b/packages/tests/cozy-lib-tests/templates/tests/resources-defaulting.yaml new file mode 100644 index 000000000..3e6d9a9eb --- /dev/null +++ b/packages/tests/cozy-lib-tests/templates/tests/resources-defaulting.yaml @@ -0,0 +1,11 @@ +{{- with .Values.defaulting }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ $.Release.Name }}-defaulting +spec: + containers: + - name: main + image: busybox + resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .preset .resources $) | nindent 6 }} +{{- end }} diff --git a/packages/tests/cozy-lib-tests/tests/resources_memory_floor_test.yaml b/packages/tests/cozy-lib-tests/tests/resources_memory_floor_test.yaml new file mode 100644 index 000000000..485829054 --- /dev/null +++ b/packages/tests/cozy-lib-tests/tests/resources_memory_floor_test.yaml @@ -0,0 +1,203 @@ +# ./tests/resources_memory_floor_test.yaml +suite: resources memory floor + +# A quantity without a unit counts bytes, so a memory of 8 typed for 8Gi would +# size the container at 8 bytes. sanitize refuses anything below 1Mi, which +# every preset clears by far. flatten renders tenant quotas without the floor, +# because a quota of 0 memory is a valid way to allow none. + +templates: + - templates/tests/resources.yaml + - templates/tests/resources-defaulting.yaml + - templates/tests/quota.yaml + +release: + name: myrelease + namespace: default + +tests: + - it: refuses a quoted number without a unit + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: "8" + asserts: + - failedTemplate: + errorMessage: memory (8) must be at least 1Mi. A value without a unit is read as bytes, so add the unit, such as 512Mi or 8Gi. + + - it: refuses an unquoted number without a unit + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 8 + asserts: + - failedTemplate: + errorPattern: '^memory \(8\) must be at least 1Mi\.' + + - it: refuses a zero memory + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 0 + asserts: + - failedTemplate: + errorPattern: '^memory \(0\) must be at least 1Mi\.' + + - it: refuses a byte count one below 1Mi + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: "1048575" + asserts: + - failedTemplate: + errorPattern: '^memory \(1048575\) must be at least 1Mi\.' + + - it: refuses a suffixed value below 1Mi + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 1023Ki + asserts: + - failedTemplate: + errorPattern: '^memory \(1023Ki\) must be at least 1Mi\.' + + - it: refuses a negative memory + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: -1Gi + asserts: + - failedTemplate: + errorPattern: '^memory \(-1Gi\) must be at least 1Mi\.' + + - it: renders exactly 1Mi + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 1Mi + asserts: + - equal: + path: spec.containers[0].resources.limits.memory + value: 1Mi + - equal: + path: spec.containers[0].resources.requests.memory + value: "1048576" + + - it: renders an unquoted byte count of exactly 1Mi + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 1048576 + asserts: + - notFailedTemplate: {} + - equal: + path: spec.containers[0].resources.requests.memory + value: "1048576" + + - it: renders 512Mi + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 512Mi + asserts: + - equal: + path: spec.containers[0].resources.limits.memory + value: 512Mi + - equal: + path: spec.containers[0].resources.requests.memory + value: "536870912" + + - it: renders 8Gi + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 8Gi + asserts: + - equal: + path: spec.containers[0].resources.limits.memory + value: 8Gi + - equal: + path: spec.containers[0].resources.requests.memory + value: "8589934592" + + - it: leaves a map without memory alone + template: templates/tests/resources.yaml + set: + resources: + ephemeral-storage: 2Gi + asserts: + - notFailedTemplate: {} + - notExists: + path: spec.containers[0].resources.limits.memory + + - it: refuses a number without a unit set over a preset + template: templates/tests/resources-defaulting.yaml + set: + defaulting: + preset: t1.nano + resources: + memory: "8" + asserts: + - failedTemplate: + errorPattern: '^memory \(8\) must be at least 1Mi\.' + + - it: renders the smallest preset unchanged + template: templates/tests/resources-defaulting.yaml + set: + defaulting: + preset: t1.nano + asserts: + - equal: + path: spec.containers[0].resources.limits.memory + value: 128Mi + - equal: + path: spec.containers[0].resources.requests.memory + value: "134217728" + + - it: renders the smallest legacy preset unchanged + template: templates/tests/resources-defaulting.yaml + set: + defaulting: + preset: nano + asserts: + - equal: + path: spec.containers[0].resources.limits.memory + value: 128Mi + + - it: renders an explicit memory with a unit over a preset + template: templates/tests/resources-defaulting.yaml + set: + defaulting: + preset: t1.nano + resources: + memory: 8Gi + asserts: + - equal: + path: spec.containers[0].resources.limits.memory + value: 8Gi + - equal: + path: spec.containers[0].resources.limits.cpu + value: 250m + + - it: still renders a tenant quota of 0 memory + template: templates/tests/quota.yaml + set: + quota: + memory: 0 + asserts: + - equal: + path: spec.hard["limits.memory"] + value: "0" + - equal: + path: spec.hard["requests.memory"] + value: "0" From 6bb0fda706e9ccaec5f433f6cd81eb2cdb7e6ef0 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:30:33 +0300 Subject: [PATCH 011/174] fix(cozystack-basics): refuse a direct delete of a tenant namespace A tenant namespace is meant to go away with its Tenant. The tenant chart's pre-delete hook deletes the applications first and the tenant modules such as etcd after them, and helm-controller deletes the namespace at the end. A direct delete hands the namespace to the namespace controller instead. It deletes the pods first and then everything else together, so the tenant etcd and its etcd-ca-tls Secret can be gone before the TenantControlPlanes that use them. Kamaji then cannot reach the datastore to clean up, the TenantControlPlane finalizer never clears and the namespace stays Terminating. The policy refuses DELETE of a namespace labelled tenant.cozystack.io/tenant-root, which the tenant chart puts on every tenant namespace below tenant-root. tenant-root carries no such label and stays with the no-delete guardrail. Two callers are let through. helm-controller and its shards run as the flux ServiceAccount in cozy-fluxcd and delete the namespace during a Tenant uninstall. The garbage collector runs in kube-system and removes a child namespace through its ownerReference once the parent namespace is gone. The namespace controller only finalizes a namespace and never sends a DELETE for it, so it is not affected. Everyone else is refused, system:masters included, as with the no-delete guardrail. The message names the Tenant to delete and the label to remove when the Tenant is already gone. The policy test helpers take the template path now, so the new test compiles the shipped template with the apiserver CEL compiler like the external TLS policy test does. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- hack/e2e-install-cozystack.bats | 37 ++++ .../tenant-namespace-delete-policy.yaml | 59 +++++++ ...ostname-policies-capability-gate_test.yaml | 4 + .../tenant-namespace-delete-policy_test.yaml | 51 ++++++ .../tests/tenant-root_test.yaml | 6 + .../application/external_tls_policy_test.go | 22 +-- .../tenant_namespace_delete_policy_test.go | 163 ++++++++++++++++++ 7 files changed, 331 insertions(+), 11 deletions(-) create mode 100644 packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml create mode 100644 packages/system/cozystack-basics/tests/tenant-namespace-delete-policy_test.yaml create mode 100644 pkg/registry/apps/application/tenant_namespace_delete_policy_test.go diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index 47315ac85..4f0884269 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -561,3 +561,40 @@ data: version: "${version}" EOF } + +@test "Tenant namespace delete policy refuses a direct delete and keeps the Tenant uninstall" { + if ! kubectl api-resources --api-group=admissionregistration.k8s.io \ + 2>/dev/null | grep -qw validatingadmissionpolicies; then + skip "ValidatingAdmissionPolicy API not available on this cluster" + fi + kubectl get validatingadmissionpolicy cozystack-tenant-namespace-delete-policy + kubectl get validatingadmissionpolicybinding cozystack-tenant-namespace-delete-policy + + kubectl apply -f - </dev/null 2>&1; do sleep 2; done' + kubectl wait hr/tenant-deleteprobe -n tenant-root --timeout=1m --for=condition=ready + + local selected output rc still_there + selected=$(kubectl get namespaces --selector tenant.cozystack.io/tenant-root --field-selector metadata.name=tenant-deleteprobe -o name) + [ "$selected" = "namespace/tenant-deleteprobe" ] + + output=$(kubectl delete namespace tenant-deleteprobe --dry-run=server 2>&1) && rc=0 || rc=$? + echo "kubectl delete exit=$rc, output=$output" + [ "$rc" -ne 0 ] + echo "$output" | grep -qF "ValidatingAdmissionPolicy 'cozystack-tenant-namespace-delete-policy'" + echo "$output" | grep -qF 'kubectl delete tenants.apps.cozystack.io deleteprobe --namespace tenant-root.' + + kubectl delete tenants.apps.cozystack.io deleteprobe -n tenant-root --timeout=5m + kubectl wait namespace/tenant-deleteprobe --for=delete --timeout=5m + if ! still_there=$(kubectl get namespace tenant-deleteprobe --ignore-not-found -o name); then + echo "FAIL: reading the tenant-deleteprobe namespace did not answer"; false + fi + if [ -n "$still_there" ]; then echo "FAIL: deleting the Tenant must remove its namespace"; false; fi +} diff --git a/packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml b/packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml new file mode 100644 index 000000000..cbc928dc4 --- /dev/null +++ b/packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml @@ -0,0 +1,59 @@ +{{- if .Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: cozystack-tenant-namespace-delete-policy + labels: + internal.cozystack.io/managed-by-cozystack: "" +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["DELETE"] + resources: ["namespaces"] + variables: + - name: platformCaller + expression: >- + has(request.userInfo.groups) && request.userInfo.groups.exists(g, + g == "system:serviceaccounts:cozy-fluxcd" || + g == "system:serviceaccounts:kube-system") + - name: name + expression: oldObject.metadata.name + - name: cut + expression: variables.name.lastIndexOf("-") + - name: tenant + expression: variables.name.substring(variables.cut + 1) + - name: parentNamespace + expression: >- + variables.cut > 0 && variables.name.substring(0, variables.cut) != "tenant" + ? variables.name.substring(0, variables.cut) + : "tenant-root" + validations: + - expression: variables.platformCaller + messageExpression: >- + "Namespace " + variables.name + " belongs to the Tenant " + variables.tenant + + " in " + variables.parentNamespace + ". Delete the Tenant instead with kubectl delete tenants.apps.cozystack.io " + + variables.tenant + " --namespace " + variables.parentNamespace + + ". Deleting the namespace directly can remove the tenant etcd before the Kubernetes clusters that use it and leave the namespace stuck in Terminating." + + " If the Tenant is already gone, remove the label first with kubectl label namespace " + variables.name + + " tenant.cozystack.io/tenant-root-" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: cozystack-tenant-namespace-delete-policy + labels: + internal.cozystack.io/managed-by-cozystack: "" +spec: + policyName: cozystack-tenant-namespace-delete-policy + validationActions: [Deny] + matchResources: + objectSelector: + matchExpressions: + - key: tenant.cozystack.io/tenant-root + operator: Exists +{{- end }} diff --git a/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml index 61b8be507..96cfdfaa3 100644 --- a/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml +++ b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml @@ -1,5 +1,6 @@ suite: cozystack-basics VAP policies are gated on the ValidatingAdmissionPolicy API templates: + - templates/tenant-namespace-delete-policy.yaml - templates/route-hostname-policy.yaml - templates/gateway-hostname-policy.yaml - templates/ingress-hostname-policy.yaml @@ -20,6 +21,9 @@ capabilities: tests: - it: none of the VAP policies render when the API is unavailable asserts: + - hasDocuments: + count: 0 + template: templates/tenant-namespace-delete-policy.yaml - hasDocuments: count: 0 template: templates/route-hostname-policy.yaml diff --git a/packages/system/cozystack-basics/tests/tenant-namespace-delete-policy_test.yaml b/packages/system/cozystack-basics/tests/tenant-namespace-delete-policy_test.yaml new file mode 100644 index 000000000..a0e53c77f --- /dev/null +++ b/packages/system/cozystack-basics/tests/tenant-namespace-delete-policy_test.yaml @@ -0,0 +1,51 @@ +suite: tenant namespace delete policy renders with its binding +templates: + - templates/tenant-namespace-delete-policy.yaml +release: + name: cozystack-basics + namespace: cozy-system +capabilities: + apiVersions: + - admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy +tests: + - it: renders the policy and a Deny binding to it + asserts: + - hasDocuments: + count: 2 + - documentIndex: 0 + equal: + path: kind + value: ValidatingAdmissionPolicy + - documentIndex: 0 + equal: + path: metadata.name + value: cozystack-tenant-namespace-delete-policy + - documentIndex: 1 + equal: + path: spec.policyName + value: cozystack-tenant-namespace-delete-policy + - documentIndex: 1 + equal: + path: spec.validationActions + value: [Deny] + + - it: matches only DELETE of core namespaces + asserts: + - documentIndex: 0 + equal: + path: spec.matchConstraints.resourceRules + value: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["DELETE"] + resources: ["namespaces"] + + - it: binds to the namespaces the tenant chart labels as descendants of tenant-root + asserts: + - documentIndex: 1 + equal: + path: spec.matchResources.objectSelector + value: + matchExpressions: + - key: tenant.cozystack.io/tenant-root + operator: Exists diff --git a/packages/system/cozystack-basics/tests/tenant-root_test.yaml b/packages/system/cozystack-basics/tests/tenant-root_test.yaml index e22dbb649..54210174c 100644 --- a/packages/system/cozystack-basics/tests/tenant-root_test.yaml +++ b/packages/system/cozystack-basics/tests/tenant-root_test.yaml @@ -79,3 +79,9 @@ tests: equal: path: spec.chartRef.name value: cozystack-tenant-application-default-tenant + + - it: tenant-root namespace carries no tenant.cozystack.io/tenant-root label, which the tenant namespace delete policy selects on + asserts: + - documentIndex: 0 + notExists: + path: metadata.labels["tenant.cozystack.io/tenant-root"] diff --git a/pkg/registry/apps/application/external_tls_policy_test.go b/pkg/registry/apps/application/external_tls_policy_test.go index 1cbc808fc..ff5704a23 100644 --- a/pkg/registry/apps/application/external_tls_policy_test.go +++ b/pkg/registry/apps/application/external_tls_policy_test.go @@ -41,14 +41,14 @@ import ( // expressions a cluster actually receives rather than a copy of them. const externalTLSPolicyPath = "../../../../packages/system/cozystack-basics/templates/database-external-tls-policy.yaml" -// loadExternalTLSPolicy returns the policy and its binding from the chart -// template. The template's only directives are the capability guard lines -// around the manifests; any other directive would mean the test is no longer -// looking at what renders, so it fails instead of guessing. -func loadExternalTLSPolicy(t *testing.T) (*admissionregistrationv1.ValidatingAdmissionPolicy, *admissionregistrationv1.ValidatingAdmissionPolicyBinding) { +// loadPolicy returns the policy and its binding from a chart template. The +// template's only directives are the capability guard lines around the +// manifests; any other directive would mean the test is no longer looking at +// what renders, so it fails instead of guessing. +func loadPolicy(t *testing.T, path string) (*admissionregistrationv1.ValidatingAdmissionPolicy, *admissionregistrationv1.ValidatingAdmissionPolicyBinding) { t.Helper() - raw, err := os.ReadFile(externalTLSPolicyPath) + raw, err := os.ReadFile(path) if err != nil { t.Fatalf("read policy template: %v", err) } @@ -92,11 +92,11 @@ func loadExternalTLSPolicy(t *testing.T) (*admissionregistrationv1.ValidatingAdm return policy, binding } -// compileExternalTLSPolicy mirrors compilePolicy in +// compilePolicy mirrors compilePolicy in // k8s.io/apiserver/pkg/admission/plugin/policy/validating, which is // unexported: the same composition environment, the same variable, match // condition, validation and message compilers. -func compileExternalTLSPolicy(t *testing.T, policy *admissionregistrationv1.ValidatingAdmissionPolicy) validating.Validator { +func compilePolicy(t *testing.T, policy *admissionregistrationv1.ValidatingAdmissionPolicy) validating.Validator { t.Helper() compiler, err := plugincel.NewCompositedCompiler(environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion())) @@ -139,7 +139,7 @@ func compileExternalTLSPolicy(t *testing.T, policy *admissionregistrationv1.Vali } func TestDatabaseExternalTLSPolicy_Binding(t *testing.T) { - policy, binding := loadExternalTLSPolicy(t) + policy, binding := loadPolicy(t, externalTLSPolicyPath) if binding.Spec.PolicyName != policy.Name { t.Errorf("binding names policy %q, want %q", binding.Spec.PolicyName, policy.Name) @@ -171,8 +171,8 @@ func TestDatabaseExternalTLSPolicy_Binding(t *testing.T) { } func TestDatabaseExternalTLSPolicy_Decisions(t *testing.T) { - policy, _ := loadExternalTLSPolicy(t) - validator := compileExternalTLSPolicy(t, policy) + policy, _ := loadPolicy(t, externalTLSPolicyPath) + validator := compilePolicy(t, policy) tlsOn := map[string]any{"enabled": true} tlsOff := map[string]any{"enabled": false} diff --git a/pkg/registry/apps/application/tenant_namespace_delete_policy_test.go b/pkg/registry/apps/application/tenant_namespace_delete_policy_test.go new file mode 100644 index 000000000..6c8870ea2 --- /dev/null +++ b/pkg/registry/apps/application/tenant_namespace_delete_policy_test.go @@ -0,0 +1,163 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package application + +import ( + "context" + "slices" + "strings" + "testing" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/admission/plugin/policy/validating" + celconfig "k8s.io/apiserver/pkg/apis/cel" + "k8s.io/apiserver/pkg/authentication/user" +) + +const tenantNamespaceDeletePolicyPath = "../../../../packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml" + +func TestTenantNamespaceDeletePolicy_Binding(t *testing.T) { + policy, binding := loadPolicy(t, tenantNamespaceDeletePolicyPath) + + if binding.Spec.PolicyName != policy.Name { + t.Errorf("binding names policy %q, want %q", binding.Spec.PolicyName, policy.Name) + } + if !slices.Equal(binding.Spec.ValidationActions, []admissionregistrationv1.ValidationAction{admissionregistrationv1.Deny}) { + t.Errorf("binding validationActions = %v, want [Deny]", binding.Spec.ValidationActions) + } + if policy.Spec.FailurePolicy == nil || *policy.Spec.FailurePolicy != admissionregistrationv1.Fail { + t.Errorf("failurePolicy must be Fail") + } + + rules := policy.Spec.MatchConstraints.ResourceRules + if len(rules) != 1 { + t.Fatalf("want one resource rule, got %d", len(rules)) + } + rule := rules[0] + if !slices.Equal(rule.APIGroups, []string{""}) || !slices.Equal(rule.APIVersions, []string{"v1"}) { + t.Errorf("rule must match core v1 only, got apiGroups %v apiVersions %v", rule.APIGroups, rule.APIVersions) + } + if !slices.Equal(rule.Operations, []admissionregistrationv1.OperationType{admissionregistrationv1.Delete}) { + t.Errorf("operations = %v, want [DELETE]", rule.Operations) + } + if !slices.Equal(rule.Resources, []string{"namespaces"}) { + t.Errorf("resources = %v, want [namespaces]", rule.Resources) + } + + if binding.Spec.MatchResources == nil || binding.Spec.MatchResources.ObjectSelector == nil { + t.Fatal("binding must select namespaces by label") + } + selector, err := metav1.LabelSelectorAsSelector(binding.Spec.MatchResources.ObjectSelector) + if err != nil { + t.Fatalf("binding objectSelector: %v", err) + } + selected := []struct { + name string + labels labels.Set + want bool + }{ + {name: "tenant namespace under tenant-root", labels: labels.Set{"tenant.cozystack.io/tenant-root": "", "tenant.cozystack.io/tenant-foo": "", "namespace.cozystack.io/host": "foo.example.org"}, want: true}, + {name: "nested tenant namespace", labels: labels.Set{"tenant.cozystack.io/tenant-root": "", "tenant.cozystack.io/tenant-foo": "", "tenant.cozystack.io/tenant-foo-bar": ""}, want: true}, + {name: "tenant-root namespace", labels: labels.Set{"platform.cozystack.io/no-delete": "true", "namespace.cozystack.io/host": "example.org", "namespace.cozystack.io/gateway": "tenant-root"}}, + {name: "namespace named tenant-* without tenant labels", labels: labels.Set{"namespace.cozystack.io/host": "case-probe.example.org"}}, + {name: "unlabelled namespace", labels: labels.Set{}}, + } + for _, tc := range selected { + if got := selector.Matches(tc.labels); got != tc.want { + t.Errorf("%s: selected = %v, want %v", tc.name, got, tc.want) + } + } +} + +func TestTenantNamespaceDeletePolicy_Decisions(t *testing.T) { + policy, _ := loadPolicy(t, tenantNamespaceDeletePolicyPath) + validator := compilePolicy(t, policy) + + fluxGroups := []string{"system:serviceaccounts", "system:serviceaccounts:cozy-fluxcd", "system:authenticated"} + kubeSystemGroups := []string{"system:serviceaccounts", "system:serviceaccounts:kube-system", "system:authenticated"} + adminGroups := []string{"system:masters", "system:authenticated"} + + cases := []struct { + name string + namespace string + user string + groups []string + denied bool + tenant string + parent string + }{ + {name: "helm-controller uninstalling a Tenant", namespace: "tenant-foo", user: "system:serviceaccount:cozy-fluxcd:flux", groups: fluxGroups}, + {name: "helm-controller uninstalling a nested Tenant", namespace: "tenant-foo-bar", user: "system:serviceaccount:cozy-fluxcd:flux", groups: fluxGroups}, + {name: "garbage collector following a deleted parent namespace", namespace: "tenant-foo-bar", user: "system:serviceaccount:kube-system:generic-garbage-collector", groups: kubeSystemGroups}, + {name: "cluster admin deleting a tenant namespace", namespace: "tenant-foo", user: "admin", groups: adminGroups, denied: true, tenant: "foo", parent: "tenant-root"}, + {name: "cluster admin deleting a nested tenant namespace", namespace: "tenant-foo-bar", user: "admin", groups: adminGroups, denied: true, tenant: "bar", parent: "tenant-foo"}, + {name: "cluster admin deleting a third-level tenant namespace", namespace: "tenant-foo-bar-baz", user: "admin", groups: adminGroups, denied: true, tenant: "baz", parent: "tenant-foo-bar"}, + {name: "cluster admin deleting a labelled namespace without a dash", namespace: "standalone", user: "admin", groups: adminGroups, denied: true, tenant: "standalone", parent: "tenant-root"}, + {name: "OIDC user bound to cluster-admin", namespace: "tenant-foo", user: "cluster-admin-user", groups: []string{"cozystack-cluster-admin", "system:authenticated"}, denied: true, tenant: "foo", parent: "tenant-root"}, + {name: "cozy-system service account", namespace: "tenant-foo", user: "system:serviceaccount:cozy-system:cozystack-api", groups: []string{"system:serviceaccounts", "system:serviceaccounts:cozy-system", "system:authenticated"}, denied: true, tenant: "foo", parent: "tenant-root"}, + {name: "tenant service account", namespace: "tenant-foo-bar", user: "system:serviceaccount:tenant-foo:default", groups: []string{"system:serviceaccounts", "system:serviceaccounts:tenant-foo", "system:authenticated"}, denied: true, tenant: "bar", parent: "tenant-foo"}, + {name: "user without groups", namespace: "tenant-foo", user: "no-groups", denied: true, tenant: "foo", parent: "tenant-root"}, + } + + gvk := schema.GroupVersionKind{Version: "v1", Kind: "Namespace"} + gvr := schema.GroupVersionResource{Version: "v1", Resource: "namespaces"} + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + oldObj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": map[string]any{ + "name": tc.namespace, + "labels": map[string]any{"tenant.cozystack.io/tenant-root": ""}, + }, + }} + var newObj runtime.Object + attrs := admission.NewAttributesRecord(newObj, oldObj, gvk, tc.namespace, tc.namespace, gvr, "", admission.Delete, &metav1.DeleteOptions{}, false, &user.DefaultInfo{Name: tc.user, Groups: tc.groups}) + versioned := &admission.VersionedAttributes{Attributes: attrs, VersionedObject: admission.NewLazyObject(newObj), VersionedOldObject: admission.NewLazyObject(oldObj), VersionedKind: gvk} + + result := validator.Validate(context.Background(), gvr, versioned, nil, nil, celconfig.RuntimeCELCostBudget, nil) + var denials []validating.PolicyDecision + for _, d := range result.Decisions { + if d.Evaluation == validating.EvalError { + t.Fatalf("evaluation error: %s", d.Message) + } + if d.Action == validating.ActionDeny { + denials = append(denials, d) + } + } + if got := len(denials) > 0; got != tc.denied { + t.Fatalf("denied = %v, want %v (decisions %+v)", got, tc.denied, result.Decisions) + } + for _, d := range denials { + for _, want := range []string{ + "kubectl delete tenants.apps.cozystack.io " + tc.tenant + " --namespace " + tc.parent + ".", + "kubectl label namespace " + tc.namespace + " tenant.cozystack.io/tenant-root-", + } { + if !strings.Contains(d.Message, want) { + t.Errorf("denial message must carry %q, got %q", want, d.Message) + } + } + } + }) + } +} From 9f6fdd77c6ca964e0adcc387e50993487f2e594e Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:52:37 +0300 Subject: [PATCH 012/174] docs(etcd): update the affinity notes for the hard member spread The affinity description and the comment above the affinity block explained the soft default by a member that would stay Pending on its node-local volume. Now that the spread is DoNotSchedule by default, the spread keeps such a member Pending as well, once its node holds another member and some other node holds none. Both texts now explain the soft default by what is left, that a required rule forbids two members on one node even when every node has one, and state the node-local case for both rules. The description also says that the operator copies affinity into each EtcdMember when it creates it, so a recreated Pod keeps the rule its member was created with. README, values.schema.json and the ApplicationDefinition are regenerated from it. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- packages/extra/etcd/README.md | 20 +++++++++---------- .../extra/etcd/templates/etcd-cluster.yaml | 14 ++++++++----- packages/extra/etcd/values.schema.json | 2 +- packages/extra/etcd/values.yaml | 2 +- packages/system/etcd-rd/cozyrds/etcd.yaml | 2 +- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/packages/extra/etcd/README.md b/packages/extra/etcd/README.md index e96c63832..91341ff32 100644 --- a/packages/extra/etcd/README.md +++ b/packages/extra/etcd/README.md @@ -34,14 +34,14 @@ coordinates). ### Common parameters -| Name | Description | Type | Value | -| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------- | -| `version` | etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image. | `string` | `3.6.11` | -| `size` | Persistent Volume size. | `quantity` | `4Gi` | -| `storageClass` | StorageClass used to store the data. | `string` | `""` | -| `replicas` | Number of etcd replicas. | `int` | `3` | -| `resources` | Resource configuration for etcd. | `object` | `{}` | -| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` | -| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` | -| `affinity` | Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively. | `object` | `{}` | +| Name | Description | Type | Value | +| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------- | +| `version` | etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image. | `string` | `3.6.11` | +| `size` | Persistent Volume size. | `quantity` | `4Gi` | +| `storageClass` | StorageClass used to store the data. | `string` | `""` | +| `replicas` | Number of etcd replicas. | `int` | `3` | +| `resources` | Resource configuration for etcd. | `object` | `{}` | +| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` | +| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` | +| `affinity` | Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones. | `object` | `{}` | diff --git a/packages/extra/etcd/templates/etcd-cluster.yaml b/packages/extra/etcd/templates/etcd-cluster.yaml index 67dd34bd9..3676f53cb 100644 --- a/packages/extra/etcd/templates/etcd-cluster.yaml +++ b/packages/extra/etcd/templates/etcd-cluster.yaml @@ -79,11 +79,15 @@ spec: affinity: {{- toYaml . | nindent 4 }} {{- else }} - # Soft anti-affinity: spread etcd members across nodes on a best-effort - # basis so a single node failure is unlikely to take out more than one - # member. Preferred (not required) because a hard rule would leave a member - # Pending when replicas exceed schedulable nodes, or when node-local storage - # pins a member to a node. Override via .Values.affinity for a hard rule. + # Soft anti-affinity, a preference on top of the spread constraint above, + # which by default is the hard rule that keeps two members off one node + # while another node has none. Preferred, not required, because a required + # rule also forbids two members on one node when every node already has + # one, so an etcd with more members than schedulable nodes could not start + # all of them. With node-local storage a recreated member Pod can only + # return to its own node, and the default spread keeps it Pending there + # while that node holds another member and some other node holds none. + # Override via .Values.affinity for a hard rule. affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: diff --git a/packages/extra/etcd/values.schema.json b/packages/extra/etcd/values.schema.json index b438ca9d7..c33219f4f 100644 --- a/packages/extra/etcd/values.schema.json +++ b/packages/extra/etcd/values.schema.json @@ -67,7 +67,7 @@ } }, "affinity": { - "description": "Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.", + "description": "Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.", "type": "object", "default": {}, "x-kubernetes-preserve-unknown-fields": true diff --git a/packages/extra/etcd/values.yaml b/packages/extra/etcd/values.yaml index 61b994d7c..c5be94909 100644 --- a/packages/extra/etcd/values.yaml +++ b/packages/extra/etcd/values.yaml @@ -23,5 +23,5 @@ resources: cpu: 1000m memory: 512Mi -## @param {object} [affinity] - Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively. +## @param {object} [affinity] - Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones. affinity: {} diff --git a/packages/system/etcd-rd/cozyrds/etcd.yaml b/packages/system/etcd-rd/cozyrds/etcd.yaml index 90ed8a80a..1ce88e8ae 100644 --- a/packages/system/etcd-rd/cozyrds/etcd.yaml +++ b/packages/system/etcd-rd/cozyrds/etcd.yaml @@ -8,7 +8,7 @@ spec: plural: etcds singular: etcd openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"version":{"description":"etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image.","type":"string","default":"3.6.11"},"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"affinity":{"description":"Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}} + {"title":"Chart Values","type":"object","properties":{"version":{"description":"etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image.","type":"string","default":"3.6.11"},"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"affinity":{"description":"Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}} release: prefix: "" labels: From 00c11c7a745db9b57d419631c3c88b7ac8c274cb Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:29:50 +0300 Subject: [PATCH 013/174] test(monitoring): check the listed alert still reaches email after the blackhole The case lists InfoInhibitor in email.alertnames but only asserted the route order up to the blackhole, so it would still pass if the email route for a listed name stopped being rendered. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- .../monitoring/tests/alertmanager_blackhole_test.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml index e987bd405..d4e366180 100644 --- a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml +++ b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml @@ -43,3 +43,9 @@ tests: matchRegex: path: stringData["alertmanager.yaml"] pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n' + - documentSelector: + path: metadata.labels.app + value: alertmanager + matchRegex: + path: stringData["alertmanager.yaml"] + pattern: '(?s)\n - receiver: blackhole\n matchers:\n - .severity="none".\n.*\n - receiver: email\n group_wait: 1m\n group_interval: 5m\n repeat_interval: .12h.\n matchers:\n - .alertname=~"\^\(InfoInhibitor\)\$".\n' From 7bda2bdbc07dd941724e5ea12e51e8c808313b1b Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:30:14 +0300 Subject: [PATCH 014/174] fix(monitoring): select the vmalert Service the operator creates The VMServiceScrape selected app.kubernetes.io/instance=vmalert, while the chart names the VMAlert vmalert- and the operator puts that name into the instance label of its Service. The scrape matched nothing, so the metrics of vmalert itself were never collected. The instance stays in the selector so the scrape keeps pointing at the one VMAlert this release renders. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- .../templates/vm/vmalert-scrape.yaml | 4 +- .../monitoring/tests/vmalert_scrape_test.yaml | 38 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 packages/system/monitoring/tests/vmalert_scrape_test.yaml diff --git a/packages/system/monitoring/templates/vm/vmalert-scrape.yaml b/packages/system/monitoring/templates/vm/vmalert-scrape.yaml index ee406a275..b467d5564 100644 --- a/packages/system/monitoring/templates/vm/vmalert-scrape.yaml +++ b/packages/system/monitoring/templates/vm/vmalert-scrape.yaml @@ -1,3 +1,4 @@ +{{- with first .Values.metricsStorages }} --- apiVersion: operator.victoriametrics.com/v1beta1 kind: VMServiceScrape @@ -13,5 +14,6 @@ spec: selector: matchLabels: app.kubernetes.io/component: monitoring - app.kubernetes.io/instance: vmalert + app.kubernetes.io/instance: vmalert-{{ .name }} app.kubernetes.io/name: vmalert +{{- end }} diff --git a/packages/system/monitoring/tests/vmalert_scrape_test.yaml b/packages/system/monitoring/tests/vmalert_scrape_test.yaml new file mode 100644 index 000000000..1dfb9ec3e --- /dev/null +++ b/packages/system/monitoring/tests/vmalert_scrape_test.yaml @@ -0,0 +1,38 @@ +suite: vmalert metrics are scraped +templates: + - templates/vm/vmalert-scrape.yaml + +release: + name: monitoring + namespace: tenant-root + +set: + _cluster: + root-host: example.org + _namespace: + host: tenant-root.example.org + ingress: tenant-root + metricsStorages: + - name: shortterm + retentionPeriod: "3d" + deduplicationInterval: "15s" + - name: longterm + retentionPeriod: "14d" + deduplicationInterval: "5m" + +tests: + - it: selects the Service of the VMAlert the chart renders + asserts: + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/component: monitoring + app.kubernetes.io/instance: vmalert-shortterm + app.kubernetes.io/name: vmalert + + - it: renders nothing without metrics storages, like the VMAlert itself + set: + metricsStorages: [] + asserts: + - hasDocuments: + count: 0 From 34766ff0b50af0801f2c94c1435dd744211bbc23 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:06:52 +0300 Subject: [PATCH 015/174] fix(vertical-pod-autoscaler): run two admission controllers spread over nodes With one admission controller and failurePolicy Ignore, a pod created while that controller is being rescheduled is admitted without its VPA recommendation, and with updateMode Initial it keeps the defaults for the rest of its life. On a v1.6.5 cluster a vmstorage pod created during a node drain got the operator defaults, a 1500Mi limit against a 3.4Gi recommendation, and was OOMKilled weeks later. A second replica on another node and a PodDisruptionBudget keep one controller serving through a drain. The controller still deletes and recreates its webhook configuration on start, so a restart leaves only that short gap instead of the whole reschedule. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- .../tests/vpa_test.yaml | 29 +++++++++++++++++++ .../vertical-pod-autoscaler/values.yaml | 12 ++++++++ 2 files changed, 41 insertions(+) diff --git a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml index 6b8c82675..1d2011330 100644 --- a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml +++ b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml @@ -242,3 +242,32 @@ tests: count: 0 + + - it: runs two admission controllers spread over nodes + template: charts/vertical-pod-autoscaler/templates/admission-controller/deployment.yaml + asserts: + - equal: + path: spec.replicas + value: 2 + - equal: + path: spec.template.spec.topologySpreadConstraints[0].topologyKey + value: kubernetes.io/hostname + - equal: + path: spec.template.spec.topologySpreadConstraints[0].labelSelector.matchLabels + value: + app.kubernetes.io/name: vertical-pod-autoscaler + app.kubernetes.io/instance: vpa + app.kubernetes.io/component: admission-controller + + - it: keeps one admission controller up through a voluntary disruption + template: charts/vertical-pod-autoscaler/templates/admission-controller/pdb.yaml + asserts: + - equal: + path: spec.minAvailable + value: 1 + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: vertical-pod-autoscaler + app.kubernetes.io/instance: vpa + app.kubernetes.io/component: admission-controller diff --git a/packages/system/vertical-pod-autoscaler/values.yaml b/packages/system/vertical-pod-autoscaler/values.yaml index 46678903e..456b99912 100644 --- a/packages/system/vertical-pod-autoscaler/values.yaml +++ b/packages/system/vertical-pod-autoscaler/values.yaml @@ -37,6 +37,18 @@ vertical-pod-autoscaler: prometheus-cadvisor-job-name: cadvisor admissionController: + replicaCount: 2 + pdb: + create: true + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: vertical-pod-autoscaler + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/component: admission-controller resources: limits: memory: 256Mi From 61289c721011af0ca4a61eb278ae2664ae7cd8e7 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:09:16 +0300 Subject: [PATCH 016/174] fix(etcd): size the defrag job for the client it runs The hourly defrag pod runs the etcd-defrag client, which only sends defragment calls to the members, yet it requested 200m CPU and 256Mi from the tenant quota. On a sandbox tenant whose quota was mostly taken by its own etcd and SeaweedFS, the job could not create a pod and the defrag never ran. The client peaked at 5.1 MiB working set over a day on a v1.6.5 cluster. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- packages/extra/etcd/templates/etcd-defrag.yaml | 8 ++++---- packages/extra/etcd/tests/defrag_test.yaml | 13 +++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/extra/etcd/templates/etcd-defrag.yaml b/packages/extra/etcd/templates/etcd-defrag.yaml index 52cc63518..c1eefa0db 100644 --- a/packages/extra/etcd/templates/etcd-defrag.yaml +++ b/packages/extra/etcd/templates/etcd-defrag.yaml @@ -29,11 +29,11 @@ spec: drop: ["ALL"] resources: requests: - cpu: 200m - memory: 256Mi + cpu: 10m + memory: 32Mi limits: - cpu: 500m - memory: 512Mi + cpu: 100m + memory: 64Mi args: # The v1alpha2 operator names the headless Service after the # cluster ({{ $.Release.Name }}) and does not use StatefulSet diff --git a/packages/extra/etcd/tests/defrag_test.yaml b/packages/extra/etcd/tests/defrag_test.yaml index 8d74e1a6b..b3acc3401 100644 --- a/packages/extra/etcd/tests/defrag_test.yaml +++ b/packages/extra/etcd/tests/defrag_test.yaml @@ -41,3 +41,16 @@ tests: documentSelector: path: kind value: EtcdCluster + + - it: asks the tenant quota only for what the defrag client uses + template: templates/etcd-defrag.yaml + asserts: + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources + value: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi From a7ec05f9eb759e0c1025cf4bf95631df338a85c0 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:15:04 +0300 Subject: [PATCH 017/174] fix(monitoring-agents): make a silent Watchdog an alarm in Alerta Alerta turns the Watchdog into a heartbeat and stores its severity in the heartbeat attributes. The alerta heartbeats --alert job of the alerta-web image takes the severity of a stale heartbeat from those attributes ahead of its HEARTBEAT_SEVERITY default, so with severity ok the HeartbeatFail it raises is a normal alert and nothing says the alerting pipeline stopped. Without a timeout label the heartbeat also took a day to go stale, while Alertmanager refreshes it every minute. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- .../alerts/general.rules.yaml | 3 +- .../tests/watchdog_heartbeat_test.yaml | 29 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml diff --git a/packages/system/monitoring-agents/alerts/general.rules.yaml b/packages/system/monitoring-agents/alerts/general.rules.yaml index 924870c40..0ac83fe72 100644 --- a/packages/system/monitoring-agents/alerts/general.rules.yaml +++ b/packages/system/monitoring-agents/alerts/general.rules.yaml @@ -33,7 +33,8 @@ spec: is working properly. expr: vector(1) labels: - severity: ok + severity: major + timeout: "300" exported_instance: global service: general.rules event: Heartbeat diff --git a/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml b/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml new file mode 100644 index 000000000..fe630a80c --- /dev/null +++ b/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml @@ -0,0 +1,29 @@ +suite: Watchdog heartbeat raises an alarm in Alerta when it stops +templates: + - templates/alerts.yaml + +release: + name: monitoring-agents + namespace: cozy-monitoring + +tests: + - it: gives the heartbeat a severity Alerta reports as an alarm + documentSelector: + path: metadata.name + value: alerts-general.rules + asserts: + - equal: + path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.severity + value: major + + - it: lets the heartbeat go stale within minutes, not a day + documentSelector: + path: metadata.name + value: alerts-general.rules + asserts: + - equal: + path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.timeout + value: "300" + - equal: + path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.event + value: Heartbeat From 70e1e8b1454db65222a0b3c0952650fa69239fdc Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Sun, 25 Oct 2026 20:33:46 +0300 Subject: [PATCH 018/174] test(vertical-pod-autoscaler): check maxSkew and whenUnsatisfiable of the spread The test only looked at the topology key and the label selector, so a change to maxSkew or to whenUnsatisfiable went unnoticed. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml index 1d2011330..0ff338b48 100644 --- a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml +++ b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml @@ -252,6 +252,12 @@ tests: - equal: path: spec.template.spec.topologySpreadConstraints[0].topologyKey value: kubernetes.io/hostname + - equal: + path: spec.template.spec.topologySpreadConstraints[0].maxSkew + value: 1 + - equal: + path: spec.template.spec.topologySpreadConstraints[0].whenUnsatisfiable + value: ScheduleAnyway - equal: path: spec.template.spec.topologySpreadConstraints[0].labelSelector.matchLabels value: From bcc5f3216a7ea4ec93bf2597ad4821ff97e855ac Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:21:32 +0300 Subject: [PATCH 019/174] fix(kubevirt): let tenant admins upload a VMDisk image with virtctl A VMDisk created with source upload waits in UploadReady for an image, but no tenant role can fill it. virtctl image-upload reads the disk PVC and CDI's PVC Prime and requests an upload token, and the tenant roles grant neither get on persistentvolumeclaims nor create on uploadtokenrequests, so the command stops on a forbidden PVC read. cozy:tenant:admin:base now grants both, so tenant admins and super-admins get them in their own namespaces through the existing RoleBindings, while the use and view tiers do not. Reading the DataVolume is already granted for the disk's own name by the vm-disk chart's dashboard role. The vm-disk notes print the command with --no-create, because the chart has already created the DataVolume, and the security self-assessment and threat model list the new grants. The rules, their tests, the notes change and the docs lines come from #3847, which also carries a console upload panel and an upload proxy certificate that are still in review there. Fixes #3759 Co-authored-by: Myasnikov Daniil Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- docs/security/self-assessment.md | 2 +- docs/security/threat-model.md | 4 +- packages/apps/vm-disk/templates/NOTES.txt | 2 +- .../apps/vm-disk/tests/upload_notes_test.yaml | 18 ++++ .../templates/clusterroles.yaml | 11 +++ .../tests/cdi_upload_rbac_test.yaml | 97 +++++++++++++++++++ 6 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 packages/apps/vm-disk/tests/upload_notes_test.yaml create mode 100644 packages/system/cozystack-basics/tests/cdi_upload_rbac_test.yaml diff --git a/docs/security/self-assessment.md b/docs/security/self-assessment.md index cb8360904..b59cbda4a 100644 --- a/docs/security/self-assessment.md +++ b/docs/security/self-assessment.md @@ -55,7 +55,7 @@ Actors are decomposed by **trust boundary**: two components are distinct actors | GitOps / Flux delivery layer | T1 — platform control plane | Reconciles HelmReleases/Kustomizations into arbitrary privileged manifests; effectively cluster-admin. | Tenants cannot create HelmReleases (RBAC); not tenant-network-reachable. | | `cozystack-api` aggregated API server | T1 — platform control plane | Converts tenant CRs into HelmReleases; can CRUD HelmReleases, Secrets, and Cilium policies cluster-wide. Acts as the policy-enforcing gateway. | Constrains tenants via fixed chart reference, name/quota/host validation, and admission. | | System operators / controllers | T1 — platform control plane | Domain-scoped cluster privileges (provision control planes, mint certs/secrets, program CNI). | Distinct ServiceAccounts; isolated from tenants by RBAC and network policy. | -| Tenant administrator (OIDC `-admin` / `-super-admin`) | T2 — tenant boundary | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` in its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) `super-admin` holds the `apps.cozystack.io/*` wildcard (and can create child `Tenant`s); `admin` has an enumerated allowlist excluding some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`). | Namespace-scoped RBAC + admission policies + quota; no HelmRelease or cross-tenant access. | +| Tenant administrator (OIDC `-admin` / `-super-admin`) | T2 — tenant boundary | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` in its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); `get` on persistentvolumeclaims and `create` on `upload.cdi.kubevirt.io/uploadtokenrequests`, the pair `virtctl image-upload` needs to upload into an existing VMDisk; no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) `super-admin` holds the `apps.cozystack.io/*` wildcard (and can create child `Tenant`s); `admin` has an enumerated allowlist excluding some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`). | Namespace-scoped RBAC + admission policies + quota; no HelmRelease or cross-tenant access. | | Tenant user (OIDC `-view` / `-use`) | T2 — tenant boundary | Read-only view plus operational verbs (VM console/vnc/portforward, start/stop/restart). Cannot create Application CRs. | Strictly dominated by tenant-admin; cannot escalate. | | Parent vs child (nested) tenant | T2 — tenant boundary | Parent ServiceAccounts bind into descendant namespaces; children do not bind upward. Quota hierarchically capped. | Ancestor→descendant trust is one-way. | | Tenant workloads / managed services | T3 — workload boundary | Run tenant-controlled code; no cluster-API rights beyond their ServiceAccount. | Namespace-scoped RBAC/identity isolate API objects; on the network, Cilium egress confines each tenant pod to its own subtree + platform system services + `world`, so unrelated tenants cannot reach each other over the pod network (enforced on egress; ingress is open to `cluster` — see Non-goals). | diff --git a/docs/security/threat-model.md b/docs/security/threat-model.md index 9790d9591..0a4ea0bb3 100644 --- a/docs/security/threat-model.md +++ b/docs/security/threat-model.md @@ -39,7 +39,7 @@ The actors below are decomposed **by trust boundary**, not by component: two com | GitOps / Flux delivery layer | T1 | Reconciles HelmReleases/Kustomizations into arbitrary privileged manifests; effectively cluster-admin. | Tenants cannot create HelmReleases/Kustomizations (RBAC); not tenant-network-reachable. | | `cozystack-api` aggregated API server (ServiceAccount) | T1 | CRUD HelmReleases, Secrets, and CiliumNetworkPolicies cluster-wide; reads Namespaces/RBAC/quotas/nodes. Acts as the policy-enforcing gateway that converts tenant CRs to HelmReleases. | A distinct, network-exposed attack surface from Flux; constrains tenants via fixed chart ref + name/quota/host validation + admission. | | System operators / controllers | T1 | Domain-scoped cluster privileges (provision control planes, mint certs/secrets, program CNI). | Distinct ServiceAccounts, but all cluster-admin-adjacent; isolated from tenants only by RBAC and network policy. | -| Tenant administrator (OIDC groups `-admin`, `-super-admin`) | T2 | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` within its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) The `-super-admin` group holds the `apps.cozystack.io/*` wildcard (and so can create child `Tenant`s); the `-admin` group has an enumerated allowlist that excludes some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`, `external-dns`, `bootbox`). | Namespace-scoped RoleBindings + admission policies + quota; no access to HelmReleases, arbitrary core writes, or other tenants. | +| Tenant administrator (OIDC groups `-admin`, `-super-admin`) | T2 | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` within its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); `get` on persistentvolumeclaims and `create` on `upload.cdi.kubevirt.io/uploadtokenrequests`, the pair `virtctl image-upload` needs to upload into an existing VMDisk; no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) The `-super-admin` group holds the `apps.cozystack.io/*` wildcard (and so can create child `Tenant`s); the `-admin` group has an enumerated allowlist that excludes some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`, `external-dns`, `bootbox`). | Namespace-scoped RoleBindings + admission policies + quota; no access to HelmReleases, arbitrary core writes, or other tenants. | | Tenant user (OIDC groups `-view`, `-use`) | T2 | Read-only view of Application CRs, pods, and services, plus operational verbs only (VM console/vnc/portforward, start/stop/restart). Cannot create Application CRs. | Strictly dominated by tenant-admin; cannot escalate to create/delete, so it is a distinct actor. | | Parent tenant vs child (nested) tenant | T2 | Parent ServiceAccounts are bound into descendant namespaces; child ServiceAccounts are not bound upward. Quota is hierarchically capped. | Ancestor→descendant only; compromising a parent compromises its children, not the reverse. | | Tenant workloads / managed services | T3 | Run tenant-controlled code; no cluster-API rights beyond their ServiceAccount. | Namespace-scoped RBAC/identity isolate their API objects. On the network, Cilium egress confines each tenant pod to its own tenant subtree, platform system services, and `world` (out-of-cluster), so unrelated tenants cannot reach each other over the pod network — enforced on the egress side (tenant-pod ingress is open to `cluster`; see Non-goals for residual caveats). | @@ -92,7 +92,7 @@ The following are explicitly out of scope for the platform's security guarantees The single most load-bearing isolation control is the combination of two facts: -1. The tenant ClusterRoles (`packages/system/cozystack-basics/templates/clusterroles.yaml`) grant **no** `helm.toolkit.fluxcd.io` access and **no** create/update of arbitrary core resources — only CRUD on `apps.cozystack.io` application kinds (super-admin holds the `*` wildcard; admin an enumerated allowlist) and `sdn.cozystack.io/securitygroups`, read access to pods/services/ingresses, and (for tenant-admins) `delete` on a fixed set of core objects (pods/services/endpoints/events; the `persistentvolumes` entry in the ClusterRole is inert under the namespace-scoped RoleBinding). Tenant isolation is enforced by the *absence* of privileged grants in RBAC, not by an admission policy that whitelists `apps.cozystack.io/*` writes. +1. The tenant ClusterRoles (`packages/system/cozystack-basics/templates/clusterroles.yaml`) grant **no** `helm.toolkit.fluxcd.io` access and **no** create/update of arbitrary core resources — only CRUD on `apps.cozystack.io` application kinds (super-admin holds the `*` wildcard; admin an enumerated allowlist) and `sdn.cozystack.io/securitygroups`, read access to pods/services/ingresses, and (for tenant-admins) `delete` on a fixed set of core objects (pods/services/endpoints/events; the `persistentvolumes` entry in the ClusterRole is inert under the namespace-scoped RoleBinding), `get` on persistentvolumeclaims and `create` on `upload.cdi.kubevirt.io/uploadtokenrequests` (for uploading an image into an existing VMDisk with `virtctl image-upload`). Tenant isolation is enforced by the *absence* of privileged grants in RBAC, not by an admission policy that whitelists `apps.cozystack.io/*` writes. 2. The Application-to-HelmRelease conversion (`pkg/registry/apps/application/rest.go`) takes the chart reference, release-name prefix, and `valuesFrom` from server-side configuration; the tenant controls only the Helm values (`app.Spec`). If a future `ApplicationDefinition`, RBAC change, or API change ever let a tenant influence the chart reference, `valuesFrom`, or write HelmReleases directly, the T2→T1 isolation would collapse. Changes touching tenant RBAC or the Application REST layer should be reviewed against this invariant. diff --git a/packages/apps/vm-disk/templates/NOTES.txt b/packages/apps/vm-disk/templates/NOTES.txt index 73d07cc59..0940e75ba 100644 --- a/packages/apps/vm-disk/templates/NOTES.txt +++ b/packages/apps/vm-disk/templates/NOTES.txt @@ -1,5 +1,5 @@ {{- if and .Values.source (hasKey .Values.source "upload") }} To upload your disk run: -virtctl image-upload dv -n {{ .Release.Namespace }} {{ .Release.Name }} --image-path +virtctl image-upload dv {{ .Release.Name }} --no-create --namespace {{ .Release.Namespace }} --image-path {{- end }} diff --git a/packages/apps/vm-disk/tests/upload_notes_test.yaml b/packages/apps/vm-disk/tests/upload_notes_test.yaml new file mode 100644 index 000000000..16a6432a3 --- /dev/null +++ b/packages/apps/vm-disk/tests/upload_notes_test.yaml @@ -0,0 +1,18 @@ +suite: VM disk upload notes +templates: + - templates/NOTES.txt + +release: + name: vm-disk-demo + namespace: tenant-root + +tests: + - it: reuses the chart-created DataVolume for local image upload + set: + source: + upload: {} + asserts: + - matchRegexRaw: + pattern: 'virtctl image-upload dv vm-disk-demo --no-create --namespace tenant-root --image-path ' + - notMatchRegexRaw: + pattern: 'virtctl image-upload dv -n' diff --git a/packages/system/cozystack-basics/templates/clusterroles.yaml b/packages/system/cozystack-basics/templates/clusterroles.yaml index 9807d32f1..5df762886 100644 --- a/packages/system/cozystack-basics/templates/clusterroles.yaml +++ b/packages/system/cozystack-basics/templates/clusterroles.yaml @@ -235,6 +235,17 @@ metadata: labels: rbac.cozystack.io/aggregate-to-tenant-admin: "true" rules: +# virtctl reads both the disk PVC and CDI's dynamically named PVC Prime. +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: + - get +- apiGroups: ["upload.cdi.kubevirt.io"] + resources: + - uploadtokenrequests + verbs: + - create - apiGroups: [""] resources: - pods diff --git a/packages/system/cozystack-basics/tests/cdi_upload_rbac_test.yaml b/packages/system/cozystack-basics/tests/cdi_upload_rbac_test.yaml new file mode 100644 index 000000000..b99df44bf --- /dev/null +++ b/packages/system/cozystack-basics/tests/cdi_upload_rbac_test.yaml @@ -0,0 +1,97 @@ +suite: tenant admin CDI upload RBAC + +templates: + - templates/clusterroles.yaml + +release: + name: cozystack-basics + namespace: cozy-system + +tests: + - it: lets tenant admins inspect upload PVCs and create CDI upload tokens + documentSelector: + path: metadata.name + value: cozy:tenant:admin:base + asserts: + - equal: + path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant-admin"] + value: "true" + - contains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - contains: + path: rules + content: + apiGroups: ["upload.cdi.kubevirt.io"] + resources: ["uploadtokenrequests"] + verbs: ["create"] + + - it: keeps the admin grant out of the use, view and service-account aggregation + documentSelector: + path: metadata.name + value: cozy:tenant:admin:base + asserts: + - notExists: + path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant-use"] + - notExists: + path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant-view"] + - notExists: + path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant"] + + - it: does not grant CDI upload permissions to the use tier + documentSelector: + path: metadata.name + value: cozy:tenant:use:base + asserts: + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: ["upload.cdi.kubevirt.io"] + resources: ["uploadtokenrequests"] + verbs: ["create"] + + - it: aggregates upload permissions through admin into super-admin + documentSelector: + path: metadata.name + value: cozy:tenant:admin + asserts: + - equal: + path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant-super-admin"] + value: "true" + - contains: + path: aggregationRule.clusterRoleSelectors + content: + matchLabels: + rbac.cozystack.io/aggregate-to-tenant-admin: "true" + + - it: selects admin permissions for super-admin + documentSelector: + path: metadata.name + value: cozy:tenant:super-admin + asserts: + - contains: + path: aggregationRule.clusterRoleSelectors + content: + matchLabels: + rbac.cozystack.io/aggregate-to-tenant-super-admin: "true" + + - it: does not grant CDI upload token creation to the view tier + documentSelector: + path: metadata.name + value: cozy:tenant:view:base + asserts: + - notContains: + path: rules + content: + apiGroups: ["upload.cdi.kubevirt.io"] + resources: ["uploadtokenrequests"] + verbs: ["create"] From 982db83dd22d6a0257beda788cf75839c738ab42 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:10:59 +0300 Subject: [PATCH 020/174] fix(postgres): accept a standard five-field backup schedule The mongodb app and backup Plans take a standard five-field cron, but the postgres chart hands backup.schedule to a CloudNativePG ScheduledBackup, which parses it with robfig/cron v1 Parse. That parser reads the first field as seconds and makes day-of-week optional, so a five-field value passes admission with only a warning and runs one field off. 0 2 * * * runs hourly and 5 * * * * every minute. A five-field schedule now gets a leading 0 seconds field. Six fields and descriptors pass through, and any other field count fails the render. A release that already sets five fields gets the standard meaning on upgrade, so 0 2 * * * goes from hourly to daily. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- api/apps/v1alpha1/postgresql/types.go | 2 +- packages/apps/postgres/README.md | 15 ++-- packages/apps/postgres/templates/backup.yaml | 19 ++++- .../postgres/tests/backup_schedule_test.yaml | 80 +++++++++++++++++++ packages/apps/postgres/values.schema.json | 2 +- packages/apps/postgres/values.yaml | 2 +- .../system/postgres-rd/cozyrds/postgres.yaml | 2 +- 7 files changed, 107 insertions(+), 15 deletions(-) create mode 100644 packages/apps/postgres/tests/backup_schedule_test.yaml diff --git a/api/apps/v1alpha1/postgresql/types.go b/api/apps/v1alpha1/postgresql/types.go index f0415f398..a1e32ed26 100644 --- a/api/apps/v1alpha1/postgresql/types.go +++ b/api/apps/v1alpha1/postgresql/types.go @@ -124,7 +124,7 @@ type Backup struct { // DEPRECATED. See `s3AccessKey`. // +kubebuilder:default:="" S3SecretKey string `json:"s3SecretKey,omitempty"` - // Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. + // Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. // +kubebuilder:default:="" Schedule string `json:"schedule,omitempty"` // Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`. diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index bdc13ea9c..49f08d441 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -20,7 +20,7 @@ together rather than picking one: | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | | **Archive plumbing** (chart, legacy) | Renders `spec.plugins` (referencing a barman-cloud `ObjectStore`) on the cnpg.io Cluster from helm-install onwards. CNPG runs the barman-cloud plugin as a WAL-archiver sidecar, every WAL switch ships to object storage, and any backup driver gets a complete WAL chain to start replay from. Renders only when `backup.enabled=true`, `useSystemBucket=false`, `destinationPath` is non-empty, AND inline-or-external creds are supplied. **Skipped in the platform `useSystemBucket=true` mode** — the CNPG backup driver SSA-applies the `ObjectStore` and patches `spec.plugins` onto the live Cluster at first BackupJob time; until that first BackupJob fires, plugin WAL archiving is not active and WAL accumulates on the PVC. Fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. | Legacy: `backup.enabled=true` plus `backup.destinationPath`, `backup.endpointURL`, and either `backup.s3AccessKey`+`backup.s3SecretKey` or `backup.s3CredentialsSecret`. Platform: `backup.enabled=true` plus `backup.useSystemBucket=true` — no S3 fields. | | **Backup orchestration** (recommended) | Drives ad-hoc and scheduled backups, retention, and restores from `backups.cozystack.io` resources that can span multiple Postgres apps in a tenant. The driver SSA-applies its own barman-cloud `ObjectStore` and patches `spec.plugins` with `ForceOwnership` (so compression / target settings flow from the strategy template, while the live Cluster's `serverName` — its WAL-archive S3 prefix — is always preserved); chart-managed values cover the same destination so there is no tug-of-war on the live Cluster. | `strategy.backups.cozystack.io/CNPG` + `BackupClass` + `Plan` (recurring) or `BackupJob` (ad-hoc), restores via `RestoreJob` | -| Legacy chart-emitted scheduled backup | The chart can also emit a `cnpg.io/ScheduledBackup` directly. Superseded by the BackupClass + Plan path, kept around for clusters that did not migrate. **Off by default** - rendered only when both `backup.enabled` and `backup.schedule` are non-empty. With `backup.schedule=""` plugin WAL archiving still runs; only the chart-emitted scheduled backup is silent. | `backup.enabled=true` plus `backup.schedule` (CNPG 6-field cron, e.g. `"0 2 * * * *"`) | +| Legacy chart-emitted scheduled backup | The chart can also emit a `cnpg.io/ScheduledBackup` directly. Superseded by the BackupClass + Plan path, kept around for clusters that did not migrate. **Off by default** - rendered only when both `backup.enabled` and `backup.schedule` are non-empty. With `backup.schedule=""` plugin WAL archiving still runs; only the chart-emitted scheduled backup is silent. | `backup.enabled=true` plus `backup.schedule` (five-field cron such as `"0 2 * * *"`, or six fields with leading seconds) | The **canonical setup** depends on whether the cluster ships the platform `cozy-default` BackupClass. @@ -87,16 +87,11 @@ field. ### How to enable chart-managed scheduled backups (legacy) -The chart can also emit a `cnpg.io/ScheduledBackup` directly, without a -BackupClass. Superseded by the BackupClass + Plan path above and kept -around for clusters that did not migrate. It does not run by default - -`backup.schedule` defaults to an empty string, which gates the chart's -ScheduledBackup template off. To turn it on, fill in a CNPG 6-field cron -expression: +The chart can also emit a `cnpg.io/ScheduledBackup` directly, without a BackupClass. Superseded by the BackupClass + Plan path above and kept around for clusters that did not migrate. It does not run by default - `backup.schedule` defaults to an empty string, which gates the chart's ScheduledBackup template off. To turn it on, set a standard five-field cron such as `0 2 * * *` for every day at 02:00, or the CloudNativePG six-field form with a leading seconds field such as `0 0 2 * * *` for the same time. The chart prepends a `0` seconds field to a five-field value, because CloudNativePG reads the first field as seconds and would otherwise run `0 2 * * *` every hour. ```yaml ## @param backup.enabled Enable plugin WAL archiving + render the chart-managed ScheduledBackup -## @param backup.schedule Cron schedule (CNPG 6-field). Empty means no chart-managed ScheduledBackup +## @param backup.schedule Cron schedule, five fields or six with leading seconds. Empty means no chart-managed ScheduledBackup ## @param backup.retentionPolicy Retention policy ## @param backup.destinationPath Path to store the backup (i.e. s3://bucket/path/to/folder) ## @param backup.endpointURL S3 Endpoint used to upload data to the cloud @@ -107,7 +102,7 @@ backup: retentionPolicy: 30d destinationPath: s3://bucket/path/to/folder/ endpointURL: http://minio-gateway-service:9000 - schedule: "0 2 * * * *" # opt in - empty (the default) means no chart-managed schedule + schedule: "0 2 * * *" # opt in - empty (the default) means no chart-managed schedule s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog ``` @@ -295,7 +290,7 @@ For `sslmode=verify-full` to work, the CA bundle retrieved above must be saved t | `backup` | Backup configuration. | `object` | `{}` | | `backup.enabled` | Enable regular backups. | `bool` | `false` | | `backup.useSystemBucket` | Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`. | `bool` | `false` | -| `backup.schedule` | Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. | `string` | `""` | +| `backup.schedule` | Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. | `string` | `""` | | `backup.retentionPolicy` | Retention policy (e.g. "30d"). | `string` | `30d` | | `backup.destinationPath` | DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility. | `string` | `s3://bucket/path/to/folder/` | | `backup.endpointURL` | DEPRECATED. See `destinationPath`. | `string` | `http://minio-gateway-service:9000` | diff --git a/packages/apps/postgres/templates/backup.yaml b/packages/apps/postgres/templates/backup.yaml index aa1d888e3..4025ca571 100644 --- a/packages/apps/postgres/templates/backup.yaml +++ b/packages/apps/postgres/templates/backup.yaml @@ -23,15 +23,32 @@ against a Cluster with no backup configuration and fail. This mirrors the ClickHouse `useSystemBucket` gates on the chart-managed backup resources. + + CNPG parses `schedule` with robfig/cron v1 `Parse`, which takes the first + field as seconds and makes day-of-week optional. A standard five-field + cron passes its webhook with only a warning and then runs one field off, + so `5 * * * *` fires every minute instead of hourly. A five-field value + therefore gets a leading `0` seconds field. Six fields and descriptors + such as `@daily` pass through as they are, and any other field count + fails the render, which CNPG would reject at admission anyway. */}} {{- if and .Values.backup.enabled .Values.backup.schedule (not .Values.backup.useSystemBucket) }} +{{- $schedule := .Values.backup.schedule }} +{{- $fields := regexFindAll `\S+` $schedule -1 }} +{{- if not (hasPrefix "@" (trim $schedule)) }} +{{- if eq (len $fields) 5 }} +{{- $schedule = printf "0 %s" (join " " $fields) }} +{{- else if ne (len $fields) 6 }} +{{- fail (printf "postgres: backup.schedule %q is not a cron schedule with 5 or 6 fields (it has %d). Use a standard five-field cron such as \"0 2 * * *\", a six-field cron with leading seconds such as \"0 0 2 * * *\", or a descriptor such as \"@daily\"." $schedule (len $fields)) }} +{{- end }} +{{- end }} --- apiVersion: postgresql.cnpg.io/v1 kind: ScheduledBackup metadata: name: {{ .Release.Name }} spec: - schedule: {{ .Values.backup.schedule | quote }} + schedule: {{ $schedule | quote }} backupOwnerReference: self method: plugin pluginConfiguration: diff --git a/packages/apps/postgres/tests/backup_schedule_test.yaml b/packages/apps/postgres/tests/backup_schedule_test.yaml new file mode 100644 index 000000000..e78526848 --- /dev/null +++ b/packages/apps/postgres/tests/backup_schedule_test.yaml @@ -0,0 +1,80 @@ +suite: chart-managed scheduled backup takes five-field and six-field cron + +# CNPG parses the ScheduledBackup schedule with robfig/cron v1 Parse, which +# takes the first field as seconds and accepts five fields with only a +# warning, so an unconverted "5 * * * *" would run every minute. + +templates: + - templates/backup.yaml + +release: + name: pg-test + namespace: tenant-test + +tests: + - it: prepends a zero seconds field to a standard five-field schedule + set: + backup: + enabled: true + schedule: "5 * * * *" + asserts: + - equal: + path: spec.schedule + value: "0 5 * * * *" + + - it: joins the fields of a five-field schedule with single spaces + set: + backup: + enabled: true + schedule: " 0 2 * *\t* " + asserts: + - equal: + path: spec.schedule + value: "0 0 2 * * *" + + - it: passes a six-field schedule through unchanged + set: + backup: + enabled: true + schedule: "0 5 * * * *" + asserts: + - equal: + path: spec.schedule + value: "0 5 * * * *" + + - it: passes a descriptor through unchanged + set: + backup: + enabled: true + schedule: "@every 6h" + asserts: + - equal: + path: spec.schedule + value: "@every 6h" + + - it: refuses a schedule with four fields + set: + backup: + enabled: true + schedule: "5 * * *" + asserts: + - failedTemplate: + errorMessage: 'postgres: backup.schedule "5 * * *" is not a cron schedule with 5 or 6 fields (it has 4). Use a standard five-field cron such as "0 2 * * *", a six-field cron with leading seconds such as "0 0 2 * * *", or a descriptor such as "@daily".' + + - it: refuses a schedule with seven fields + set: + backup: + enabled: true + schedule: "0 0 2 * * * 2026" + asserts: + - failedTemplate: + errorMessage: 'postgres: backup.schedule "0 0 2 * * * 2026" is not a cron schedule with 5 or 6 fields (it has 7). Use a standard five-field cron such as "0 2 * * *", a six-field cron with leading seconds such as "0 0 2 * * *", or a descriptor such as "@daily".' + + - it: refuses a schedule of only spaces + set: + backup: + enabled: true + schedule: " " + asserts: + - failedTemplate: + errorMessage: 'postgres: backup.schedule " " is not a cron schedule with 5 or 6 fields (it has 0). Use a standard five-field cron such as "0 2 * * *", a six-field cron with leading seconds such as "0 0 2 * * *", or a descriptor such as "@daily".' diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index 91747ba24..799b05776 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -427,7 +427,7 @@ "default": "" }, "schedule": { - "description": "Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.", + "description": "Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.", "type": "string", "default": "" }, diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index 83b658a64..a938497d2 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -321,7 +321,7 @@ databases: {} ## @typedef {struct} Backup - Backup configuration. ## @field {bool} enabled - Enable regular backups. ## @field {bool} [useSystemBucket] - Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`. -## @field {string} [schedule] - Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. +## @field {string} [schedule] - Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. ## @field {string} [retentionPolicy] - Retention policy (e.g. "30d"). ## @field {string} [destinationPath] - DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility. ## @field {string} [endpointURL] - DEPRECATED. See `destinationPath`. diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml index 39adc0f58..aeea68e12 100644 --- a/packages/system/postgres-rd/cozyrds/postgres.yaml +++ b/packages/system/postgres-rd/cozyrds/postgres.yaml @@ -30,7 +30,7 @@ spec: singular: postgres plural: postgreses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas. Under active autoscaling this is the resting/seed count, not the live count — KEDA drives the live count via `/scale`. Disabling autoscaling does NOT automatically bring the live count back to `replicas`: the client-side merge writes `instances` only when the rendered value changes, so a cluster KEDA grew past `replicas` stays there with no autoscaler left to reduce it — lower `replicas` to the count you want and it rebases down (shedding the surplus replicas + PVCs). Lowering it below the live count, or disabling/dry-running while it sits below the live count, sheds in one step past the scaleDown pacing; stage it to the live count first (suspend the HelmRelease and `kubectl scale` if needed) to avoid a transient. Must stay greater than `quorum.maxSyncReplicas`.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"flavor":{"description":"PostgreSQL image flavor, fixed when the cluster is created. `postgresql` uses the CloudNativePG PostgreSQL image. `postgis` uses the CloudNativePG PostGIS image, which adds postgis, postgis_raster, postgis_topology, postgis_sfcgal, postgis_tiger_geocoder and address_standardizer; enable them per database in `databases..extensions`. Changing the flavor of an existing cluster is refused, and a backup restores only into a cluster of the flavor it was taken from: to change flavor, create a new cluster and move the data with a logical dump (pg_dump / pg_restore).","type":"string","default":"postgresql","enum":["postgresql","postgis"]},"tls":{"description":"TLS configuration for server connections.","type":"object","default":{},"properties":{"enabled":{"description":"Tri-state switch controlling whether the chart injects the external hostnames into the operator-managed CNPG cert via spec.certificates.serverAltDNSNames. When omitted, the chart injects them if `external: true` and skips them otherwise. Set explicitly to `true` to inject regardless of `external` (no-op when `external: false` since there is no external hostname to add). Set to `false` to skip injection. Note that CNPG keeps its built-in TLS on the wire regardless of this flag — this toggle only controls the chart-side SAN injection; to disable PostgreSQL TLS entirely set `postgresql.parameters.ssl = \"off\"` at the CNPG layer.","type":"boolean"},"required":{"description":"Tri-state switch controlling whether the server refuses sessions that are not encrypted. When omitted, it is on if `external: true` and off otherwise. When on, the chart renders `spec.postgresql.pg_hba` with `hostssl all all all scram-sha-256` (`md5` on PostgreSQL 13, matching CloudNativePG's default rule) followed by `host all all all reject`, so a client that connects with `sslmode=disable` is refused instead of getting a plaintext session; in-cluster clients are held to the same rule, since the pod network cannot be told apart from an external client. Set to `false` to keep accepting plaintext sessions on an external instance, or to `true` to require TLS on an in-cluster one. Rendering fails if `postgresql.parameters.ssl` is off while this is in effect.","type":"boolean"}}},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters. Values may be strings or integers; integers are coerced to strings by the template (e.g. both `max_connections: 100` and `max_connections: \"100\"` are accepted). BLOCKED (enable arbitrary code execution): archive_command, restore_command, ssl_passphrase_command, archive_cleanup_command, recovery_end_command, dynamic_library_path, local_preload_libraries, session_preload_libraries, shared_preload_libraries. Do NOT override CloudNativePG-managed parameters: archive_mode, primary_conninfo, wal_level, max_replication_slots.","type":"object","default":{"max_connections":"100"},"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}},"sharedPreloadLibraries":{"description":"Libraries loaded at server start, restricted to an allowlist of modules already shipped in the PostgreSQL image. CloudNativePG also preloads `pg_stat_statements`, `auto_explain` and `pgaudit` by itself whenever `parameters` sets a key under their prefix (e.g. `pg_stat_statements.track`), so removing one of them from this list does not unload it while such a key is set; `pg_prewarm` is preloaded only through this list. Changing the list restarts PostgreSQL on every instance, the primary included, so writes pause briefly.","type":"array","items":{"type":"string","enum":["pg_stat_statements","auto_explain","pgaudit","pg_prewarm"]}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas). Under autoscaling it also sets the scale-down floor (the effective minimum is `max(minReplicas, maxSyncReplicas+1, 2)`); the chart fails to render if that floor exceeds `autoscaling.maxReplicas`.","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"autoscaling":{"description":"Read-replica autoscaling configuration.","type":"object","default":{},"required":["dryRun","enabled","maxReplicas","maxReplicationLagSeconds","metric","minReplicas","serverAddress","target","transition"],"properties":{"dryRun":{"description":"Recommendation mode: keep the static count and render the ScaledObject with both scaling directions paused, so KEDA keeps the HPA and keeps serving its trigger metric but actuates neither way. Note both scaling policies are disabled, so the HPA's desiredReplicas is frozen to the current count - read the recommendation from the matching read-load dashboard panel for your metric (connections or CPU): desired = 1 + ceil(read load / target), clamped to [effectiveMin..maxReplicas] where effectiveMin = max(minReplicas, maxSyncReplicas+1, 2), not from the desired-replicas line. Like disabling, flipping a LIVE autoscaled cluster into dryRun re-renders `instances: replicas` (the raw static value, which can be below the autoscaling floor); if `replicas` is below the live count, stage it to the live count first (see the enablement note) or the cluster sheds down to `replicas`. No shed occurs when the rendered value is unchanged (replicas already equals the active seed) — client-side merge is then a no-op and the live count is preserved. Permanent, unlike `transition`.","type":"boolean","default":false},"enabled":{"description":"Enable horizontal autoscaling of read replicas. Requires the platform `keda` package to be enabled by an administrator first; the chart fails to render (rather than break the release) if KEDA is not installed. Prerequisite: the queried vmselect must hold BOTH the CNPG metrics (cnpg_backends_total / container CPU) AND kube_pod_labels (kube-state-metrics). This holds for a tenant on the shared root monitoring stack (`monitoring: false`, both land in tenant-root). A tenant running its OWN Monitoring app (`monitoring: true`) has the CNPG metrics but not kube_pod_labels (KSM is root-only), so it must set `serverAddress` to a vmselect carrying both series or autoscaling never actuates.","type":"boolean","default":false},"maxReplicas":{"description":"Maximum total instances. Must be >= minReplicas and >= the synchronous-quorum floor (maxSyncReplicas+1); the chart fails to render (rather than silently run an unbounded instance count) if the effective floor exceeds it - raise maxReplicas. HAZARD: lowering maxReplicas below the live count makes the HPA cut straight to the new maximum in one step (the current>max branch bypasses the scale-down pacing), shedding the surplus replicas + PVCs at once; to lower it deliberately, suspend the HelmRelease and pin the live count with `kubectl scale` first.","type":"integer","default":6},"maxReplicationLagSeconds":{"description":"Freeze scaling while replication lag exceeds this (seconds) and the primary is writing. 0 disables the brake. Default 0: the freeze branch has only been validated on a live cluster in its non-braking (pass-through) form, so the lag brake is opt-in until the braking path is exercised under load; set a positive value (e.g. 30) to enable it.","type":"integer","default":0},"metric":{"description":"Read-load signal to scale on.","type":"string","default":"ReadConnections","enum":["ReadConnections","ReadCPUUtilization"]},"minReplicas":{"description":"Minimum total instances; raised to the synchronous-quorum floor (`maxSyncReplicas + 1`) and to 2 when either is higher.","type":"integer","default":2},"serverAddress":{"description":"Override the Prometheus/vmselect URL the ScaledObject queries. Empty (default) derives `http://vmselect-shortterm..svc:8481/select/0/prometheus`. Set it when the derived vmselect does not hold both the CNPG metrics and kube_pod_labels (e.g. `monitoring: true` tenants, or a non-default `metricsStorages` name). SSRF constraint: the host must be an in-cluster Service DNS name (`..svc[.]`) whose namespace is within the tenant's own subtree (its namespace, its parents, or `tenant-root`) — the cluster-privileged KEDA operator fetches this URL every poll, so it is refused a metadata endpoint, an external host, or a sibling/system namespace. Residual surface (accepted trade-off): `tenant-root` is always allowed because the derived default points there, so a tenant can still aim the operator at any Service in `tenant-root` on any port/path (a blind in-cluster reachability probe — the operator reads no response body back to the tenant); the egress NetworkPolicy does not narrow that.","type":"string","default":""},"target":{"description":"Target read load per read-serving replica (unit depends on `metric`: active connections, or CPU millicores). Dimensionless integer - a resource.Quantity here would accept `150m`, which PromQL reads as 150 minutes and KEDA cannot use as a threshold.","type":"integer","default":150},"transition":{"description":"Migration phase 1: render `.spec.instances` from `replicas` (which the operator stages to the current live count) AND pause the ScaledObject in both directions, so KEDA stands up with its HPA present but not actuating. Flip to false (phase 2) to hand the runtime count to KEDA. Because the both-direction pause keeps the HPA (not the HPA-deleting full pause), KEDA is genuinely warm at phase 2 — no HPA recreate — and because the active seed is max(replicas, effectiveMin), phase 2 re-renders the same instance count phase 1 did (when `replicas` is staged at/above the floor), so the flip is a merge no-op with no dip. The HPA's scaleUp stabilization window still smooths the first actuation.","type":"boolean","default":false}}},"users":{"description":"Users configuration map. Passwords are always auto-generated and stored in the `-credentials` Secret; they cannot be set from values — read a user's password from that Secret. A `password` left over in values from before the field was removed is ignored by the render and draws an admission warning, but it is not inert on an upgraded release: the chart preserves whatever password is already in the Secret, which on the first upgrade is the value that was set before removal, so that value stays the live credential until it is rotated (dedicated rotation is tracked in cozystack/community#72). The live password lives only in the Secret.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"bypassRls":{"description":"Whether the user can bypass row-level security policies (BYPASSRLS). A role can only create other roles carrying BYPASSRLS if it holds the attribute itself, so this is required alongside `createRole` to provision a privileged service role.","type":"boolean"},"createRole":{"description":"Whether the user can create and manage roles (CREATEROLE). Requires PostgreSQL 16 or newer. Setting it back to `false` withdraws the attribute only: the roles the user created, and its ADMIN and SET memberships in them, stay. Once any user has held it, the chart no longer adopts an existing role it did not create, so a user or database declared later under the name of such a role fails the init job until a platform administrator drops that role.","type":"boolean"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility.","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointCA":{"description":"DEPRECATED. Pre-existing Secret with the CA bundle the barman-cloud plugin should trust when reaching a self-signed S3 endpoint. Used for both backup and bootstrap recovery in the legacy chart-managed flow.","type":"object","default":{},"properties":{"key":{"description":"Key within the Secret containing the CA bundle. Defaults to `ca.crt`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means no endpointCA is emitted (the plugin uses the system trust store).","type":"string","default":""}}},"endpointURL":{"description":"DEPRECATED. See `destinationPath`.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"DEPRECATED. Tenants no longer supply S3 keys; the system Bucket Secret is projected into the tenant namespace by the backup controller. Ignored when `s3CredentialsSecret.name` is set or `useSystemBucket` is true. The chart skips materialising `-s3-creds` whenever this field is empty so a default install does not leak placeholder credentials into the tenant namespace.","type":"string","default":""},"s3CredentialsSecret":{"description":"DEPRECATED. Pre-existing Secret with S3 credentials. Use the platform-managed `cozy-default` BackupClass instead. When set, the chart references this Secret directly (legacy chart-managed flow). The CNPG backup driver writes this field on restore so credentials never land in the CR `.spec`.","type":"object","default":{},"properties":{"accessKeyIDKey":{"description":"Key in the Secret holding the access key ID. Defaults to `AWS_ACCESS_KEY_ID`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means the chart materialises `-s3-creds` from `s3AccessKey`/`s3SecretKey`.","type":"string","default":""},"secretAccessKeyKey":{"description":"Key in the Secret holding the secret access key. Defaults to `AWS_SECRET_ACCESS_KEY`.","type":"string","default":""}}},"s3SecretKey":{"description":"DEPRECATED. See `s3AccessKey`.","type":"string","default":""},"schedule":{"description":"Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.","type":"string","default":""},"useSystemBucket":{"description":"Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`.","type":"boolean","default":false}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"backupID":{"description":"Barman ID of the base backup to recover from, passed as `recoveryTarget.backupID`. Empty lets the barman-cloud plugin pick one: the newest backup when `recoveryTime` is empty, otherwise the newest one ending at or before it, on any timeline. The CNPG backup driver sets this on restore.","type":"string","default":""},"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"newServerName":{"description":"WAL-archive server name (S3 path prefix) the RESTORED cluster writes to. Must differ from `serverName` (the recovery source) so a restored cluster archives to a fresh, empty prefix and barman-cloud-check-wal-archive passes while it recovers from the source's prefix. The CNPG backup driver sets this on restore; empty means the cluster archives under its own name.","type":"string","default":""},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Server name (S3 path prefix) used by the original cluster when writing backups; passed to the barman-cloud plugin via `externalClusters[].plugin.parameters.serverName`. Defaults to `bootstrap.oldName`. Set this only when the original cluster wrote backups under an explicit server name that differed from its Kubernetes resource name.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas. Under active autoscaling this is the resting/seed count, not the live count — KEDA drives the live count via `/scale`. Disabling autoscaling does NOT automatically bring the live count back to `replicas`: the client-side merge writes `instances` only when the rendered value changes, so a cluster KEDA grew past `replicas` stays there with no autoscaler left to reduce it — lower `replicas` to the count you want and it rebases down (shedding the surplus replicas + PVCs). Lowering it below the live count, or disabling/dry-running while it sits below the live count, sheds in one step past the scaleDown pacing; stage it to the live count first (suspend the HelmRelease and `kubectl scale` if needed) to avoid a transient. Must stay greater than `quorum.maxSyncReplicas`.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"flavor":{"description":"PostgreSQL image flavor, fixed when the cluster is created. `postgresql` uses the CloudNativePG PostgreSQL image. `postgis` uses the CloudNativePG PostGIS image, which adds postgis, postgis_raster, postgis_topology, postgis_sfcgal, postgis_tiger_geocoder and address_standardizer; enable them per database in `databases..extensions`. Changing the flavor of an existing cluster is refused, and a backup restores only into a cluster of the flavor it was taken from: to change flavor, create a new cluster and move the data with a logical dump (pg_dump / pg_restore).","type":"string","default":"postgresql","enum":["postgresql","postgis"]},"tls":{"description":"TLS configuration for server connections.","type":"object","default":{},"properties":{"enabled":{"description":"Tri-state switch controlling whether the chart injects the external hostnames into the operator-managed CNPG cert via spec.certificates.serverAltDNSNames. When omitted, the chart injects them if `external: true` and skips them otherwise. Set explicitly to `true` to inject regardless of `external` (no-op when `external: false` since there is no external hostname to add). Set to `false` to skip injection. Note that CNPG keeps its built-in TLS on the wire regardless of this flag — this toggle only controls the chart-side SAN injection; to disable PostgreSQL TLS entirely set `postgresql.parameters.ssl = \"off\"` at the CNPG layer.","type":"boolean"},"required":{"description":"Tri-state switch controlling whether the server refuses sessions that are not encrypted. When omitted, it is on if `external: true` and off otherwise. When on, the chart renders `spec.postgresql.pg_hba` with `hostssl all all all scram-sha-256` (`md5` on PostgreSQL 13, matching CloudNativePG's default rule) followed by `host all all all reject`, so a client that connects with `sslmode=disable` is refused instead of getting a plaintext session; in-cluster clients are held to the same rule, since the pod network cannot be told apart from an external client. Set to `false` to keep accepting plaintext sessions on an external instance, or to `true` to require TLS on an in-cluster one. Rendering fails if `postgresql.parameters.ssl` is off while this is in effect.","type":"boolean"}}},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters. Values may be strings or integers; integers are coerced to strings by the template (e.g. both `max_connections: 100` and `max_connections: \"100\"` are accepted). BLOCKED (enable arbitrary code execution): archive_command, restore_command, ssl_passphrase_command, archive_cleanup_command, recovery_end_command, dynamic_library_path, local_preload_libraries, session_preload_libraries, shared_preload_libraries. Do NOT override CloudNativePG-managed parameters: archive_mode, primary_conninfo, wal_level, max_replication_slots.","type":"object","default":{"max_connections":"100"},"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}},"sharedPreloadLibraries":{"description":"Libraries loaded at server start, restricted to an allowlist of modules already shipped in the PostgreSQL image. CloudNativePG also preloads `pg_stat_statements`, `auto_explain` and `pgaudit` by itself whenever `parameters` sets a key under their prefix (e.g. `pg_stat_statements.track`), so removing one of them from this list does not unload it while such a key is set; `pg_prewarm` is preloaded only through this list. Changing the list restarts PostgreSQL on every instance, the primary included, so writes pause briefly.","type":"array","items":{"type":"string","enum":["pg_stat_statements","auto_explain","pgaudit","pg_prewarm"]}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas). Under autoscaling it also sets the scale-down floor (the effective minimum is `max(minReplicas, maxSyncReplicas+1, 2)`); the chart fails to render if that floor exceeds `autoscaling.maxReplicas`.","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"autoscaling":{"description":"Read-replica autoscaling configuration.","type":"object","default":{},"required":["dryRun","enabled","maxReplicas","maxReplicationLagSeconds","metric","minReplicas","serverAddress","target","transition"],"properties":{"dryRun":{"description":"Recommendation mode: keep the static count and render the ScaledObject with both scaling directions paused, so KEDA keeps the HPA and keeps serving its trigger metric but actuates neither way. Note both scaling policies are disabled, so the HPA's desiredReplicas is frozen to the current count - read the recommendation from the matching read-load dashboard panel for your metric (connections or CPU): desired = 1 + ceil(read load / target), clamped to [effectiveMin..maxReplicas] where effectiveMin = max(minReplicas, maxSyncReplicas+1, 2), not from the desired-replicas line. Like disabling, flipping a LIVE autoscaled cluster into dryRun re-renders `instances: replicas` (the raw static value, which can be below the autoscaling floor); if `replicas` is below the live count, stage it to the live count first (see the enablement note) or the cluster sheds down to `replicas`. No shed occurs when the rendered value is unchanged (replicas already equals the active seed) — client-side merge is then a no-op and the live count is preserved. Permanent, unlike `transition`.","type":"boolean","default":false},"enabled":{"description":"Enable horizontal autoscaling of read replicas. Requires the platform `keda` package to be enabled by an administrator first; the chart fails to render (rather than break the release) if KEDA is not installed. Prerequisite: the queried vmselect must hold BOTH the CNPG metrics (cnpg_backends_total / container CPU) AND kube_pod_labels (kube-state-metrics). This holds for a tenant on the shared root monitoring stack (`monitoring: false`, both land in tenant-root). A tenant running its OWN Monitoring app (`monitoring: true`) has the CNPG metrics but not kube_pod_labels (KSM is root-only), so it must set `serverAddress` to a vmselect carrying both series or autoscaling never actuates.","type":"boolean","default":false},"maxReplicas":{"description":"Maximum total instances. Must be >= minReplicas and >= the synchronous-quorum floor (maxSyncReplicas+1); the chart fails to render (rather than silently run an unbounded instance count) if the effective floor exceeds it - raise maxReplicas. HAZARD: lowering maxReplicas below the live count makes the HPA cut straight to the new maximum in one step (the current>max branch bypasses the scale-down pacing), shedding the surplus replicas + PVCs at once; to lower it deliberately, suspend the HelmRelease and pin the live count with `kubectl scale` first.","type":"integer","default":6},"maxReplicationLagSeconds":{"description":"Freeze scaling while replication lag exceeds this (seconds) and the primary is writing. 0 disables the brake. Default 0: the freeze branch has only been validated on a live cluster in its non-braking (pass-through) form, so the lag brake is opt-in until the braking path is exercised under load; set a positive value (e.g. 30) to enable it.","type":"integer","default":0},"metric":{"description":"Read-load signal to scale on.","type":"string","default":"ReadConnections","enum":["ReadConnections","ReadCPUUtilization"]},"minReplicas":{"description":"Minimum total instances; raised to the synchronous-quorum floor (`maxSyncReplicas + 1`) and to 2 when either is higher.","type":"integer","default":2},"serverAddress":{"description":"Override the Prometheus/vmselect URL the ScaledObject queries. Empty (default) derives `http://vmselect-shortterm..svc:8481/select/0/prometheus`. Set it when the derived vmselect does not hold both the CNPG metrics and kube_pod_labels (e.g. `monitoring: true` tenants, or a non-default `metricsStorages` name). SSRF constraint: the host must be an in-cluster Service DNS name (`..svc[.]`) whose namespace is within the tenant's own subtree (its namespace, its parents, or `tenant-root`) — the cluster-privileged KEDA operator fetches this URL every poll, so it is refused a metadata endpoint, an external host, or a sibling/system namespace. Residual surface (accepted trade-off): `tenant-root` is always allowed because the derived default points there, so a tenant can still aim the operator at any Service in `tenant-root` on any port/path (a blind in-cluster reachability probe — the operator reads no response body back to the tenant); the egress NetworkPolicy does not narrow that.","type":"string","default":""},"target":{"description":"Target read load per read-serving replica (unit depends on `metric`: active connections, or CPU millicores). Dimensionless integer - a resource.Quantity here would accept `150m`, which PromQL reads as 150 minutes and KEDA cannot use as a threshold.","type":"integer","default":150},"transition":{"description":"Migration phase 1: render `.spec.instances` from `replicas` (which the operator stages to the current live count) AND pause the ScaledObject in both directions, so KEDA stands up with its HPA present but not actuating. Flip to false (phase 2) to hand the runtime count to KEDA. Because the both-direction pause keeps the HPA (not the HPA-deleting full pause), KEDA is genuinely warm at phase 2 — no HPA recreate — and because the active seed is max(replicas, effectiveMin), phase 2 re-renders the same instance count phase 1 did (when `replicas` is staged at/above the floor), so the flip is a merge no-op with no dip. The HPA's scaleUp stabilization window still smooths the first actuation.","type":"boolean","default":false}}},"users":{"description":"Users configuration map. Passwords are always auto-generated and stored in the `-credentials` Secret; they cannot be set from values — read a user's password from that Secret. A `password` left over in values from before the field was removed is ignored by the render and draws an admission warning, but it is not inert on an upgraded release: the chart preserves whatever password is already in the Secret, which on the first upgrade is the value that was set before removal, so that value stays the live credential until it is rotated (dedicated rotation is tracked in cozystack/community#72). The live password lives only in the Secret.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"bypassRls":{"description":"Whether the user can bypass row-level security policies (BYPASSRLS). A role can only create other roles carrying BYPASSRLS if it holds the attribute itself, so this is required alongside `createRole` to provision a privileged service role.","type":"boolean"},"createRole":{"description":"Whether the user can create and manage roles (CREATEROLE). Requires PostgreSQL 16 or newer. Setting it back to `false` withdraws the attribute only: the roles the user created, and its ADMIN and SET memberships in them, stay. Once any user has held it, the chart no longer adopts an existing role it did not create, so a user or database declared later under the name of such a role fails the init job until a platform administrator drops that role.","type":"boolean"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility.","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointCA":{"description":"DEPRECATED. Pre-existing Secret with the CA bundle the barman-cloud plugin should trust when reaching a self-signed S3 endpoint. Used for both backup and bootstrap recovery in the legacy chart-managed flow.","type":"object","default":{},"properties":{"key":{"description":"Key within the Secret containing the CA bundle. Defaults to `ca.crt`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means no endpointCA is emitted (the plugin uses the system trust store).","type":"string","default":""}}},"endpointURL":{"description":"DEPRECATED. See `destinationPath`.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"DEPRECATED. Tenants no longer supply S3 keys; the system Bucket Secret is projected into the tenant namespace by the backup controller. Ignored when `s3CredentialsSecret.name` is set or `useSystemBucket` is true. The chart skips materialising `-s3-creds` whenever this field is empty so a default install does not leak placeholder credentials into the tenant namespace.","type":"string","default":""},"s3CredentialsSecret":{"description":"DEPRECATED. Pre-existing Secret with S3 credentials. Use the platform-managed `cozy-default` BackupClass instead. When set, the chart references this Secret directly (legacy chart-managed flow). The CNPG backup driver writes this field on restore so credentials never land in the CR `.spec`.","type":"object","default":{},"properties":{"accessKeyIDKey":{"description":"Key in the Secret holding the access key ID. Defaults to `AWS_ACCESS_KEY_ID`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means the chart materialises `-s3-creds` from `s3AccessKey`/`s3SecretKey`.","type":"string","default":""},"secretAccessKeyKey":{"description":"Key in the Secret holding the secret access key. Defaults to `AWS_SECRET_ACCESS_KEY`.","type":"string","default":""}}},"s3SecretKey":{"description":"DEPRECATED. See `s3AccessKey`.","type":"string","default":""},"schedule":{"description":"Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.","type":"string","default":""},"useSystemBucket":{"description":"Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`.","type":"boolean","default":false}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"backupID":{"description":"Barman ID of the base backup to recover from, passed as `recoveryTarget.backupID`. Empty lets the barman-cloud plugin pick one: the newest backup when `recoveryTime` is empty, otherwise the newest one ending at or before it, on any timeline. The CNPG backup driver sets this on restore.","type":"string","default":""},"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"newServerName":{"description":"WAL-archive server name (S3 path prefix) the RESTORED cluster writes to. Must differ from `serverName` (the recovery source) so a restored cluster archives to a fresh, empty prefix and barman-cloud-check-wal-archive passes while it recovers from the source's prefix. The CNPG backup driver sets this on restore; empty means the cluster archives under its own name.","type":"string","default":""},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Server name (S3 path prefix) used by the original cluster when writing backups; passed to the barman-cloud plugin via `externalClusters[].plugin.parameters.serverName`. Defaults to `bootstrap.oldName`. Set this only when the original cluster wrote backups under an explicit server name that differed from its Kubernetes resource name.","type":"string","default":""}}}}} release: prefix: postgres- labels: From c0c5f2580fb3d878efd34b91c9e088c9aef4db8d Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:20:45 +0300 Subject: [PATCH 021/174] docs(postgres): note how an existing backup schedule changes on upgrade A release that already sets backup.schedule to "0 2 * * *" runs hourly at minute 2 today and daily at 02:00 after this change. The README now says so and gives "0 2 * * * *" for keeping the hourly run. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- packages/apps/postgres/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index 49f08d441..9185b35d5 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -89,6 +89,8 @@ field. The chart can also emit a `cnpg.io/ScheduledBackup` directly, without a BackupClass. Superseded by the BackupClass + Plan path above and kept around for clusters that did not migrate. It does not run by default - `backup.schedule` defaults to an empty string, which gates the chart's ScheduledBackup template off. To turn it on, set a standard five-field cron such as `0 2 * * *` for every day at 02:00, or the CloudNativePG six-field form with a leading seconds field such as `0 0 2 * * *` for the same time. The chart prepends a `0` seconds field to a five-field value, because CloudNativePG reads the first field as seconds and would otherwise run `0 2 * * *` every hour. +On upgrade, an existing `backup.schedule: "0 2 * * *"` changes from hourly at minute 2 to daily at 02:00. To keep the previous hourly schedule, set `backup.schedule: "0 2 * * * *"` before upgrading. + ```yaml ## @param backup.enabled Enable plugin WAL archiving + render the chart-managed ScheduledBackup ## @param backup.schedule Cron schedule, five fields or six with leading seconds. Empty means no chart-managed ScheduledBackup From 9d33037566a35979ecdb95ae256f6f209afb078d Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:40:54 +0300 Subject: [PATCH 022/174] fix(workloadmonitor): keep a bucket not ready until its claim is provisioned A bucket WorkloadMonitor has no pods and minReplicas 0, so it stayed operational, and the Bucket app stayed Ready, while its BucketClaim was never provisioned. A tenant whose nearest S3 ancestor runs no SeaweedFS gets a claim for a bucket class that does not exist, and the dashboard showed a ready bucket that was never created. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- api/v1alpha1/workloadmonitor_types.go | 4 + .../controller/workloadmonitor_controller.go | 35 ++++- .../workloadmonitor_controller_test.go | 132 ++++++++++++++++++ 3 files changed, 168 insertions(+), 3 deletions(-) diff --git a/api/v1alpha1/workloadmonitor_types.go b/api/v1alpha1/workloadmonitor_types.go index 0a4aa04cc..b5089652a 100644 --- a/api/v1alpha1/workloadmonitor_types.go +++ b/api/v1alpha1/workloadmonitor_types.go @@ -39,6 +39,10 @@ type WorkloadMonitorSpec struct { // that is not operational because a DataVolume it selects is not ready. const WorkloadMonitorReasonDataVolumeNotReady = "DataVolumeNotReady" +// WorkloadMonitorReasonBucketClaimNotReady is the status Reason of a monitor +// that is not operational because a BucketClaim it selects is not provisioned. +const WorkloadMonitorReasonBucketClaimNotReady = "BucketClaimNotReady" + // WorkloadMonitorStatus defines the observed state of WorkloadMonitor type WorkloadMonitorStatus struct { // Operational indicates if the workload meets all operational requirements diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 984ab7b21..d5d30e4e4 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -374,6 +374,25 @@ func (r *WorkloadMonitorReconciler) isBucketClaimReady(bc *cosiv1alpha1.BucketCl return bc.Status.BucketReady } +// bucketClaimsMessage names every BucketClaim that has not been provisioned, +// with its bucket class, and is empty when there is none. +func (r *WorkloadMonitorReconciler) bucketClaimsMessage(claims []cosiv1alpha1.BucketClaim) string { + var pending []string + for i := range claims { + bc := &claims[i] + if r.isBucketClaimReady(bc) { + continue + } + entry := fmt.Sprintf("BucketClaim %s is not provisioned", bc.Name) + if bc.Spec.BucketClassName != "" { + entry = fmt.Sprintf("BucketClaim %s of bucket class %s is not provisioned", bc.Name, bc.Spec.BucketClassName) + } + pending = append(pending, entry) + } + sort.Strings(pending) + return strings.Join(pending, "; ") +} + // isServiceReady checks if the service has an external IP bound func (r *WorkloadMonitorReconciler) isServiceReady(svc *corev1.Service) bool { return len(svc.Status.LoadBalancer.Ingress) > 0 @@ -1157,6 +1176,8 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } + bucketsMessage := r.bucketClaimsMessage(bucketClaimList.Items) + // Update WorkloadMonitor status based on observed pods monitor.Status.ObservedReplicas = observedReplicas monitor.Status.AvailableReplicas = availableReplicas @@ -1178,15 +1199,23 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ if fresh.Spec.MinReplicas != nil && availableReplicas < *fresh.Spec.MinReplicas { fresh.Status.Operational = pointer.Bool(false) } - // Only the DataVolume check writes Message and Reason, so the stored ones - // are the last DataVolume verdict: kept while no reader is installed yet, so a - // controller restart does not report a stuck disk as ready. + // A stored DataVolume verdict is the last one: kept while no reader is + // installed yet, so a controller restart does not report a stuck disk as + // ready. BucketClaims are listed on every pass, so their verdict is + // recomputed and only fills fields no DataVolume verdict holds. if dataVolumesRead { fresh.Status.Message = dataVolumesMessage fresh.Status.Reason = "" if dataVolumesMessage != "" { fresh.Status.Reason = cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady } + } else if fresh.Status.Reason == cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady { + fresh.Status.Message = "" + fresh.Status.Reason = "" + } + if fresh.Status.Message == "" && bucketsMessage != "" { + fresh.Status.Message = bucketsMessage + fresh.Status.Reason = cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady } if fresh.Status.Message != "" { fresh.Status.Operational = pointer.Bool(false) diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index 353066c4b..f8c761da5 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -2975,3 +2975,135 @@ func TestReconcileBucketApp_LegacySizeWinsOverOwnerlessNewWorkload(t *testing.T) t.Errorf("size = %s, want the legacy 689Gi over the stale 10Gi", got.String()) } } + +func newBucketClaim(name string, labels map[string]string, class string, ready bool) *cosiv1alpha1.BucketClaim { + return &cosiv1alpha1.BucketClaim{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Labels: labels}, + Spec: cosiv1alpha1.BucketClaimSpec{ + BucketClassName: class, + Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3}, + }, + Status: cosiv1alpha1.BucketClaimStatus{BucketReady: ready}, + } +} + +func reconcileBucketMonitor(t *testing.T, c client.Client, s *runtime.Scheme, withReader bool) *cozyv1alpha1.WorkloadMonitor { + t.Helper() + reconciler := &WorkloadMonitorReconciler{Client: c, Scheme: s} + if withReader { + reconciler.DataVolumeReader = c + } + req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "my-bucket", Namespace: "default"}} + if _, err := reconciler.Reconcile(context.TODO(), req); err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + updated := &cozyv1alpha1.WorkloadMonitor{} + if err := c.Get(context.TODO(), req.NamespacedName, updated); err != nil { + t.Fatalf("Failed to get updated WorkloadMonitor: %v", err) + } + if updated.Status.Operational == nil { + t.Fatal("Expected Operational to be set, got nil") + } + return updated +} + +func newBucketMonitorClient(s *runtime.Scheme, objs ...client.Object) client.Client { + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{Name: "my-bucket", Namespace: "default"}, + Spec: cozyv1alpha1.WorkloadMonitorSpec{ + Kind: "bucket", + Type: "s3", + Selector: map[string]string{"app.kubernetes.io/instance": "my-bucket"}, + Replicas: ptr.To[int32](0), + MinReplicas: ptr.To[int32](0), + }, + } + return fake.NewClientBuilder(). + WithScheme(s). + WithObjects(append([]client.Object{monitor}, objs...)...). + WithStatusSubresource(monitor). + Build() +} + +func TestReconcile_BucketClaimDecidesOperational(t *testing.T) { + selected := map[string]string{"app.kubernetes.io/instance": "my-bucket"} + cases := []struct { + name string + claim *cosiv1alpha1.BucketClaim + operational bool + message string + }{ + {"provisioned", newBucketClaim("my-bucket", selected, "tenant-root", true), true, ""}, + {"not provisioned", newBucketClaim("my-bucket", selected, "tenant-root", false), false, "BucketClaim my-bucket of bucket class tenant-root is not provisioned"}, + {"not provisioned without a class", newBucketClaim("my-bucket", selected, "", false), false, "BucketClaim my-bucket is not provisioned"}, + {"outside the selector", newBucketClaim("other", map[string]string{"app.kubernetes.io/instance": "other"}, "tenant-root", false), true, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := newTestScheme() + got := reconcileBucketMonitor(t, newBucketMonitorClient(s, tc.claim), s, true) + if *got.Status.Operational != tc.operational { + t.Errorf("Operational = %v, want %v", *got.Status.Operational, tc.operational) + } + if got.Status.Message != tc.message { + t.Errorf("Message = %q, want %q", got.Status.Message, tc.message) + } + wantReason := "" + if tc.message != "" { + wantReason = cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady + } + if got.Status.Reason != wantReason { + t.Errorf("Reason = %q, want %q", got.Status.Reason, wantReason) + } + }) + } +} + +func TestReconcile_ProvisionedBucketClaimClearsTheVerdict(t *testing.T) { + for _, withReader := range []bool{true, false} { + t.Run(fmt.Sprintf("dataVolumeReader=%v", withReader), func(t *testing.T) { + s := newTestScheme() + claim := newBucketClaim("my-bucket", map[string]string{"app.kubernetes.io/instance": "my-bucket"}, "tenant-root", false) + c := newBucketMonitorClient(s, claim) + + got := reconcileBucketMonitor(t, c, s, withReader) + if *got.Status.Operational || got.Status.Reason != cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady { + t.Fatalf("before provisioning: Operational = %v, Reason = %q", *got.Status.Operational, got.Status.Reason) + } + + stored := &cosiv1alpha1.BucketClaim{} + if err := c.Get(context.TODO(), types.NamespacedName{Name: "my-bucket", Namespace: "default"}, stored); err != nil { + t.Fatalf("Failed to get BucketClaim: %v", err) + } + stored.Status.BucketReady = true + if err := c.Update(context.TODO(), stored); err != nil { + t.Fatalf("Failed to update BucketClaim: %v", err) + } + + got = reconcileBucketMonitor(t, c, s, withReader) + if !*got.Status.Operational || got.Status.Message != "" || got.Status.Reason != "" { + t.Errorf("after provisioning: Operational = %v, Message = %q, Reason = %q", *got.Status.Operational, got.Status.Message, got.Status.Reason) + } + }) + } +} + +func TestReconcile_DataVolumeVerdictOutranksBucketClaim(t *testing.T) { + s := newTestScheme() + selected := map[string]string{"app.kubernetes.io/instance": "my-bucket"} + phase := "ImportInProgress" + c := newBucketMonitorClient(s, + newBucketClaim("my-bucket", selected, "tenant-root", false), + newDataVolume("my-bucket", selected, &phase), + ) + got := reconcileBucketMonitor(t, c, s, true) + if *got.Status.Operational { + t.Error("Operational = true, want false") + } + if got.Status.Reason != cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady { + t.Errorf("Reason = %q, want %q", got.Status.Reason, cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady) + } + if got.Status.Message != "DataVolume my-bucket is ImportInProgress" { + t.Errorf("Message = %q", got.Status.Message) + } +} From e96ceedd7ebd6a8e138e8b363ef36a2ffbf7e20f Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:25:58 +0700 Subject: [PATCH 023/174] feat(dashboard): add a console client for the observability proxy The observability proxy serves metrics and logs under Kubernetes-shaped paths but answers in native Prometheus JSON and VictoriaLogs NDJSON, so the typed K8sClient methods cannot read it. A raw fetch on the client keeps the same credentials and 401 handling the rest of the console uses, and the proxy module maps each documented status to its own error kind so a cluster without the proxy (404) can be told apart from a denied request or a store that is down. Assisted-by: LLM Signed-off-by: ZverGuy --- .../__tests__/k8s-client/client.raw.test.ts | 47 ++++ .../src/lib/observability/proxy.test.ts | 179 +++++++++++++++ .../console/src/lib/observability/proxy.ts | 203 ++++++++++++++++++ .../console/packages/k8s-client/src/client.ts | 18 ++ 4 files changed, 447 insertions(+) create mode 100644 packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts diff --git a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts new file mode 100644 index 000000000..ab6c056f5 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts @@ -0,0 +1,47 @@ +import { describe, it, expect, vi, afterEach } from "vitest" +import { K8sClient } from "@cozystack/k8s-client" + +afterEach(() => { + vi.unstubAllGlobals() + vi.restoreAllMocks() +}) + +describe("K8sClient.raw", () => { + it("returns the response untouched, including a non-2xx one", async () => { + const res = new Response("not found", { status: 404 }) + vi.stubGlobal("fetch", vi.fn(async () => res)) + const client = new K8sClient() + + await expect(client.raw("/x")).resolves.toBe(res) + }) + + it("prefixes the base URL and attaches the bearer token", async () => { + const fetchMock = vi.fn(async () => new Response("{}")) + vi.stubGlobal("fetch", fetchMock) + const client = new K8sClient({ baseUrl: "/base", getToken: async () => "t0k" }) + + await client.raw("/apis/x", { headers: { Accept: "application/json" } }) + + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(url).toBe("/base/apis/x") + expect(init.headers).toEqual({ Accept: "application/json", Authorization: "Bearer t0k" }) + }) + + it("routes a 401 through the client's unauthorized handler", async () => { + const onUnauthorized = vi.fn() + vi.stubGlobal("fetch", vi.fn(async () => new Response("", { status: 401 }))) + const client = new K8sClient({ onUnauthorized }) + + const res = await client.raw("/x") + + expect(res.status).toBe(401) + expect(onUnauthorized).toHaveBeenCalledTimes(1) + }) + + it("does not swallow a network failure", async () => { + vi.stubGlobal("fetch", vi.fn(async () => { throw new TypeError("offline") })) + const client = new K8sClient() + + await expect(client.raw("/x")).rejects.toThrow("offline") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts new file mode 100644 index 000000000..f06835ff7 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts @@ -0,0 +1,179 @@ +import { describe, it, expect, vi } from "vitest" +import { K8sClient } from "@cozystack/k8s-client" +import { + ProxyError, + errorKindForStatus, + fieldValues, + logsRoot, + metricsRoot, + queryLogs, + queryRange, + tailLogs, +} from "./proxy.ts" + +const app = { namespace: "tenant-a", plural: "postgreses", name: "db" } +const ns = { namespace: "tenant-a" } + +function clientReturning(...responses: Response[]) { + const client = new K8sClient({ onUnauthorized: () => {} }) + const raw = vi.spyOn(client, "raw") + for (const r of responses) raw.mockResolvedValueOnce(r) + return { client, raw } +} + +function calledUrl(raw: ReturnType["raw"], call = 0): URL { + return new URL(raw.mock.calls[call][0], "http://x") +} + +describe("roots", () => { + it("builds the application roots under apps.cozystack.io", () => { + expect(metricsRoot(app)).toBe("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/") + expect(logsRoot(app)).toBe("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/") + }) + + it("builds the namespace roots under cozystack.io", () => { + expect(metricsRoot(ns)).toBe("/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/") + expect(logsRoot(ns)).toBe("/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/") + }) +}) + +describe("errorKindForStatus", () => { + it.each([ + [400, "bad-request"], + [401, "unauthorized"], + [403, "forbidden"], + [404, "not-available"], + [422, "limit"], + [429, "rate-limited"], + [502, "store-unreachable"], + [503, "unavailable"], + [504, "timeout"], + [500, "unknown"], + ])("maps %i to %s", (status, kind) => { + expect(errorKindForStatus(status)).toBe(kind) + }) +}) + +describe("queryRange", () => { + it("GETs query_range under the root and returns the matrix", async () => { + const body = { status: "success", data: { resultType: "matrix", result: [{ metric: { pod: "p" }, values: [[1, "2"]] }] } } + const { client, raw } = clientReturning(Response.json(body)) + + const result = await queryRange(client, metricsRoot(app), { query: "up", start: 10, end: 20, step: 5 }) + + const url = calledUrl(raw) + expect(url.pathname).toBe(`${metricsRoot(app)}api/v1/query_range`) + expect(Object.fromEntries(url.searchParams)).toEqual({ query: "up", start: "10", end: "20", step: "5" }) + expect(result).toEqual(body.data.result) + }) + + it("throws a ProxyError carrying the Prometheus error text", async () => { + const { client } = clientReturning( + Response.json({ status: "error", error: "range too long" }, { status: 422 }), + ) + + const err = await queryRange(client, metricsRoot(app), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e) + + expect(err).toBeInstanceOf(ProxyError) + expect(err).toMatchObject({ kind: "limit", status: 422, message: "range too long" }) + }) + + it("reports a fetch failure as a network error", async () => { + const client = new K8sClient() + vi.spyOn(client, "raw").mockRejectedValue(new TypeError("offline")) + + const err = await queryRange(client, metricsRoot(app), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e) + + expect(err).toMatchObject({ kind: "network" }) + }) +}) + +describe("queryLogs", () => { + it("parses NDJSON lines and sorts them by _time ascending", async () => { + const ndjson = [ + JSON.stringify({ _time: "2026-11-04T10:00:02Z", _msg: "b" }), + "", + JSON.stringify({ _time: "2026-11-04T10:00:01Z", _msg: "a" }), + ].join("\n") + const { client, raw } = clientReturning(new Response(ndjson)) + + const entries = await queryLogs(client, logsRoot(ns), { query: "*", start: "s", end: "e", limit: 100 }) + + expect(calledUrl(raw).pathname).toBe(`${logsRoot(ns)}select/logsql/query`) + expect(Object.fromEntries(calledUrl(raw).searchParams)).toEqual({ query: "*", start: "s", end: "e", limit: "100" }) + expect(entries.map((e) => e._msg)).toEqual(["a", "b"]) + }) + + it("keeps a plain-text error body as the message", async () => { + const { client } = clientReturning(new Response("cannot parse query", { status: 400 })) + + const err = await queryLogs(client, logsRoot(ns), { query: "(", start: "s", end: "e", limit: 1 }).catch((e) => e) + + expect(err).toMatchObject({ kind: "bad-request", message: "cannot parse query" }) + }) +}) + +describe("fieldValues", () => { + it("uses stream_field_values for stream fields and returns the values", async () => { + const { client, raw } = clientReturning( + Response.json({ values: [{ value: "pod-b", hits: 1 }, { value: "pod-a", hits: 9 }] }), + ) + + const values = await fieldValues(client, logsRoot(app), { + field: "kubernetes_pod_name", + stream: true, + query: "*", + start: "s", + end: "e", + }) + + expect(calledUrl(raw).pathname).toBe(`${logsRoot(app)}select/logsql/stream_field_values`) + expect(calledUrl(raw).searchParams.get("field")).toBe("kubernetes_pod_name") + expect(values).toEqual(["pod-a", "pod-b"]) + }) + + it("uses field_values for other fields", async () => { + const { client, raw } = clientReturning(Response.json({ values: [] })) + + await fieldValues(client, logsRoot(app), { field: "level", stream: false, query: "*", start: "s", end: "e" }) + + expect(calledUrl(raw).pathname).toBe(`${logsRoot(app)}select/logsql/field_values`) + }) +}) + +describe("tailLogs", () => { + it("delivers every line as it arrives, including one split across chunks", async () => { + const enc = new TextEncoder() + const chunks = ['{"_time":"t1","_msg":"a"}\n{"_ti', 'me":"t2","_msg":"b"}\n'] + const body = new ReadableStream({ + start(c) { + for (const ch of chunks) c.enqueue(enc.encode(ch)) + c.close() + }, + }) + const { client, raw } = clientReturning(new Response(body)) + const seen: string[] = [] + + await tailLogs(client, logsRoot(ns), { query: "*", since: "t0" }, (e) => seen.push(e._msg)) + + expect(calledUrl(raw).pathname).toBe(`${logsRoot(ns)}select/logsql/tail`) + expect(calledUrl(raw).searchParams.get("since")).toBe("t0") + expect(seen).toEqual(["a", "b"]) + }) + + it("omits since on the first connect", async () => { + const { client, raw } = clientReturning(new Response("")) + + await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}) + + expect(calledUrl(raw).searchParams.has("since")).toBe(false) + }) + + it("throws a ProxyError when the tail is refused", async () => { + const { client } = clientReturning(new Response("", { status: 403 })) + + const err = await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}).catch((e) => e) + + expect(err).toMatchObject({ kind: "forbidden" }) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts new file mode 100644 index 000000000..a257825c9 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts @@ -0,0 +1,203 @@ +import type { K8sClient } from "@cozystack/k8s-client" + +export type ObservabilityScope = + | { namespace: string } + | { namespace: string; plural: string; name: string } + +function appBase(scope: ObservabilityScope): string | null { + if (!("plural" in scope)) return null + return `/apis/apps.cozystack.io/v1alpha1/namespaces/${scope.namespace}/${scope.plural}/${scope.name}` +} + +export function metricsRoot(scope: ObservabilityScope): string { + const app = appBase(scope) + return app ? `${app}/metrics/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/metrics/` +} + +export function logsRoot(scope: ObservabilityScope): string { + const app = appBase(scope) + return app ? `${app}/log/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/logs/` +} + +export type ProxyErrorKind = + | "bad-request" + | "unauthorized" + | "forbidden" + | "not-available" + | "limit" + | "rate-limited" + | "store-unreachable" + | "unavailable" + | "timeout" + | "network" + | "unknown" + +const KIND_BY_STATUS: Record = { + 400: "bad-request", + 401: "unauthorized", + 403: "forbidden", + 404: "not-available", + 422: "limit", + 429: "rate-limited", + 502: "store-unreachable", + 503: "unavailable", + 504: "timeout", +} + +export function errorKindForStatus(status: number): ProxyErrorKind { + return KIND_BY_STATUS[status] ?? "unknown" +} + +export class ProxyError extends Error { + kind: ProxyErrorKind + status: number + + constructor(kind: ProxyErrorKind, status: number, message: string) { + super(message) + this.name = "ProxyError" + this.kind = kind + this.status = status + } +} + +// Prometheus answers {"error": "..."}, VictoriaLogs and nginx answer plain text. +function errorMessage(text: string, status: number): string { + try { + const parsed: unknown = JSON.parse(text) + if (parsed && typeof parsed === "object" && "error" in parsed && typeof parsed.error === "string") { + return parsed.error + } + } catch { + // not JSON + } + const trimmed = text.trim() + return trimmed && trimmed.length < 300 && !trimmed.startsWith("<") ? trimmed : `Server returned ${status}` +} + +async function proxyGet( + client: K8sClient, + path: string, + params: Record, + signal?: AbortSignal, +): Promise { + const search = new URLSearchParams() + for (const [k, v] of Object.entries(params)) if (v !== undefined) search.set(k, String(v)) + let res: Response + try { + res = await client.raw(`${path}?${search}`, { signal }) + } catch (err) { + if (signal?.aborted) throw err + throw new ProxyError("network", 0, "Unable to connect to the server") + } + if (!res.ok) { + const text = await res.text().catch(() => "") + throw new ProxyError(errorKindForStatus(res.status), res.status, errorMessage(text, res.status)) + } + return res +} + +export interface PromSeries { + metric: Record + values: [number, string][] +} + +export interface RangeParams { + query: string + start: number + end: number + step: number +} + +export async function queryRange( + client: K8sClient, + root: string, + params: RangeParams, + signal?: AbortSignal, +): Promise { + const res = await proxyGet(client, `${root}api/v1/query_range`, { ...params }, signal) + const body = (await res.json()) as { data?: { result?: PromSeries[] } } + return body.data?.result ?? [] +} + +export interface LogEntry { + _time: string + _msg: string + _stream_id?: string + [field: string]: string | undefined +} + +function parseLine(line: string): LogEntry | null { + if (!line.trim()) return null + try { + const value: unknown = JSON.parse(line) + if (value && typeof value === "object" && "_time" in value) return value as LogEntry + } catch { + // a torn or non-JSON line carries nothing to show + } + return null +} + +export interface LogsQueryParams { + query: string + start: string + end: string + limit: number +} + +export async function queryLogs( + client: K8sClient, + root: string, + params: LogsQueryParams, + signal?: AbortSignal, +): Promise { + const res = await proxyGet(client, `${root}select/logsql/query`, { ...params }, signal) + const entries = (await res.text()).split("\n").map(parseLine).filter((e) => e !== null) + return entries.sort((a, b) => (a._time < b._time ? -1 : a._time > b._time ? 1 : 0)) +} + +export interface FieldValuesParams { + field: string + stream: boolean + query: string + start: string + end: string +} + +export async function fieldValues( + client: K8sClient, + root: string, + { stream, ...params }: FieldValuesParams, + signal?: AbortSignal, +): Promise { + const endpoint = stream ? "stream_field_values" : "field_values" + const res = await proxyGet(client, `${root}select/logsql/${endpoint}`, { ...params }, signal) + const body = (await res.json()) as { values?: { value: string }[] } + return (body.values ?? []).map((v) => v.value).sort() +} + +export async function tailLogs( + client: K8sClient, + root: string, + params: { query: string; since?: string }, + onEntry: (entry: LogEntry) => void, + signal?: AbortSignal, +): Promise { + const res = await proxyGet(client, `${root}select/logsql/tail`, params, signal) + if (!res.body) return + const reader = res.body.getReader() + const decoder = new TextDecoder() + let buffer = "" + for (;;) { + const { done, value } = await reader.read() + if (done) break + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split("\n") + buffer = lines.pop() ?? "" + for (const line of lines) { + const entry = parseLine(line) + if (entry) onEntry(entry) + } + } + const last = parseLine(buffer) + if (last) onEntry(last) +} diff --git a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts index 12f50e0fd..04c528760 100644 --- a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts +++ b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts @@ -241,6 +241,24 @@ export class K8sClient { }) } + /** + * Fetch a same-origin path with the client's credentials and 401 handling, + * leaving status and body to the caller. For endpoints that do not speak + * Kubernetes JSON, such as the observability proxy's streamed NDJSON. + */ + async raw(path: string, init?: RequestInit): Promise { + const headers: Record = { + ...(init?.headers as Record), + } + if (this.getToken) { + headers["Authorization"] = `Bearer ${await this.getToken()}` + } + const res = await fetch(`${this.baseUrl}${path}`, { ...init, headers }) + if (res.status === 401) this.handleUnauthorized() + else if (res.ok) this.unauthorizedHandled = false + return res + } + getApiGroups(): Promise { return this.request("/apis") } From 063e56ede57a481b2db223a37641e601a2d51a05 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:30:03 +0700 Subject: [PATCH 024/174] feat(dashboard): show resource consumption on application pages Tenants had no way to see what their applications consume. A Metrics tab on every application detail page (except VM disks, which run no containers) charts CPU, memory and network for a selectable range through the observability proxy's application root, which injects the scope server-side, so the queries stay kind-agnostic and no per-kind UI is needed. Charts use uPlot: about 25 KB gzipped with no dependencies, built for time series, and loaded only with the Metrics tab so other pages do not pay for it. A cluster without the proxy answers 404 and the tab says metrics are not available there instead of showing an error. Assisted-by: LLM Signed-off-by: ZverGuy --- .../images/console/apps/console/package.json | 3 +- .../observability/MetricsView.test.tsx | 88 +++++++++++++ .../components/observability/MetricsView.tsx | 72 +++++++++++ .../observability/ProxyNotice.test.tsx | 43 +++++++ .../components/observability/ProxyNotice.tsx | 49 +++++++ .../components/observability/RangePicker.tsx | 30 +++++ .../observability/TimeSeriesChart.test.tsx | 46 +++++++ .../observability/TimeSeriesChart.tsx | 51 ++++++++ .../src/lib/observability/metrics.test.ts | 107 ++++++++++++++++ .../console/src/lib/observability/metrics.ts | 121 ++++++++++++++++++ .../src/lib/observability/proxy.test.ts | 10 ++ .../console/src/lib/observability/proxy.ts | 8 ++ .../detail/ApplicationDetailPage.test.tsx | 28 ++++ .../routes/detail/ApplicationDetailPage.tsx | 12 ++ .../console/src/routes/detail/MetricsTab.tsx | 15 +++ .../dashboard/images/console/pnpm-lock.yaml | 8 ++ 16 files changed, 690 insertions(+), 1 deletion(-) create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx diff --git a/packages/system/dashboard/images/console/apps/console/package.json b/packages/system/dashboard/images/console/apps/console/package.json index fcddb7023..254195f28 100644 --- a/packages/system/dashboard/images/console/apps/console/package.json +++ b/packages/system/dashboard/images/console/apps/console/package.json @@ -31,7 +31,8 @@ "react": "^19.2.4", "react-dom": "^19.2.4", "react-router": "^7.6.1", - "simple-icons": "^16.22.0" + "simple-icons": "^16.22.0", + "uplot": "^1.6.32" }, "devDependencies": { "@tailwindcss/vite": "^4.2.2", diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx new file mode 100644 index 000000000..f09e15201 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx @@ -0,0 +1,88 @@ +import { describe, it, expect, vi } from "vitest" +import { screen, waitFor } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { K8sClient } from "@cozystack/k8s-client" +import { renderWithK8sProvider } from "../../test-utils/render.tsx" +import type { ChartData } from "../../lib/observability/metrics.ts" + +vi.mock("./TimeSeriesChart.tsx", () => ({ + TimeSeriesChart: ({ data }: { data: ChartData }) => ( +
{data.series.map((s) => s.label).join(",")}
+ ), +})) + +const { MetricsView } = await import("./MetricsView.tsx") + +const scope = { namespace: "tenant-a", plural: "postgreses", name: "db" } + +function matrix(pod: string) { + return Response.json({ + status: "success", + data: { resultType: "matrix", result: [{ metric: { pod }, values: [[1, "1"]] }] }, + }) +} + +function clientWith(handler: (url: URL) => Response) { + const client = new K8sClient() + const raw = vi.spyOn(client, "raw").mockImplementation(async (path) => handler(new URL(path, "http://x"))) + return { client, raw } +} + +describe("MetricsView", () => { + it("draws CPU, memory and network from the application root", async () => { + const { client, raw } = clientWith(() => matrix("db-0")) + + renderWithK8sProvider(, { client }) + + await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3)) + expect(screen.getByText("CPU (cores)")).toBeInTheDocument() + expect(screen.getByText("Memory (working set)")).toBeInTheDocument() + expect(screen.getByText("Network")).toBeInTheDocument() + for (const [path] of raw.mock.calls) { + expect(path.startsWith("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query_range?")).toBe(true) + } + }) + + it("requeries with a coarser step when a longer range is picked", async () => { + const { client, raw } = clientWith(() => matrix("db-0")) + renderWithK8sProvider(, { client }) + await waitFor(() => expect(raw).toHaveBeenCalled()) + + await userEvent.click(screen.getByRole("button", { name: "7d" })) + + await waitFor(() => + expect(raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("step") === "2520")).toBe(true), + ) + }) + + it("shows one quiet notice instead of three errors when the proxy is absent", async () => { + const { client } = clientWith(() => new Response("404 page not found", { status: 404 })) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("Metrics are not available on this cluster.")).toBeInTheDocument() + expect(screen.getAllByRole("status")).toHaveLength(1) + expect(screen.queryByText("CPU (cores)")).toBeNull() + }) + + it("keeps the other panels when one query hits a limit", async () => { + const { client } = clientWith((url) => + url.searchParams.get("query")?.includes("memory") + ? Response.json({ status: "error", error: "too many series" }, { status: 422 }) + : matrix("db-0"), + ) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("too many series. Choose a shorter range.")).toBeInTheDocument() + await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(2)) + }) + + it("says so when a panel has no data", async () => { + const { client } = clientWith(() => Response.json({ status: "success", data: { resultType: "matrix", result: [] } })) + + renderWithK8sProvider(, { client }) + + await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(3)) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx new file mode 100644 index 000000000..35c8e8417 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx @@ -0,0 +1,72 @@ +import { useState } from "react" +import { useQueries } from "@tanstack/react-query" +import { useK8sClient } from "@cozystack/k8s-client" +import { Section, Spinner } from "@cozystack/ui" +import { + METRIC_PANELS, + RANGES, + rangeWindow, + toChartData, + type RangeKey, +} from "../../lib/observability/metrics.ts" +import { isScopeWide, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts" +import { ProxyNotice } from "./ProxyNotice.tsx" +import { TimeSeriesChart } from "./TimeSeriesChart.tsx" +import { RangePicker } from "./RangePicker.tsx" + +const REFRESH_MS = 60_000 + +export function MetricsView({ scope }: { scope: ObservabilityScope }) { + const client = useK8sClient() + const root = metricsRoot(scope) + const [range, setRange] = useState("1h") + + const results = useQueries({ + queries: METRIC_PANELS.map((panel) => ({ + queryKey: ["observability", "metrics", root, panel.id, range], + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const window = rangeWindow(range, Math.floor(Date.now() / 1000)) + const series = await Promise.all( + panel.queries(window).map(async (q) => ({ + label: q.label, + series: await queryRange(client, root, { query: q.expr, ...window }, signal), + })), + ) + return toChartData(series) + }, + retry: false, + refetchInterval: REFRESH_MS, + })), + }) + + const wide = results.find((r) => isScopeWide(r.error))?.error + + return ( +
+
+

Resource consumption

+ +
+ {wide ? ( + + ) : ( + METRIC_PANELS.map((panel, i) => { + const { data, error, isLoading } = results[i] + return ( +
+ {error ? ( + + ) : isLoading || !data ? ( + + ) : data.series.length === 0 ? ( +

No data for this range.

+ ) : ( + + )} +
+ ) + }) + )} +
+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx new file mode 100644 index 000000000..fd001bcae --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx @@ -0,0 +1,43 @@ +import { describe, it, expect } from "vitest" +import { render, screen } from "@testing-library/react" +import { ProxyError } from "../../lib/observability/proxy.ts" +import { ProxyNotice } from "./ProxyNotice.tsx" + +describe("ProxyNotice", () => { + it("renders a 404 as not available on this cluster, not as an error", () => { + render() + + expect(screen.getByRole("status")).toHaveTextContent("Metrics are not available on this cluster.") + expect(screen.queryByRole("alert")).toBeNull() + }) + + it("renders a 502 as a quiet notice about the store", () => { + render() + + expect(screen.getByRole("status")).toHaveTextContent("The logs store is unreachable right now.") + }) + + it.each([ + ["forbidden", 403, "You do not have permission to read metrics here."], + ["limit", 422, "too many points. Choose a shorter range."], + ["rate-limited", 429, "Too many requests. Try again in a moment."], + ["timeout", 504, "The store did not answer in time. Choose a shorter range or a narrower filter."], + ["unavailable", 503, "The service is temporarily unavailable. Try again in a moment."], + ] as const)("explains %s", (kind, status, text) => { + render() + + expect(screen.getByRole("alert")).toHaveTextContent(text) + }) + + it("shows the server message for a rejected query", () => { + render() + + expect(screen.getByRole("alert")).toHaveTextContent("cannot parse query") + }) + + it("falls back to the message of an unexpected error", () => { + render() + + expect(screen.getByRole("alert")).toHaveTextContent("boom") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx new file mode 100644 index 000000000..a60b831b0 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx @@ -0,0 +1,49 @@ +import { ProxyError } from "../../lib/observability/proxy.ts" + +interface ProxyNoticeProps { + what: "Metrics" | "Logs" + error: unknown +} + +function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: boolean; text: string } { + const lower = what.toLowerCase() + if (!(error instanceof ProxyError)) { + return { quiet: false, text: error instanceof Error ? error.message : String(error) } + } + switch (error.kind) { + case "not-available": + return { quiet: true, text: `${what} are not available on this cluster.` } + case "store-unreachable": + return { quiet: true, text: `The ${lower} store is unreachable right now.` } + case "unauthorized": + return { quiet: false, text: "Your session has expired. Sign in again." } + case "forbidden": + return { quiet: false, text: `You do not have permission to read ${lower} here.` } + case "limit": + return { quiet: false, text: `${error.message}. Choose a shorter range.` } + case "rate-limited": + return { quiet: false, text: "Too many requests. Try again in a moment." } + case "timeout": + return { quiet: false, text: "The store did not answer in time. Choose a shorter range or a narrower filter." } + case "unavailable": + return { quiet: false, text: "The service is temporarily unavailable. Try again in a moment." } + default: + return { quiet: false, text: error.message } + } +} + +export function ProxyNotice({ what, error }: ProxyNoticeProps) { + const { quiet, text } = describe(what, error) + if (quiet) { + return ( +

+ {text} +

+ ) + } + return ( +

+ {text} +

+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx new file mode 100644 index 000000000..22030c706 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx @@ -0,0 +1,30 @@ +import { cn } from "@cozystack/ui" + +export function RangePicker({ + options, + value, + onChange, +}: { + options: readonly K[] + value: K + onChange: (value: K) => void +}) { + return ( +
+ {options.map((key) => ( + + ))} +
+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx new file mode 100644 index 000000000..57cf6fb99 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx @@ -0,0 +1,46 @@ +import { describe, it, expect, vi, beforeEach } from "vitest" +import { render } from "@testing-library/react" + +const created: { opts: { series: { label?: string }[] }; data: unknown[]; destroy: ReturnType }[] = [] + +vi.mock("uplot", () => ({ + default: class { + destroy = vi.fn() + constructor(opts: { series: { label?: string }[] }, data: unknown[]) { + created.push({ opts, data, destroy: this.destroy }) + } + setSize() {} + }, +})) + +const { TimeSeriesChart } = await import("./TimeSeriesChart.tsx") + +beforeEach(() => { + created.length = 0 +}) + +describe("TimeSeriesChart", () => { + const data = { + timestamps: [1, 2], + series: [ + { label: "db-0", values: [1, 2] }, + { label: "db-1", values: [null, 3] }, + ], + } + + it("hands uPlot the timestamps first and one labelled series per input", () => { + render() + + expect(created).toHaveLength(1) + expect(created[0].data).toEqual([[1, 2], [1, 2], [null, 3]]) + expect(created[0].opts.series.slice(1).map((s) => s.label)).toEqual(["db-0", "db-1"]) + }) + + it("destroys the chart on unmount", () => { + const { unmount } = render() + + unmount() + + expect(created[0].destroy).toHaveBeenCalled() + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx new file mode 100644 index 000000000..6b596e17f --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx @@ -0,0 +1,51 @@ +import { useEffect, useRef } from "react" +import uPlot from "uplot" +import "uplot/dist/uPlot.min.css" +import type { ChartData } from "../../lib/observability/metrics.ts" + +const COLORS = ["#2563eb", "#16a34a", "#d97706", "#9333ea", "#dc2626", "#0891b2", "#4b5563", "#db2777"] +const HEIGHT = 200 + +interface TimeSeriesChartProps { + data: ChartData + format: (value: number) => string +} + +export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) { + const ref = useRef(null) + + useEffect(() => { + const el = ref.current + if (!el) return + const fmt = (v: number | null) => (v === null ? "–" : format(v)) + const chart = new uPlot( + { + width: el.clientWidth || 600, + height: HEIGHT, + scales: { x: { time: true } }, + legend: { live: true }, + series: [ + {}, + ...data.series.map((s, i) => ({ + label: s.label, + stroke: COLORS[i % COLORS.length], + width: 1.5, + value: (_: uPlot, v: number | null) => fmt(v), + })), + ], + axes: [{}, { size: 70, values: (_: uPlot, splits: number[]) => splits.map(format) }], + }, + [data.timestamps, ...data.series.map((s) => s.values)], + el, + ) + if (typeof ResizeObserver === "undefined") return () => chart.destroy() + const observer = new ResizeObserver(() => chart.setSize({ width: el.clientWidth, height: HEIGHT })) + observer.observe(el) + return () => { + observer.disconnect() + chart.destroy() + } + }, [data, format]) + + return
+} diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts new file mode 100644 index 000000000..0803940bf --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts @@ -0,0 +1,107 @@ +import { describe, it, expect } from "vitest" +import { + METRIC_PANELS, + formatBytes, + formatCores, + rangeWindow, + toChartData, +} from "./metrics.ts" + +describe("rangeWindow", () => { + it("ends now and spans the range in about 240 points", () => { + const w = rangeWindow("6h", 1_000_000) + + expect(w.end).toBe(1_000_000) + expect(w.start).toBe(1_000_000 - 6 * 3600) + expect(w.step).toBe(90) + }) + + it("never steps finer than 15s", () => { + expect(rangeWindow("1h", 0).step).toBe(15) + }) +}) + +describe("METRIC_PANELS", () => { + const window = { start: 0, end: 3600, step: 15 } + + it("rates CPU over at least five minutes so sparse scrapes still draw", () => { + const [query] = METRIC_PANELS.find((p) => p.id === "cpu")!.queries(window) + + expect(query.expr).toBe( + 'sum by (pod) (rate(container_cpu_usage_seconds_total{container!="",container!="POD"}[300s]))', + ) + }) + + it("widens the rate window with the step on long ranges", () => { + const [query] = METRIC_PANELS.find((p) => p.id === "cpu")!.queries({ start: 0, end: 1, step: 2520 }) + + expect(query.expr).toContain("[5040s]") + }) + + it("reads memory as working set per pod", () => { + const [query] = METRIC_PANELS.find((p) => p.id === "memory")!.queries(window) + + expect(query.expr).toBe('sum by (pod) (container_memory_working_set_bytes{container!="",container!="POD"})') + }) + + it("draws network receive and transmit as two named series", () => { + const queries = METRIC_PANELS.find((p) => p.id === "network")!.queries(window) + + expect(queries.map((q) => q.label)).toEqual(["receive", "transmit"]) + expect(queries[0].expr).toBe("sum(rate(container_network_receive_bytes_total[300s]))") + expect(queries[1].expr).toBe("sum(rate(container_network_transmit_bytes_total[300s]))") + }) + + it("names no application kind in any query", () => { + for (const panel of METRIC_PANELS) { + for (const q of panel.queries(window)) expect(q.expr).not.toMatch(/apps_cozystack_io|namespace=/) + } + }) +}) + +describe("toChartData", () => { + it("aligns series on the union of timestamps and leaves gaps as null", () => { + const data = toChartData([ + { label: "a", series: [{ metric: {}, values: [[10, "1"], [20, "2"]] }] }, + { label: "b", series: [{ metric: {}, values: [[20, "5"], [30, "6"]] }] }, + ]) + + expect(data.timestamps).toEqual([10, 20, 30]) + expect(data.series).toEqual([ + { label: "a", values: [1, 2, null] }, + { label: "b", values: [null, 5, 6] }, + ]) + }) + + it("labels each series by its pod, falling back to the query label", () => { + const data = toChartData([ + { + label: "cpu", + series: [ + { metric: { pod: "db-1" }, values: [[1, "0.5"]] }, + { metric: {}, values: [[1, "0.1"]] }, + ], + }, + ]) + + expect(data.series.map((s) => s.label)).toEqual(["db-1", "cpu"]) + }) + + it("drops NaN samples instead of drawing them as zero", () => { + const data = toChartData([{ label: "x", series: [{ metric: {}, values: [[1, "NaN"]] }] }]) + + expect(data.series[0].values).toEqual([null]) + }) +}) + +describe("formatters", () => { + it("formats CPU in millicores below one core", () => { + expect(formatCores(0.25)).toBe("250m") + expect(formatCores(2)).toBe("2") + }) + + it("formats bytes with binary units", () => { + expect(formatBytes(512)).toBe("512 B") + expect(formatBytes(3 * 1024 * 1024)).toBe("3.0 MiB") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts new file mode 100644 index 000000000..b23a51e27 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts @@ -0,0 +1,121 @@ +import type { PromSeries } from "./proxy.ts" + +export const RANGES = { + "1h": 3600, + "6h": 6 * 3600, + "24h": 24 * 3600, + "7d": 7 * 24 * 3600, +} as const + +export type RangeKey = keyof typeof RANGES + +export interface RangeWindow { + start: number + end: number + step: number +} + +const POINTS = 240 +const MIN_STEP = 15 + +export function rangeWindow(range: RangeKey, nowSeconds: number): RangeWindow { + const span = RANGES[range] + return { start: nowSeconds - span, end: nowSeconds, step: Math.max(MIN_STEP, Math.round(span / POINTS)) } +} + +// A rate needs several scrapes inside its window, and on long ranges the window +// must also cover the step or samples between points are skipped. +function rateWindow(step: number): string { + return `${Math.max(300, step * 2)}s` +} + +const CONTAINERS = 'container!="",container!="POD"' + +export interface MetricQuery { + label: string + expr: string +} + +export interface MetricPanel { + id: "cpu" | "memory" | "network" + title: string + format: (value: number) => string + queries: (window: RangeWindow) => MetricQuery[] +} + +export function formatCores(value: number): string { + if (value === 0) return "0" + if (Math.abs(value) < 1) return `${Math.round(value * 1000)}m` + return `${Number(value.toFixed(2))}` +} + +const BYTE_UNITS = ["B", "KiB", "MiB", "GiB", "TiB"] + +export function formatBytes(value: number): string { + let v = value + let unit = 0 + while (Math.abs(v) >= 1024 && unit < BYTE_UNITS.length - 1) { + v /= 1024 + unit++ + } + return unit === 0 ? `${Math.round(v)} B` : `${v.toFixed(1)} ${BYTE_UNITS[unit]}` +} + +// The proxy injects the namespace or application scope, so these stay kind-agnostic. +export const METRIC_PANELS: MetricPanel[] = [ + { + id: "cpu", + title: "CPU (cores)", + format: formatCores, + queries: ({ step }) => [ + { + label: "cpu", + expr: `sum by (pod) (rate(container_cpu_usage_seconds_total{${CONTAINERS}}[${rateWindow(step)}]))`, + }, + ], + }, + { + id: "memory", + title: "Memory (working set)", + format: formatBytes, + queries: () => [ + { label: "memory", expr: `sum by (pod) (container_memory_working_set_bytes{${CONTAINERS}})` }, + ], + }, + { + id: "network", + title: "Network", + format: (v) => `${formatBytes(v)}/s`, + queries: ({ step }) => [ + { label: "receive", expr: `sum(rate(container_network_receive_bytes_total[${rateWindow(step)}]))` }, + { label: "transmit", expr: `sum(rate(container_network_transmit_bytes_total[${rateWindow(step)}]))` }, + ], + }, +] + +export interface ChartSeries { + label: string + values: (number | null)[] +} + +export interface ChartData { + timestamps: number[] + series: ChartSeries[] +} + +export function toChartData(results: { label: string; series: PromSeries[] }[]): ChartData { + const flat = results.flatMap(({ label, series }) => + series.map((s) => ({ label: s.metric.pod ?? label, points: new Map(s.values.map(([t, v]) => [t, Number(v)])) })), + ) + const timestamps = [...new Set(flat.flatMap((s) => [...s.points.keys()]))].sort((a, b) => a - b) + return { + timestamps, + series: flat.map(({ label, points }) => ({ + label, + values: timestamps.map((t) => { + const v = points.get(t) + return v === undefined || Number.isNaN(v) ? null : v + }), + })), + } +} diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts index f06835ff7..f8cf44656 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts @@ -4,6 +4,7 @@ import { ProxyError, errorKindForStatus, fieldValues, + isScopeWide, logsRoot, metricsRoot, queryLogs, @@ -177,3 +178,12 @@ describe("tailLogs", () => { expect(err).toMatchObject({ kind: "forbidden" }) }) }) + +describe("isScopeWide", () => { + it("is true for answers that do not depend on the query", () => { + expect(isScopeWide(new ProxyError("not-available", 404, ""))).toBe(true) + expect(isScopeWide(new ProxyError("forbidden", 403, ""))).toBe(true) + expect(isScopeWide(new ProxyError("limit", 422, ""))).toBe(false) + expect(isScopeWide(new Error("x"))).toBe(false) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts index a257825c9..40b451b0b 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts @@ -60,6 +60,14 @@ export class ProxyError extends Error { } } +// These answers hold for every query in the scope, so one notice replaces the whole view. +export function isScopeWide(error: unknown): boolean { + return ( + error instanceof ProxyError && + (error.kind === "not-available" || error.kind === "forbidden" || error.kind === "unauthorized") + ) +} + // Prometheus answers {"error": "..."}, VictoriaLogs and nginx answer plain text. function errorMessage(text: string, status: number): string { try { diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx index a015a98bf..40b923f62 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx @@ -59,6 +59,7 @@ vi.mock("./EventsTab.tsx", () => ({ EventsTab: () => null })) vi.mock("./VncTab.tsx", () => ({ VncTab: () => null })) vi.mock("./SerialTab.tsx", () => ({ SerialTab: () => null })) vi.mock("./VMPowerControls.tsx", () => ({ VMPowerControls: () => null })) +vi.mock("./MetricsTab.tsx", () => ({ MetricsTab: () => null })) const { ApplicationDetailPage } = await import("./ApplicationDetailPage.tsx") @@ -111,11 +112,38 @@ describe("ApplicationDetailPage tabs for a virtual machine", () => { "VNC", "Workloads", "Services", + "Metrics", "Events", ]) }) }) +describe("ApplicationDetailPage observability tabs", () => { + function showKind(kind: string, plural: string) { + h.appKind = kind + h.appPlural = plural + h.get = { + data: { kind, metadata: { name: "demo", namespace: "tenant-test" } }, + isLoading: false, + error: undefined, + } + h.configMaps = [] + h.configMapsError = undefined + h.tabLabels = [] + renderPage() + } + + it("offers Metrics on any application kind, ahead of Events", () => { + showKind("Postgres", "postgreses") + expect(h.tabLabels.slice(-3)).toEqual(["Secrets", "Metrics", "Events"]) + }) + + it("leaves Metrics off a VM disk, which runs no containers", () => { + showKind("VMDisk", "vmdisks") + expect(h.tabLabels).not.toContain("Metrics") + }) +}) + describe("ApplicationDetailPage configuration", () => { it("offers ConfigMaps declared by the application's resource map", () => { h.appKind = "FoundationDB" diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx index 55fa684b8..4520244d5 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx @@ -37,6 +37,8 @@ import { VncTab } from "./VncTab.tsx" // fetched only when a VM's console is opened. Every dashboard page reaches this // file, and most of them never touch a VM. const SerialTab = lazy(() => import("./SerialTab.tsx").then((m) => ({ default: m.SerialTab }))) +// Same for the charting library behind the Metrics tab. +const MetricsTab = lazy(() => import("./MetricsTab.tsx").then((m) => ({ default: m.MetricsTab }))) import { VMPowerControls } from "./VMPowerControls.tsx" import { useResourceBasePath } from "../../lib/portal.ts" import { useResourcePresence } from "./use-resource-presence.ts" @@ -143,6 +145,7 @@ export function ApplicationDetailPage() { { to: `${base}/vnc`, label: "VNC", end: false }, { to: `${base}/workloads`, label: "Workloads", end: false }, { to: `${base}/services`, label: "Services", end: false }, + { to: `${base}/metrics`, label: "Metrics", end: false }, { to: `${base}/events`, label: "Events", end: false }, ) } else { @@ -163,6 +166,7 @@ export function ApplicationDetailPage() { } tabs.push( { to: `${base}/secrets`, label: "Secrets", end: false }, + { to: `${base}/metrics`, label: "Metrics", end: false }, { to: `${base}/events`, label: "Events", end: false }, ) } @@ -262,6 +266,14 @@ export function ApplicationDetailPage() { } /> } /> + }> + + + } + />
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx new file mode 100644 index 000000000..a928e633d --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx @@ -0,0 +1,15 @@ +import type { ApplicationDefinition, ApplicationInstance } from "@cozystack/types" +import { useTenantContext } from "../../lib/tenant-context.tsx" +import { MetricsView } from "../../components/observability/MetricsView.tsx" + +interface MetricsTabProps { + ad: ApplicationDefinition + instance: ApplicationInstance +} + +export function MetricsTab({ ad, instance }: MetricsTabProps) { + const { tenantNamespace } = useTenantContext() + const plural = ad.spec?.application.plural + if (!tenantNamespace || !plural) return null + return +} diff --git a/packages/system/dashboard/images/console/pnpm-lock.yaml b/packages/system/dashboard/images/console/pnpm-lock.yaml index 9d90b0c5c..4661c444a 100644 --- a/packages/system/dashboard/images/console/pnpm-lock.yaml +++ b/packages/system/dashboard/images/console/pnpm-lock.yaml @@ -92,6 +92,9 @@ importers: simple-icons: specifier: ^16.22.0 version: 16.22.0 + uplot: + specifier: ^1.6.32 + version: 1.6.32 devDependencies: '@tailwindcss/vite': specifier: ^4.2.2 @@ -1773,6 +1776,9 @@ packages: peerDependencies: browserslist: '>= 4.21.0' + uplot@1.6.32: + resolution: {integrity: sha512-KIMVnG68zvu5XXUbC4LQEPnhwOxBuLyW1AHtpm6IKTXImkbLgkMy+jabjLgSLMasNuGGzQm/ep3tOkyTxpiQIw==} + uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} @@ -3405,6 +3411,8 @@ snapshots: escalade: 3.2.0 picocolors: 1.1.1 + uplot@1.6.32: {} + uri-js@4.4.1: dependencies: punycode: 2.3.1 From b5a0ab8de6db7e68a6f4396f01becd8cb98b1dff Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:34:01 +0700 Subject: [PATCH 025/174] feat(dashboard): add a log viewer for the observability proxy Tenants could not read their applications' logs without Grafana. The viewer reads VictoriaLogs' native responses through a proxy root: a time-ranged LogsQL query, pod and container pickers fed by the stream field values of the same scope, and a follow mode over the streamed tail. The picked pod and container wrap the filter part of the query rather than being appended to it, because AND binds tighter than OR in LogsQL and the user's pipes must still run last. A dropped tail resumes from the last line's _time; the proxy may replay lines around that point, so the client drops replayed lines it already holds by stream and time, which VictoriaLogs does not guarantee unique and is therefore applied only at the resume boundary. Follow keeps the newest 1000 lines so a chatty stream cannot grow the page without bound. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observability/LogsView.test.tsx | 111 +++++++++++++ .../src/components/observability/LogsView.tsx | 152 ++++++++++++++++++ .../src/lib/observability/logs.test.ts | 65 ++++++++ .../console/src/lib/observability/logs.ts | 73 +++++++++ .../console/src/lib/observability/proxy.ts | 3 +- .../lib/observability/use-log-tail.test.tsx | 65 ++++++++ .../src/lib/observability/use-log-tail.ts | 95 +++++++++++ 7 files changed, 563 insertions(+), 1 deletion(-) create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx new file mode 100644 index 000000000..693adf70a --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -0,0 +1,111 @@ +import { describe, it, expect, vi } from "vitest" +import { screen, waitFor } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { K8sClient } from "@cozystack/k8s-client" +import { renderWithK8sProvider } from "../../test-utils/render.tsx" +import { LogsView } from "./LogsView.tsx" + +const app = { namespace: "tenant-a", plural: "postgreses", name: "db" } +const ROOT = "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/" + +const line = (time: string, msg: string, pod = "db-0") => + JSON.stringify({ _time: time, _msg: msg, _stream_id: "s", kubernetes_pod_name: pod, kubernetes_container_name: "postgres" }) + +function clientWith(handler: (endpoint: string, params: URLSearchParams) => Response | Promise) { + const client = new K8sClient() + const raw = vi.spyOn(client, "raw").mockImplementation(async (path) => { + const url = new URL(path, "http://x") + return handler(url.pathname, url.searchParams) + }) + return { client, raw } +} + +const values = (...v: string[]) => Response.json({ values: v.map((value) => ({ value, hits: 1 })) }) + +function defaultHandler(endpoint: string, params?: URLSearchParams): Response | Promise { + if (endpoint.endsWith("stream_field_values")) { + return params?.get("field") === "kubernetes_pod_name" ? values("db-0", "db-1") : values("postgres") + } + if (endpoint.endsWith("query")) return new Response(`${line("2026-11-04T10:00:01Z", "ready to accept connections")}\n`) + return new Promise(() => {}) +} + +describe("LogsView", () => { + it("shows recent lines from the application root", async () => { + const { client, raw } = clientWith(defaultHandler) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + const call = raw.mock.calls.find(([p]) => p.includes("select/logsql/query?"))! + expect(call[0].startsWith(`${ROOT}select/logsql/query?`)).toBe(true) + const params = new URL(call[0], "http://x").searchParams + expect(params.get("query")).toBe("*") + expect(params.get("limit")).toBe("500") + }) + + it("narrows to the picked pod with a stream filter", async () => { + const { client, raw } = clientWith(defaultHandler) + renderWithK8sProvider(, { client }) + await screen.findByRole("option", { name: "db-1" }) + + await userEvent.selectOptions(screen.getByLabelText("Pod"), "db-1") + + await waitFor(() => + expect( + raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("query") === '{kubernetes_pod_name="db-1"} (*)'), + ).toBe(true), + ) + }) + + it("applies the LogsQL filter on submit", async () => { + const { client, raw } = clientWith(defaultHandler) + renderWithK8sProvider(, { client }) + + await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}") + + await waitFor(() => + expect(raw.mock.calls.some(([p]) => p.includes("select/logsql/query?") && p.includes("query=error"))).toBe(true), + ) + }) + + it("follows the live tail when Follow is pressed", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("tail") ? new Response(`${line("2026-11-04T10:00:05Z", "checkpoint complete")}\n`) : defaultHandler(endpoint, params), + ) + renderWithK8sProvider(, { client }) + + await userEvent.click(screen.getByRole("button", { name: "Follow" })) + + expect(await screen.findByText("checkpoint complete")).toBeInTheDocument() + expect(screen.getByRole("button", { name: "Follow" })).toHaveAttribute("aria-pressed", "true") + }) + + it("renders a cluster without the proxy as not available", async () => { + const { client } = clientWith(() => new Response("404 page not found", { status: 404 })) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("Logs are not available on this cluster.")).toBeInTheDocument() + expect(screen.queryByLabelText("LogsQL filter")).toBeNull() + }) + + it("keeps the controls and explains a rejected query", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") ? new Response("cannot parse query", { status: 400 }) : defaultHandler(endpoint, params), + ) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("cannot parse query")).toBeInTheDocument() + expect(screen.getByLabelText("LogsQL filter")).toBeInTheDocument() + }) + + it("names the source pod and container of each line", async () => { + const { client } = clientWith(defaultHandler) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("db-0/postgres")).toBeInTheDocument() + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx new file mode 100644 index 000000000..03235cda3 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -0,0 +1,152 @@ +import { useEffect, useRef, useState } from "react" +import { useQuery } from "@tanstack/react-query" +import { useK8sClient } from "@cozystack/k8s-client" +import { Button, Spinner } from "@cozystack/ui" +import { + CONTAINER_FIELD, + LOG_RANGES, + POD_FIELD, + buildLogsQuery, + type LogRangeKey, +} from "../../lib/observability/logs.ts" +import { + fieldValues, + isScopeWide, + logsRoot, + queryLogs, + type LogEntry, + type ObservabilityScope, +} from "../../lib/observability/proxy.ts" +import { useLogTail } from "../../lib/observability/use-log-tail.ts" +import { ProxyNotice } from "./ProxyNotice.tsx" +import { RangePicker } from "./RangePicker.tsx" + +const HISTORY_LIMIT = 500 +const FOLLOW_CAP = 1000 + +function windowFor(range: LogRangeKey) { + const end = Date.now() + return { start: new Date(end - LOG_RANGES[range] * 1000).toISOString(), end: new Date(end).toISOString() } +} + +function Picker({ label, value, options, onChange }: { label: string; value: string; options: string[]; onChange: (v: string) => void }) { + return ( + + ) +} + +function LogLines({ entries, follow }: { entries: LogEntry[]; follow: boolean }) { + const ref = useRef(null) + useEffect(() => { + if (follow && ref.current) ref.current.scrollTop = ref.current.scrollHeight + }, [entries, follow]) + if (entries.length === 0) return

No log lines for this range and filter.

+ return ( +
    + {entries.map((e, i) => ( +
  1. + {new Date(e._time).toLocaleTimeString()}{" "} + {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`}{" "} + {e._msg} +
  2. + ))} +
+ ) +} + +export function LogsView({ scope }: { scope: ObservabilityScope }) { + const client = useK8sClient() + const root = logsRoot(scope) + const [range, setRange] = useState("1h") + const [draft, setDraft] = useState("") + const [filter, setFilter] = useState("") + const [pod, setPod] = useState("") + const [container, setContainer] = useState("") + const [follow, setFollow] = useState(false) + const query = buildLogsQuery({ filter, pod, container }) + + const pickerQuery = (field: string, narrowTo?: string) => ({ + queryKey: ["observability", "logs-fields", root, field, range, narrowTo ?? ""], + queryFn: ({ signal }: { signal: AbortSignal }) => + fieldValues(client, root, { + field, + stream: true, + query: buildLogsQuery({ filter: "", pod: narrowTo }), + ...windowFor(range), + }, signal), + retry: false, + }) + const pods = useQuery(pickerQuery(POD_FIELD)) + const containers = useQuery(pickerQuery(CONTAINER_FIELD, pod || undefined)) + + const history = useQuery({ + queryKey: ["observability", "logs", root, query, range], + queryFn: ({ signal }) => queryLogs(client, root, { query, limit: HISTORY_LIMIT, ...windowFor(range) }, signal), + enabled: !follow, + retry: false, + }) + const tail = useLogTail(root, query, follow, { cap: FOLLOW_CAP }) + + const wide = [pods.error, history.error, tail.error].find(isScopeWide) + if (wide) { + return ( +
+ +
+ ) + } + + const error = follow ? tail.error : history.error + const entries = follow ? tail.entries : (history.data ?? []) + + return ( +
+
+
{ + e.preventDefault() + setFilter(draft) + }} + > + setDraft(e.target.value)} + placeholder='LogsQL filter, e.g. error or "connection refused"' + className="w-full rounded-md border border-slate-200 px-2.5 py-1.5 font-mono text-xs" + /> +
+ { setPod(v); setContainer("") }} /> + + {!follow && } + +
+ {follow && ( +

+ {tail.state === "live" ? "Live" : tail.state === "reconnecting" ? "Reconnecting…" : "Connecting…"} + {` · newest ${FOLLOW_CAP} lines kept`} +

+ )} + {error ? : null} + {!follow && history.isLoading ? : } +
+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts new file mode 100644 index 000000000..25d26a23a --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect } from "vitest" +import { buildLogsQuery, compareTime, mergeTail } from "./logs.ts" + +describe("buildLogsQuery", () => { + it("passes the filter through when nothing is picked", () => { + expect(buildLogsQuery({ filter: "error or warn" })).toBe("error or warn") + expect(buildLogsQuery({ filter: " " })).toBe("*") + }) + + it("ANDs the picked pod and container as a stream filter around the whole filter", () => { + expect(buildLogsQuery({ filter: "error or warn", pod: "db-0", container: "postgres" })).toBe( + '{kubernetes_pod_name="db-0",kubernetes_container_name="postgres"} (error or warn)', + ) + }) + + it("keeps the user's pipes after the scoped filter", () => { + expect(buildLogsQuery({ filter: 'a or "x|y" | fields _msg', pod: "db-0" })).toBe( + '{kubernetes_pod_name="db-0"} (a or "x|y") | fields _msg', + ) + }) + + it("escapes quotes in picked values", () => { + expect(buildLogsQuery({ filter: "", pod: 'a"b' })).toBe('{kubernetes_pod_name="a\\"b"} (*)') + }) +}) + +describe("compareTime", () => { + it("orders RFC3339 times with differing fraction lengths", () => { + expect(compareTime("2026-11-04T10:00:01Z", "2026-11-04T10:00:01.1Z")).toBeLessThan(0) + expect(compareTime("2026-11-04T10:00:01.5Z", "2026-11-04T10:00:01.123Z")).toBeGreaterThan(0) + expect(compareTime("2026-11-04T10:00:01.10Z", "2026-11-04T10:00:01.1Z")).toBe(0) + }) +}) + +const e = (time: string, msg: string, stream = "s1") => ({ _time: time, _msg: msg, _stream_id: stream }) + +describe("mergeTail", () => { + it("appends and keeps only the newest lines over the cap", () => { + const out = mergeTail([e("t1", "a"), e("t2", "b")], [e("t3", "c")], 2) + + expect(out.map((x) => x._msg)).toEqual(["b", "c"]) + }) + + it("drops replayed lines at or before the resume point", () => { + const buffer = [e("2026-01-01T00:00:01Z", "a"), e("2026-01-01T00:00:02Z", "b")] + + const out = mergeTail(buffer, [e("2026-01-01T00:00:02Z", "b"), e("2026-01-01T00:00:03Z", "c")], 10, "2026-01-01T00:00:02Z") + + expect(out.map((x) => x._msg)).toEqual(["a", "b", "c"]) + }) + + it("keeps a line at the resume point from another stream", () => { + const buffer = [e("2026-01-01T00:00:02Z", "b", "s1")] + + const out = mergeTail(buffer, [e("2026-01-01T00:00:02Z", "other", "s2")], 10, "2026-01-01T00:00:02Z") + + expect(out).toHaveLength(2) + }) + + it("does not deduplicate outside a resume", () => { + const out = mergeTail([e("t1", "a")], [e("t1", "a")], 10) + + expect(out).toHaveLength(2) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts new file mode 100644 index 000000000..a6255e541 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts @@ -0,0 +1,73 @@ +import type { LogEntry } from "./proxy.ts" + +export const LOG_RANGES = { + "15m": 15 * 60, + "1h": 3600, + "6h": 6 * 3600, + "24h": 24 * 3600, +} as const + +export type LogRangeKey = keyof typeof LOG_RANGES + +export const POD_FIELD = "kubernetes_pod_name" +export const CONTAINER_FIELD = "kubernetes_container_name" + +function quote(value: string): string { + return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"` +} + +function splitAtFirstPipe(query: string): [string, string] { + let quoteChar = "" + for (let i = 0; i < query.length; i++) { + const c = query[i] + if (quoteChar) { + if (c === "\\") i++ + else if (c === quoteChar) quoteChar = "" + } else if (c === '"' || c === "'" || c === "`") { + quoteChar = c + } else if (c === "|") { + return [query.slice(0, i), query.slice(i + 1)] + } + } + return [query, ""] +} + +// The picker narrows the filter part only: AND binds tighter than OR in +// LogsQL, and the user's pipes must still run after the narrowed filter. +export function buildLogsQuery({ filter, pod, container }: { filter: string; pod?: string; container?: string }): string { + const trimmed = filter.trim() || "*" + const matchers = [ + pod ? `${POD_FIELD}=${quote(pod)}` : "", + container ? `${CONTAINER_FIELD}=${quote(container)}` : "", + ].filter(Boolean) + if (matchers.length === 0) return trimmed + const [head, pipes] = splitAtFirstPipe(trimmed) + const scoped = `{${matchers.join(",")}} (${head.trim() || "*"})` + return pipes.trim() ? `${scoped} | ${pipes.trim()}` : scoped +} + +// VictoriaLogs trims trailing zeros from the fraction, so the strings do not sort lexically. +function padFraction(time: string): string { + const m = /^([^.Z]+)(?:\.(\d+))?Z$/.exec(time) + return m ? `${m[1]}.${(m[2] ?? "").padEnd(9, "0")}Z` : time +} + +export function compareTime(a: string, b: string): number { + const x = padFraction(a) + const y = padFraction(b) + return x < y ? -1 : x > y ? 1 : 0 +} + +const entryKey = (e: LogEntry) => `${e._stream_id ?? ""}\u0000${e._time}` + +// (_stream_id, _time) is not unique in VictoriaLogs, so dedup applies only to +// lines a resumed tail replays at or before the resume point. +export function mergeTail(buffer: LogEntry[], incoming: LogEntry[], cap: number, resumedFrom?: string): LogEntry[] { + let fresh = incoming + if (resumedFrom) { + const seen = new Set(buffer.map(entryKey)) + fresh = incoming.filter((e) => compareTime(e._time, resumedFrom) > 0 || !seen.has(entryKey(e))) + } + const merged = buffer.concat(fresh) + return merged.length > cap ? merged.slice(merged.length - cap) : merged +} diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts index 40b451b0b..345d49815 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts @@ -1,4 +1,5 @@ import type { K8sClient } from "@cozystack/k8s-client" +import { compareTime } from "./logs.ts" export type ObservabilityScope = | { namespace: string } @@ -160,7 +161,7 @@ export async function queryLogs( ): Promise { const res = await proxyGet(client, `${root}select/logsql/query`, { ...params }, signal) const entries = (await res.text()).split("\n").map(parseLine).filter((e) => e !== null) - return entries.sort((a, b) => (a._time < b._time ? -1 : a._time > b._time ? 1 : 0)) + return entries.sort((a, b) => compareTime(a._time, b._time)) } export interface FieldValuesParams { diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx new file mode 100644 index 000000000..5c420314a --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx @@ -0,0 +1,65 @@ +import { describe, it, expect, vi } from "vitest" +import { renderHook, waitFor } from "@testing-library/react" +import type { ReactNode } from "react" +import { K8sClient, K8sProvider } from "@cozystack/k8s-client" +import { useLogTail } from "./use-log-tail.ts" + +const line = (time: string, msg: string) => JSON.stringify({ _time: time, _msg: msg, _stream_id: "s" }) + +function ndjson(...lines: string[]): Response { + return new Response(lines.map((l) => `${l}\n`).join("")) +} + +function setup(...responses: Response[]) { + const client = new K8sClient() + const raw = vi.spyOn(client, "raw") + for (const r of responses) raw.mockResolvedValueOnce(r) + raw.mockImplementation(() => new Promise(() => {})) + const wrapper = ({ children }: { children: ReactNode }) => {children} + return { raw, wrapper } +} + +const T1 = "2026-11-04T10:00:01Z" +const T2 = "2026-11-04T10:00:02Z" +const T3 = "2026-11-04T10:00:03Z" + +describe("useLogTail", () => { + it("streams lines and resumes after a disconnect from the last _time without duplicates", async () => { + const { raw, wrapper } = setup(ndjson(line(T1, "a"), line(T2, "b")), ndjson(line(T2, "b"), line(T3, "c"))) + + const { result } = renderHook(() => useLogTail("/root/", "*", true, { cap: 100, retryDelayMs: 1 }), { wrapper }) + + await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a", "b", "c"])) + expect(new URL(raw.mock.calls[0][0], "http://x").searchParams.has("since")).toBe(false) + expect(new URL(raw.mock.calls[1][0], "http://x").searchParams.get("since")).toBe(T2) + }) + + it("stops and reports an error the proxy will repeat on every retry", async () => { + const { raw, wrapper } = setup(new Response("", { status: 403 })) + + const { result } = renderHook(() => useLogTail("/root/", "*", true, { cap: 100, retryDelayMs: 1 }), { wrapper }) + + await waitFor(() => expect(result.current.state).toBe("stopped")) + expect(result.current.error).toMatchObject({ kind: "forbidden" }) + expect(raw).toHaveBeenCalledTimes(1) + }) + + it("does not connect while disabled", () => { + const { raw, wrapper } = setup() + + renderHook(() => useLogTail("/root/", "*", false), { wrapper }) + + expect(raw).not.toHaveBeenCalled() + }) + + it("aborts the stream on unmount", async () => { + const { raw, wrapper } = setup() + const { unmount } = renderHook(() => useLogTail("/root/", "*", true), { wrapper }) + await waitFor(() => expect(raw).toHaveBeenCalled()) + + unmount() + + const init = raw.mock.calls[0][1] as RequestInit + expect(init.signal?.aborted).toBe(true) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts new file mode 100644 index 000000000..cb9222f48 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts @@ -0,0 +1,95 @@ +import { useEffect, useState } from "react" +import { useK8sClient } from "@cozystack/k8s-client" +import { mergeTail } from "./logs.ts" +import { ProxyError, tailLogs, type LogEntry } from "./proxy.ts" + +export type TailState = "connecting" | "live" | "reconnecting" | "stopped" + +interface TailSnapshot { + key: string + entries: LogEntry[] + state: TailState + error: unknown +} + +const FLUSH_MS = 100 +const MAX_RETRY_MS = 30_000 +const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forbidden", "not-available", "limit"]) + +const sleep = (ms: number, signal: AbortSignal) => + new Promise((resolve) => { + const t = setTimeout(resolve, ms) + signal.addEventListener("abort", () => { + clearTimeout(t) + resolve() + }) + }) + +export function useLogTail( + root: string, + query: string, + enabled: boolean, + { cap = 1000, retryDelayMs = 2000 }: { cap?: number; retryDelayMs?: number } = {}, +): Omit { + const client = useK8sClient() + const key = `${root}\n${query}\n${enabled}` + const [snap, setSnap] = useState({ key, entries: [], state: "connecting", error: null }) + + useEffect(() => { + if (!enabled) return + const controller = new AbortController() + const { signal } = controller + let pending: LogEntry[] = [] + let timer: ReturnType | undefined + let resumedFrom: string | undefined + let lastTime: string | undefined + + const update = (patch: Partial, batch: LogEntry[] = [], from = resumedFrom) => + setSnap((prev) => { + const base = prev.key === key ? prev : { key, entries: [], state: "connecting" as const, error: null } + return { ...base, ...patch, entries: batch.length ? mergeTail(base.entries, batch, cap, from) : base.entries } + }) + const flush = () => { + clearTimeout(timer) + timer = undefined + if (pending.length === 0 || signal.aborted) return + const batch = pending + pending = [] + update({ state: "live", error: null }, batch) + } + + const run = async () => { + for (let attempt = 0; !signal.aborted; attempt++) { + resumedFrom = lastTime + try { + await tailLogs(client, root, { query, since: lastTime }, (entry) => { + attempt = 0 + lastTime = entry._time + pending.push(entry) + timer ??= setTimeout(flush, FLUSH_MS) + }, signal) + flush() + } catch (err) { + flush() + if (signal.aborted) return + if (err instanceof ProxyError && FATAL.has(err.kind)) { + update({ state: "stopped", error: err }) + return + } + update({ error: err }) + } + if (signal.aborted) return + update({ state: "reconnecting" }) + await sleep(Math.min(retryDelayMs * 2 ** attempt, MAX_RETRY_MS), signal) + } + } + void run() + return () => { + controller.abort() + clearTimeout(timer) + } + }, [client, root, query, enabled, key, cap, retryDelayMs]) + + const current = snap.key === key ? snap : { key, entries: [], state: "connecting" as const, error: null } + return { entries: current.entries, state: current.state, error: current.error } +} From f9ed2afad749646bb5c1de0a56b4e8d17e773bc1 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:34:01 +0700 Subject: [PATCH 026/174] feat(dashboard): show logs on application pages and per tenant A Logs tab on application detail pages reads the application root of the observability proxy, and a Logs page under a new Observability sidebar group reads the tenant namespace root, so lines from before the application identity labels existed stay reachable there. Assisted-by: LLM Signed-off-by: ZverGuy --- .../apps/console/src/routes/ConsolePage.tsx | 2 ++ .../apps/console/src/routes/LogsPage.test.tsx | 20 +++++++++++++++++++ .../apps/console/src/routes/LogsPage.tsx | 15 ++++++++++++++ .../detail/ApplicationDetailPage.test.tsx | 9 ++++++--- .../routes/detail/ApplicationDetailPage.tsx | 4 ++++ .../console/src/routes/detail/LogsTab.tsx | 15 ++++++++++++++ .../src/routes/sidebar-sections.test.tsx | 10 ++++++++++ .../console/src/routes/sidebar-sections.tsx | 8 +++++++- 8 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx index 37caae92f..5e204e84f 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx @@ -15,6 +15,7 @@ import { MigrationSourceEditPage } from "./MigrationSourceEditPage.tsx" import { MigrationTaskCreatePage } from "./MigrationTaskCreatePage.tsx" import { MigrationTaskDetailPage } from "./MigrationTaskDetailPage.tsx" import { ApplicationOrderPage } from "./ApplicationOrderPage.tsx" +import { LogsPage } from "./LogsPage.tsx" export function ConsolePage() { return ( @@ -96,6 +97,7 @@ export function ConsolePage() { path="migration/vmimporttasks/:name" element={} /> + } /> } /> } /> } /> diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx new file mode 100644 index 000000000..1f4263cf1 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx @@ -0,0 +1,20 @@ +import { describe, it, expect, vi } from "vitest" +import { render, screen } from "@testing-library/react" + +vi.mock("../lib/tenant-context.tsx", () => ({ + useTenantContext: () => ({ tenantNamespace: "tenant-a" }), +})) +vi.mock("../components/observability/LogsView.tsx", () => ({ + LogsView: ({ scope }: { scope: object }) =>
{JSON.stringify(scope)}
, +})) + +const { LogsPage } = await import("./LogsPage.tsx") + +describe("LogsPage", () => { + it("reads logs from the selected tenant's namespace root", () => { + render() + + expect(screen.getByRole("heading", { name: "Logs" })).toBeInTheDocument() + expect(JSON.parse(screen.getByTestId("scope").textContent ?? "")).toEqual({ namespace: "tenant-a" }) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx new file mode 100644 index 000000000..7612dfbb2 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx @@ -0,0 +1,15 @@ +import { useTenantContext } from "../lib/tenant-context.tsx" +import { LogsView } from "../components/observability/LogsView.tsx" + +export function LogsPage() { + const { tenantNamespace } = useTenantContext() + return ( +
+
+

Logs

+

Every application in {tenantNamespace ?? "this tenant"}

+
+ {tenantNamespace && } +
+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx index 40b923f62..a0c3a6b84 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx @@ -60,6 +60,7 @@ vi.mock("./VncTab.tsx", () => ({ VncTab: () => null })) vi.mock("./SerialTab.tsx", () => ({ SerialTab: () => null })) vi.mock("./VMPowerControls.tsx", () => ({ VMPowerControls: () => null })) vi.mock("./MetricsTab.tsx", () => ({ MetricsTab: () => null })) +vi.mock("./LogsTab.tsx", () => ({ LogsTab: () => null })) const { ApplicationDetailPage } = await import("./ApplicationDetailPage.tsx") @@ -113,6 +114,7 @@ describe("ApplicationDetailPage tabs for a virtual machine", () => { "Workloads", "Services", "Metrics", + "Logs", "Events", ]) }) @@ -133,14 +135,15 @@ describe("ApplicationDetailPage observability tabs", () => { renderPage() } - it("offers Metrics on any application kind, ahead of Events", () => { + it("offers Metrics and Logs on any application kind, ahead of Events", () => { showKind("Postgres", "postgreses") - expect(h.tabLabels.slice(-3)).toEqual(["Secrets", "Metrics", "Events"]) + expect(h.tabLabels.slice(-4)).toEqual(["Secrets", "Metrics", "Logs", "Events"]) }) - it("leaves Metrics off a VM disk, which runs no containers", () => { + it("leaves Metrics and Logs off a VM disk, which runs no containers", () => { showKind("VMDisk", "vmdisks") expect(h.tabLabels).not.toContain("Metrics") + expect(h.tabLabels).not.toContain("Logs") }) }) diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx index 4520244d5..ca499aac1 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx @@ -44,6 +44,7 @@ import { useResourceBasePath } from "../../lib/portal.ts" import { useResourcePresence } from "./use-resource-presence.ts" import { useApplicationConfigMaps } from "./use-app-configmaps.ts" import { ConfigMapsTab } from "./ConfigMapsTab.tsx" +import { LogsTab } from "./LogsTab.tsx" export function ApplicationDetailPage() { const { plural, name } = useParams<{ plural: string; name: string }>() @@ -146,6 +147,7 @@ export function ApplicationDetailPage() { { to: `${base}/workloads`, label: "Workloads", end: false }, { to: `${base}/services`, label: "Services", end: false }, { to: `${base}/metrics`, label: "Metrics", end: false }, + { to: `${base}/logs`, label: "Logs", end: false }, { to: `${base}/events`, label: "Events", end: false }, ) } else { @@ -167,6 +169,7 @@ export function ApplicationDetailPage() { tabs.push( { to: `${base}/secrets`, label: "Secrets", end: false }, { to: `${base}/metrics`, label: "Metrics", end: false }, + { to: `${base}/logs`, label: "Logs", end: false }, { to: `${base}/events`, label: "Events", end: false }, ) } @@ -274,6 +277,7 @@ export function ApplicationDetailPage() { } /> + } /> diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx new file mode 100644 index 000000000..4ac5f878d --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx @@ -0,0 +1,15 @@ +import type { ApplicationDefinition, ApplicationInstance } from "@cozystack/types" +import { useTenantContext } from "../../lib/tenant-context.tsx" +import { LogsView } from "../../components/observability/LogsView.tsx" + +interface LogsTabProps { + ad: ApplicationDefinition + instance: ApplicationInstance +} + +export function LogsTab({ ad, instance }: LogsTabProps) { + const { tenantNamespace } = useTenantContext() + const plural = ad.spec?.application.plural + if (!tenantNamespace || !plural) return null + return +} diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx index efb69dc7b..da40081bc 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx @@ -110,6 +110,16 @@ describe("useConsoleSidebarSections — admin areas moved out", () => { expect(findItem(result.current, "Imports")?.to).toBe("/console/migration/vmimporttasks") expect(result.current.some((s) => s.title === "Migration")).toBe(true) }) + + it("offers the tenant's namespace logs in an Observability group", async () => { + const client = makeClient({ nodes: true, backupclasses: true }) + const { result } = renderHook(() => useConsoleSidebarSections(), { + wrapper: makeWrapper(client), + }) + await waitFor(() => expect(result.current.length).toBeGreaterThan(0)) + expect(findItem(result.current, "Logs")?.to).toBe("/console/logs") + expect(result.current.find((s) => s.title === "Observability")?.items).toHaveLength(1) + }) }) function appDef(name: string, kind: string, category: string, module = false) { diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx index d5d100813..4cec3a94c 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx @@ -11,6 +11,7 @@ import { Layers, Network, Package, + ScrollText, Plug, Server, ToyBrick, @@ -118,7 +119,12 @@ export function useConsoleSidebarSections(): SidebarSection[] { ], } - return [...categorySections, backupsSection, migrationSection] + const observabilitySection: SidebarSection = { + title: "Observability", + items: [{ label: "Logs", to: "/console/logs", icon: ScrollText }], + } + + return [...categorySections, backupsSection, migrationSection, observabilitySection] }, [grouped]) } From c5b57bffd173ea5e4bb371edb75a13a60f549d96 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:39:27 +0700 Subject: [PATCH 027/174] fix(dashboard): open log history at the newest line A historical query lists lines oldest first, so the viewer opened at the top of a 500-line window while the line a user looks for is usually the latest. The chart's value axis also clipped wide labels such as "476.8 MiB". Assisted-by: LLM Signed-off-by: ZverGuy --- .../console/src/components/observability/LogsView.tsx | 8 ++++---- .../src/components/observability/TimeSeriesChart.tsx | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx index 03235cda3..707b037b9 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -50,11 +50,11 @@ function Picker({ label, value, options, onChange }: { label: string; value: str ) } -function LogLines({ entries, follow }: { entries: LogEntry[]; follow: boolean }) { +function LogLines({ entries }: { entries: LogEntry[] }) { const ref = useRef(null) useEffect(() => { - if (follow && ref.current) ref.current.scrollTop = ref.current.scrollHeight - }, [entries, follow]) + if (ref.current) ref.current.scrollTop = ref.current.scrollHeight + }, [entries]) if (entries.length === 0) return

No log lines for this range and filter.

return (
    @@ -146,7 +146,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {

    )} {error ? : null} - {!follow && history.isLoading ? : } + {!follow && history.isLoading ? : } ) } diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx index 6b596e17f..683df2dff 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx @@ -33,7 +33,7 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) { value: (_: uPlot, v: number | null) => fmt(v), })), ], - axes: [{}, { size: 70, values: (_: uPlot, splits: number[]) => splits.map(format) }], + axes: [{}, { size: 84, values: (_: uPlot, splits: number[]) => splits.map(format) }], }, [data.timestamps, ...data.series.map((s) => s.values)], el, From 97dcf4c4f02d78dc2dcd4ec862cdaa77631de4f3 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:45:42 +0700 Subject: [PATCH 028/174] fix(dashboard): treat an apiserver answer on a proxy path as no proxy Until the console nginx routes the proxy roots, those paths reach kube-apiserver, which RBAC-denies the namespace root with 403 for a tenant, so the Logs page told every tenant they lacked permission. The proxy itself answers in plain text, so a Kubernetes Status body is a reliable sign that the path did not reach it. Assisted-by: LLM Signed-off-by: ZverGuy --- .../components/observability/LogsView.test.tsx | 10 ++++++++++ .../console/src/lib/observability/proxy.test.ts | 17 +++++++++++++++++ .../apps/console/src/lib/observability/proxy.ts | 14 +++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx index 693adf70a..fae3f6a4b 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -90,6 +90,16 @@ describe("LogsView", () => { expect(screen.queryByLabelText("LogsQL filter")).toBeNull() }) + it("renders the tenant logs page as not available when kube-apiserver answers instead of the proxy", async () => { + const { client } = clientWith(() => + Response.json({ kind: "Status", apiVersion: "v1", status: "Failure", reason: "Forbidden", code: 403 }, { status: 403 }), + ) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("Logs are not available on this cluster.")).toBeInTheDocument() + }) + it("keeps the controls and explains a rejected query", async () => { const { client } = clientWith((endpoint, params) => endpoint.endsWith("query") ? new Response("cannot parse query", { status: 400 }) : defaultHandler(endpoint, params), diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts index f8cf44656..69a8c7c8c 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts @@ -79,6 +79,23 @@ describe("queryRange", () => { expect(err).toMatchObject({ kind: "limit", status: 422, message: "range too long" }) }) + it("treats a Kubernetes Status 403 as a cluster without the proxy route", async () => { + const status = { kind: "Status", apiVersion: "v1", status: "Failure", reason: "Forbidden", code: 403, message: "logs.cozystack.io \"select\" is forbidden" } + const { client } = clientReturning(Response.json(status, { status: 403 })) + + const err = await queryRange(client, metricsRoot(ns), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e) + + expect(err).toMatchObject({ kind: "not-available", status: 403 }) + }) + + it("keeps the proxy's own plain-text 403 as forbidden", async () => { + const { client } = clientReturning(new Response("forbidden\n", { status: 403 })) + + const err = await queryRange(client, metricsRoot(ns), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e) + + expect(err).toMatchObject({ kind: "forbidden" }) + }) + it("reports a fetch failure as a network error", async () => { const client = new K8sClient() vi.spyOn(client, "raw").mockRejectedValue(new TypeError("offline")) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts index 345d49815..c9fce3d4f 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts @@ -69,6 +69,17 @@ export function isScopeWide(error: unknown): boolean { ) } +// The proxy answers in plain text; a Kubernetes Status body means nginx sent +// the path to kube-apiserver, which RBAC-denies or 404s a route it does not serve. +function isKubernetesStatus(text: string): boolean { + try { + const parsed: unknown = JSON.parse(text) + return !!parsed && typeof parsed === "object" && "kind" in parsed && parsed.kind === "Status" + } catch { + return false + } +} + // Prometheus answers {"error": "..."}, VictoriaLogs and nginx answer plain text. function errorMessage(text: string, status: number): string { try { @@ -100,7 +111,8 @@ async function proxyGet( } if (!res.ok) { const text = await res.text().catch(() => "") - throw new ProxyError(errorKindForStatus(res.status), res.status, errorMessage(text, res.status)) + const kind = isKubernetesStatus(text) ? "not-available" : errorKindForStatus(res.status) + throw new ProxyError(kind, res.status, errorMessage(text, res.status)) } return res } From 4b44971d125cce2c2fd06457c68f23cf1645ac87 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:47:13 +0700 Subject: [PATCH 029/174] fix(dashboard): start each log follow run with its own buffer Turning Follow off and on again reused the previous run's buffer, so new lines were appended after a silent gap and, on a quick toggle, duplicated. A follow run is now identified by a session value, so the buffer starts empty each time. A resume now uses the newest _time seen rather than the last line received, so an out-of-order line cannot move since backwards, and the backoff sleep no longer leaves an abort listener per retry. Assisted-by: LLM Signed-off-by: ZverGuy --- .../lib/observability/use-log-tail.test.tsx | 31 ++++++++++++++++--- .../src/lib/observability/use-log-tail.ts | 23 ++++++++------ 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx index 5c420314a..46df47970 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx @@ -27,7 +27,7 @@ describe("useLogTail", () => { it("streams lines and resumes after a disconnect from the last _time without duplicates", async () => { const { raw, wrapper } = setup(ndjson(line(T1, "a"), line(T2, "b")), ndjson(line(T2, "b"), line(T3, "c"))) - const { result } = renderHook(() => useLogTail("/root/", "*", true, { cap: 100, retryDelayMs: 1 }), { wrapper }) + const { result } = renderHook(() => useLogTail("/root/", "*", 1, { cap: 100, retryDelayMs: 1 }), { wrapper }) await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a", "b", "c"])) expect(new URL(raw.mock.calls[0][0], "http://x").searchParams.has("since")).toBe(false) @@ -37,24 +37,47 @@ describe("useLogTail", () => { it("stops and reports an error the proxy will repeat on every retry", async () => { const { raw, wrapper } = setup(new Response("", { status: 403 })) - const { result } = renderHook(() => useLogTail("/root/", "*", true, { cap: 100, retryDelayMs: 1 }), { wrapper }) + const { result } = renderHook(() => useLogTail("/root/", "*", 1, { cap: 100, retryDelayMs: 1 }), { wrapper }) await waitFor(() => expect(result.current.state).toBe("stopped")) expect(result.current.error).toMatchObject({ kind: "forbidden" }) expect(raw).toHaveBeenCalledTimes(1) }) + it("starts a new follow session with an empty buffer", async () => { + const { wrapper } = setup(ndjson(line(T1, "first")), ndjson(line(T3, "second"))) + const { result, rerender } = renderHook( + ({ session }: { session: number | null }) => useLogTail("/root/", "*", session, { retryDelayMs: 60_000 }), + { wrapper, initialProps: { session: 1 as number | null } }, + ) + await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["first"])) + + rerender({ session: null }) + rerender({ session: 2 }) + + await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["second"])) + }) + + it("resumes from the newest line seen, not the last one received", async () => { + const { raw, wrapper } = setup(ndjson(line(T3, "c"), line(T1, "a")), ndjson()) + + renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper }) + + await waitFor(() => expect(raw.mock.calls.length).toBeGreaterThan(1)) + expect(new URL(raw.mock.calls[1][0], "http://x").searchParams.get("since")).toBe(T3) + }) + it("does not connect while disabled", () => { const { raw, wrapper } = setup() - renderHook(() => useLogTail("/root/", "*", false), { wrapper }) + renderHook(() => useLogTail("/root/", "*", null), { wrapper }) expect(raw).not.toHaveBeenCalled() }) it("aborts the stream on unmount", async () => { const { raw, wrapper } = setup() - const { unmount } = renderHook(() => useLogTail("/root/", "*", true), { wrapper }) + const { unmount } = renderHook(() => useLogTail("/root/", "*", 1), { wrapper }) await waitFor(() => expect(raw).toHaveBeenCalled()) unmount() diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts index cb9222f48..ddc835821 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts @@ -1,6 +1,6 @@ import { useEffect, useState } from "react" import { useK8sClient } from "@cozystack/k8s-client" -import { mergeTail } from "./logs.ts" +import { compareTime, mergeTail } from "./logs.ts" import { ProxyError, tailLogs, type LogEntry } from "./proxy.ts" export type TailState = "connecting" | "live" | "reconnecting" | "stopped" @@ -18,25 +18,30 @@ const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forb const sleep = (ms: number, signal: AbortSignal) => new Promise((resolve) => { - const t = setTimeout(resolve, ms) - signal.addEventListener("abort", () => { + const onAbort = () => { clearTimeout(t) resolve() - }) + } + const t = setTimeout(() => { + signal.removeEventListener("abort", onAbort) + resolve() + }, ms) + signal.addEventListener("abort", onAbort, { once: true }) }) +// Each non-null session is one follow run with its own buffer; null stops following. export function useLogTail( root: string, query: string, - enabled: boolean, + session: number | null, { cap = 1000, retryDelayMs = 2000 }: { cap?: number; retryDelayMs?: number } = {}, ): Omit { const client = useK8sClient() - const key = `${root}\n${query}\n${enabled}` + const key = `${root}\n${query}\n${session}` const [snap, setSnap] = useState({ key, entries: [], state: "connecting", error: null }) useEffect(() => { - if (!enabled) return + if (session === null) return const controller = new AbortController() const { signal } = controller let pending: LogEntry[] = [] @@ -64,7 +69,7 @@ export function useLogTail( try { await tailLogs(client, root, { query, since: lastTime }, (entry) => { attempt = 0 - lastTime = entry._time + if (!lastTime || compareTime(entry._time, lastTime) > 0) lastTime = entry._time pending.push(entry) timer ??= setTimeout(flush, FLUSH_MS) }, signal) @@ -88,7 +93,7 @@ export function useLogTail( controller.abort() clearTimeout(timer) } - }, [client, root, query, enabled, key, cap, retryDelayMs]) + }, [client, root, query, session, key, cap, retryDelayMs]) const current = snap.key === key ? snap : { key, entries: [], state: "connecting" as const, error: null } return { entries: current.entries, state: current.state, error: current.error } From 290490769eacf8906256290d3078bba02e4e9d04 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:47:13 +0700 Subject: [PATCH 030/174] fix(dashboard): let the reader scroll up while logs stream The viewer jumped to the newest line on every 100 ms flush, so a line could not be read on a busy application. It now follows the bottom only while the reader is already there, and says when a refused tail has stopped instead of showing it as connecting. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observability/LogLines.test.tsx | 34 +++++++++++++++++++ .../src/components/observability/LogLines.tsx | 34 +++++++++++++++++++ .../observability/LogsView.test.tsx | 12 +++++++ .../src/components/observability/LogsView.tsx | 33 +++++------------- 4 files changed, 88 insertions(+), 25 deletions(-) create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx new file mode 100644 index 000000000..775c5fcc6 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx @@ -0,0 +1,34 @@ +import { describe, it, expect } from "vitest" +import { fireEvent, render, screen } from "@testing-library/react" +import { LogLines } from "./LogLines.tsx" + +const entry = (n: number) => ({ _time: `2026-11-04T10:00:0${n}Z`, _msg: `line ${n}`, _stream_id: "s" }) + +function sized(list: HTMLElement, scrollHeight: number) { + Object.defineProperty(list, "scrollHeight", { configurable: true, value: scrollHeight }) + Object.defineProperty(list, "clientHeight", { configurable: true, value: 100 }) +} + +describe("LogLines", () => { + it("keeps following the newest line while the reader is at the bottom", () => { + const { rerender } = render() + const list = screen.getByRole("list") + sized(list, 500) + + rerender() + + expect(list.scrollTop).toBe(500) + }) + + it("leaves the position alone once the reader scrolls up", () => { + const { rerender } = render() + const list = screen.getByRole("list") + sized(list, 500) + list.scrollTop = 120 + fireEvent.scroll(list) + + rerender() + + expect(list.scrollTop).toBe(120) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx new file mode 100644 index 000000000..a6f4b5861 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx @@ -0,0 +1,34 @@ +import { useEffect, useRef } from "react" +import { CONTAINER_FIELD, POD_FIELD } from "../../lib/observability/logs.ts" +import type { LogEntry } from "../../lib/observability/proxy.ts" + +const BOTTOM_SLACK_PX = 24 + +export function LogLines({ entries }: { entries: LogEntry[] }) { + const ref = useRef(null) + const pinned = useRef(true) + + useEffect(() => { + if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight + }, [entries]) + + if (entries.length === 0) return

    No log lines for this range and filter.

    + return ( +
      { + const el = e.currentTarget + pinned.current = el.scrollHeight - el.scrollTop - el.clientHeight <= BOTTOM_SLACK_PX + }} + className="max-h-[60vh] overflow-auto rounded-md bg-slate-950 p-3 font-mono text-xs leading-5 text-slate-100" + > + {entries.map((e, i) => ( +
    1. + {new Date(e._time).toLocaleTimeString()}{" "} + {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`}{" "} + {e._msg} +
    2. + ))} +
    + ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx index fae3f6a4b..526efeb23 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -81,6 +81,18 @@ describe("LogsView", () => { expect(screen.getByRole("button", { name: "Follow" })).toHaveAttribute("aria-pressed", "true") }) + it("says the tail stopped when the proxy refuses the query", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("tail") ? new Response("pipe not allowed in tail", { status: 400 }) : defaultHandler(endpoint, params), + ) + renderWithK8sProvider(, { client }) + + await userEvent.click(screen.getByRole("button", { name: "Follow" })) + + expect(await screen.findByText(/Stopped/)).toBeInTheDocument() + expect(screen.getByText("pipe not allowed in tail")).toBeInTheDocument() + }) + it("renders a cluster without the proxy as not available", async () => { const { client } = clientWith(() => new Response("404 page not found", { status: 404 })) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx index 707b037b9..f21f2eb86 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState } from "react" +import { useState } from "react" import { useQuery } from "@tanstack/react-query" import { useK8sClient } from "@cozystack/k8s-client" import { Button, Spinner } from "@cozystack/ui" @@ -14,15 +14,16 @@ import { isScopeWide, logsRoot, queryLogs, - type LogEntry, type ObservabilityScope, } from "../../lib/observability/proxy.ts" import { useLogTail } from "../../lib/observability/use-log-tail.ts" import { ProxyNotice } from "./ProxyNotice.tsx" import { RangePicker } from "./RangePicker.tsx" +import { LogLines } from "./LogLines.tsx" const HISTORY_LIMIT = 500 const FOLLOW_CAP = 1000 +const TAIL_STATE_LABEL = { connecting: "Connecting…", live: "Live", reconnecting: "Reconnecting…", stopped: "Stopped" } function windowFor(range: LogRangeKey) { const end = Date.now() @@ -50,25 +51,6 @@ function Picker({ label, value, options, onChange }: { label: string; value: str ) } -function LogLines({ entries }: { entries: LogEntry[] }) { - const ref = useRef(null) - useEffect(() => { - if (ref.current) ref.current.scrollTop = ref.current.scrollHeight - }, [entries]) - if (entries.length === 0) return

    No log lines for this range and filter.

    - return ( -
      - {entries.map((e, i) => ( -
    1. - {new Date(e._time).toLocaleTimeString()}{" "} - {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`}{" "} - {e._msg} -
    2. - ))} -
    - ) -} - export function LogsView({ scope }: { scope: ObservabilityScope }) { const client = useK8sClient() const root = logsRoot(scope) @@ -77,7 +59,8 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) { const [filter, setFilter] = useState("") const [pod, setPod] = useState("") const [container, setContainer] = useState("") - const [follow, setFollow] = useState(false) + const [followRun, setFollowRun] = useState(null) + const follow = followRun !== null const query = buildLogsQuery({ filter, pod, container }) const pickerQuery = (field: string, narrowTo?: string) => ({ @@ -100,7 +83,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) { enabled: !follow, retry: false, }) - const tail = useLogTail(root, query, follow, { cap: FOLLOW_CAP }) + const tail = useLogTail(root, query, followRun, { cap: FOLLOW_CAP }) const wide = [pods.error, history.error, tail.error].find(isScopeWide) if (wide) { @@ -135,13 +118,13 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) { { setPod(v); setContainer("") }} /> {!follow && } - {follow && (

    - {tail.state === "live" ? "Live" : tail.state === "reconnecting" ? "Reconnecting…" : "Connecting…"} + {TAIL_STATE_LABEL[tail.state]} {` · newest ${FOLLOW_CAP} lines kept`}

    )} From b05323f6bee913256c3e4ae6eedc764689ec508c Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:47:22 +0700 Subject: [PATCH 031/174] docs(dashboard): note the observability proxy client in the console guide The next session needs to know why one module bypasses the typed Kubernetes client and where the charting dependency is loaded. Assisted-by: LLM Signed-off-by: ZverGuy --- packages/system/dashboard/images/console/CLAUDE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/system/dashboard/images/console/CLAUDE.md b/packages/system/dashboard/images/console/CLAUDE.md index 2acf80504..96419d118 100644 --- a/packages/system/dashboard/images/console/CLAUDE.md +++ b/packages/system/dashboard/images/console/CLAUDE.md @@ -32,6 +32,8 @@ Implications: - **RJSF** (`@rjsf/core` + `validator-ajv8`) for application config forms. - **Vitest + jsdom + @testing-library/react** for tests. - **Monaco** for YAML view, **noVNC** for the VM console tab. +- **uPlot** for the Metrics tab charts, lazy-loaded with that tab. +- Metrics and logs come from the observability proxy, not kube-apiserver: `lib/observability/proxy.ts` reads its native Prometheus JSON and VictoriaLogs NDJSON through `K8sClient.raw`, under the paths nginx routes to it ahead of `/apis`. ## Dev loop From 84ed3aaefd34159549534dd8ff6f7e0e3111c3c9 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:50:33 +0700 Subject: [PATCH 032/174] fix(dashboard): follow the newest line again after the log list clears A reader who scrolled up stayed unpinned across a new follow run or a new filter, so the fresh list sat at its top while lines streamed in below. An empty list now means the reader is back at the bottom. Assisted-by: LLM Signed-off-by: ZverGuy --- .../components/observability/LogLines.test.tsx | 16 ++++++++++++++++ .../src/components/observability/LogLines.tsx | 3 ++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx index 775c5fcc6..7110fe464 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx @@ -31,4 +31,20 @@ describe("LogLines", () => { expect(list.scrollTop).toBe(120) }) + + it("follows again once the list is cleared for a new run", () => { + const { rerender } = render() + const first = screen.getByRole("list") + sized(first, 500) + first.scrollTop = 120 + fireEvent.scroll(first) + + rerender() + rerender() + const second = screen.getByRole("list") + sized(second, 300) + rerender() + + expect(second.scrollTop).toBe(300) + }) }) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx index a6f4b5861..452faac72 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx @@ -9,7 +9,8 @@ export function LogLines({ entries }: { entries: LogEntry[] }) { const pinned = useRef(true) useEffect(() => { - if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight + if (entries.length === 0) pinned.current = true + else if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight }, [entries]) if (entries.length === 0) return

    No log lines for this range and filter.

    From 1b4ce7433e1acbb7d5b1d5ec6a4634fe2cd1b730 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:50:33 +0700 Subject: [PATCH 033/174] fix(dashboard): keep retrying a log tail refused for load The proxy answers a tail over its rate or concurrent-tail cap with a 4xx, and a reconnect can arrive before the previous tail's slot is freed. Treating that as final left Follow stopped for good, while the existing backoff already waits long enough for the cap to clear. Assisted-by: LLM Signed-off-by: ZverGuy --- .../console/src/lib/observability/use-log-tail.test.tsx | 8 ++++++++ .../apps/console/src/lib/observability/use-log-tail.ts | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx index 46df47970..794f30216 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx @@ -67,6 +67,14 @@ describe("useLogTail", () => { expect(new URL(raw.mock.calls[1][0], "http://x").searchParams.get("since")).toBe(T3) }) + it("retries a tail refused for load instead of stopping", async () => { + const { wrapper } = setup(new Response("too many concurrent requests", { status: 422 }), ndjson(line(T1, "a"))) + + const { result } = renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper }) + + await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a"])) + }) + it("does not connect while disabled", () => { const { raw, wrapper } = setup() diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts index ddc835821..292f93e9e 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts @@ -14,7 +14,8 @@ interface TailSnapshot { const FLUSH_MS = 100 const MAX_RETRY_MS = 30_000 -const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forbidden", "not-available", "limit"]) +// A 422 or 429 on tail is load (rate or concurrent-tail caps), which backoff outlasts. +const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forbidden", "not-available"]) const sleep = (ms: number, signal: AbortSignal) => new Promise((resolve) => { From ebdd74ce4e010cc347d1ddae312b2c20dfeb39fa Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:53:23 +0700 Subject: [PATCH 034/174] fix(dashboard): name the proxy's real load status for a refused tail The proxy caps rate and concurrent tails with 429, so the retry reasoning and its test now say so. Leaving Follow also remounts the log list, so the history it falls back to opens at the newest line even when the reader had scrolled up while following. Assisted-by: LLM Signed-off-by: ZverGuy --- .../apps/console/src/components/observability/LogsView.tsx | 2 +- .../apps/console/src/lib/observability/use-log-tail.test.tsx | 2 +- .../console/apps/console/src/lib/observability/use-log-tail.ts | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx index f21f2eb86..a27ca41c5 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -129,7 +129,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {

    )} {error ? : null} - {!follow && history.isLoading ? : } + {!follow && history.isLoading ? : } ) } diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx index 794f30216..0c2306613 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx @@ -68,7 +68,7 @@ describe("useLogTail", () => { }) it("retries a tail refused for load instead of stopping", async () => { - const { wrapper } = setup(new Response("too many concurrent requests", { status: 422 }), ndjson(line(T1, "a"))) + const { wrapper } = setup(new Response("too many concurrent requests", { status: 429 }), ndjson(line(T1, "a"))) const { result } = renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper }) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts index 292f93e9e..53cbaf856 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts @@ -14,7 +14,8 @@ interface TailSnapshot { const FLUSH_MS = 100 const MAX_RETRY_MS = 30_000 -// A 422 or 429 on tail is load (rate or concurrent-tail caps), which backoff outlasts. +// Load refusals retry: the proxy caps rate and concurrent tails with 429 (422 in +// its first builds), and a reconnect can race the previous tail's freed slot. const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forbidden", "not-available"]) const sleep = (ms: number, signal: AbortSignal) => From 2b823ab79fa178f22b338ad654962bc9c334a30e Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:53:50 +0700 Subject: [PATCH 035/174] fix(dashboard): rate over at least 5m and four steps Long metric ranges are served by the longterm store, which keeps one sample per 5m, so a rate window narrower than that can hold no samples and the chart comes back empty. Four steps also keep every sample between two points inside some window. Assisted-by: LLM Signed-off-by: ZverGuy --- .../src/lib/observability/metrics.test.ts | 17 +++++++++++++---- .../console/src/lib/observability/metrics.ts | 6 +++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts index 0803940bf..0598c0c7f 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts @@ -32,10 +32,19 @@ describe("METRIC_PANELS", () => { ) }) - it("widens the rate window with the step on long ranges", () => { - const [query] = METRIC_PANELS.find((p) => p.id === "cpu")!.queries({ start: 0, end: 1, step: 2520 }) - - expect(query.expr).toContain("[5040s]") + // Long ranges are served by the longterm store, deduplicated at 5m, where a + // narrower window can hold no samples and the rate comes back empty. + it.each([ + [15, "300s"], + [75, "300s"], + [90, "360s"], + [2520, "10080s"], + ])("rates over max(5m, 4 x step) for step %is", (step, window) => { + for (const panel of METRIC_PANELS) { + for (const q of panel.queries({ start: 0, end: 1, step })) { + if (q.expr.includes("rate(")) expect(q.expr).toContain(`[${window}]`) + } + } }) it("reads memory as working set per pod", () => { diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts index b23a51e27..d7182b8d6 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts @@ -23,10 +23,10 @@ export function rangeWindow(range: RangeKey, nowSeconds: number): RangeWindow { return { start: nowSeconds - span, end: nowSeconds, step: Math.max(MIN_STEP, Math.round(span / POINTS)) } } -// A rate needs several scrapes inside its window, and on long ranges the window -// must also cover the step or samples between points are skipped. +// The longterm store keeps one sample per 5m, so a narrower window can be empty; +// 4 x step also keeps samples between points from being skipped. function rateWindow(step: number): string { - return `${Math.max(300, step * 2)}s` + return `${Math.max(300, step * 4)}s` } const CONTAINERS = 'container!="",container!="POD"' From 3b1302ba8e8d44a5294b28302a775f805bd40e2a Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 18:45:36 +0700 Subject: [PATCH 036/174] feat(dashboard): add an accessible Select to the console The console has no select primitive: form widgets fall back to the native element, and the shared Dropdown has no listbox roles, no keyboard navigation and no label association. Base UI, already a dependency, ships one; this wraps it the same way dialog.tsx wraps the Base UI dialog so pickers get a labelled, keyboard-driven listbox that matches the console's buttons. Assisted-by: LLM Signed-off-by: ZverGuy --- .../console/src/components/ui/select.test.tsx | 64 +++++++++++ .../apps/console/src/components/ui/select.tsx | 107 ++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx new file mode 100644 index 000000000..fb5b84e6f --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx @@ -0,0 +1,64 @@ +import { describe, it, expect, vi } from "vitest" +import { render, screen } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { SelectItem, SelectLabel, SelectPopup, SelectRoot, SelectTrigger, SelectValue } from "./select.tsx" + +const items = [ + { value: "", label: "All" }, + { value: "db-0", label: "db-0" }, + { value: "db-1", label: "db-1" }, +] + +function PodSelect({ value = "", onValueChange = () => {} }: { value?: string; onValueChange?: (v: string) => void }) { + return ( + onValueChange(v ?? "")}> + Pod + + + + + {items.map((i) => ( + + {i.label} + + ))} + + + ) +} + +describe("Select", () => { + it("names the trigger by its label and shows the chosen item", () => { + render() + + expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("db-1") + }) + + it("shows the label of an empty-string value", () => { + render() + + expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("All") + }) + + it("lists the items and reports the one clicked", async () => { + const onValueChange = vi.fn() + render() + + await userEvent.click(screen.getByRole("combobox", { name: "Pod" })) + await userEvent.click(await screen.findByRole("option", { name: "db-1" })) + + expect(onValueChange).toHaveBeenCalledWith("db-1") + }) + + it("can be driven from the keyboard", async () => { + const onValueChange = vi.fn() + render() + + screen.getByRole("combobox", { name: "Pod" }).focus() + await userEvent.keyboard("{ArrowDown}") + expect(await screen.findByRole("listbox")).toBeInTheDocument() + await userEvent.keyboard("{ArrowDown}{Enter}") + + expect(onValueChange).toHaveBeenCalledWith("db-0") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx new file mode 100644 index 000000000..f256dba45 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx @@ -0,0 +1,107 @@ +import { Select as SelectPrimitive } from "@base-ui/react/select" +import { Check, ChevronDown } from "lucide-react" +import { cn } from "@cozystack/ui" + +const SelectRoot = SelectPrimitive.Root + +function SelectLabel({ + className, + ...props +}: SelectPrimitive.Label.Props) { + return ( + + ) +} + +function SelectTrigger({ + className, + children, + ...props +}: SelectPrimitive.Trigger.Props) { + return ( + + {children} + + + + + ) +} + +function SelectValue({ + className, + ...props +}: SelectPrimitive.Value.Props) { + return ( + + ) +} + +function SelectPopup({ + className, + children, + ...props +}: SelectPrimitive.Popup.Props) { + return ( + + + + {children} + + + + ) +} + +function SelectItem({ + className, + children, + ...props +}: SelectPrimitive.Item.Props) { + return ( + + + + + {children} + + ) +} + +export { + SelectRoot, + SelectLabel, + SelectTrigger, + SelectValue, + SelectPopup, + SelectItem, +} From f500aa955ac65384b527bf61167e77bf5dd02a3e Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 18:46:15 +0700 Subject: [PATCH 037/174] feat(dashboard): add a text Input matching the console select Every text field in the console restyles a bare input by hand, each a little differently. A toolbar that mixes a text filter with the new Select needs the two at the same height, border and focus ring, so the styling lives in one place next to the Select instead of being copied. Assisted-by: LLM Signed-off-by: ZverGuy --- .../console/src/components/ui/input.test.tsx | 21 +++++++++++++++++++ .../apps/console/src/components/ui/input.tsx | 17 +++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx new file mode 100644 index 000000000..e65a70a3a --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx @@ -0,0 +1,21 @@ +import { describe, it, expect, vi } from "vitest" +import { render, screen } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { Input } from "./input.tsx" + +describe("Input", () => { + it("is a labelled text box that reports what is typed", async () => { + const onValueChange = vi.fn() + render() + + await userEvent.type(screen.getByRole("textbox", { name: "Filter" }), "x") + + expect(onValueChange).toHaveBeenCalledWith("x", expect.anything()) + }) + + it("keeps the caller's classes next to its own", () => { + render() + + expect(screen.getByRole("textbox", { name: "Filter" })).toHaveClass("font-mono", "rounded-lg") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx new file mode 100644 index 000000000..b079e6b35 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx @@ -0,0 +1,17 @@ +import { Input as InputPrimitive } from "@base-ui/react/input" +import { cn } from "@cozystack/ui" + +function Input({ className, ...props }: InputPrimitive.Props) { + return ( + + ) +} + +export { Input } From b88ad09678a234ff351173a1cc0674b9721d795f Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 18:47:25 +0700 Subject: [PATCH 038/174] fix(dashboard): keep the log console on screen when it has no lines With no matching lines, while loading, and while Follow waits for the first line, the dark log console vanished and left a line of small grey text or a bare spinner, so the page jumped and an empty result looked like a broken view. The console frame now keeps a fixed height in every state and says inside it what is going on. The pod and container pickers move to the console Select and the filter to the console Input, so the toolbar is keyboard and screen reader friendly and a long pod name truncates instead of widening it. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observability/EmptyState.test.tsx | 17 +++++ .../components/observability/EmptyState.tsx | 25 ++++++++ .../observability/LogLines.test.tsx | 35 ++++++++--- .../src/components/observability/LogLines.tsx | 44 +++++++------ .../observability/LogsView.test.tsx | 50 ++++++++++++++- .../src/components/observability/LogsView.tsx | 62 +++++++++++++------ 6 files changed, 184 insertions(+), 49 deletions(-) create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx new file mode 100644 index 000000000..8872f4d43 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx @@ -0,0 +1,17 @@ +import { describe, it, expect } from "vitest" +import { render, screen } from "@testing-library/react" +import { EmptyState } from "./EmptyState.tsx" + +describe("EmptyState", () => { + it("announces its message as a status", () => { + render() + + expect(screen.getByRole("status")).toHaveTextContent("No data for this range.Pick a longer range.") + }) + + it("marks itself busy while something is loading", () => { + render() + + expect(screen.getByRole("status")).toHaveAttribute("aria-busy", "true") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx new file mode 100644 index 000000000..0cf58224b --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx @@ -0,0 +1,25 @@ +import type { ReactNode } from "react" +import { cn, Spinner } from "@cozystack/ui" + +interface EmptyStateProps { + title: ReactNode + hint?: ReactNode + busy?: boolean + tone?: "light" | "dark" + className?: string +} + +export function EmptyState({ title, hint, busy, tone = "light", className }: EmptyStateProps) { + const dark = tone === "dark" + return ( +
    + {busy && } +

    {title}

    + {hint &&

    {hint}

    } +
    + ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx index 7110fe464..dd8c503eb 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx @@ -10,40 +10,59 @@ function sized(list: HTMLElement, scrollHeight: number) { } describe("LogLines", () => { + it("keeps the console frame and shows the placeholder inside it when there are no lines", () => { + render() + + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(frame).toHaveTextContent("Waiting for new log lines…") + expect(screen.queryByRole("list")).toBeNull() + }) + + it("puts the lines in the same frame once they arrive", () => { + const { rerender } = render() + const frame = screen.getByRole("region", { name: "Log lines" }) + + rerender() + + expect(screen.getByRole("region", { name: "Log lines" })).toBe(frame) + expect(frame).toContainElement(screen.getByRole("list")) + expect(frame).not.toHaveTextContent("Waiting") + }) + it("keeps following the newest line while the reader is at the bottom", () => { - const { rerender } = render() + const { rerender } = render() const list = screen.getByRole("list") sized(list, 500) - rerender() + rerender() expect(list.scrollTop).toBe(500) }) it("leaves the position alone once the reader scrolls up", () => { - const { rerender } = render() + const { rerender } = render() const list = screen.getByRole("list") sized(list, 500) list.scrollTop = 120 fireEvent.scroll(list) - rerender() + rerender() expect(list.scrollTop).toBe(120) }) it("follows again once the list is cleared for a new run", () => { - const { rerender } = render() + const { rerender } = render() const first = screen.getByRole("list") sized(first, 500) first.scrollTop = 120 fireEvent.scroll(first) - rerender() - rerender() + rerender() + rerender() const second = screen.getByRole("list") sized(second, 300) - rerender() + rerender() expect(second.scrollTop).toBe(300) }) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx index 452faac72..14b941ab2 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx @@ -1,10 +1,10 @@ -import { useEffect, useRef } from "react" +import { useEffect, useRef, type ReactNode } from "react" import { CONTAINER_FIELD, POD_FIELD } from "../../lib/observability/logs.ts" import type { LogEntry } from "../../lib/observability/proxy.ts" const BOTTOM_SLACK_PX = 24 -export function LogLines({ entries }: { entries: LogEntry[] }) { +export function LogLines({ entries, placeholder }: { entries: LogEntry[]; placeholder: ReactNode }) { const ref = useRef(null) const pinned = useRef(true) @@ -13,23 +13,31 @@ export function LogLines({ entries }: { entries: LogEntry[] }) { else if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight }, [entries]) - if (entries.length === 0) return

    No log lines for this range and filter.

    return ( -
      { - const el = e.currentTarget - pinned.current = el.scrollHeight - el.scrollTop - el.clientHeight <= BOTTOM_SLACK_PX - }} - className="max-h-[60vh] overflow-auto rounded-md bg-slate-950 p-3 font-mono text-xs leading-5 text-slate-100" +
      - {entries.map((e, i) => ( -
    1. - {new Date(e._time).toLocaleTimeString()}{" "} - {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`}{" "} - {e._msg} -
    2. - ))} -
    + {entries.length === 0 ? ( + placeholder + ) : ( +
      { + const el = e.currentTarget + pinned.current = el.scrollHeight - el.scrollTop - el.clientHeight <= BOTTOM_SLACK_PX + }} + className="h-full overflow-auto p-3" + > + {entries.map((e, i) => ( +
    1. + {new Date(e._time).toLocaleTimeString()}{" "} + {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`}{" "} + {e._msg} +
    2. + ))} +
    + )} + ) } diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx index 526efeb23..31c6fac0e 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -1,5 +1,5 @@ import { describe, it, expect, vi } from "vitest" -import { screen, waitFor } from "@testing-library/react" +import { screen, waitFor, within } from "@testing-library/react" import userEvent from "@testing-library/user-event" import { K8sClient } from "@cozystack/k8s-client" import { renderWithK8sProvider } from "../../test-utils/render.tsx" @@ -47,9 +47,9 @@ describe("LogsView", () => { it("narrows to the picked pod with a stream filter", async () => { const { client, raw } = clientWith(defaultHandler) renderWithK8sProvider(, { client }) - await screen.findByRole("option", { name: "db-1" }) - await userEvent.selectOptions(screen.getByLabelText("Pod"), "db-1") + await userEvent.click(screen.getByRole("combobox", { name: "Pod" })) + await userEvent.click(await screen.findByRole("option", { name: "db-1" })) await waitFor(() => expect( @@ -58,6 +58,50 @@ describe("LogsView", () => { ) }) + it("offers All next to every pod and starts on it", async () => { + const { client } = clientWith(defaultHandler) + renderWithK8sProvider(, { client }) + const pod = screen.getByRole("combobox", { name: "Pod" }) + expect(pod).toHaveTextContent("All") + + await userEvent.click(pod) + + expect(await screen.findByRole("option", { name: "db-1" })).toBeInTheDocument() + expect(screen.getAllByRole("option").map((o) => o.textContent)).toEqual(["All", "db-0", "db-1"]) + }) + + it("keeps the log console and says so inside it when nothing matches", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") ? new Response("") : defaultHandler(endpoint, params), + ) + + renderWithK8sProvider(, { client }) + + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(await within(frame).findByText("No log lines for this range and filter.")).toBeInTheDocument() + }) + + it("shows loading inside the log console", () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") ? new Promise(() => {}) : defaultHandler(endpoint, params), + ) + + renderWithK8sProvider(, { client }) + + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(within(frame).getByRole("status")).toHaveAttribute("aria-busy", "true") + }) + + it("says it is waiting while Follow has no line yet", async () => { + const { client } = clientWith(defaultHandler) + renderWithK8sProvider(, { client }) + + await userEvent.click(screen.getByRole("button", { name: "Follow" })) + + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(within(frame).getByText("Waiting for new log lines…")).toBeInTheDocument() + }) + it("applies the LogsQL filter on submit", async () => { const { client, raw } = clientWith(defaultHandler) renderWithK8sProvider(, { client }) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx index a27ca41c5..d2a000010 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -1,7 +1,9 @@ import { useState } from "react" import { useQuery } from "@tanstack/react-query" import { useK8sClient } from "@cozystack/k8s-client" -import { Button, Spinner } from "@cozystack/ui" +import { Button } from "@cozystack/ui" +import { Input } from "../ui/input.tsx" +import { SelectItem, SelectLabel, SelectPopup, SelectRoot, SelectTrigger, SelectValue } from "../ui/select.tsx" import { CONTAINER_FIELD, LOG_RANGES, @@ -20,6 +22,7 @@ import { useLogTail } from "../../lib/observability/use-log-tail.ts" import { ProxyNotice } from "./ProxyNotice.tsx" import { RangePicker } from "./RangePicker.tsx" import { LogLines } from "./LogLines.tsx" +import { EmptyState } from "./EmptyState.tsx" const HISTORY_LIMIT = 500 const FOLLOW_CAP = 1000 @@ -30,27 +33,42 @@ function windowFor(range: LogRangeKey) { return { start: new Date(end - LOG_RANGES[range] * 1000).toISOString(), end: new Date(end).toISOString() } } +const ALL = "" + function Picker({ label, value, options, onChange }: { label: string; value: string; options: string[]; onChange: (v: string) => void }) { + const items = [{ value: ALL, label: "All" }, ...options.map((o) => ({ value: o, label: o }))] return ( - + + ) } +function placeholderFor({ follow, stopped, loading, failed }: { follow: boolean; stopped: boolean; loading: boolean; failed: boolean }) { + if (follow) { + return stopped ? ( + + ) : ( + + ) + } + if (loading) return + if (failed) return + return +} + export function LogsView({ scope }: { scope: ObservabilityScope }) { const client = useK8sClient() const root = logsRoot(scope) @@ -107,12 +125,12 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) { setFilter(draft) }} > - setDraft(e.target.value)} + onValueChange={setDraft} placeholder='LogsQL filter, e.g. error or "connection refused"' - className="w-full rounded-md border border-slate-200 px-2.5 py-1.5 font-mono text-xs" + className="font-mono" /> { setPod(v); setContainer("") }} /> @@ -129,7 +147,11 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {

    )} {error ? : null} - {!follow && history.isLoading ? : } + ) } From f5ebb43c4bdce908887029568065eb27785ff833 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 18:48:43 +0700 Subject: [PATCH 039/174] fix(dashboard): match the range picker to the log toolbar controls Next to the new Select and Input the range picker was shorter, lighter bordered and had no focus ring, so the toolbar looked uneven and a keyboard user could not see which range had focus. It now shares their height, border and focus ring, and its buttons are grouped under one accessible name. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observability/RangePicker.test.tsx | 23 +++++++++++++++++++ .../components/observability/RangePicker.tsx | 4 ++-- 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx new file mode 100644 index 000000000..9787963b2 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx @@ -0,0 +1,23 @@ +import { describe, it, expect, vi } from "vitest" +import { render, screen, within } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { RangePicker } from "./RangePicker.tsx" + +describe("RangePicker", () => { + it("groups the ranges under one name and marks the chosen one pressed", () => { + render( {}} />) + + const group = screen.getByRole("group", { name: "Time range" }) + expect(within(group).getByRole("button", { name: "6h" })).toHaveAttribute("aria-pressed", "true") + expect(within(group).getByRole("button", { name: "1h" })).toHaveAttribute("aria-pressed", "false") + }) + + it("reports the range clicked", async () => { + const onChange = vi.fn() + render() + + await userEvent.click(screen.getByRole("button", { name: "1h" })) + + expect(onChange).toHaveBeenCalledWith("1h") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx index 22030c706..95d245176 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx @@ -10,7 +10,7 @@ export function RangePicker({ onChange: (value: K) => void }) { return ( -
    +
    {options.map((key) => ( @@ -150,7 +153,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) {
    ) From 9498c9bff520d73fe200f5043bc7efce9eb901ef Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 19:05:42 +0700 Subject: [PATCH 046/174] fix(dashboard): keep the proxy's own advice on a limit refusal The notice appended "Choose a shorter range." to every 422, so a message that already ended in "use a shorter range" said it twice and one that ended in "raise step" got advice that contradicted it. The console's own advice is now added only when the proxy gave none. Assisted-by: LLM Signed-off-by: ZverGuy --- .../src/components/observability/ProxyNotice.test.tsx | 9 +++++++++ .../console/src/components/observability/ProxyNotice.tsx | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx index fd001bcae..6e85ced33 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx @@ -29,6 +29,15 @@ describe("ProxyNotice", () => { expect(screen.getByRole("alert")).toHaveTextContent(text) }) + it.each([ + "the application has more than 50 pods in this window; use a shorter range", + "1200 points per series exceed the limit of 1000; raise step", + ])("does not add advice to a limit message that already carries its own: %s", (message) => { + render() + + expect(screen.getByRole("alert").textContent).toBe(`${message}.`) + }) + it("shows the server message for a rejected query", () => { render() diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx index a60b831b0..ee68e4633 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx @@ -20,7 +20,8 @@ function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: bool case "forbidden": return { quiet: false, text: `You do not have permission to read ${lower} here.` } case "limit": - return { quiet: false, text: `${error.message}. Choose a shorter range.` } + // The proxy puts its own remedy after "; " (e.g. "use a shorter range", "raise step"). + return { quiet: false, text: error.message.includes("; ") ? `${error.message}.` : `${error.message}. Choose a shorter range.` } case "rate-limited": return { quiet: false, text: "Too many requests. Try again in a moment." } case "timeout": From 76441871b8c3ae73f40ecf29fd454a468f1e0573 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 19:06:59 +0700 Subject: [PATCH 047/174] fix(dashboard): keep caller headers of any shape in the k8s client The client spread init.headers as a plain object, so a Headers instance or a list of pairs passed by a caller was silently dropped and the request went out without them. Headers are now built with the Headers constructor, which accepts every shape fetch does. Assisted-by: LLM Signed-off-by: ZverGuy --- .../k8s-client/client.headers.test.ts | 31 +++++++++++++++++++ .../__tests__/k8s-client/client.raw.test.ts | 20 +++++++++++- .../console/packages/k8s-client/src/client.ts | 14 +++------ 3 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts diff --git a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts new file mode 100644 index 000000000..37985fca5 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts @@ -0,0 +1,31 @@ +import { describe, it, expect, vi, afterEach } from "vitest" +import { K8sClient } from "@cozystack/k8s-client" + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe("K8sClient request headers", () => { + it("lets a method's own content type replace the JSON default", async () => { + const fetchMock = vi.fn(async () => new Response("{}")) + vi.stubGlobal("fetch", fetchMock) + const client = new K8sClient({ getToken: async () => "t0k" }) + + await client.patch("apps", "v1", "deployments", "web", { spec: {} }, "default", "merge") + + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + const headers = new Headers(init.headers) + expect(headers.get("Content-Type")).toBe("application/merge-patch+json") + expect(headers.get("Authorization")).toBe("Bearer t0k") + }) + + it("sends JSON by default", async () => { + const fetchMock = vi.fn(async () => new Response("{}")) + vi.stubGlobal("fetch", fetchMock) + + await new K8sClient().getApiGroups() + + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(new Headers(init.headers).get("Content-Type")).toBe("application/json") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts index ab6c056f5..0ac0d75ae 100644 --- a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts +++ b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts @@ -24,7 +24,25 @@ describe("K8sClient.raw", () => { const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] expect(url).toBe("/base/apis/x") - expect(init.headers).toEqual({ Accept: "application/json", Authorization: "Bearer t0k" }) + const headers = new Headers(init.headers) + expect(headers.get("Accept")).toBe("application/json") + expect(headers.get("Authorization")).toBe("Bearer t0k") + }) + + it.each([ + ["a Headers instance", new Headers({ Accept: "application/x-ndjson" })], + ["a tuple list", [["Accept", "application/x-ndjson"]] as [string, string][]], + ])("keeps caller headers passed as %s", async (_, given) => { + const fetchMock = vi.fn(async () => new Response("{}")) + vi.stubGlobal("fetch", fetchMock) + const client = new K8sClient({ getToken: async () => "t0k" }) + + await client.raw("/x", { headers: given }) + + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + const headers = new Headers(init.headers) + expect(headers.get("Accept")).toBe("application/x-ndjson") + expect(headers.get("Authorization")).toBe("Bearer t0k") }) it("routes a 401 through the client's unauthorized handler", async () => { diff --git a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts index 04c528760..64af1131d 100644 --- a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts +++ b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts @@ -50,14 +50,12 @@ export class K8sClient { } private async request(path: string, init?: RequestInit): Promise { - const headers: Record = { - "Content-Type": "application/json", - ...(init?.headers as Record), - } + const headers = new Headers(init?.headers) + if (!headers.has("Content-Type")) headers.set("Content-Type", "application/json") if (this.getToken) { const token = await this.getToken() - headers["Authorization"] = `Bearer ${token}` + headers.set("Authorization", `Bearer ${token}`) } let res: Response @@ -247,11 +245,9 @@ export class K8sClient { * Kubernetes JSON, such as the observability proxy's streamed NDJSON. */ async raw(path: string, init?: RequestInit): Promise { - const headers: Record = { - ...(init?.headers as Record), - } + const headers = new Headers(init?.headers) if (this.getToken) { - headers["Authorization"] = `Bearer ${await this.getToken()}` + headers.set("Authorization", `Bearer ${await this.getToken()}`) } const res = await fetch(`${this.baseUrl}${path}`, { ...init, headers }) if (res.status === 401) this.handleUnauthorized() From 2a473086cf6cd1d245e014ed56ed44f9594038cd Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 19:11:05 +0700 Subject: [PATCH 048/174] fix(dashboard): advise a shorter range unless the proxy already did Any "; " in a 422 message was taken as the proxy's own remedy, but the proxy passes vmselect's 422 bodies through, and their advice (raise the -search.max* flags) is for operators. Those lost the one thing a tenant can do. Only the remedies the proxy itself appends now replace the console's advice. Assisted-by: LLM Signed-off-by: ZverGuy --- .../src/components/observability/ProxyNotice.test.tsx | 7 +++++++ .../src/components/observability/ProxyNotice.tsx | 10 ++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx index 6e85ced33..c4b4e89c1 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx @@ -38,6 +38,13 @@ describe("ProxyNotice", () => { expect(screen.getByRole("alert").textContent).toBe(`${message}.`) }) + it("still advises a shorter range for a store limit whose own advice is for operators", () => { + const message = "the number of matching timeseries exceeds 300000; either narrow down the search or increase -search.max* command-line flag values" + render() + + expect(screen.getByRole("alert").textContent).toBe(`${message}. Choose a shorter range.`) + }) + it("shows the server message for a rejected query", () => { render() diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx index ee68e4633..1fb30c8ff 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx @@ -5,6 +5,10 @@ interface ProxyNoticeProps { error: unknown } +// Remedies the observability proxy itself appends; vmselect's 422 text carries +// advice for operators (raise -search.max* flags), so it still gets ours. +const PROXY_REMEDIES = ["; use a shorter range", "; raise step"] + function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: boolean; text: string } { const lower = what.toLowerCase() if (!(error instanceof ProxyError)) { @@ -20,8 +24,10 @@ function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: bool case "forbidden": return { quiet: false, text: `You do not have permission to read ${lower} here.` } case "limit": - // The proxy puts its own remedy after "; " (e.g. "use a shorter range", "raise step"). - return { quiet: false, text: error.message.includes("; ") ? `${error.message}.` : `${error.message}. Choose a shorter range.` } + return { + quiet: false, + text: PROXY_REMEDIES.some((r) => error.message.endsWith(r)) ? `${error.message}.` : `${error.message}. Choose a shorter range.`, + } case "rate-limited": return { quiet: false, text: "Too many requests. Try again in a moment." } case "timeout": From a042f335900b0a2340d27d3a235bb7ea2748c0ec Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 4 Nov 2026 15:49:35 +0000 Subject: [PATCH 049/174] fix(mariadb): record the dump a Backup was taken as Every run of a MariaDB application writes its dump under the same storage prefix, and the Backup did not say which of those dumps is its own. A restore had nothing to give mariadb-operator but the operator Backup, which resolves to the newest dump under the prefix. The Backup now records the dump's file name and the timestamp in that name in driverMetadata, taken from the match the size probe already makes. Nothing is recorded when that match cannot prove the dump belongs to the operator Backup, so a Backup never names another run's dump. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- .../backupcontroller/mariadb_artifact_size.go | 16 ++-- .../mariadb_artifact_size_test.go | 95 ++++++++++++++++--- .../mariadbstrategy_controller.go | 14 ++- 3 files changed, 104 insertions(+), 21 deletions(-) diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index 6397325ca..950303d78 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -39,19 +39,19 @@ type mariadbDump struct { LastModified time.Time } -// mariadbArtifactSizeBytes measures the dump mariadb-operator uploaded for -// mdbBackup. Returns 0 - leaving Artifact.SizeBytes absent - when the dump -// cannot be found or told apart: an unknown size must not be reported as a -// measured one. -func (r *BackupJobReconciler) mariadbArtifactSizeBytes(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) int64 { +// attributedMariaDBDump returns the dump mariadb-operator uploaded for +// mdbBackup. ok is false when the dump cannot be found or told apart, and +// the Backup then reports neither a size nor a dump: an unknown dump must +// not be reported as a measured one. +func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (dump mariadbDump, ok bool) { ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout) defer cancel() dump, err := findMariaDBDump(ctx, r.objectStore(), mdbBackup, completedAt) if err != nil { - getLogger(ctx).Info("MariaDB backup size not measured; leaving it unset", "backup", mdbBackup.Name, "error", err) - return 0 + getLogger(ctx).Info("MariaDB dump not attributed; leaving its size and name unset", "backup", mdbBackup.Name, "error", err) + return mariadbDump{}, false } - return dump.Size + return dump, true } // findMariaDBDump lists the bucket mdbBackup wrote to and returns the one diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index 9d649c15c..80e9e4617 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -29,6 +29,14 @@ func dumpKey(prefix string, ts time.Time, compression string) string { return fmt.Sprintf("%sbackup.%s.%s.sql", prefix, ts.UTC().Format(time.RFC3339), compression) } +// probedSize is the size a Backup reports for a probe result. +func probedSize(dump mariadbDump, attributed bool) int64 { + if !attributed { + return 0 + } + return dump.Size +} + // mariadbBackupSucceeded is the Complete condition mariadb-operator sets on a // Backup whose Job succeeded. A failed Job also gets Status=True, with // Reason=JobFailed, so a fixture without the reason is not a success. @@ -105,7 +113,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)}, {Key: prefix + "0-backup-target.txt", Size: 7}, }} - got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed) + got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)) if got != 4242 { t.Errorf("size = %d, want 4242", got) } @@ -130,7 +138,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { } l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 3, LastModified: created.Add(time.Minute)}}} r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), ca) - if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed); got != 3 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed)); got != 3 { t.Errorf("size = %d, want 3", got) } if !l.conn.Secure || string(l.conn.CAPEM) != "PEM" { @@ -148,7 +156,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { } l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4, LastModified: created.Add(time.Minute)}}} r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), sts) - if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed); got != 4 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed)); got != 4 { t.Errorf("size = %d, want 4", got) } if l.conn.SessionToken != "TOKEN" { @@ -160,7 +168,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9, LastModified: created.Add(time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 9 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 9 { t.Errorf("size = %d, want 9", got) } }) @@ -173,7 +181,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(-4*time.Minute), "gzip"), Size: 7, LastModified: created.Add(time.Minute)}, {Key: dumpKey(prefix, created.Add(30*time.Second), "gzip"), Size: 666, LastModified: created.Add(-4 * time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0: no dump is inside the window on both clocks", got) } }) @@ -182,7 +190,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, completed.Add(30*time.Second), "gzip"), Size: 5, LastModified: completed.Add(30 * time.Second)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 5 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 5 { t.Errorf("size = %d, want 5", got) } }) @@ -191,7 +199,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds")) r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } withShortProbeTimeout(t, func() { - if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -202,7 +210,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1, LastModified: created.Add(time.Minute)}, {Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2, LastModified: created.Add(2 * time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0 (ambiguous)", got) } }) @@ -211,14 +219,14 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1, LastModified: completed.Add(time.Hour)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) t.Run("list error leaves size unset", func(t *testing.T) { l := &fakeS3Lister{err: errors.New("AccessDenied")} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -226,7 +234,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { t.Run("missing credentials Secret leaves size unset", func(t *testing.T) { c := newMariaDBStrategyTestClient(t) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} - if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -481,7 +489,7 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { s3.Prefix = "tenant/db/" s3.Region = "us-east-1" - if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute)); got != 31337 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute))); got != 31337 { t.Fatalf("size = %d, want 31337", got) } if gotPath != "/bkt/" || gotPrefix != "tenant/db/" || gotDelimiter != "/" { @@ -523,6 +531,69 @@ func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { } } +// TestCreateMariaDBBackupArtifact_RecordsDump pins that the Backup names +// its dump only when the probe attributed one to the operator Backup. +func TestCreateMariaDBBackupArtifact_RecordsDump(t *testing.T) { + apps := mariadbapp.GroupName + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + completed := metav1.NewTime(created.Add(3 * time.Minute)) + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "src/" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + own := s3ObjectInfo{Key: "src/backup.2026-10-21T10:01:00Z.gzip.sql", Size: 2048, LastModified: created.Add(2 * time.Minute)} + second := s3ObjectInfo{Key: "src/backup.2026-10-21T10:02:00Z.gzip.sql", Size: 1, LastModified: created.Add(2 * time.Minute)} + offset := s3ObjectInfo{Key: "src/backup.2026-10-21T14:01:00+04:00.sql", Size: 7, LastModified: created.Add(2 * time.Minute)} + + cases := []struct { + name string + lister *fakeS3Lister + wantFile, wantTime string + }{ + {name: "an attributed dump is recorded", lister: &fakeS3Lister{objects: []s3ObjectInfo{own}}, + wantFile: "backup.2026-10-21T10:01:00Z.gzip.sql", wantTime: "2026-10-21T10:01:00Z"}, + {name: "a dump named with an offset is timed in UTC", lister: &fakeS3Lister{objects: []s3ObjectInfo{offset}}, + wantFile: "backup.2026-10-21T14:01:00+04:00.sql", wantTime: "2026-10-21T10:01:00Z"}, + {name: "two dumps in the lifetime record none", lister: &fakeS3Lister{objects: []s3ObjectInfo{own, second}}}, + {name: "no dump in the lifetime records none", lister: &fakeS3Lister{}}, + {name: "a failed listing records none", lister: &fakeS3Lister{err: errors.New("AccessDenied")}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mdbBackup := operatorBackupStoringTo(t, rendered.Storage.S3, created) + mdbBackup.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(completed)} + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-dump"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + r := reconcilerWithLister(t, tc.lister, job, s3CredsSecret("tenant", "creds")) + if _, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, rendered); err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + persisted := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "bj-dump"}, persisted); err != nil { + t.Fatalf("get Backup: %v", err) + } + md := persisted.Spec.DriverMetadata + file, hasFile := md[mariadbDumpFileKey] + at, hasTime := md[mariadbDumpTimeKey] + if tc.wantFile == "" { + if hasFile || hasTime { + t.Errorf("driverMetadata = %v; want no dump recorded", md) + } + return + } + if file != tc.wantFile || at != tc.wantTime { + t.Errorf("recorded dump %q at %q; want %q at %q", file, at, tc.wantFile, tc.wantTime) + } + }) + } +} + // TestCreateMariaDBBackupArtifact_StrategyEditedMidBackup pins that the // probe reads where the operator Backup wrote its dump, not where the // strategy, edited while the backup ran, now points. diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 4d88ab8a1..50c871db3 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -47,6 +47,14 @@ const ( mariadbBackupNamespaceKey = "k8s.mariadb.com/backup-namespace" mariadbStorageURIKey = "k8s.mariadb.com/storage-uri" + // The dump the operator Backup wrote: its file name under the storage + // prefix, and the RFC3339 timestamp in that name. Every run of an + // application writes under the same prefix, so only this name tells its + // dump from the others. Recorded only when findMariaDBDump proved the + // dump belongs to the operator Backup. + mariadbDumpFileKey = "k8s.mariadb.com/dump-file" + mariadbDumpTimeKey = "k8s.mariadb.com/dump-time" + // Polling cadence for the operator Backup/Restore lifecycle. mariadbPollInterval = 5 * time.Second @@ -398,7 +406,11 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( if uri := driverMD[mariadbStorageURIKey]; uri != "" { status.Artifact = &backupsv1alpha1.BackupArtifact{URI: uri} if probe { - status.Artifact.SizeBytes = r.mariadbArtifactSizeBytes(ctx, mdbBackup, takenAt.Time) + if dump, ok := r.attributedMariaDBDump(ctx, mdbBackup, takenAt.Time); ok { + status.Artifact.SizeBytes = dump.Size + driverMD[mariadbDumpFileKey] = dump.Name + driverMD[mariadbDumpTimeKey] = dump.Timestamp.UTC().Format(time.RFC3339) + } } } backup := &backupsv1alpha1.Backup{ From 21d1e9f02b4dafeefab3acece372cc8f077b7701 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 4 Nov 2026 15:53:38 +0000 Subject: [PATCH 050/174] fix(mariadb): restore the dump a Backup records A RestoreJob created a mariadb-operator Restore that named only the operator Backup. The operator then lists the prefix that Backup wrote to and restores the dump nearest to the moment the restore started: the application's newest dump, not the one the Backup was taken as. When the Backup records its dump, the Restore now carries the timestamp in that dump's name as targetRecoveryTime. mariadb-operator returns the dump whose timestamp equals the target before it looks for the nearest one, and the nearest one is exactly what it would restore if the recorded dump were gone. So the controller first checks, with its own bounded call, that the dump is still in the operator Backup's storage. A dump that is gone fails the RestoreJob, naming the dump, instead of restoring a neighbour in its place. A check that cannot be made keeps the RestoreJob waiting with Ready=False until the restore deadline, then fails it with the cause: the Backup could record its dump only because the controller reached that storage when it was taken, so an unreachable storage is not a reason to guess. The dump is chosen once, as the operator Restore is created; a Restore that already exists is followed as before. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- internal/backupcontroller/artifact_size.go | 4 + .../backupcontroller/mariadb_artifact_size.go | 52 ++- .../mariadb_restore_dump_test.go | 304 ++++++++++++++++++ .../mariadbstrategy_controller.go | 111 ++++++- .../mariadbstrategy_controller_test.go | 19 +- .../backupcontroller/restorejob_controller.go | 5 + 6 files changed, 467 insertions(+), 28 deletions(-) create mode 100644 internal/backupcontroller/mariadb_restore_dump_test.go diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 941b00323..9197f07be 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -90,6 +90,10 @@ func (r *BackupJobReconciler) objectStore() objectStore { return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} } +func (r *RestoreJobReconciler) objectStore() objectStore { + return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} +} + func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { return r.objectStore().lister(conn) } diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index 950303d78..a61ec4a43 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -60,22 +60,14 @@ func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBack // from can have been edited since, and pointing at another bucket now says // nothing about where this dump went. func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { - s3 := mdbBackup.Spec.Storage.S3 - if s3 == nil { - return mariadbDump{}, errors.New("the operator Backup does not store to S3") + lister, s3, err := mariadbStorage(ctx, store, mdbBackup) + if err != nil { + return mariadbDump{}, err } others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup) if err != nil { return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) } - conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3) - if err != nil { - return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err) - } - lister, err := store.lister(conn) - if err != nil { - return mariadbDump{}, fmt.Errorf("build S3 client: %w", err) - } // Non-recursive, as mariadb-operator lists it: the default prefix is also // the default CNPG destinationPath, so an app of each kind with the same // name shares it and a recursive listing would walk the WAL archive. @@ -88,6 +80,44 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt return matchMariaDBDump(objects, prefix, own, others) } +// mariadbDumpExists reports whether the dump file name is still in the +// storage mdbBackup wrote to. +func mariadbDumpExists(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, name string) (bool, error) { + lister, s3, err := mariadbStorage(ctx, store, mdbBackup) + if err != nil { + return false, err + } + key := mariadbOperatorS3Prefix(s3.Prefix) + name + objects, err := lister.ListObjects(ctx, s3.Bucket, key, false) + if err != nil { + return false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) + } + for _, obj := range objects { + if obj.Key == key { + return true, nil + } + } + return false, nil +} + +// mariadbStorage opens the bucket mdbBackup wrote to, with the credentials +// its own spec names. +func mariadbStorage(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup) (s3ObjectLister, *mariadbtypes.S3Storage, error) { + s3 := mdbBackup.Spec.Storage.S3 + if s3 == nil { + return nil, nil, errors.New("the operator Backup does not store to S3") + } + conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3) + if err != nil { + return nil, nil, fmt.Errorf("resolve S3 credentials: %w", err) + } + lister, err := store.lister(conn) + if err != nil { + return nil, nil, fmt.Errorf("build S3 client: %w", err) + } + return lister, s3, nil +} + // mariadbRun is the span in which an operator Backup can have written its // dump: its lifetime, widened by mariadbSizeProbeClockSkew on each side. type mariadbRun struct { diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go new file mode 100644 index 000000000..0375def33 --- /dev/null +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +// The source run of the restore fixture: operator Backup op-bk of +// application src, alive from mdbRestoreT0 to mdbRestoreT0+40s, writing +// under tenant/src/ with mdbRestoreDump as its dump. +var mdbRestoreT0 = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + +const ( + mdbRestorePrefix = "tenant/src/" + mdbRestoreDump = "backup.2026-10-21T10:00:05Z.gzip.sql" + mdbRestoreTime = "2026-10-21T10:00:05Z" + mdbNewerDump = "backup.2026-10-21T11:00:05Z.gzip.sql" +) + +type mariadbRestoreFixture struct { + c client.Client + r *RestoreJobReconciler + rj *backupsv1alpha1.RestoreJob + backup *backupsv1alpha1.Backup + src *mariadbtypes.Backup + lister *fakeS3Lister + events *record.FakeRecorder +} + +// newMariaDBRestoreFixture seeds a RestoreJob of Backup src-bk, whose +// driverMetadata is driverMD plus the operator Backup's name, into the +// existing application target. extra objects are seeded too. +func newMariaDBRestoreFixture(t *testing.T, driverMD map[string]string, lister *fakeS3Lister, extra ...client.Object) *mariadbRestoreFixture { + t.Helper() + apps := mariadbapp.GroupName + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/src" + src := operatorBackupStoringTo(t, s3, mdbRestoreT0) + src.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)))} + + md := map[string]string{mariadbBackupNameKey: src.Name} + for k, v := range driverMD { + md[k] = v + } + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-bk"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + TakenAt: metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)), + DriverMetadata: md, + }, + } + started := metav1.Now() + rj := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"}, + Spec: backupsv1alpha1.RestoreJobSpec{ + BackupRef: corev1.LocalObjectReference{Name: backup.Name}, + TargetApplicationRef: &corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "target", APIGroup: &apps}, + }, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &started, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + target := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("target")}} + objs := append([]client.Object{src, backup, rj, target}, extra...) + c := newMariaDBStrategyTestClient(t, objs...) + fetched := &backupsv1alpha1.RestoreJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), fetched); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + events := record.NewFakeRecorder(10) + return &mariadbRestoreFixture{ + c: c, + rj: fetched, + backup: backup, + src: src, + lister: lister, + events: events, + r: &RestoreJobReconciler{ + Client: c, + Recorder: events, + newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) { + lister.conn = conn + return lister, nil + }, + }, + } +} + +func (f *mariadbRestoreFixture) reconcile(t *testing.T) ctrl.Result { + t.Helper() + res, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + return res +} + +func (f *mariadbRestoreFixture) restores(t *testing.T) []mariadbtypes.Restore { + t.Helper() + list := &mariadbtypes.RestoreList{} + if err := f.c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { + t.Fatalf("list operator Restores: %v", err) + } + return list.Items +} + +// wantFailed asserts the RestoreJob failed with a message holding every +// fragment, and that no operator Restore was created. +func (f *mariadbRestoreFixture) wantFailed(t *testing.T, fragments ...string) { + t.Helper() + if f.rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("phase = %q, want Failed", f.rj.Status.Phase) + } + for _, s := range fragments { + if !strings.Contains(f.rj.Status.Message, s) { + t.Errorf("message %q does not contain %q", f.rj.Status.Message, s) + } + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want none", len(got)) + } +} + +// wantWaiting asserts the RestoreJob is kept waiting on the dump check +// with Ready=False saying why, and that no operator Restore was created. +func (f *mariadbRestoreFixture) wantWaiting(t *testing.T, res ctrl.Result, fragments ...string) { + t.Helper() + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed: %s", f.rj.Status.Message) + } + if res.RequeueAfter != mariadbDumpCheckRetryInterval { + t.Errorf("RequeueAfter = %s, want %s", res.RequeueAfter, mariadbDumpCheckRetryInterval) + } + persisted := &backupsv1alpha1.RestoreJob{} + if err := f.c.Get(context.Background(), client.ObjectKeyFromObject(f.rj), persisted); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready") + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "DumpCheckPending" { + t.Fatalf("Ready = %+v, want False/DumpCheckPending", cond) + } + for _, s := range fragments { + if !strings.Contains(cond.Message, s) { + t.Errorf("Ready message %q does not contain %q", cond.Message, s) + } + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want none", len(got)) + } +} + +// wantRestore asserts exactly one operator Restore of op-bk into target, +// asking for targetTime. +func (f *mariadbRestoreFixture) wantRestore(t *testing.T, targetTime string) { + t.Helper() + got := f.restores(t) + if len(got) != 1 { + t.Fatalf("created %d operator Restores; want one", len(got)) + } + spec := got[0].Spec + if spec.BackupRef == nil || spec.BackupRef.Name != f.src.Name || spec.MariaDBRef.Name != mariadbNameForApp("target") { + t.Errorf("Restore spec = %+v, want backupRef %s into %s", spec, f.src.Name, mariadbNameForApp("target")) + } + if spec.TargetRecoveryTime != targetTime { + t.Errorf("targetRecoveryTime = %q, want %q", spec.TargetRecoveryTime, targetTime) + } +} + +func recordedDump(name string) map[string]string { + return map[string]string{mariadbDumpFileKey: name, mariadbDumpTimeKey: mdbRestoreTime} +} + +func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + + t.Run("restores the recorded dump by its own timestamp", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}, + {Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}, + }} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds) + f.reconcile(t) + f.wantRestore(t, mdbRestoreTime) + if l.bucket != "bkt" || l.prefix != mdbRestorePrefix+mdbRestoreDump || l.conn.AccessKey != "AKID" { + t.Errorf("checked %s/%s as %q, want bkt/%s%s with the operator Backup's credentials", l.bucket, l.prefix, l.conn.AccessKey, mdbRestorePrefix, mdbRestoreDump) + } + }) + + t.Run("a recorded dump gone from storage fails the RestoreJob", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}}} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds) + f.reconcile(t) + f.wantFailed(t, mdbRestoreDump, "no longer exists in storage", "s3://bkt/"+mdbRestorePrefix) + }) + + t.Run("a storage that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds) + f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "Access Denied.") + }) + + t.Run("missing credentials keep the RestoreJob waiting", func(t *testing.T) { + l := &fakeS3Lister{} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l) + f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "resolve S3 credentials") + if l.calls != 0 { + t.Errorf("S3 listed %d times without credentials", l.calls) + } + }) + + t.Run("a hanging endpoint is given up at the check timeout", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{}, creds) + f.r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + var res ctrl.Result + var err error + withShortDumpCheckTimeout(t, func() { res, err = f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) }) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + f.wantWaiting(t, res, mdbRestoreDump, context.DeadlineExceeded.Error()) + }) + + t.Run("a storage still unreadable at the deadline fails the RestoreJob with the cause", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds) + long := metav1.NewTime(time.Now().Add(-mariadbDefaultRestoreDeadline - time.Minute)) + f.rj.Status.StartedAt = &long + f.reconcile(t) + f.wantFailed(t, mdbRestoreDump, "Access Denied.", "gave up after") + }) + + t.Run("a recorded name that is not a dump fails the RestoreJob unchecked", func(t *testing.T) { + l := &fakeS3Lister{} + f := newMariaDBRestoreFixture(t, recordedDump("dump.sql"), l, creds) + f.reconcile(t) + f.wantFailed(t, `"dump.sql"`) + if l.calls != 0 { + t.Errorf("S3 listed %d times for a name that is not a dump", l.calls) + } + }) + + t.Run("an operator Restore created earlier is followed without a check", func(t *testing.T) { + l := &fakeS3Lister{err: errors.New("Access Denied.")} + existing := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds, existing) + res := f.reconcile(t) + if l.calls != 0 { + t.Errorf("S3 listed %d times for a RestoreJob whose operator Restore exists", l.calls) + } + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed || res.RequeueAfter != mariadbPollInterval { + t.Errorf("phase %q, requeue %s; want the existing Restore polled", f.rj.Status.Phase, res.RequeueAfter) + } + if got := f.restores(t); len(got) != 1 { + t.Errorf("%d operator Restores; want only the existing one", len(got)) + } + }) +} + +func TestReconcileMariaDBRestore_NoRecordedDump(t *testing.T) { + l := &fakeS3Lister{err: errors.New("Access Denied.")} + f := newMariaDBRestoreFixture(t, nil, l, s3CredsSecret("tenant", "creds")) + f.reconcile(t) + f.wantRestore(t, "") + if l.calls != 0 { + t.Errorf("S3 listed %d times for a Backup that records no dump", l.calls) + } +} + +// withShortDumpCheckTimeout shortens the restore's dump check timeout and +// fails the test if fn is not back well before the default would expire. +func withShortDumpCheckTimeout(t *testing.T, fn func()) { + t.Helper() + saved := mariadbRestoreDumpCheckTimeout + mariadbRestoreDumpCheckTimeout = 50 * time.Millisecond + defer func() { mariadbRestoreDumpCheckTimeout = saved }() + done := make(chan struct{}) + go func() { + fn() + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the dump check did not give up at its timeout") + } +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 50c871db3..86b42d33f 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -4,6 +4,7 @@ package backupcontroller import ( "context" "encoding/json" + "errors" "fmt" "time" @@ -50,8 +51,9 @@ const ( // The dump the operator Backup wrote: its file name under the storage // prefix, and the RFC3339 timestamp in that name. Every run of an // application writes under the same prefix, so only this name tells its - // dump from the others. Recorded only when findMariaDBDump proved the - // dump belongs to the operator Backup. + // dump from the others, and a restore asks for exactly that dump. + // Recorded only when findMariaDBDump proved the dump belongs to the + // operator Backup. mariadbDumpFileKey = "k8s.mariadb.com/dump-file" mariadbDumpTimeKey = "k8s.mariadb.com/dump-time" @@ -68,6 +70,12 @@ const ( // to terminate. Tenants override via spec.options.restoreTimeoutSeconds. mariadbDefaultRestoreDeadline = 30 * time.Minute + // Retry cadence while the dump a restore asks for cannot be checked. + // Longer than mariadbPollInterval because each attempt can hold the + // RestoreJob controller's single worker for + // mariadbRestoreDumpCheckTimeout. + mariadbDumpCheckRetryInterval = 30 * time.Second + // mariadbBackupSnapshotKind is the Kind stamped onto the snapshot // persisted in Backup.status.underlyingResources. The same shape carries // the storage descriptor and the rendered strategy parameters used at @@ -81,6 +89,12 @@ const ( // existing API surface. var mariadbBackupSnapshotAPIVersion = backupsv1alpha1.GroupVersion.String() +// mariadbRestoreDumpCheckTimeout bounds the object-store round trips a +// restore makes before it creates the operator Restore. The RestoreJob +// controller has a single worker, so an unreachable endpoint stalls every +// other RestoreJob for this long. A variable so tests can shorten it. +var mariadbRestoreDumpCheckTimeout = 10 * time.Second + // mariadbNameForApp returns the k8s.mariadb.com/MariaDB CR name for a // cozystack MariaDB application instance. The mapping mirrors the mariadb // ApplicationDefinition (release.prefix=mariadb-). @@ -582,10 +596,25 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest return ctrl.Result{}, err } - mdbRestore, err := r.ensureMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName) + mdbRestore, err := r.findMariaDBRestoreForJob(ctx, restoreJob) if err != nil { return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) } + // The dump is chosen once, as the operator Restore is created; one that + // already exists has made its choice. + if mdbRestore == nil { + targetTime, err := r.mariadbRestoreTargetTime(ctx, backup, srcBackup) + var refused mariadbRestoreRefused + if errors.As(err, &refused) { + return r.markRestoreJobFailed(ctx, restoreJob, refused.Error()) + } + if err != nil { + return r.awaitMariaDBDumpCheck(ctx, restoreJob, options, err) + } + if mdbRestore, err = r.createMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName, targetTime); err != nil { + return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) + } + } cond := apimeta.FindStatusCondition(mdbRestore.Status.Conditions, mariadbtypes.ConditionTypeComplete) switch { @@ -626,10 +655,68 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest } } -// ensureMariaDBRestore creates a k8s.mariadb.com/Restore CR labelled with -// the RestoreJob, or returns the existing one if a previous reconcile -// already created it. -func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName string) (*mariadbtypes.Restore, error) { +// mariadbRestoreRefused is why the dump a RestoreJob asks for cannot be +// restored. It fails the RestoreJob, where any other error from choosing +// the dump is retried. +type mariadbRestoreRefused string + +func (e mariadbRestoreRefused) Error() string { return string(e) } + +// mariadbRestoreTargetTime returns the targetRecoveryTime of the operator +// Restore for backup: the timestamp in the name of the dump backup +// records. mariadb-operator restores the dump whose timestamp equals the +// target and, when there is none, the nearest one, so the dump is first +// checked to still exist. Empty when backup records no dump, which leaves +// the operator to take the newest dump under the prefix. +func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) { + name := backup.Spec.DriverMetadata[mariadbDumpFileKey] + if name == "" { + return "", nil + } + ts, ok := parseMariaDBLogicalBackupFile(name) + if !ok { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)) + } + ctx, cancel := context.WithTimeout(ctx, mariadbRestoreDumpCheckTimeout) + defer cancel() + exists, err := mariadbDumpExists(ctx, r.objectStore(), src, name) + if err != nil { + return "", fmt.Errorf("cannot check that dump %s of k8s.mariadb.com/Backup %s/%s still exists in storage: %w", name, src.Namespace, src.Name, err) + } + if !exists { + s3 := src.Spec.Storage.S3 + return "", mariadbRestoreRefused(fmt.Sprintf( + "dump %s of k8s.mariadb.com/Backup %s/%s no longer exists in storage (s3://%s/%s); restoring now would bring back a different dump", + name, src.Namespace, src.Name, s3.Bucket, mariadbOperatorS3Prefix(s3.Prefix))) + } + return ts.UTC().Format(time.RFC3339), nil +} + +// awaitMariaDBDumpCheck keeps the RestoreJob waiting, with Ready=False +// saying why, while the dump it asks for cannot be checked, and fails it +// with the cause once the restore deadline has passed. +func (r *RestoreJobReconciler) awaitMariaDBDumpCheck(ctx context.Context, rj *backupsv1alpha1.RestoreJob, options MariaDBRestoreOptions, cause error) (ctrl.Result, error) { + deadline := options.effectiveRestoreDeadline() + if rj.Status.StartedAt != nil && time.Since(rj.Status.StartedAt.Time) > deadline { + return r.markRestoreJobFailed(ctx, rj, fmt.Sprintf( + "%v; gave up after %s (override via spec.options.restoreTimeoutSeconds)", cause, deadline)) + } + getLogger(ctx).Info("cannot check the MariaDB dump to restore; retrying", "restorejob", rj.Name, "error", cause) + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "DumpCheckPending", + Message: cause.Error(), + }) + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: mariadbDumpCheckRetryInterval}, nil +} + +// findMariaDBRestoreForJob returns the k8s.mariadb.com/Restore labelled +// with the RestoreJob, if a previous reconcile already created one. +func (r *RestoreJobReconciler) findMariaDBRestoreForJob(ctx context.Context, rj *backupsv1alpha1.RestoreJob) (*mariadbtypes.Restore, error) { list := &mariadbtypes.RestoreList{} if err := r.List(ctx, list, client.InNamespace(rj.Namespace), @@ -640,10 +727,15 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac ); err != nil { return nil, err } - if len(list.Items) > 0 { - return &list.Items[0], nil + if len(list.Items) == 0 { + return nil, nil } + return &list.Items[0], nil +} +// createMariaDBRestore creates the k8s.mariadb.com/Restore of the +// RestoreJob, labelled with it. targetRecoveryTime is RFC3339, or empty. +func (r *RestoreJobReconciler) createMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName, targetRecoveryTime string) (*mariadbtypes.Restore, error) { obj := &mariadbtypes.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: rj.Namespace, @@ -659,6 +751,7 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac Kind: mariadbtypes.BackupKind, Name: sourceBackupName, }, + TargetRecoveryTime: targetRecoveryTime, }, } if err := r.Create(ctx, obj); err != nil { diff --git a/internal/backupcontroller/mariadbstrategy_controller_test.go b/internal/backupcontroller/mariadbstrategy_controller_test.go index 0e4f63b5e..fa033fb22 100644 --- a/internal/backupcontroller/mariadbstrategy_controller_test.go +++ b/internal/backupcontroller/mariadbstrategy_controller_test.go @@ -313,7 +313,7 @@ func TestEnsureMariaDBBackup_IdempotentByLabel(t *testing.T) { // Restore-side ensure idempotency + target rewrite // --------------------------------------------------------------------------- -func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { +func TestFindMariaDBRestoreForJob_FindsTheCreatedRestore(t *testing.T) { c := newMariaDBStrategyTestClient(t) r := &RestoreJobReconciler{Client: c, Scheme: c.Scheme()} rj := newMariaDBRestoreJob("rj-1", "tenant") @@ -321,16 +321,16 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { t.Fatalf("seed RestoreJob: %v", err) } - first, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target") - if err != nil { - t.Fatalf("first ensureMariaDBRestore: %v", err) + if none, err := r.findMariaDBRestoreForJob(context.Background(), rj); err != nil || none != nil { + t.Fatalf("findMariaDBRestoreForJob before create = %v, %v; want none", none, err) } - second, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target") + created, err := r.createMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target", "2026-10-21T10:00:05Z") if err != nil { - t.Fatalf("second ensureMariaDBRestore: %v", err) + t.Fatalf("createMariaDBRestore: %v", err) } - if first.Name != second.Name { - t.Errorf("expected idempotent reuse: first=%q second=%q", first.Name, second.Name) + found, err := r.findMariaDBRestoreForJob(context.Background(), rj) + if err != nil || found == nil || found.Name != created.Name { + t.Fatalf("findMariaDBRestoreForJob = %v, %v; want the Restore just created", found, err) } list := &mariadbtypes.RestoreList{} if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { @@ -346,6 +346,9 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { if got.Spec.BackupRef == nil || got.Spec.BackupRef.Name != "src-backup" || got.Spec.BackupRef.Kind != mariadbtypes.BackupKind { t.Errorf("BackupRef mismatch: %#v", got.Spec.BackupRef) } + if got.Spec.TargetRecoveryTime != "2026-10-21T10:00:05Z" { + t.Errorf("TargetRecoveryTime: got %q want 2026-10-21T10:00:05Z", got.Spec.TargetRecoveryTime) + } } // --------------------------------------------------------------------------- diff --git a/internal/backupcontroller/restorejob_controller.go b/internal/backupcontroller/restorejob_controller.go index 1a61f0800..db7348923 100644 --- a/internal/backupcontroller/restorejob_controller.go +++ b/internal/backupcontroller/restorejob_controller.go @@ -57,6 +57,11 @@ type RestoreJobReconciler struct { // see a peer's StartedAt before the cache does. Wired in // SetupWithManager. APIReader client.Reader + + // newS3ObjectLister overrides the S3 client used to check that a + // backup's object still exists; nil means a real minio client. Set by + // unit tests. + newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error) } func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { From 195de033e8156876885d685cd1cced3d2dfb1142 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 4 Nov 2026 15:55:36 +0000 Subject: [PATCH 051/174] fix(mariadb): pin legacy Backups to a proven dump, refuse failed runs A Backup taken before Backups recorded their dump gives a restore nothing to pin. Refusing such Backups would leave every existing one unrestorable, so a restore of one now makes the same ownership-proven match the backup side makes, against the operator Backup it names, and restores that dump by its timestamp. When no dump can be proven to be the run's own, or the storage cannot be read, mariadb-operator restores the newest dump under the prefix as it always did, and a BestEffortDump Warning on the RestoreJob says so and why. An operator Backup whose Job failed is Complete=True with reason JobFailed, and a BackupJob that took Complete=True for success made a Backup of it. That run left no dump of its own, and restoring it would quietly bring back another run's dump, so the RestoreJob now fails instead. The condition is read by its reason, because its status does not tell a failure from a success. Assisted-by: LLM Signed-off-by: Myasnikov Daniil --- .../mariadb_restore_dump_test.go | 129 +++++++++++++++++- .../mariadbstrategy_controller.go | 41 +++++- 2 files changed, 156 insertions(+), 14 deletions(-) diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go index 0375def33..6c8202adf 100644 --- a/internal/backupcontroller/mariadb_restore_dump_test.go +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -274,13 +274,128 @@ func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) { }) } -func TestReconcileMariaDBRestore_NoRecordedDump(t *testing.T) { - l := &fakeS3Lister{err: errors.New("Access Denied.")} - f := newMariaDBRestoreFixture(t, nil, l, s3CredsSecret("tenant", "creds")) - f.reconcile(t) - f.wantRestore(t, "") - if l.calls != 0 { - t.Errorf("S3 listed %d times for a Backup that records no dump", l.calls) +// TestReconcileMariaDBRestore_LegacyBackup drives Backups that record no +// dump: one proven to belong to the source run is restored by its +// timestamp, and anything else restores the newest dump with a Warning. +func TestReconcileMariaDBRestore_LegacyBackup(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + own := s3ObjectInfo{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)} + newer := s3ObjectInfo{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)} + second := s3ObjectInfo{Key: mdbRestorePrefix + "backup.2026-10-21T10:00:30Z.gzip.sql", LastModified: mdbRestoreT0.Add(35 * time.Second)} + + t.Run("a dump proven to be the run's own is restored by its timestamp", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{objects: []s3ObjectInfo{own, newer}}, creds) + f.reconcile(t) + f.wantRestore(t, mdbRestoreTime) + if ev := drainEvents(f.events); len(ev) != 0 { + t.Errorf("events %q; want none for a proven dump", ev) + } + }) + + cases := []struct { + name string + lister *fakeS3Lister + cause string + }{ + {name: "two dumps in the run's lifetime", lister: &fakeS3Lister{objects: []s3ObjectInfo{own, second, newer}}, cause: "cannot be told apart"}, + {name: "no dump in the run's lifetime", lister: &fakeS3Lister{objects: []s3ObjectInfo{newer}}, cause: "no dump in the bucket belongs"}, + {name: "a storage that cannot be read", lister: &fakeS3Lister{err: errors.New("Access Denied.")}, cause: "Access Denied."}, + } + wantBestEffort := func(t *testing.T, f *mariadbRestoreFixture, cause string) { + t.Helper() + f.wantRestore(t, "") + ev := drainEvents(f.events) + if len(ev) != 1 || !strings.HasPrefix(ev[0], "Warning BestEffortDump ") || !strings.Contains(ev[0], cause) || !strings.Contains(ev[0], "src-bk records no dump") { + t.Errorf("events %q; want one BestEffortDump warning naming the Backup and %q", ev, cause) + } + } + for _, tc := range cases { + t.Run(tc.name+" restores the newest dump with a warning", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, nil, tc.lister, creds) + f.reconcile(t) + wantBestEffort(t, f, tc.cause) + }) + } + + t.Run("a hanging endpoint is given up at the check timeout", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{}, creds) + f.r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + var err error + withShortDumpCheckTimeout(t, func() { _, err = f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) }) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + wantBestEffort(t, f, context.DeadlineExceeded.Error()) + }) +} + +// TestReconcileMariaDBRestore_FailedBackupRun drives a Backup made of an +// operator Backup whose Job failed: there is no dump of its own to +// restore, whatever the Backup records. +func TestReconcileMariaDBRestore_FailedBackupRun(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + failed := func(status metav1.ConditionStatus) []metav1.Condition { + return []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: status, Reason: mariadbtypes.ConditionReasonJobFailed, + Message: "Failed", LastTransitionTime: metav1.NewTime(mdbRestoreT0.Add(3 * time.Minute)), + }} + } + listing := []s3ObjectInfo{{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}} + + cases := []struct { + name string + driverMD map[string]string + status metav1.ConditionStatus + }{ + {name: "Complete=True/JobFailed with no recorded dump", status: metav1.ConditionTrue}, + {name: "Complete=True/JobFailed with a recorded dump", driverMD: recordedDump(mdbRestoreDump), status: metav1.ConditionTrue}, + {name: "Complete=False/JobFailed", status: metav1.ConditionFalse}, + } + for _, tc := range cases { + t.Run(tc.name+" fails the RestoreJob", func(t *testing.T) { + l := &fakeS3Lister{objects: listing} + f := newMariaDBRestoreFixture(t, tc.driverMD, l, creds) + f.src.Status.Conditions = failed(tc.status) + if err := f.c.Update(context.Background(), f.src); err != nil { + t.Fatalf("update operator Backup: %v", err) + } + f.reconcile(t) + f.wantFailed(t, "k8s.mariadb.com/Backup tenant/op-bk failed", "left no dump to restore") + if l.calls != 0 { + t.Errorf("S3 listed %d times for a failed backup run", l.calls) + } + }) + } + + t.Run("an operator Restore created earlier is followed", func(t *testing.T) { + existing := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }} + f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{objects: listing}, creds, existing) + f.src.Status.Conditions = failed(metav1.ConditionTrue) + if err := f.c.Update(context.Background(), f.src); err != nil { + t.Fatalf("update operator Backup: %v", err) + } + f.reconcile(t) + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Errorf("RestoreJob failed although its operator Restore was already created: %s", f.rj.Status.Message) + } + }) +} + +func drainEvents(r *record.FakeRecorder) []string { + var out []string + for { + select { + case e := <-r.Events: + out = append(out, e) + default: + return out + } } } diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 86b42d33f..6d6312eb4 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -603,7 +603,7 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest // The dump is chosen once, as the operator Restore is created; one that // already exists has made its choice. if mdbRestore == nil { - targetTime, err := r.mariadbRestoreTargetTime(ctx, backup, srcBackup) + targetTime, err := r.mariadbRestoreTargetTime(ctx, restoreJob, backup, srcBackup) var refused mariadbRestoreRefused if errors.As(err, &refused) { return r.markRestoreJobFailed(ctx, restoreJob, refused.Error()) @@ -666,19 +666,26 @@ func (e mariadbRestoreRefused) Error() string { return string(e) } // Restore for backup: the timestamp in the name of the dump backup // records. mariadb-operator restores the dump whose timestamp equals the // target and, when there is none, the nearest one, so the dump is first -// checked to still exist. Empty when backup records no dump, which leaves -// the operator to take the newest dump under the prefix. -func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) { +// checked to still exist. A Backup that records no dump is resolved by +// legacyMariaDBRestoreTargetTime. +func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) { + // A failed Job is Complete=True too, with reason JobFailed, and a + // BackupJob that took Complete=True for success made a Backup of it. + if cond := apimeta.FindStatusCondition(src.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed { + return "", mariadbRestoreRefused(fmt.Sprintf( + "the backup run of k8s.mariadb.com/Backup %s/%s failed (Complete=%s/%s) and left no dump to restore; restoring would bring back another run's dump", + src.Namespace, src.Name, cond.Status, cond.Reason)) + } + ctx, cancel := context.WithTimeout(ctx, mariadbRestoreDumpCheckTimeout) + defer cancel() name := backup.Spec.DriverMetadata[mariadbDumpFileKey] if name == "" { - return "", nil + return r.legacyMariaDBRestoreTargetTime(ctx, rj, backup, src), nil } ts, ok := parseMariaDBLogicalBackupFile(name) if !ok { return "", mariadbRestoreRefused(fmt.Sprintf("Backup driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)) } - ctx, cancel := context.WithTimeout(ctx, mariadbRestoreDumpCheckTimeout) - defer cancel() exists, err := mariadbDumpExists(ctx, r.objectStore(), src, name) if err != nil { return "", fmt.Errorf("cannot check that dump %s of k8s.mariadb.com/Backup %s/%s still exists in storage: %w", name, src.Namespace, src.Name, err) @@ -692,6 +699,26 @@ func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, bac return ts.UTC().Format(time.RFC3339), nil } +// legacyMariaDBRestoreTargetTime chooses the dump of a Backup taken before +// Backups recorded theirs: the one findMariaDBDump proves belongs to src, +// with the completion time the size probe was given, which takenAt keeps. +// When none is proven, for any reason, it returns empty, and the operator +// restores the newest dump under the prefix as such a restore always did: +// refusing would leave these Backups unrestorable, and the controller may +// never have been able to reach their storage. A Warning event says the +// dump is a best effort. +func (r *RestoreJobReconciler) legacyMariaDBRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) string { + dump, err := findMariaDBDump(ctx, r.objectStore(), src, backup.Spec.TakenAt.Time) + if err != nil { + getLogger(ctx).Info("no dump proven for a MariaDB Backup that records none; restoring the newest", "restorejob", rj.Name, "backup", backup.Name, "error", err) + r.Recorder.Eventf(rj, corev1.EventTypeWarning, "BestEffortDump", + "Backup %s records no dump and none in storage could be proven to belong to k8s.mariadb.com/Backup %s/%s (%v); mariadb-operator restores the newest dump under its prefix as a best effort, which may be another backup run's", + backup.Name, src.Namespace, src.Name, err) + return "" + } + return dump.Timestamp.UTC().Format(time.RFC3339) +} + // awaitMariaDBDumpCheck keeps the RestoreJob waiting, with Ready=False // saying why, while the dump it asks for cannot be checked, and fails it // with the cause once the restore deadline has passed. From 1e1fdb740c229f7233e3be4993d063c2fd5cc5fb Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:14:24 +0700 Subject: [PATCH 052/174] fix(dashboard): show log rows a pipe stripped of their timestamp A LogsQL pipe such as `fields _msg` or `stats` returns rows without _time, and the console dropped every one of them, so a successful query read as "no lines matched". Such rows are now kept in the order the store sent them, shown without a timestamp, and never deduplicated on a tail resume. A row without _msg shows its remaining fields instead of an empty line. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observability/LogLines.test.tsx | 12 +++++++ .../src/components/observability/LogLines.tsx | 33 +++++++++++++++---- .../observability/LogsView.test.tsx | 13 ++++++++ .../src/lib/observability/logs.test.ts | 6 ++++ .../console/src/lib/observability/logs.ts | 5 +-- .../src/lib/observability/proxy.test.ts | 20 ++++++++++- .../console/src/lib/observability/proxy.ts | 10 +++--- .../src/lib/observability/use-log-tail.ts | 2 +- 8 files changed, 86 insertions(+), 15 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx index dd8c503eb..a0f519852 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx @@ -29,6 +29,18 @@ describe("LogLines", () => { expect(frame).not.toHaveTextContent("Waiting") }) + it("shows a line without _time with no timestamp and no empty source", () => { + render() + + expect(screen.getByRole("listitem")).toHaveTextContent(/^visible message$/) + }) + + it("shows the remaining fields of a row without _msg", () => { + render() + + expect(screen.getByRole("listitem")).toHaveTextContent(/^db-0\/ rows=42$/) + }) + it("keeps following the newest line while the reader is at the bottom", () => { const { rerender } = render() const list = screen.getByRole("list") diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx index 14b941ab2..f29af017c 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx @@ -3,6 +3,22 @@ import { CONTAINER_FIELD, POD_FIELD } from "../../lib/observability/logs.ts" import type { LogEntry } from "../../lib/observability/proxy.ts" const BOTTOM_SLACK_PX = 24 +const SHOWN_APART = new Set(["_time", "_msg", "_stream_id", POD_FIELD, CONTAINER_FIELD]) + +function source(e: LogEntry): string | null { + const pod = e[POD_FIELD] + const container = e[CONTAINER_FIELD] + return pod || container ? `${pod ?? ""}/${container ?? ""}` : null +} + +// A `fields` or `stats` pipe can leave a row without _msg; its other fields are then the content. +function message(e: LogEntry): string { + if (e._msg !== undefined) return e._msg + return Object.entries(e) + .filter(([k, v]) => v !== undefined && !SHOWN_APART.has(k)) + .map(([k, v]) => `${k}=${v}`) + .join(" ") +} export function LogLines({ entries, placeholder }: { entries: LogEntry[]; placeholder: ReactNode }) { const ref = useRef(null) @@ -29,13 +45,16 @@ export function LogLines({ entries, placeholder }: { entries: LogEntry[]; placeh }} className="h-full overflow-auto p-3" > - {entries.map((e, i) => ( -
  1. - {new Date(e._time).toLocaleTimeString()}{" "} - {`${e[POD_FIELD] ?? ""}/${e[CONTAINER_FIELD] ?? ""}`}{" "} - {e._msg} -
  2. - ))} + {entries.map((e, i) => { + const from = source(e) + return ( +
  3. + {e._time !== undefined && <>{new Date(e._time).toLocaleTimeString()}{" "}} + {from !== null && <>{from}{" "}} + {message(e)} +
  4. + ) + })}
)} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx index 1b18dd208..770c8a263 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -199,6 +199,19 @@ describe("LogsView", () => { expect(screen.getAllByRole("option").map((o) => o.textContent)).toEqual(["All", "db-0"]) }) + it("shows rows of a query whose pipe removed _time", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") ? new Response('{"_msg":"visible message"}\n') : defaultHandler(endpoint, params), + ) + renderWithK8sProvider(, { client }) + + await userEvent.type(screen.getByLabelText("LogsQL filter"), "* | fields _msg{Enter}") + + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(await within(frame).findByText("visible message")).toBeInTheDocument() + expect(within(frame).queryByText(/No log lines/)).toBeNull() + }) + it("names the source pod and container of each line", async () => { const { client } = clientWith(defaultHandler) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts index 25d26a23a..62d1ac249 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts @@ -57,6 +57,12 @@ describe("mergeTail", () => { expect(out).toHaveLength(2) }) + it("never deduplicates lines without _time, even on a resume", () => { + const out = mergeTail([{ _msg: "a" }], [{ _msg: "a" }], 10, "2026-01-01T00:00:02Z") + + expect(out).toHaveLength(2) + }) + it("does not deduplicate outside a resume", () => { const out = mergeTail([e("t1", "a")], [e("t1", "a")], 10) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts index a6255e541..eb24a5f96 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts @@ -61,12 +61,13 @@ export function compareTime(a: string, b: string): number { const entryKey = (e: LogEntry) => `${e._stream_id ?? ""}\u0000${e._time}` // (_stream_id, _time) is not unique in VictoriaLogs, so dedup applies only to -// lines a resumed tail replays at or before the resume point. +// lines a resumed tail replays at or before the resume point, and never to +// lines a pipe stripped of _time. export function mergeTail(buffer: LogEntry[], incoming: LogEntry[], cap: number, resumedFrom?: string): LogEntry[] { let fresh = incoming if (resumedFrom) { const seen = new Set(buffer.map(entryKey)) - fresh = incoming.filter((e) => compareTime(e._time, resumedFrom) > 0 || !seen.has(entryKey(e))) + fresh = incoming.filter((e) => e._time === undefined || compareTime(e._time, resumedFrom) > 0 || !seen.has(entryKey(e))) } const merged = buffer.concat(fresh) return merged.length > cap ? merged.slice(merged.length - cap) : merged diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts index 2809905d8..f2ba8d7c3 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts @@ -122,6 +122,15 @@ describe("queryLogs", () => { expect(entries.map((e) => e._msg)).toEqual(["a", "b"]) }) + it("keeps rows a pipe stripped of _time, in the order the store sent them", async () => { + const ndjson = ['{"_msg":"visible message"}', '{"_msg":"second"}', "[1]", "null", "{torn"].join("\n") + const { client } = clientReturning(new Response(ndjson)) + + const entries = await queryLogs(client, logsRoot(ns), { query: "* | fields _msg", start: "s", end: "e", limit: 100 }) + + expect(entries).toEqual([{ _msg: "visible message" }, { _msg: "second" }]) + }) + it("keeps a plain-text error body as the message", async () => { const { client } = clientReturning(new Response("cannot parse query", { status: 400 })) @@ -170,7 +179,7 @@ describe("tailLogs", () => { }, }) const { client, raw } = clientReturning(new Response(body)) - const seen: string[] = [] + const seen: (string | undefined)[] = [] await tailLogs(client, logsRoot(ns), { query: "*", since: "t0" }, (e) => seen.push(e._msg)) @@ -179,6 +188,15 @@ describe("tailLogs", () => { expect(seen).toEqual(["a", "b"]) }) + it("delivers a line without _time", async () => { + const { client } = clientReturning(new Response('{"_msg":"untimed"}\n')) + const seen: (string | undefined)[] = [] + + await tailLogs(client, logsRoot(ns), { query: "* | fields _msg" }, (e) => seen.push(e._msg)) + + expect(seen).toEqual(["untimed"]) + }) + it("omits since on the first connect", async () => { const { client, raw } = clientReturning(new Response("")) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts index 3b5567f6b..c28bd12d4 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts @@ -140,9 +140,10 @@ export async function queryRange( return body.data?.result ?? [] } +// A pipe such as `fields _msg` or `stats` can drop _time and _msg. export interface LogEntry { - _time: string - _msg: string + _time?: string + _msg?: string _stream_id?: string [field: string]: string | undefined } @@ -151,7 +152,7 @@ function parseLine(line: string): LogEntry | null { if (!line.trim()) return null try { const value: unknown = JSON.parse(line) - if (value && typeof value === "object" && "_time" in value) return value as LogEntry + if (value && typeof value === "object" && !Array.isArray(value)) return value as LogEntry } catch { // a torn or non-JSON line carries nothing to show } @@ -173,7 +174,8 @@ export async function queryLogs( ): Promise { const res = await proxyGet(client, `${root}select/logsql/query`, { ...params }, signal) const entries = (await res.text()).split("\n").map(parseLine).filter((e) => e !== null) - return entries.sort((a, b) => compareTime(a._time, b._time)) + if (entries.some((e) => e._time === undefined)) return entries + return entries.sort((a, b) => compareTime(a._time ?? "", b._time ?? "")) } export interface FieldValuesParams { diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts index 2e3b4d706..d55bd5708 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts @@ -73,7 +73,7 @@ export function useLogTail( let delay = retryDelayMs try { await tailLogs(client, root, { query, since: lastTime ?? openedAt }, (entry) => { - if (!lastTime || compareTime(entry._time, lastTime) > 0) lastTime = entry._time + if (entry._time && (!lastTime || compareTime(entry._time, lastTime) > 0)) lastTime = entry._time pending.push(entry) timer ??= setTimeout(flush, FLUSH_MS) }, signal, () => { From 8e6048eb4a2556cc9af587b4df3259cface49b37 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:16:04 +0700 Subject: [PATCH 053/174] fix(dashboard): make metric samples readable without a pointer The charts are a canvas and their legend showed dashes until a point was hovered, so keyboard and screen-reader users could not read any sample. The legend now shows each series' latest value when nothing is hovered, and every panel has a Show data toggle that opens a focusable table of all samples, newest first, in the chart's units. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observability/MetricsView.test.tsx | 27 ++++++++++- .../components/observability/MetricsView.tsx | 36 ++++++++++++-- .../observability/SeriesTable.test.tsx | 37 +++++++++++++++ .../components/observability/SeriesTable.tsx | 47 +++++++++++++++++++ .../observability/TimeSeriesChart.test.tsx | 17 ++++++- .../observability/TimeSeriesChart.tsx | 4 +- 6 files changed, 160 insertions(+), 8 deletions(-) create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx create mode 100644 packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx index d3f1fc1dd..8e7829ed5 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx @@ -1,5 +1,5 @@ import { describe, it, expect, vi } from "vitest" -import { screen, waitFor } from "@testing-library/react" +import { screen, waitFor, within } from "@testing-library/react" import userEvent from "@testing-library/user-event" import { K8sClient } from "@cozystack/k8s-client" import { renderWithK8sProvider } from "../../test-utils/render.tsx" @@ -98,6 +98,31 @@ describe("MetricsView", () => { expect(screen.queryByText("Try a longer range.")).toBeNull() }) + it("shows a panel's samples as a table on request", async () => { + const { client } = clientWith(() => matrix("db-0")) + renderWithK8sProvider(, { client }) + await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3)) + const toggle = screen.getByRole("button", { name: "Show data for CPU (cores)" }) + expect(toggle).toHaveAttribute("aria-expanded", "false") + + await userEvent.click(toggle) + + const table = screen.getByRole("region", { name: "CPU (cores) data" }) + expect(toggle).toHaveAttribute("aria-expanded", "true") + expect(toggle).toHaveAttribute("aria-controls", table.id) + expect(toggle).toHaveTextContent("Hide data") + expect(within(table).getByRole("columnheader", { name: "db-0" })).toBeInTheDocument() + expect(screen.queryByRole("region", { name: "Memory (working set) data" })).toBeNull() + }) + + it("offers no table while a panel has no samples", () => { + const { client } = clientWith(() => new Promise(() => {})) + + renderWithK8sProvider(, { client }) + + expect(screen.getByRole("button", { name: "Show data for Network" })).toBeDisabled() + }) + it("holds each chart's place while it loads", () => { const { client } = clientWith(() => new Promise(() => {})) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx index fac2a6b91..4663b2ae3 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx @@ -1,7 +1,7 @@ -import { useState } from "react" +import { useId, useState } from "react" import { useQueries } from "@tanstack/react-query" import { useK8sClient } from "@cozystack/k8s-client" -import { Section } from "@cozystack/ui" +import { Button, Section } from "@cozystack/ui" import { METRIC_PANELS, RANGES, @@ -14,6 +14,7 @@ import { ProxyNotice } from "./ProxyNotice.tsx" import { TimeSeriesChart } from "./TimeSeriesChart.tsx" import { RangePicker } from "./RangePicker.tsx" import { EmptyState } from "./EmptyState.tsx" +import { SeriesTable } from "./SeriesTable.tsx" const REFRESH_MS = 60_000 // The uPlot canvas plus its one-row legend, so a placeholder takes the chart's place without a jump. @@ -23,6 +24,8 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) { const client = useK8sClient() const root = metricsRoot(scope) const [range, setRange] = useState("1h") + const [tables, setTables] = useState>(new Set()) + const idBase = useId() const rangeKeys = Object.keys(RANGES) as RangeKey[] const longestRange = range === rangeKeys[rangeKeys.length - 1] @@ -57,8 +60,30 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) { ) : ( METRIC_PANELS.map((panel, i) => { const { data, error, isLoading } = results[i] + const hasSeries = !error && !isLoading && !!data && data.series.length > 0 + const tableOpen = hasSeries && tables.has(panel.id) + const tableId = `${idBase}-${panel.id}-data` + const toggle = ( + + ) return ( -
+
{error ? ( ) : isLoading || !data ? ( @@ -66,7 +91,10 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) { ) : data.series.length === 0 ? ( ) : ( - + <> + + {tableOpen && } + )}
) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx new file mode 100644 index 000000000..0b3364268 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx @@ -0,0 +1,37 @@ +import { describe, it, expect } from "vitest" +import { render, screen, within } from "@testing-library/react" +import { SeriesTable } from "./SeriesTable.tsx" + +const data = { + timestamps: [1_700_000_000, 1_700_000_060], + series: [ + { label: "db-0", values: [1, 2] }, + { label: "db-1", values: [null, 3] }, + ], +} + +describe("SeriesTable", () => { + it("is a named region the keyboard can reach", () => { + render() + + const region = screen.getByRole("region", { name: "Memory data" }) + expect(region).toHaveAttribute("id", "t") + expect(region).toHaveAttribute("tabindex", "0") + }) + + it("has a time column and one column per series", () => { + render() + + expect(screen.getAllByRole("columnheader").map((h) => h.textContent)).toEqual(["Time", "db-0", "db-1"]) + }) + + it("lists every sample newest first in the chart's units", () => { + render( `${v} B`} />) + + const rows = screen.getAllByRole("row").slice(1) + expect(rows).toHaveLength(2) + expect(within(rows[0]).getAllByRole("cell").map((c) => c.textContent)).toEqual(["2 B", "3 B"]) + expect(within(rows[1]).getAllByRole("cell").map((c) => c.textContent)).toEqual(["1 B", "–"]) + expect(within(rows[0]).getByRole("rowheader")).toHaveTextContent(new Date(1_700_000_060_000).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" })) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx new file mode 100644 index 000000000..57cf755a0 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx @@ -0,0 +1,47 @@ +import type { ChartData } from "../../lib/observability/metrics.ts" + +interface SeriesTableProps { + id: string + title: string + data: ChartData + format: (value: number) => string +} + +const formatTime = (seconds: number) => + new Date(seconds * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit" }) + +export function SeriesTable({ id, title, data, format }: SeriesTableProps) { + const newestFirst = data.timestamps.map((t, i) => ({ t, i })).reverse() + return ( +
+ + + + + + {data.series.map((s, j) => ( + + ))} + + + + {newestFirst.map(({ t, i }) => ( + + + {data.series.map((s, j) => { + const v = s.values[i] + return + })} + + ))} + +
{`${title}, newest first`}
Time{s.label}
{formatTime(t)}{v === null ? "–" : format(v)}
+
+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx index 57cf6fb99..3df8125e3 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx @@ -1,12 +1,15 @@ import { describe, it, expect, vi, beforeEach } from "vitest" import { render } from "@testing-library/react" -const created: { opts: { series: { label?: string }[] }; data: unknown[]; destroy: ReturnType }[] = [] +type LegendValue = (u: unknown, v: number | null, sidx: number, idx: number | null) => string +type Opts = { series: { label?: string; value?: LegendValue }[] } + +const created: { opts: Opts; data: unknown[]; destroy: ReturnType }[] = [] vi.mock("uplot", () => ({ default: class { destroy = vi.fn() - constructor(opts: { series: { label?: string }[] }, data: unknown[]) { + constructor(opts: Opts, data: unknown[]) { created.push({ opts, data, destroy: this.destroy }) } setSize() {} @@ -36,6 +39,16 @@ describe("TimeSeriesChart", () => { expect(created[0].opts.series.slice(1).map((s) => s.label)).toEqual(["db-0", "db-1"]) }) + it("shows each series' latest value in the legend while nothing is hovered", () => { + render( `${v} B`} />) + + const legend = created[0].opts.series[1].value! + + expect(legend(null, null, 1, null)).toBe("2 B") + expect(legend(null, 1, 1, 0)).toBe("1 B") + expect(legend(null, null, 1, 2)).toBe("–") + }) + it("destroys the chart on unmount", () => { const { unmount } = render() diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx index 683df2dff..a6251d40f 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx @@ -18,6 +18,7 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) { const el = ref.current if (!el) return const fmt = (v: number | null) => (v === null ? "–" : format(v)) + const latest = data.series.map((s) => s.values.findLast((v) => v !== null) ?? null) const chart = new uPlot( { width: el.clientWidth || 600, @@ -30,7 +31,8 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) { label: s.label, stroke: COLORS[i % COLORS.length], width: 1.5, - value: (_: uPlot, v: number | null) => fmt(v), + // Without a hovered point the legend shows the latest sample instead of a dash. + value: (_: uPlot, v: number | null, sidx: number, idx: number | null) => fmt(idx === null ? latest[sidx - 1] : v), })), ], axes: [{}, { size: 84, values: (_: uPlot, splits: number[]) => splits.map(format) }], From e65d827cf9eca51f742200e76172b62224003871 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:16:49 +0700 Subject: [PATCH 054/174] fix(dashboard): move the tenant logs page under observability/ The fixed /console/logs route came before the generic :plural route, so an application kind with the plural "logs" would have its list page replaced by the logs page. A two-segment path cannot shadow an application list, the same way the backups and migration pages avoid it. Assisted-by: LLM Signed-off-by: ZverGuy --- .../src/routes/ConsolePage.routing.test.tsx | 24 +++++++++++++++++++ .../apps/console/src/routes/ConsolePage.tsx | 2 +- .../src/routes/sidebar-sections.test.tsx | 2 +- .../console/src/routes/sidebar-sections.tsx | 2 +- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx index 7c6d1209c..cf6982823 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx @@ -70,4 +70,28 @@ describe("ConsolePage routing", () => { await waitFor(() => expect(client.list).toHaveBeenCalled()) expect(screen.queryByText(/Cluster-scoped capacity/i)).toBeNull() }) + + it("serves the tenant logs page under observability/", async () => { + renderWithK8sProvider( + + + , + { client: makeClient(), initialRoute: "/observability/logs" }, + ) + + expect(await screen.findByText(/^Every application in/)).toBeInTheDocument() + }) + + it("leaves /logs to an application kind whose plural is logs", async () => { + const client = makeClient() + renderWithK8sProvider( + + + , + { client, initialRoute: "/logs" }, + ) + + await waitFor(() => expect(client.list).toHaveBeenCalled()) + expect(screen.queryByText(/^Every application in/)).toBeNull() + }) }) diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx index 5e204e84f..2b2aed3e4 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx @@ -97,7 +97,7 @@ export function ConsolePage() { path="migration/vmimporttasks/:name" element={} /> - } /> + } /> } /> } /> } /> diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx index da40081bc..1fcc3cde4 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx @@ -117,7 +117,7 @@ describe("useConsoleSidebarSections — admin areas moved out", () => { wrapper: makeWrapper(client), }) await waitFor(() => expect(result.current.length).toBeGreaterThan(0)) - expect(findItem(result.current, "Logs")?.to).toBe("/console/logs") + expect(findItem(result.current, "Logs")?.to).toBe("/console/observability/logs") expect(result.current.find((s) => s.title === "Observability")?.items).toHaveLength(1) }) }) diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx index 4cec3a94c..0cf1ce089 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx @@ -121,7 +121,7 @@ export function useConsoleSidebarSections(): SidebarSection[] { const observabilitySection: SidebarSection = { title: "Observability", - items: [{ label: "Logs", to: "/console/logs", icon: ScrollText }], + items: [{ label: "Logs", to: "/console/observability/logs", icon: ScrollText }], } return [...categorySections, backupsSection, migrationSection, observabilitySection] From 52087d92dcaae1316463973643ccbeaf37869a43 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:18:25 +0700 Subject: [PATCH 055/174] fix(dashboard): keep the logs view when only the pod list is refused The pod picker's values query counted as a scope-wide refusal, so a proxy that refused only that path replaced the whole view, filter and lines included, with a notice even though the log query itself worked. A real scope-wide refusal still reaches the history or tail query and shows the notice from there. Assisted-by: LLM Signed-off-by: ZverGuy --- .../src/components/observability/LogsView.test.tsx | 12 ++++++++++++ .../src/components/observability/LogsView.tsx | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx index 770c8a263..86428733d 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -212,6 +212,18 @@ describe("LogsView", () => { expect(within(frame).queryByText(/No log lines/)).toBeNull() }) + it("keeps the view when only the pod list is refused", async () => { + const status = JSON.stringify({ kind: "Status", apiVersion: "v1", status: "Failure", code: 404 }) + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("stream_field_values") ? new Response(status, { status: 404 }) : defaultHandler(endpoint, params), + ) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + expect(screen.queryByText("Logs are not available on this cluster.")).toBeNull() + }) + it("names the source pod and container of each line", async () => { const { client } = clientWith(defaultHandler) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx index a0a73cc43..c7a1f5d58 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -106,7 +106,8 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) { }) const tail = useLogTail(root, query, followRun, { cap: FOLLOW_CAP }) - const wide = [pods.error, history.error, tail.error].find(isScopeWide) + // The pod list is optional: refusing it alone must not hide lines the query can still read. + const wide = [history.error, tail.error].find(isScopeWide) if (wide) { return (
From ef7312579d9872272baf407f622dea7e5fd05944 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:18:33 +0700 Subject: [PATCH 056/174] fix(dashboard): offer a retry when the log query fails The console does not refetch on window focus and must not poll, so a failed history query stayed failed until the filter or range changed, even after the store recovered. The failed state now has a Retry button that runs the same query again. Assisted-by: LLM Signed-off-by: ZverGuy --- .../src/components/observability/EmptyState.tsx | 4 +++- .../src/components/observability/LogsView.test.tsx | 13 +++++++++++++ .../src/components/observability/LogsView.tsx | 14 +++++++++++--- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx index 0cf58224b..5b41ef13d 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx @@ -4,12 +4,13 @@ import { cn, Spinner } from "@cozystack/ui" interface EmptyStateProps { title: ReactNode hint?: ReactNode + action?: ReactNode busy?: boolean tone?: "light" | "dark" className?: string } -export function EmptyState({ title, hint, busy, tone = "light", className }: EmptyStateProps) { +export function EmptyState({ title, hint, action, busy, tone = "light", className }: EmptyStateProps) { const dark = tone === "dark" return (
}

{title}

{hint &&

{hint}

} + {action &&
{action}
}
) } diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx index 86428733d..8b60b7d35 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -224,6 +224,19 @@ describe("LogsView", () => { expect(screen.queryByText("Logs are not available on this cluster.")).toBeNull() }) + it("retries a failed history query on request", async () => { + let calls = 0 + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") && calls++ === 0 ? new Response("upstream down", { status: 502 }) : defaultHandler(endpoint, params), + ) + renderWithK8sProvider(, { client }) + + await userEvent.click(await screen.findByRole("button", { name: "Retry" })) + + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + expect(screen.queryByRole("button", { name: "Retry" })).toBeNull() + }) + it("names the source pod and container of each line", async () => { const { client } = clientWith(defaultHandler) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx index c7a1f5d58..b8d77003a 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -59,7 +59,7 @@ function Picker({ label, value, options, onChange }: { label: string; value: str const RANGE_KEYS = Object.keys(LOG_RANGES) as LogRangeKey[] const LONGEST_RANGE = RANGE_KEYS[RANGE_KEYS.length - 1] -function placeholderFor({ follow, stopped, loading, failed, longestRange }: { follow: boolean; stopped: boolean; loading: boolean; failed: boolean; longestRange: boolean }) { +function placeholderFor({ follow, stopped, loading, failed, longestRange, retry }: { follow: boolean; stopped: boolean; loading: boolean; failed: boolean; longestRange: boolean; retry: () => void }) { if (follow) { return stopped ? ( @@ -68,7 +68,15 @@ function placeholderFor({ follow, stopped, loading, failed, longestRange }: { fo ) } if (loading) return - if (failed) return + if (failed) { + return ( + Retry} + /> + ) + } return } @@ -154,7 +162,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) { void history.refetch() })} />
) From 7d43c3bcd6470c1282a539e05609e9a96de25067 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:18:38 +0700 Subject: [PATCH 057/174] fix(dashboard): open each new log result at its newest line The console keeps a query's previous data as placeholder while the next one loads, so the log list was never remounted between queries. After a reader scrolled up, a new filter or range opened at the old scroll offset instead of the newest lines. The list is now keyed on the query and range as well. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observability/LogsView.test.tsx | 25 ++++++++++++++++--- .../src/components/observability/LogsView.tsx | 3 ++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx index 8b60b7d35..1a703cceb 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -86,13 +86,13 @@ describe("LogsView", () => { endpoint.endsWith("query") ? new Response("") : defaultHandler(endpoint, params), ) renderWithK8sProvider(, { client }) - const frame = screen.getByRole("region", { name: "Log lines" }) - expect(await within(frame).findByText("Try a longer range or a broader filter.")).toBeInTheDocument() + const frame = () => screen.getByRole("region", { name: "Log lines" }) + expect(await within(frame()).findByText("Try a longer range or a broader filter.")).toBeInTheDocument() await userEvent.click(screen.getByRole("button", { name: "24h" })) - expect(await within(frame).findByText("Try a broader filter.")).toBeInTheDocument() - expect(within(frame).queryByText(/longer range/)).toBeNull() + await waitFor(() => expect(within(frame()).getByText("Try a broader filter.")).toBeInTheDocument()) + expect(within(frame()).queryByText(/longer range/)).toBeNull() }) it("shows loading inside the log console", () => { @@ -237,6 +237,23 @@ describe("LogsView", () => { expect(screen.queryByRole("button", { name: "Retry" })).toBeNull() }) + it("opens a new history result at its newest line, not the old scroll position", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") + ? new Response(`${line("2026-11-04T10:00:01Z", `result for ${params.get("query")}`)}\n`) + : defaultHandler(endpoint, params), + ) + const { queryClient } = renderWithK8sProvider(, { client }) + queryClient.setDefaultOptions({ queries: { ...queryClient.getDefaultOptions().queries, placeholderData: (prev: unknown) => prev } }) + await screen.findByText("result for *") + const before = screen.getByRole("list") + + await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}") + + await screen.findByText("result for error") + expect(screen.getByRole("list")).not.toBe(before) + }) + it("names the source pod and container of each line", async () => { const { client } = clientWith(defaultHandler) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx index b8d77003a..1a25d4229 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -160,7 +160,8 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) { )} {error ? : null} void history.refetch() })} /> From 601c9ece3f0ce24792d1dc1f9dce862a6826e876 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:18:47 +0700 Subject: [PATCH 058/174] docs(dashboard): note why the metrics panels refresh on an interval The console rule against refetchInterval assumes a watch is available, which holds for Kubernetes lists but not for the proxy's Prometheus query API. Record the exception so the metrics panels' refresh does not read as a rule violation. Assisted-by: LLM Signed-off-by: ZverGuy --- packages/system/dashboard/images/console/CLAUDE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/system/dashboard/images/console/CLAUDE.md b/packages/system/dashboard/images/console/CLAUDE.md index 96419d118..036e527d6 100644 --- a/packages/system/dashboard/images/console/CLAUDE.md +++ b/packages/system/dashboard/images/console/CLAUDE.md @@ -81,6 +81,7 @@ the apiserver checks that user's RBAC. to a chunked-encoding WATCH against the same `resourceVersion`. Don't poll. Don't add `refetchInterval`. If you need a one-shot, pass `{ watch: false }`. + The one exception is the metrics panels in `components/observability`: the observability proxy serves the Prometheus query API, which has no watch, so they refresh on a fixed interval instead. Log lines stream through the tail endpoint and are not polled. 4. **Tenant scoping.** Most resources live in `tenant-` namespaces. Pull the active tenant from `useTenantContext()` — never read the namespace from a URL param or guess it. From 046df7fe969f2b0b6d62ced127ed601463d9dfc8 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:53:06 +0700 Subject: [PATCH 059/174] fix(dashboard): say the legend shows the latest sample In the browser the legend's time still read "--" next to the latest values, and the data table repeated the same minute on every row at the one-hour range, whose step is 15 seconds. The legend's time now reads "latest" until a point is hovered, and the table shows seconds. Assisted-by: LLM Signed-off-by: ZverGuy --- .../src/components/observability/SeriesTable.test.tsx | 2 +- .../src/components/observability/SeriesTable.tsx | 2 +- .../components/observability/TimeSeriesChart.test.tsx | 10 ++++++++++ .../src/components/observability/TimeSeriesChart.tsx | 3 ++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx index 0b3364268..e85041c0a 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx @@ -32,6 +32,6 @@ describe("SeriesTable", () => { expect(rows).toHaveLength(2) expect(within(rows[0]).getAllByRole("cell").map((c) => c.textContent)).toEqual(["2 B", "3 B"]) expect(within(rows[1]).getAllByRole("cell").map((c) => c.textContent)).toEqual(["1 B", "–"]) - expect(within(rows[0]).getByRole("rowheader")).toHaveTextContent(new Date(1_700_000_060_000).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" })) + expect(within(rows[0]).getByRole("rowheader")).toHaveTextContent(new Date(1_700_000_060_000).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" })) }) }) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx index 57cf755a0..4cc382781 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx @@ -8,7 +8,7 @@ interface SeriesTableProps { } const formatTime = (seconds: number) => - new Date(seconds * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit" }) + new Date(seconds * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit", second: "2-digit" }) export function SeriesTable({ id, title, data, format }: SeriesTableProps) { const newestFirst = data.timestamps.map((t, i) => ({ t, i })).reverse() diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx index 3df8125e3..13cbe933d 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx @@ -13,6 +13,7 @@ vi.mock("uplot", () => ({ created.push({ opts, data, destroy: this.destroy }) } setSize() {} + static fmtDate = () => (d: Date) => d.toISOString() }, })) @@ -49,6 +50,15 @@ describe("TimeSeriesChart", () => { expect(legend(null, null, 1, 2)).toBe("–") }) + it("labels the legend's time as latest while nothing is hovered", () => { + render() + + const time = created[0].opts.series[0].value! + + expect(time(null, null, 0, null)).toBe("latest") + expect(time(null, 2, 0, 1)).toBe(new Date(2000).toISOString()) + }) + it("destroys the chart on unmount", () => { const { unmount } = render() diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx index a6251d40f..063b42b12 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx @@ -19,6 +19,7 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) { if (!el) return const fmt = (v: number | null) => (v === null ? "–" : format(v)) const latest = data.series.map((s) => s.values.findLast((v) => v !== null) ?? null) + const stamp = uPlot.fmtDate("{M}/{D}/{YY} {h}:{mm}:{ss} {AA}") const chart = new uPlot( { width: el.clientWidth || 600, @@ -26,7 +27,7 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) { scales: { x: { time: true } }, legend: { live: true }, series: [ - {}, + { value: (_: uPlot, v: number | null, _sidx: number, idx: number | null) => (idx === null || v === null ? "latest" : stamp(new Date(v * 1000))) }, ...data.series.map((s, i) => ({ label: s.label, stroke: COLORS[i % COLORS.length], From bfa85e57bfe9a619d6c6bcba31c842171a4d848d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:56:26 +0700 Subject: [PATCH 060/174] fix(dashboard): show sample times the same way in legend and table The legend used a fixed US date format while the data table used the reader's locale, so one panel showed the same instant two ways. Both now use one locale-aware format. A test also pins that the data toggle stays disabled for a panel with no series or an error. Assisted-by: LLM Signed-off-by: ZverGuy --- .../components/observability/MetricsView.test.tsx | 15 +++++++++++++++ .../src/components/observability/SeriesTable.tsx | 7 ++----- .../observability/TimeSeriesChart.test.tsx | 4 ++-- .../components/observability/TimeSeriesChart.tsx | 5 ++--- .../apps/console/src/lib/observability/metrics.ts | 4 ++++ 5 files changed, 25 insertions(+), 10 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx index 8e7829ed5..ece4d70ba 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx @@ -123,6 +123,21 @@ describe("MetricsView", () => { expect(screen.getByRole("button", { name: "Show data for Network" })).toBeDisabled() }) + it("offers no table for a panel with no series or an error", async () => { + const { client } = clientWith((url) => + url.searchParams.get("query")?.includes("memory") + ? Response.json({ status: "error", error: "too many series" }, { status: 422 }) + : Response.json({ status: "success", data: { resultType: "matrix", result: [] } }), + ) + renderWithK8sProvider(, { client }) + + await screen.findByText("too many series. Choose a shorter range.") + await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(2)) + for (const title of ["CPU (cores)", "Memory (working set)", "Network"]) { + expect(screen.getByRole("button", { name: `Show data for ${title}` })).toBeDisabled() + } + }) + it("holds each chart's place while it loads", () => { const { client } = clientWith(() => new Promise(() => {})) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx index 4cc382781..5e9d88ba1 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx @@ -1,4 +1,4 @@ -import type { ChartData } from "../../lib/observability/metrics.ts" +import { formatSampleTime, type ChartData } from "../../lib/observability/metrics.ts" interface SeriesTableProps { id: string @@ -7,9 +7,6 @@ interface SeriesTableProps { format: (value: number) => string } -const formatTime = (seconds: number) => - new Date(seconds * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit", second: "2-digit" }) - export function SeriesTable({ id, title, data, format }: SeriesTableProps) { const newestFirst = data.timestamps.map((t, i) => ({ t, i })).reverse() return ( @@ -33,7 +30,7 @@ export function SeriesTable({ id, title, data, format }: SeriesTableProps) { {newestFirst.map(({ t, i }) => ( - {formatTime(t)} + {formatSampleTime(t)} {data.series.map((s, j) => { const v = s.values[i] return {v === null ? "–" : format(v)} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx index 13cbe933d..bb76edb5a 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx @@ -13,11 +13,11 @@ vi.mock("uplot", () => ({ created.push({ opts, data, destroy: this.destroy }) } setSize() {} - static fmtDate = () => (d: Date) => d.toISOString() }, })) const { TimeSeriesChart } = await import("./TimeSeriesChart.tsx") +const { formatSampleTime } = await import("../../lib/observability/metrics.ts") beforeEach(() => { created.length = 0 @@ -56,7 +56,7 @@ describe("TimeSeriesChart", () => { const time = created[0].opts.series[0].value! expect(time(null, null, 0, null)).toBe("latest") - expect(time(null, 2, 0, 1)).toBe(new Date(2000).toISOString()) + expect(time(null, 2, 0, 1)).toBe(formatSampleTime(2)) }) it("destroys the chart on unmount", () => { diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx index 063b42b12..607d44593 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx @@ -1,7 +1,7 @@ import { useEffect, useRef } from "react" import uPlot from "uplot" import "uplot/dist/uPlot.min.css" -import type { ChartData } from "../../lib/observability/metrics.ts" +import { formatSampleTime, type ChartData } from "../../lib/observability/metrics.ts" const COLORS = ["#2563eb", "#16a34a", "#d97706", "#9333ea", "#dc2626", "#0891b2", "#4b5563", "#db2777"] const HEIGHT = 200 @@ -19,7 +19,6 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) { if (!el) return const fmt = (v: number | null) => (v === null ? "–" : format(v)) const latest = data.series.map((s) => s.values.findLast((v) => v !== null) ?? null) - const stamp = uPlot.fmtDate("{M}/{D}/{YY} {h}:{mm}:{ss} {AA}") const chart = new uPlot( { width: el.clientWidth || 600, @@ -27,7 +26,7 @@ export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) { scales: { x: { time: true } }, legend: { live: true }, series: [ - { value: (_: uPlot, v: number | null, _sidx: number, idx: number | null) => (idx === null || v === null ? "latest" : stamp(new Date(v * 1000))) }, + { value: (_: uPlot, v: number | null, _sidx: number, idx: number | null) => (idx === null || v === null ? "latest" : formatSampleTime(v)) }, ...data.series.map((s, i) => ({ label: s.label, stroke: COLORS[i % COLORS.length], diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts index d7182b8d6..2f77e251f 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts @@ -93,6 +93,10 @@ export const METRIC_PANELS: MetricPanel[] = [ }, ] +export function formatSampleTime(seconds: number): string { + return new Date(seconds * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit", second: "2-digit" }) +} + export interface ChartSeries { label: string values: (number | null)[] From 311cb1c90b751bcc24cf962e68676a6997680c4d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:10:02 +0700 Subject: [PATCH 061/174] fix(dashboard): skip tail dedup for rows without a stream ID A pipe such as `delete _stream_id` leaves rows from different streams that share a timestamp with the same dedup key, so a resumed tail dropped a record it had never shown. Only rows that still carry both _stream_id and _time are deduplicated now, as the accepted logs design specifies. Assisted-by: LLM Signed-off-by: ZverGuy --- .../console/src/lib/observability/logs.test.ts | 18 ++++++++++++++++++ .../apps/console/src/lib/observability/logs.ts | 15 ++++++++++----- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts index 62d1ac249..278da6890 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts @@ -63,6 +63,24 @@ describe("mergeTail", () => { expect(out).toHaveLength(2) }) + it("never deduplicates lines a pipe stripped of _stream_id, even on a resume", () => { + const t = "2026-01-01T00:00:02Z" + const buffer = [{ _time: t, _msg: "from stream 1" }] + + const out = mergeTail(buffer, [{ _time: t, _msg: "from stream 2" }], 10, t) + + expect(out.map((x) => x._msg)).toEqual(["from stream 1", "from stream 2"]) + }) + + it("deduplicates only the complete rows of a mixed resume", () => { + const t = "2026-01-01T00:00:02Z" + const buffer = [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }] + + const out = mergeTail(buffer, [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }], 10, t) + + expect(out.map((x) => x._msg)).toEqual(["seen", "no stream", "no time", "no stream", "no time"]) + }) + it("does not deduplicate outside a resume", () => { const out = mergeTail([e("t1", "a")], [e("t1", "a")], 10) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts index eb24a5f96..b1005a1cd 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts @@ -58,16 +58,21 @@ export function compareTime(a: string, b: string): number { return x < y ? -1 : x > y ? 1 : 0 } -const entryKey = (e: LogEntry) => `${e._stream_id ?? ""}\u0000${e._time}` +// Null when a pipe removed either identity field: such a row cannot be told from another stream's. +const entryKey = (e: LogEntry) => + e._stream_id !== undefined && e._time !== undefined ? `${e._stream_id}\u0000${e._time}` : null // (_stream_id, _time) is not unique in VictoriaLogs, so dedup applies only to -// lines a resumed tail replays at or before the resume point, and never to -// lines a pipe stripped of _time. +// lines a resumed tail replays at or before the resume point, and only to +// lines that still carry both fields. export function mergeTail(buffer: LogEntry[], incoming: LogEntry[], cap: number, resumedFrom?: string): LogEntry[] { let fresh = incoming if (resumedFrom) { - const seen = new Set(buffer.map(entryKey)) - fresh = incoming.filter((e) => e._time === undefined || compareTime(e._time, resumedFrom) > 0 || !seen.has(entryKey(e))) + const seen = new Set(buffer.map(entryKey).filter((k) => k !== null)) + fresh = incoming.filter((e) => { + const key = entryKey(e) + return key === null || compareTime(e._time ?? "", resumedFrom) > 0 || !seen.has(key) + }) } const merged = buffer.concat(fresh) return merged.length > cap ? merged.slice(merged.length - cap) : merged From 9c111b3a95f77731b8296310cfd0d32a431b50d6 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:11:34 +0700 Subject: [PATCH 062/174] fix(dashboard): keep placeholder results within one observability scope The provider keeps a query's previous result while its key changes, so switching tenants showed the first tenant's log lines and pod names under the second tenant's heading until its request returned. Every observability query now keeps the previous result only while the proxy root is the same, so a new filter or range still shows the old lines, but a new tenant or application starts empty. Tests render with the provider's own query defaults. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observability/LogsView.test.tsx | 49 ++++++++++++++++++- .../src/components/observability/LogsView.tsx | 3 ++ .../observability/MetricsView.test.tsx | 20 +++++++- .../components/observability/MetricsView.tsx | 3 +- .../console/src/lib/observability/proxy.ts | 8 +++ 5 files changed, 78 insertions(+), 5 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx index 1a703cceb..fe73e4164 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -1,7 +1,7 @@ import { describe, it, expect, vi } from "vitest" -import { screen, waitFor, within } from "@testing-library/react" +import { render, screen, waitFor, within } from "@testing-library/react" import userEvent from "@testing-library/user-event" -import { K8sClient } from "@cozystack/k8s-client" +import { K8sClient, K8sProvider } from "@cozystack/k8s-client" import { renderWithK8sProvider } from "../../test-utils/render.tsx" import { LogsView } from "./LogsView.tsx" @@ -30,6 +30,51 @@ function defaultHandler(endpoint: string, params?: URLSearchParams): Response | return new Promise(() => {}) } +// The provider's own QueryClient keeps the previous result while a key changes; it is +// shared across tests, so each test below uses namespaces no other test reads. +function renderWithProductionDefaults(client: K8sClient, namespace: string) { + const view = (ns: string) => ( + + + + ) + const result = render(view(namespace)) + return { switchTo: (ns: string) => result.rerender(view(ns)) } +} + +describe("LogsView with the production query defaults", () => { + it("drops the previous tenant's lines and pods while the next tenant loads", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.includes("/tenant-switch-b/") ? new Promise(() => {}) : defaultHandler(endpoint, params), + ) + const { switchTo } = renderWithProductionDefaults(client, "tenant-switch-a") + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + await userEvent.click(screen.getByRole("combobox", { name: "Pod" })) + await screen.findByRole("option", { name: "db-1" }) + await userEvent.keyboard("{Escape}") + + switchTo("tenant-switch-b") + + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(within(frame).getByText("Loading log lines…")).toBeInTheDocument() + expect(screen.queryByText("ready to accept connections")).toBeNull() + await userEvent.click(screen.getByRole("combobox", { name: "Pod" })) + expect((await screen.findAllByRole("option")).map((o) => o.textContent)).toEqual(["All"]) + }) + + it("keeps the current lines while a new filter in the same tenant loads", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") && params.get("query") === "slow" ? new Promise(() => {}) : defaultHandler(endpoint, params), + ) + renderWithProductionDefaults(client, "tenant-filter-a") + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + + await userEvent.type(screen.getByLabelText("LogsQL filter"), "slow{Enter}") + + expect(screen.getByText("ready to accept connections")).toBeInTheDocument() + }) +}) + describe("LogsView", () => { it("shows recent lines from the application root", async () => { const { client, raw } = clientWith(defaultHandler) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx index 1a25d4229..c2dc8ca5c 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -14,6 +14,7 @@ import { import { fieldValues, isScopeWide, + keepWithinScope, logsRoot, queryLogs, type ObservabilityScope, @@ -101,6 +102,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) { query: buildLogsQuery({ filter: "", pod: narrowTo }), ...windowFor(range), }, signal), + placeholderData: keepWithinScope(root), retry: false, }) const pods = useQuery(pickerQuery(POD_FIELD)) @@ -110,6 +112,7 @@ export function LogsView({ scope }: { scope: ObservabilityScope }) { queryKey: ["observability", "logs", root, query, range], queryFn: ({ signal }) => queryLogs(client, root, { query, limit: HISTORY_LIMIT, ...windowFor(range) }, signal), enabled: !follow, + placeholderData: keepWithinScope(root), retry: false, }) const tail = useLogTail(root, query, followRun, { cap: FOLLOW_CAP }) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx index ece4d70ba..fccb6f43d 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx @@ -1,7 +1,7 @@ import { describe, it, expect, vi } from "vitest" -import { screen, waitFor, within } from "@testing-library/react" +import { render, screen, waitFor, within } from "@testing-library/react" import userEvent from "@testing-library/user-event" -import { K8sClient } from "@cozystack/k8s-client" +import { K8sClient, K8sProvider } from "@cozystack/k8s-client" import { renderWithK8sProvider } from "../../test-utils/render.tsx" import type { ChartData } from "../../lib/observability/metrics.ts" @@ -138,6 +138,22 @@ describe("MetricsView", () => { } }) + it("drops the previous application's charts while the next one loads, with the production query defaults", async () => { + const { client } = clientWith((url) => (url.pathname.includes("/switch-b/") ? new Promise(() => {}) : matrix("db-0"))) + const view = (name: string) => ( + + + + ) + const { rerender } = render(view("switch-a")) + await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3)) + + rerender(view("switch-b")) + + expect(screen.queryAllByTestId("chart")).toHaveLength(0) + expect(screen.getAllByText("Loading…")).toHaveLength(3) + }) + it("holds each chart's place while it loads", () => { const { client } = clientWith(() => new Promise(() => {})) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx index 4663b2ae3..07980aacf 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx @@ -9,7 +9,7 @@ import { toChartData, type RangeKey, } from "../../lib/observability/metrics.ts" -import { isScopeWide, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts" +import { isScopeWide, keepWithinScope, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts" import { ProxyNotice } from "./ProxyNotice.tsx" import { TimeSeriesChart } from "./TimeSeriesChart.tsx" import { RangePicker } from "./RangePicker.tsx" @@ -42,6 +42,7 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) { ) return toChartData(series) }, + placeholderData: keepWithinScope(root), retry: false, refetchInterval: REFRESH_MS, })), diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts index c28bd12d4..3986c6a3d 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts @@ -20,6 +20,14 @@ export function logsRoot(scope: ObservabilityScope): string { return app ? `${app}/log/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/logs/` } +// The provider keeps a query's previous result while its key changes; across +// scopes that would show one tenant's or application's data under another. +// Every observability query key carries its root at index 2. +export function keepWithinScope(root: string) { + return (previous: T | undefined, previousQuery?: { queryKey: readonly unknown[] }): T | undefined => + previousQuery?.queryKey[2] === root ? previous : undefined +} + export type ProxyErrorKind = | "bad-request" | "unauthorized" From 711a82eaaf3b073c4e69b6bca94c235adf9a891f Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:15:06 +0700 Subject: [PATCH 063/174] refactor(dashboard): drop the scoped placeholder from metrics panels useQueries creates a new observer for every new key, so the metrics panels never carried a previous result across scopes and the scoped placeholder there did nothing. The switch test for the panels stays and passes without it; the helper's comment now says which queries it is for. Assisted-by: LLM Signed-off-by: ZverGuy --- .../console/src/components/observability/MetricsView.tsx | 3 +-- .../console/apps/console/src/lib/observability/proxy.ts | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx index 07980aacf..4663b2ae3 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx @@ -9,7 +9,7 @@ import { toChartData, type RangeKey, } from "../../lib/observability/metrics.ts" -import { isScopeWide, keepWithinScope, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts" +import { isScopeWide, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts" import { ProxyNotice } from "./ProxyNotice.tsx" import { TimeSeriesChart } from "./TimeSeriesChart.tsx" import { RangePicker } from "./RangePicker.tsx" @@ -42,7 +42,6 @@ export function MetricsView({ scope }: { scope: ObservabilityScope }) { ) return toChartData(series) }, - placeholderData: keepWithinScope(root), retry: false, refetchInterval: REFRESH_MS, })), diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts index 3986c6a3d..d33052f70 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts @@ -20,8 +20,9 @@ export function logsRoot(scope: ObservabilityScope): string { return app ? `${app}/log/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/logs/` } -// The provider keeps a query's previous result while its key changes; across -// scopes that would show one tenant's or application's data under another. +// The provider's default keeps a useQuery's previous result while its key +// changes; across scopes that would show one tenant's or application's data +// under another. (useQueries starts a new observer per key, so it never does.) // Every observability query key carries its root at index 2. export function keepWithinScope(root: string) { return (previous: T | undefined, previousQuery?: { queryKey: readonly unknown[] }): T | undefined => From 89194b4f88e4a7399709a48946419745be7326ec Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:17:48 +0700 Subject: [PATCH 064/174] fix(dashboard): reset the log filters when the tenant or app changes LogsView kept its filter, pod and container across a scope switch, so the first request for the new tenant still asked for the previous tenant's pod before the picker fell back to All. The view's state is now keyed on the proxy root, so a new tenant or application starts with no filter and All selected. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observability/LogsView.test.tsx | 20 +++++++++++++++++++ .../src/components/observability/LogsView.tsx | 7 ++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx index fe73e4164..a0df67c68 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -62,6 +62,26 @@ describe("LogsView with the production query defaults", () => { expect((await screen.findAllByRole("option")).map((o) => o.textContent)).toEqual(["All"]) }) + it("starts the next tenant with no filter, pod or container of the previous one", async () => { + const { client, raw } = clientWith(defaultHandler) + const { switchTo } = renderWithProductionDefaults(client, "tenant-reset-a") + await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}") + await userEvent.click(screen.getByRole("combobox", { name: "Pod" })) + await userEvent.click(await screen.findByRole("option", { name: "db-1" })) + await waitFor(() => + expect(raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("query") === '{kubernetes_pod_name="db-1"} (error)')).toBe(true), + ) + + switchTo("tenant-reset-b") + + await waitFor(() => expect(raw.mock.calls.some(([p]) => p.includes("/tenant-reset-b/") && p.includes("select/logsql/query?"))).toBe(true)) + const first = raw.mock.calls.find(([p]) => p.includes("/tenant-reset-b/") && p.includes("select/logsql/query?"))! + expect(new URL(first[0], "http://x").searchParams.get("query")).toBe("*") + expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("All") + expect(screen.getByRole("combobox", { name: "Container" })).toHaveTextContent("All") + expect(screen.getByLabelText("LogsQL filter")).toHaveValue("") + }) + it("keeps the current lines while a new filter in the same tenant loads", async () => { const { client } = clientWith((endpoint, params) => endpoint.endsWith("query") && params.get("query") === "slow" ? new Promise(() => {}) : defaultHandler(endpoint, params), diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx index c2dc8ca5c..8c05a8743 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -81,9 +81,14 @@ function placeholderFor({ follow, stopped, loading, failed, longestRange, retry return } +// Filter, pod and container belong to one scope; a new tenant or application starts clean. export function LogsView({ scope }: { scope: ObservabilityScope }) { - const client = useK8sClient() const root = logsRoot(scope) + return +} + +function ScopedLogsView({ root }: { root: string }) { + const client = useK8sClient() const [range, setRange] = useState("1h") const [draft, setDraft] = useState("") const [filter, setFilter] = useState("") From 32a466fb141b27833737fa6bc634f428376ad4ce Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 18:10:35 +0700 Subject: [PATCH 065/174] fix(dashboard): scope the logs and metrics tabs to the selected app The provider keeps a query's previous result while its key changes, so while the next application's GET is pending the detail page still holds the previous instance. The Logs and Metrics tabs took the application name from that instance, so their proxy root did not change and the previous application's lines and series stayed under the new heading until the GET completed. Take the name from the route instead, which is the identity the user selected, and cover the pending GET at the route level with the real page, router and production query defaults. Assisted-by: LLM Signed-off-by: ZverGuy --- ...plicationDetailPage.observability.test.tsx | 113 ++++++++++++++++++ .../routes/detail/ApplicationDetailPage.tsx | 4 +- .../console/src/routes/detail/LogsTab.tsx | 9 +- .../console/src/routes/detail/MetricsTab.tsx | 9 +- 4 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.observability.test.tsx diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.observability.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.observability.test.tsx new file mode 100644 index 000000000..e16bf1665 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.observability.test.tsx @@ -0,0 +1,113 @@ +import { describe, it, expect, vi, beforeEach } from "vitest" +import { act, render, screen, waitFor } from "@testing-library/react" +import { createMemoryRouter, RouterProvider } from "react-router" +import { K8sProvider } from "@cozystack/k8s-client" +import { TenantContext, type TenantContextValue } from "../../lib/tenant-context.tsx" +import type { ChartData } from "../../lib/observability/metrics.ts" + +// Keep the consoles' and the chart's browser-only libraries out of jsdom; the +// Logs and Metrics tabs, the page and the providers stay real. +vi.mock("./VncTab.tsx", () => ({ VncTab: () => null })) +vi.mock("./SerialTab.tsx", () => ({ SerialTab: () => null })) +vi.mock("../../components/observability/TimeSeriesChart.tsx", () => ({ + TimeSeriesChart: ({ data }: { data: ChartData }) => ( +
{data.series.map((s) => s.label).join(",")}
+ ), +})) + +const { ApplicationDetailPage } = await import("./ApplicationDetailPage.tsx") + +const NS = "tenant-test" +const APPS = `/apis/apps.cozystack.io/v1alpha1/namespaces/${NS}/postgreses` + +const tenant: TenantContextValue = { + tenants: [], + selectedTenant: "test", + selectTenant: () => {}, + tenantNamespace: NS, + isLoading: false, + error: null, +} + +const definitions = { + items: [{ metadata: { name: "postgres" }, spec: { application: { plural: "postgreses", kind: "Postgres" } } }], +} + +const instance = (name: string) => ({ + apiVersion: "apps.cozystack.io/v1alpha1", + kind: "Postgres", + metadata: { name, namespace: NS }, +}) + +const logLine = (name: string) => + `${JSON.stringify({ _time: "2026-11-04T10:00:01Z", _msg: `${name} is ready`, _stream_id: "s", kubernetes_pod_name: `${name}-0`, kubernetes_container_name: "postgres" })}\n` + +const matrix = (name: string) => + Response.json({ status: "success", data: { resultType: "matrix", result: [{ metric: { pod: `${name}-0` }, values: [[1, "1"]] }] } }) + +// app-b's own GET never answers, so the page keeps the placeholder of app-a +// that the production query defaults hand it; its proxy reads do answer. +function handle(url: URL): Response | Promise { + const path = url.pathname + if (path.endsWith("/applicationdefinitions")) return Response.json(definitions) + if (path === `${APPS}/app-a`) return Response.json(instance("app-a")) + if (path === `${APPS}/app-b`) return new Promise(() => {}) + const app = path.match(/\/postgreses\/(app-[ab])\//)?.[1] + if (app && path.endsWith("/select/logsql/query")) return new Response(logLine(app)) + if (app && path.endsWith("/stream_field_values")) return Response.json({ values: [] }) + if (app && path.includes("/metrics/api/v1/query_range")) return matrix(app) + return Response.json({ items: [] }) +} + +const fetchMock = vi.fn(async (input: RequestInfo | URL) => handle(new URL(String(input), "http://x"))) + +function renderAt(path: string) { + const router = createMemoryRouter( + [{ path: "/console/:plural/:name/*", element: }], + { initialEntries: [path] }, + ) + // No queryClient: the provider's production defaults keep a query's previous + // result while its key changes, which is what exposes the stale instance. + render( + + + + + , + ) + return { go: (to: string) => act(() => router.navigate(to)) } +} + +const requested = (prefix: string) => + fetchMock.mock.calls.some(([input]) => new URL(String(input), "http://x").pathname.startsWith(prefix)) + +beforeEach(() => { + vi.stubGlobal("fetch", fetchMock) + fetchMock.mockClear() +}) + +describe("ApplicationDetailPage observability while the next application loads", () => { + it("shows the selected application's logs, not the previous one's", async () => { + const { go } = renderAt("/console/postgreses/app-a/logs") + expect(await screen.findByText("app-a is ready")).toBeInTheDocument() + + await go("/console/postgreses/app-b/logs") + + expect(screen.getByRole("heading", { name: "app-b" })).toBeInTheDocument() + expect(screen.queryByText("app-a is ready")).toBeNull() + await waitFor(() => expect(requested(`${APPS}/app-b/log/`)).toBe(true)) + expect(await screen.findByText("app-b is ready")).toBeInTheDocument() + }) + + it("shows the selected application's metrics, not the previous one's", async () => { + const { go } = renderAt("/console/postgreses/app-a/metrics") + await waitFor(() => expect(screen.getAllByTestId("chart")[0]).toHaveTextContent("app-a-0")) + + await go("/console/postgreses/app-b/metrics") + + expect(screen.getByRole("heading", { name: "app-b" })).toBeInTheDocument() + expect(screen.queryAllByText("app-a-0", { exact: false })).toEqual([]) + await waitFor(() => expect(requested(`${APPS}/app-b/metrics/`)).toBe(true)) + await waitFor(() => expect(screen.getAllByTestId("chart")[0]).toHaveTextContent("app-b-0")) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx index ca499aac1..ab68f7c49 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx @@ -273,11 +273,11 @@ export function ApplicationDetailPage() { path="metrics" element={ }> - + } /> - } /> + } /> diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx index 4ac5f878d..64cbcc89e 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx @@ -1,15 +1,16 @@ -import type { ApplicationDefinition, ApplicationInstance } from "@cozystack/types" +import type { ApplicationDefinition } from "@cozystack/types" import { useTenantContext } from "../../lib/tenant-context.tsx" import { LogsView } from "../../components/observability/LogsView.tsx" interface LogsTabProps { ad: ApplicationDefinition - instance: ApplicationInstance + // The route's name: while the next application loads, the page still holds the previous instance. + name: string } -export function LogsTab({ ad, instance }: LogsTabProps) { +export function LogsTab({ ad, name }: LogsTabProps) { const { tenantNamespace } = useTenantContext() const plural = ad.spec?.application.plural if (!tenantNamespace || !plural) return null - return + return } diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx index a928e633d..992cfc89b 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx @@ -1,15 +1,16 @@ -import type { ApplicationDefinition, ApplicationInstance } from "@cozystack/types" +import type { ApplicationDefinition } from "@cozystack/types" import { useTenantContext } from "../../lib/tenant-context.tsx" import { MetricsView } from "../../components/observability/MetricsView.tsx" interface MetricsTabProps { ad: ApplicationDefinition - instance: ApplicationInstance + // The route's name: while the next application loads, the page still holds the previous instance. + name: string } -export function MetricsTab({ ad, instance }: MetricsTabProps) { +export function MetricsTab({ ad, name }: MetricsTabProps) { const { tenantNamespace } = useTenantContext() const plural = ad.spec?.application.plural if (!tenantNamespace || !plural) return null - return + return } From 55f1fd33a8dd8b73c196d4b45bb328e4c1791a74 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Fri, 6 Nov 2026 16:04:04 +0300 Subject: [PATCH 066/174] fix(backups): let the controller delete operator MariaDB Backups Deleting a MariaDB Backup now deletes the k8s.mariadb.com/Backup that wrote its dump, once the dump itself is gone, so the controller needs the delete verb on that resource. Restores stay create-only. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../templates/rbac.yaml | 11 ++++---- .../tests/mariadb_cleanup_rbac_test.yaml | 28 +++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml diff --git a/packages/system/backupstrategy-controller/templates/rbac.yaml b/packages/system/backupstrategy-controller/templates/rbac.yaml index 8d90980a8..e9e35b066 100644 --- a/packages/system/backupstrategy-controller/templates/rbac.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac.yaml @@ -146,15 +146,16 @@ rules: # MariaDB strategy: read the operator-side MariaDB CR to surface "not yet # rendered" preconditions, create one-shot k8s.mariadb.com/Backup CRs on # the BackupJob path, and create k8s.mariadb.com/Restore CRs on the -# RestoreJob path. The driver only ever Creates/Lists/Gets these objects - -# it does not patch or delete them; their lifecycle is owned by -# mariadb-operator and (for cleanup) by tenants / explicit teardown flows, -# never by this controller. +# RestoreJob path. A Backup is deleted once the dump it wrote is, when its +# Cozystack Backup is deleted; a Restore is never deleted. - apiGroups: ["k8s.mariadb.com"] resources: ["mariadbs"] verbs: ["get", "list", "watch"] - apiGroups: ["k8s.mariadb.com"] - resources: ["backups", "restores"] + resources: ["backups"] + verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: ["k8s.mariadb.com"] + resources: ["restores"] verbs: ["get", "list", "watch", "create"] # MongoDB strategy: read the operator-side PerconaServerMongoDB CR to gate on # the backup preconditions, create one-shot PerconaServerMongoDBBackup CRs on the diff --git a/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml b/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml new file mode 100644 index 000000000..21ac6e807 --- /dev/null +++ b/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml @@ -0,0 +1,28 @@ +suite: RBAC of the MariaDB dump cleanup + +# Deleting a MariaDB Backup deletes its operator Backup once the dump is gone. +# The Go unit tests run with no authorization, so a missing verb only shows up +# on a cluster, where every such Backup would stay Terminating. + +templates: + - templates/rbac.yaml + +release: + name: backupstrategy-controller + namespace: cozy-backup-controller + +tests: + - it: "ClusterRole can delete operator Backups but not Restores" + asserts: + - contains: + path: rules + content: + apiGroups: ["k8s.mariadb.com"] + resources: ["backups"] + verbs: ["get", "list", "watch", "create", "delete"] + - contains: + path: rules + content: + apiGroups: ["k8s.mariadb.com"] + resources: ["restores"] + verbs: ["get", "list", "watch", "create"] From f147b22d86343eb25aa8ff88651efc5ce7993a95 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Fri, 6 Nov 2026 16:04:11 +0300 Subject: [PATCH 067/174] fix(backups): delete a MariaDB dump when its Backup is deleted Deleting a MariaDB Backup left its dump in the bucket for good: mariadb-operator's maxRetention prunes only from inside the next run of the same prefix, and a Plan never deletes a Backup, so on the platform bucket nothing else ever removes it. Every run of an application writes under one prefix, so the key deleted has to be proven to be this Backup's. It is the dump the Backup records, or, for a Backup taken before dumps were recorded, the one the existing ownership match attributes to its operator Backup. That match is recorded on the Backup before the delete, so a retry deletes the same key by name instead of matching again against a bucket that no longer holds it. Without proof the Backup is released with a Warning and nothing is deleted, the operator Backup included, since any later attribution starts from it. The operator Backup is deleted only after its dump, and the delete waits while a RestoreJob still restores from the Backup. The delete runs in the controller through the same minio client the size probe uses, so it verifies TLS the way mariadb-operator did when it wrote the dump. A dump on the platform bucket, which the operator Backup names by its bucket and by the projected credentials Secret, is reached through the platform's current endpoint and keys, still trusting a CA the strategy pinned, so a Backup taken before the endpoint moved is still deleted. Platform credentials that cannot be read fall back to the ones the operator Backup names. A failed delete holds the Backup and retries with a Warning naming the skip-artifact-cleanup annotation, except in a terminating namespace, where the Backup is released with a Warning so it does not hold the namespace. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/artifact_size.go | 38 +- .../backupcontroller/backup_controller.go | 33 +- .../backup_controller_test.go | 7 +- .../backupcontroller/mariadb_artifact_size.go | 15 +- .../mariadbstrategy_cleanup.go | 268 ++++++ .../mariadbstrategy_cleanup_test.go | 886 ++++++++++++++++++ 6 files changed, 1223 insertions(+), 24 deletions(-) create mode 100644 internal/backupcontroller/mariadbstrategy_cleanup.go create mode 100644 internal/backupcontroller/mariadbstrategy_cleanup_test.go diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 9197f07be..40d827c62 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -38,6 +38,11 @@ type s3ObjectLister interface { ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) } +type s3ObjectRemover interface { + // RemoveObject deletes key; a key that does not exist is not an error. + RemoveObject(ctx context.Context, bucket, key string) error +} + // s3Connection is what a driver resolves from its rendered strategy to // read the bucket with the same identity the backup was written with. type s3Connection struct { @@ -63,8 +68,10 @@ type objectStore struct { // reader reads Secrets, which the manager's client fetches uncached, // and lists operator Backups, which it serves from the cache. reader client.Reader - // newLister overrides the S3 client; nil means a real minio client. - newLister func(conn s3Connection) (s3ObjectLister, error) + // newLister and newRemover override the S3 client; nil means a real + // minio client. + newLister func(conn s3Connection) (s3ObjectLister, error) + newRemover func(conn s3Connection) (s3ObjectRemover, error) } func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { @@ -74,6 +81,17 @@ func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { return newMinioObjectLister(conn) } +func (s objectStore) remover(conn s3Connection) (s3ObjectRemover, error) { + if s.newRemover != nil { + return s.newRemover(conn) + } + c, err := newMinioObjectClient(conn) + if err != nil { + return nil, err + } + return c, nil +} + func (s objectStore) secretValue(ctx context.Context, namespace, name, key string) (string, error) { secret := &corev1.Secret{} if err := s.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { @@ -94,6 +112,10 @@ func (r *RestoreJobReconciler) objectStore() objectStore { return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} } +func (r *BackupReconciler) objectStore() objectStore { + return objectStore{reader: r.Client, newLister: r.newS3ObjectLister, newRemover: r.newS3ObjectRemover} +} + func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { return r.objectStore().lister(conn) } @@ -103,6 +125,14 @@ type minioObjectLister struct { } func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { + c, err := newMinioObjectClient(conn) + if err != nil { + return nil, err + } + return c, nil +} + +func newMinioObjectClient(conn s3Connection) (*minioObjectLister, error) { opts := &minio.Options{ Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken), Region: conn.Region, @@ -152,6 +182,10 @@ func (l *minioObjectLister) ListObjects(ctx context.Context, bucket, prefix stri return collectObjects(ctx, l.client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive})) } +func (l *minioObjectLister) RemoveObject(ctx context.Context, bucket, key string) error { + return l.client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{}) +} + // collectObjects drains a minio listing. minio-go ends a listing whose // context is cancelled between pages by closing the channel without an // error item, so a cancelled context is checked once the channel closes: diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index 1a8ec0b40..f817e696b 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -39,9 +39,16 @@ type BackupReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - // CredentialsConfig lets the Rabbitmq cleanup path project cozy-backups-creds - // into the Backup's namespace before spawning its best-effort delete Job. + // CredentialsConfig locates the platform backup credentials: the Job-based + // cleanups project them into the Backup's namespace, and the MariaDB + // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig + + // newS3ObjectLister and newS3ObjectRemover override the S3 client the + // MariaDB cleanup deletes a dump with; nil means a real minio client. + // Set by unit tests. + newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error) + newS3ObjectRemover func(conn s3Connection) (s3ObjectRemover, error) } func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -126,22 +133,18 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // orchestration" for tenant-facing guidance. return ctrl.Result{}, nil case strategyv1alpha1.MariaDBStrategyKind: - // Cozystack Backup deletion does NOT delete the operator-side - // k8s.mariadb.com/Backup CR or the backing S3/PVC archive. - // Lifecycle of that CR is owned by mariadb-operator's - // MaxRetention setting and by tenants / explicit teardown flows; - // the controller's RBAC has no delete verb on - // k8s.mariadb.com/backups for that reason. Surfacing this as an - // explicit branch (rather than falling through to the Velero - // default) keeps the contract honest: the MariaDB driver does - // not own the archive, so it does not delete it on CR removal. - // Mirrors the Altinity branch's "we do not own S3" stance. - return ctrl.Result{}, nil + // mariadb-operator's maxRetention prunes only from inside the next + // backup run of the same prefix and a Plan never deletes a Backup, so + // a dump on the platform bucket is deleted here or never. A dump in + // a tenant's own bucket is deleted too: the Backup proves it owns + // it, and the tenant deleting the Backup is asking for exactly that + // (see cleanupMariaDBBackup). + return r.cleanupMariaDBBackup(ctx, backup) case strategyv1alpha1.MongoDBStrategyKind: // Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the // tenant's own bucket and its lifecycle stays the tenant's / psmdb // operator's — that branch is a no-op, the same "we do not own the - // archive" contract as Altinity / MariaDB / FDB. A useSystemBucket backup + // archive" contract as Altinity / FDB. A useSystemBucket backup // lands in the shared cozy-backups bucket with no psmdb task retention to // prune it, so the driver OWNS it: this deletes the operator // PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer @@ -154,7 +157,7 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // is governed by the next BackupJob's stopOtherFoundationDBBackups // call, by manual operator action, or by // examples/backups/foundationdb/cleanup.sh. Same "we do not own - // the archive" contract as Altinity / MariaDB — the explicit + // the archive" contract as Altinity — the explicit // branch guards the seam against a future driver refactor that // might incidentally stamp velero.io/backup-name onto FDB // driverMetadata via a shared helper. diff --git a/internal/backupcontroller/backup_controller_test.go b/internal/backupcontroller/backup_controller_test.go index 579cb5fe6..d77f5d208 100644 --- a/internal/backupcontroller/backup_controller_test.go +++ b/internal/backupcontroller/backup_controller_test.go @@ -176,11 +176,8 @@ func TestBackupCleanup_Altinity_DoesNotFallThroughToVelero(t *testing.T) { // TestBackupCleanup_MariaDB_DoesNotFallThroughToVelero locks in the same // invariant as the Altinity test above for the MariaDB strategy. The -// dispatcher must explicitly route MariaDB-strategy Backups to a no-op -// cleanup rather than fall through to the Velero default - the MariaDB -// driver does not own the operator-side k8s.mariadb.com/Backup CR or the -// S3/PVC archive behind it (rbac.yaml grants no delete verb on -// k8s.mariadb.com/backups for that reason). The test seeds the Velero +// dispatcher must route MariaDB-strategy Backups to their own cleanup +// rather than fall through to the Velero default. The test seeds the Velero // metadata keys plus a matching velero.io/Backup; with the explicit branch // no DeleteBackupRequest is synthesised, and the Velero Backup survives. // Without the branch the default falls through to cleanupVeleroBackup, diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index a61ec4a43..f9b9973ad 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -60,11 +60,22 @@ func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBack // from can have been edited since, and pointing at another bucket now says // nothing about where this dump went. func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { - lister, s3, err := mariadbStorage(ctx, store, mdbBackup) + lister, _, err := mariadbStorage(ctx, store, mdbBackup) if err != nil { return mariadbDump{}, err } - others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup) + return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt) +} + +// findMariaDBDumpWith is findMariaDBDump reading the bucket through lister, +// for a caller that reaches the bucket by another endpoint than the one +// mdbBackup names. +func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { + s3 := mdbBackup.Spec.Storage.S3 + if s3 == nil { + return mariadbDump{}, errors.New("the operator Backup does not store to S3") + } + others, err := finishedMariaDBRuns(ctx, reader, mdbBackup) if err != nil { return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) } diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go new file mode 100644 index 000000000..483ded502 --- /dev/null +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +const ( + mariadbSkipArtifactCleanupAnnotation = "backups.cozystack.io/skip-artifact-cleanup" + + // Retry cadence while a dump cannot be deleted or a restore from the + // Backup is still running. + mariadbCleanupRetryInterval = 30 * time.Second +) + +// mariadbDumpCleanupTimeout bounds the object-store round trips of one +// cleanup attempt. The Backup controller has a single worker, so an +// unreachable endpoint stalls every other Backup for this long. A variable +// so tests can shorten it. +var mariadbDumpCleanupTimeout = 10 * time.Second + +// cleanupMariaDBBackup deletes the dump a MariaDB Backup was taken as, then +// the operator Backup that wrote it. Every run of an application writes +// under one prefix, so the only key deleted is one proven to be this +// Backup's: the dump it records, or, for a Backup taken before dumps were +// recorded, the one findMariaDBDump attributes to its operator Backup. When +// no dump is proven the Backup is released with a Warning and nothing is +// deleted, the operator Backup included, since it is what any later +// attribution would start from. A failed delete holds the Backup and +// retries, as the Redis and Rabbitmq cleanups do. +func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + logger := getLogger(ctx) + if backup.Annotations[mariadbSkipArtifactCleanupAnnotation] == "true" { + logger.Debug("skipping MariaDB artifact cleanup per annotation", "backup", backup.Name, "annotation", mariadbSkipArtifactCleanupAnnotation) + return ctrl.Result{}, nil + } + + opName := backup.Spec.DriverMetadata[mariadbBackupNameKey] + if opName == "" { + return r.releaseMariaDBCleanup(ctx, backup, "the dump", "the Backup records no k8s.mariadb.com/Backup"), nil + } + + // The operator Restore reads the dump through the operator Backup, so + // neither may go while a restore from this Backup runs. + restore, err := r.activeRestoreOf(ctx, backup) + if err != nil { + return ctrl.Result{}, err + } + if restore != "" { + logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restorejob", restore) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactCleanupWaiting", + "RestoreJob %s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump", + restore, mariadbSkipArtifactCleanupAnnotation) + } + return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil + } + + op := &mariadbtypes.Backup{} + if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: opName}, op); err != nil { + if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) { + return r.releaseMariaDBCleanup(ctx, backup, "the dump", + fmt.Sprintf("k8s.mariadb.com/Backup %s is gone, so the storage and credentials its dump was written with cannot be read", opName)), nil + } + return ctrl.Result{}, err + } + + s3 := op.Spec.Storage.S3 + if s3 == nil { + return r.releaseMariaDBCleanup(ctx, backup, fmt.Sprintf("the dump on the storage volume of k8s.mariadb.com/Backup %s", opName), + "PVC and Volume storage is not reachable from the controller"), nil + } + prefix := mariadbOperatorS3Prefix(s3.Prefix) + location := fmt.Sprintf("s3://%s/%s", s3.Bucket, prefix) + + // A failed Job is Complete=True too, with reason JobFailed, and wrote no + // dump: whatever its window holds is another run's. + if cond := apimeta.FindStatusCondition(op.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond == nil || + cond.Status != metav1.ConditionTrue || cond.Reason != mariadbtypes.ConditionReasonJobComplete { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("k8s.mariadb.com/Backup %s did not complete successfully, so no dump is its own", opName)), nil + } + + name := backup.Spec.DriverMetadata[mariadbDumpFileKey] + if name != "" { + if _, ok := parseMariaDBLogicalBackupFile(name); !ok { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)), nil + } + } + + conn, err := r.mariadbCleanupConnection(ctx, op) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("resolve S3 credentials: %w", err)) + } + + s3ctx, cancel := context.WithTimeout(ctx, mariadbDumpCleanupTimeout) + defer cancel() + store := r.objectStore() + if name == "" { + lister, err := store.lister(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time) + if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil + } + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, err) + } + name = dump.Name + // Recorded first: once the dump is gone a retry could not match it + // again, and could match a neighbour's instead. + base := backup.DeepCopy() + backup.Spec.DriverMetadata[mariadbDumpFileKey] = name + backup.Spec.DriverMetadata[mariadbDumpTimeKey] = dump.Timestamp.UTC().Format(time.RFC3339) + if err := r.Patch(ctx, backup, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, fmt.Errorf("record dump %s on Backup %s: %w", name, backup.Name, err) + } + } + + remover, err := store.remover(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + if err := remover.RemoveObject(s3ctx, s3.Bucket, prefix+name); err != nil { + return r.retryMariaDBCleanup(ctx, backup, location+name, err) + } + policy := metav1.DeletePropagationBackground + if err := r.Delete(ctx, op, &client.DeleteOptions{PropagationPolicy: &policy}); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete k8s.mariadb.com/Backup %s/%s: %w", op.Namespace, op.Name, err) + } + logger.Info("deleted MariaDB dump and its operator Backup", "backup", backup.Name, "dump", location+name, "operatorBackup", opName) + return ctrl.Result{}, nil +} + +// activeRestoreOf returns the name of a RestoreJob that restores from +// backup and has not finished, or "". +func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { + list := &backupsv1alpha1.RestoreJobList{} + if err := r.List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { + return "", fmt.Errorf("list RestoreJobs: %w", err) + } + for i := range list.Items { + rj := &list.Items[i] + if rj.Spec.BackupRef.Name != backup.Name { + continue + } + if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded && rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + return rj.Name, nil + } + } + return "", nil +} + +// mariadbCleanupConnection returns how to reach the bucket op wrote to. A +// dump on the platform bucket, which op names by its bucket and by the +// Secret the platform credentials are projected to, is reached as the +// platform reaches it now, so a Backup taken before the endpoint moved is +// still deleted. Any other bucket, or the platform bucket while the +// platform credentials cannot be read, is reached as op recorded it; a +// delete that fails there is handled by retryMariaDBCleanup like any other. +func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) { + if r.CredentialsConfig.IsEnabled() { + conn, bucket, err := r.platformS3Connection(ctx) + switch { + case err != nil: + getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err) + case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName: + // A strategy can pin a private CA for the platform bucket; the + // platform credentials carry none. + if tls := op.Spec.Storage.S3.TLS; conn.Secure && tls != nil && tls.CASecretKeyRef != nil { + ca, err := r.objectStore().secretValue(ctx, op.Namespace, tls.CASecretKeyRef.Name, tls.CASecretKeyRef.Key) + if err != nil { + return s3Connection{}, err + } + conn.CAPEM = []byte(ca) + } + return conn, nil + } + } + return mariadbS3Connection(ctx, r.objectStore(), op.Namespace, op.Spec.Storage.S3) +} + +// platformS3Connection reads the platform's backup bucket from the Secret +// cozy-backups-creds is projected from, and its TLS from the scheme of +// BACKUP_STORAGE_ENDPOINT, which is how the default strategy sets +// tls.enabled. It reads the source rather than projecting it: projecting +// writes a Secret, which a terminating namespace refuses. +func (r *BackupReconciler) platformS3Connection(ctx context.Context) (s3Connection, string, error) { + cfg := r.CredentialsConfig + src := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Namespace: cfg.SourceNamespace, Name: cfg.SourceSecretName}, src); err != nil { + if apierrors.IsNotFound(err) { + return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMissing, + Message: fmt.Sprintf("source credentials Secret %s/%s not found", cfg.SourceNamespace, cfg.SourceSecretName)} + } + return s3Connection{}, "", &ProjectionError{Reason: ReasonAPIError, Message: fmt.Sprintf("read source credentials Secret: %v", err)} + } + creds, err := parseSourceSecret(src) + if err != nil { + return s3Connection{}, "", err + } + if creds.endpoint == "" { + creds.endpoint = stripScheme(cfg.Endpoint) + } + if creds.endpoint == "" { + return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMalformed, + Message: fmt.Sprintf("endpoint is empty in both source Secret %s/%s and BACKUP_STORAGE_ENDPOINT", cfg.SourceNamespace, cfg.SourceSecretName)} + } + if creds.region == "" { + creds.region = cfg.Region + } + return s3Connection{ + Endpoint: creds.endpoint, + Secure: strings.HasPrefix(cfg.Endpoint, "https://"), + Region: creds.region, + AccessKey: creds.accessKey, + SecretKey: creds.secretKey, + }, creds.bucket, nil +} + +// retryMariaDBCleanup holds the Backup after a failed attempt and says so, +// unless its namespace is going away, where nothing it needs can be read +// any longer and holding it would hold the namespace. +func (r *BackupReconciler) retryMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location string, cause error) (ctrl.Result, error) { + terminating, err := r.namespaceTerminating(ctx, backup.Namespace) + if err != nil { + return ctrl.Result{}, err + } + if terminating { + return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("namespace is terminating and the dump cannot be deleted: %v", cause)), nil + } + getLogger(ctx).Info("cannot delete MariaDB dump; retrying", "backup", backup.Name, "location", location, "error", cause) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactCleanupFailed", + "cannot delete the dump under %s: %v; retrying. Annotate the Backup %s=true to release it and leave the dump", + location, cause, mariadbSkipArtifactCleanupAnnotation) + } + return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil +} + +// releaseMariaDBCleanup lets the Backup go without deleting anything further +// and records a Warning naming what was left, so the give-up is visible. +func (r *BackupReconciler) releaseMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location, reason string) ctrl.Result { + getLogger(ctx).Info("releasing Backup without deleting its dump", "backup", backup.Name, "location", location, "reason", reason) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactNotDeleted", "left %s behind: %s", location, reason) + } + return ctrl.Result{} +} diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go new file mode 100644 index 000000000..745f7aef4 --- /dev/null +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -0,0 +1,886 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +// fakeS3Bucket stands in for the object store on both the listing and the +// delete path, and records what was asked of it. +type fakeS3Bucket struct { + objects []s3ObjectInfo + listErr error + removeErr error + + conns []s3Connection + removed []string +} + +func (f *fakeS3Bucket) ListObjects(_ context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) { + return f.objects, f.listErr +} + +func (f *fakeS3Bucket) RemoveObject(_ context.Context, bucket, key string) error { + if f.removeErr != nil { + return f.removeErr + } + f.removed = append(f.removed, bucket+"/"+key) + return nil +} + +var ( + cleanupCreated = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + cleanupCompleted = cleanupCreated.Add(3 * time.Minute) + cleanupDumpName = "backup." + cleanupCreated.Add(time.Minute).Format(time.RFC3339) + ".gzip.sql" +) + +const cleanupPrefix = "tenant/db/" + +// cleanupOperatorBackup is the operator Backup a MariaDB Backup was made of: +// it stored to bkt under tenant/db with the tenant's creds Secret, and its +// Job succeeded. +func cleanupOperatorBackup(t *testing.T) *mariadbtypes.Backup { + t.Helper() + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/db" + op := operatorBackupStoringTo(t, s3, cleanupCreated) + op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(cleanupCompleted))} + return op +} + +// cleanupBackup is a MariaDB Backup being deleted, made of operator Backup +// op-bk, recording dumpFile when it is not empty. +func cleanupBackup(dumpFile string) *backupsv1alpha1.Backup { + apiGroup := strategyv1alpha1.GroupVersion.Group + md := map[string]string{ + mariadbBackupNameKey: "op-bk", + mariadbBackupNamespaceKey: "tenant", + } + if dumpFile != "" { + md[mariadbDumpFileKey] = dumpFile + } + now := metav1.Now() + return &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", + Name: "bk", + DeletionTimestamp: &now, + Finalizers: []string{backupFinalizer}, + }, + Spec: backupsv1alpha1.BackupSpec{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: &apiGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: "cozy-default-mariadb", + }, + TakenAt: metav1.NewTime(cleanupCompleted), + DriverMetadata: md, + }, + } +} + +func tenantNamespace() *corev1.Namespace { + return &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tenant"}} +} + +func cleanupReconciler(t *testing.T, bucket *fakeS3Bucket, objs ...client.Object) (*BackupReconciler, *record.FakeRecorder) { + t.Helper() + return cleanupReconcilerWithClient(newMariaDBStrategyTestClient(t, objs...), bucket) +} + +func cleanupReconcilerWithClient(c client.Client, bucket *fakeS3Bucket) (*BackupReconciler, *record.FakeRecorder) { + rec := record.NewFakeRecorder(20) + return &BackupReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: rec, + newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) { + bucket.conns = append(bucket.conns, conn) + return bucket, nil + }, + newS3ObjectRemover: func(conn s3Connection) (s3ObjectRemover, error) { + bucket.conns = append(bucket.conns, conn) + return bucket, nil + }, + }, rec +} + +func operatorBackupExists(t *testing.T, c client.Client) bool { + t.Helper() + err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "op-bk"}, &mariadbtypes.Backup{}) + if err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("get operator Backup: %v", err) + } + return err == nil +} + +func requeues(res ctrl.Result) bool { + return res.Requeue || res.RequeueAfter > 0 +} + +func hasEvent(events []string, prefix string) bool { + for _, e := range events { + if strings.HasPrefix(e, prefix) { + return true + } + } + return false +} + +// assertReleasedUntouched checks a Backup that is let go with nothing +// deleted: no S3 delete, the operator Backup kept, and a Warning saying so. +func assertReleasedUntouched(t *testing.T, r *BackupReconciler, rec *record.FakeRecorder, bucket *fakeS3Bucket, res ctrl.Result, err error) { + t.Helper() + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if len(bucket.removed) != 0 { + t.Errorf("removed %v, want nothing deleted from the bucket", bucket.removed) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted, want it kept") + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } +} + +func TestCleanupMariaDBBackup_DeletesRecordedDumpThenOperatorBackup(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + cleanupPrefix + cleanupDumpName}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } + if len(bucket.conns) == 0 || bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connections = %+v, want the operator Backup's endpoint and Secret", bucket.conns) + } + if events := drainEvents(rec); len(events) != 0 { + t.Errorf("events = %v, want none", events) + } +} + +func TestCleanupMariaDBBackup_FailedDeleteKeepsOperatorBackupAndRetries(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("503 SlowDown")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil { + t.Fatalf("err = %v", err) + } + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted although its dump was not") + } + events := drainEvents(rec) + if !hasEvent(events, "Warning ArtifactCleanupFailed") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed naming the skip annotation", events) + } +} + +func TestCleanupMariaDBBackup_FailedOperatorBackupDeleteRetries(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil { + t.Fatal("err = nil, want the failed operator Backup delete returned so the Backup is held") + } +} + +// A legacy Backup records the dump it proved before deleting it, so a retry +// after a failed operator Backup delete finds the key by name instead of +// matching again against a bucket that no longer holds it. +func TestBackupReconcile_LegacyBackupRetryAfterAFailedOperatorBackupDelete(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + backup := cleanupBackup("") + failDelete := true + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok && failDelete { + return errors.New("apiserver unavailable") + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)} + + if _, err := r.Reconcile(context.Background(), req); err == nil { + t.Fatal("first pass: err = nil, want the failed operator Backup delete returned") + } + got := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), req.NamespacedName, got); err != nil { + t.Fatalf("Backup gone after a failed operator Backup delete: %v", err) + } + if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName { + t.Errorf("driverMetadata = %v, want the proven dump recorded before its delete", got.Spec.DriverMetadata) + } + + // The dump is gone now, as the first pass deleted it. + bucket.objects = nil + failDelete = false + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("second pass: %v", err) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted on the retry") + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) { + t.Errorf("Backup still present (err=%v), want its finalizer removed", err) + } + if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want no claim that the deleted dump was left behind", events) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupThatCannotRecordItsDumpDeletesNothing(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + backup := cleanupBackup("") + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + + if _, err := r.cleanupMariaDBBackup(context.Background(), backup); err == nil { + t.Fatal("err = nil, want the failed record returned so the Backup is held") + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted before the dump is recorded", bucket.removed, operatorBackupExists(t, r.Client)) + } +} + +func TestCleanupMariaDBBackup_RejectsARecordedNameThatIsNotADump(t *testing.T) { + for _, name := range []string{"../other/backup.2026-10-21T10:01:00Z.sql", "x.sql", "sub/" + cleanupDumpName} { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(name)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupDeletesTheProvenDump(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)}, + {Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}, + }} + backup := cleanupBackup("") + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } + if events := drainEvents(rec); len(events) != 0 { + t.Errorf("events = %v, want none", events) + } + got := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), client.ObjectKeyFromObject(backup), got); err != nil { + t.Fatal(err) + } + if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName || got.Spec.DriverMetadata[mariadbDumpTimeKey] != cleanupCreated.Add(time.Minute).Format(time.RFC3339) { + t.Errorf("driverMetadata = %v, want the proven dump and its time recorded", got.Spec.DriverMetadata) + } +} + +// The window is the operator Backup's lifetime, closed by the Backup's +// takenAt: a dump stamped after takenAt plus the skew is not this run's. +func TestCleanupMariaDBBackup_LegacyWindowEndsAtTakenAt(t *testing.T) { + late := cleanupCompleted.Add(mariadbSizeProbeClockSkew + time.Minute) + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, late, "gzip"), LastModified: late}, + }} + op := cleanupOperatorBackup(t) + // The operator's own Complete transition is not what bounds the window. + op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(late.Add(time.Hour)))} + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_LegacyBackupWithoutAProvenDumpIsReleased(t *testing.T) { + inWindow := cleanupCreated.Add(time.Minute) + cases := map[string][]s3ObjectInfo{ + "no dump in its lifetime": { + {Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)}, + }, + "two dumps in its lifetime": { + {Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow}, + {Key: dumpKey(cleanupPrefix, inWindow.Add(30*time.Second), "gzip"), LastModified: inWindow.Add(30 * time.Second)}, + }, + "a dump placed by one clock only": { + {Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow.Add(time.Hour)}, + }, + } + for name, objects := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: objects} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpANeighbourCouldOwnIsReleased(t *testing.T) { + inWindow := cleanupCreated.Add(time.Minute) + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow}}} + neighbour := cleanupOperatorBackup(t) + neighbour.Name = "op-neighbour" + neighbour.CreationTimestamp = metav1.NewTime(cleanupCreated.Add(30 * time.Second)) + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), neighbour, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_LegacyListingFailureRetries(t *testing.T) { + bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted, want it kept") + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_ReleasesWhatItCannotAttribute(t *testing.T) { + t.Run("operator Backup is gone", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } + }) + t.Run("Backup records no operator Backup", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), tenantNamespace()) + backup := cleanupBackup(cleanupDumpName) + delete(backup.Spec.DriverMetadata, mariadbBackupNameKey) + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup's Job failed", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions[0].Reason = mariadbtypes.ConditionReasonJobFailed + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup has no Complete condition", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup never completed", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions[0].Status = metav1.ConditionFalse + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("dump is on a volume", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage = mariadbtypes.BackupStorage{} + r, rec := cleanupReconciler(t, bucket, op, tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) +} + +func TestCleanupMariaDBBackup_SkipAnnotationDeletesNothing(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + backup := cleanupBackup(cleanupDumpName) + backup.Annotations = map[string]string{mariadbSkipArtifactCleanupAnnotation: "true"} + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } +} + +func restoreJobOf(name, backupName string, phase backupsv1alpha1.RestoreJobPhase) *backupsv1alpha1.RestoreJob { + return &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: name}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backupName}}, + Status: backupsv1alpha1.RestoreJobStatus{Phase: phase}, + } +} + +func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { + for _, phase := range []backupsv1alpha1.RestoreJobPhase{backupsv1alpha1.RestoreJobPhaseEmpty, backupsv1alpha1.RestoreJobPhasePending, backupsv1alpha1.RestoreJobPhaseRunning} { + t.Run(string(phase), func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("rj", "bk", phase)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held while the restore runs", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted during the restore", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Normal ArtifactCleanupWaiting") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) { + t.Errorf("events = %v, want a Normal ArtifactCleanupWaiting naming the skip annotation", events) + } + }) + } + t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("done", "bk", backupsv1alpha1.RestoreJobPhaseSucceeded), + restoreJobOf("failed", "bk", backupsv1alpha1.RestoreJobPhaseFailed), + restoreJobOf("other", "other-bk", backupsv1alpha1.RestoreJobPhaseRunning)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + }) +} + +// platformCredsConfig points the reconciler at the platform's credentials +// Secret, the source the per-tenant Secret is projected from. The projected +// Secret is "creds", the one cleanupOperatorBackup names, as the default +// strategy names cozy-backups-creds. +func platformCredsConfig(endpoint string) BackupCredentialsConfig { + return BackupCredentialsConfig{ + SourceNamespace: "cozy-system", + SourceSecretName: "cozy-backups-credentials", + TargetSecretName: "creds", + Endpoint: endpoint, + Region: "us-east-1", + } +} + +func platformCredsSecret(bucketName, endpoint string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "cozy-system", Name: "cozy-backups-credentials"}, + Data: map[string][]byte{ + "accessKey": []byte("PLATFORM"), + "secretKey": []byte("platform-secret"), + "endpoint": []byte(endpoint), + "bucketName": []byte(bucketName), + }, + } +} + +func TestCleanupMariaDBBackup_SystemBucketUsesTheCurrentEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + for _, conn := range bucket.conns { + if conn.Endpoint != "s3.new.example" || !conn.Secure || conn.AccessKey != "PLATFORM" || conn.Region != "us-east-1" { + t.Errorf("connection = %+v, want the platform's current endpoint over TLS with its keys", conn) + } + } +} + +func TestCleanupMariaDBBackup_OtherBucketUsesTheOperatorBackupsEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("another-bucket", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + for _, conn := range bucket.conns { + if conn.Endpoint != "s3.example:9000" || conn.AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", conn) + } + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) { + for _, bucketName := range []string{"bkt", "tenant-own-bucket"} { + t.Run(bucketName, func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.Bucket = bucketName + // No platform Secret: the projection source is missing. + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted with the operator Backup's own credentials", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0]) + } + }) + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteHold(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held", res, err) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted although its dump was not") + } + events := drainEvents(rec) + if hasEvent(events, "Warning ArtifactNotDeleted") || !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed and the Backup not released", events) + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteReleaseInATerminatingNamespace(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} + ns := tenantNamespace() + now := metav1.Now() + ns.DeletionTimestamp = &now + ns.Finalizers = []string{"kubernetes"} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_SystemBucketKeepsTheOperatorBackupsCA(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + ca := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"}, Data: map[string][]byte{"ca.crt": []byte("PEM")}} + r, _ := cleanupReconciler(t, bucket, op, ca, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if string(bucket.conns[0].CAPEM) != "PEM" || bucket.conns[0].Endpoint != "s3.new.example" { + t.Errorf("connection = %+v, want the current endpoint trusting the operator Backup's CA", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_SystemBucketWithAnUnreadableCAIsHeld(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_PlainHTTPPlatformEndpointNeedsNoCA(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("http://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted without reading a CA plain HTTP does not use", res, err, bucket.removed) + } +} + +func TestCleanupMariaDBBackup_PlatformEndpointFallsBackToTheConfiguredOne(t *testing.T) { + bucket := &fakeS3Bucket{} + platform := platformCredsSecret("bkt", "") + delete(platform.Data, "endpoint") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform) + r.CredentialsConfig = platformCredsConfig("https://s3.configured.example") + + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err != nil { + t.Fatal(err) + } + if bucket.conns[0].Endpoint != "s3.configured.example" { + t.Errorf("connection = %+v, want BACKUP_STORAGE_ENDPOINT without its scheme", bucket.conns[0]) + } +} + +// blockingBucket answers only when its context ends, as an unreachable +// endpoint does. +type blockingBucket struct{ fakeS3Bucket } + +func (b *blockingBucket) RemoveObject(ctx context.Context, _, _ string) error { + <-ctx.Done() + return ctx.Err() +} + +func TestCleanupMariaDBBackup_BoundsAnUnreachableEndpoint(t *testing.T) { + prev := mariadbDumpCleanupTimeout + mariadbDumpCleanupTimeout = 50 * time.Millisecond + t.Cleanup(func() { mariadbDumpCleanupTimeout = prev }) + bucket := &blockingBucket{} + c := newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r := &BackupReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(20), + newS3ObjectRemover: func(s3Connection) (s3ObjectRemover, error) { return bucket, nil }} + + done := make(chan ctrl.Result, 1) + go func() { + res, _ := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + done <- res + }() + select { + case res := <-done: + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + case <-time.After(5 * time.Second): + t.Fatal("cleanup did not give up on an unreachable endpoint") + } +} + +func TestCleanupMariaDBBackup_ReleasesWhenTheOperatorCRDIsGone(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return &apimeta.NoKindMatchError{GroupKind: mariadbtypes.GroupVersion.WithKind("Backup").GroupKind()} + } + return c.Get(ctx, key, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } +} + +func TestCleanupMariaDBBackup_OperatorBackupAlreadyDeletedIsDone(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return apierrors.NewNotFound(mariadbtypes.GroupVersion.WithResource("backups").GroupResource(), obj.GetName()) + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } +} + +// A strategy of its own that happens to use the platform bucket's name, as +// an admin-chosen name on external S3 can, is reached as recorded: only the +// projected Secret marks a Backup as written to the platform bucket. +func TestCleanupMariaDBBackup_SameBucketNameWithOwnCredentialsUsesTheOperatorBackupsEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name = "own-creds" + op.Spec.Storage.S3.SecretAccessKeySecretKeyRef.Name = "own-creds" + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "own-creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0]) + } +} + +// With no endpoint anywhere the platform credentials are unusable, which +// falls back like any other unreadable credentials. +func TestCleanupMariaDBBackup_PlatformCredentialsWithoutAnEndpointFallBack(t *testing.T) { + bucket := &fakeS3Bucket{} + platform := platformCredsSecret("bkt", "") + delete(platform.Data, "endpoint") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform) + r.CredentialsConfig = platformCredsConfig("") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_MalformedPlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) { + bucket := &fakeS3Bucket{} + malformed := platformCredsSecret("bkt", "s3.new.example") + delete(malformed.Data, "secretKey") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), malformed) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_TerminatingNamespaceReleasesOnS3Failure(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("connection refused")} + ns := tenantNamespace() + now := metav1.Now() + ns.DeletionTimestamp = &now + ns.Finalizers = []string{"kubernetes"} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +// Reconcile keeps the finalizer while the dump cannot be deleted, and drops +// it once both the dump and the operator Backup are gone. +func TestBackupReconcile_MariaDBFinalizerFollowsTheDumpDelete(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("503")} + backup := cleanupBackup(cleanupDumpName) + r, _ := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)} + + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); err != nil { + t.Fatalf("Backup gone while its dump could not be deleted: %v", err) + } + + bucket.removeErr = nil + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) { + t.Errorf("Backup still present (err=%v), want its finalizer removed", err) + } +} + +// The minio adapter deletes the exact key, with the colons of the dump's +// timestamp escaped in the path; SeaweedFS refuses them raw. +func TestMinioObjectRemover_DeletesTheEscapedKey(t *testing.T) { + var method, rawPath, auth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + method, rawPath, auth = req.Method, req.URL.EscapedPath(), req.Header.Get("Authorization") + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"}) + if err != nil { + t.Fatal(err) + } + if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "tenant#a/db/backup.2026-10-21T10:01:00Z.gzip.sql"); err != nil { + t.Fatalf("RemoveObject: %v", err) + } + if method != http.MethodDelete || rawPath != "/bkt/tenant%23a/db/backup.2026-10-21T10%3A01%3A00Z.gzip.sql" { + t.Errorf("request %s %s, want DELETE of the escaped key", method, rawPath) + } + if !strings.Contains(auth, "Credential=AKID/") { + t.Errorf("request not signed with the access key: %q", auth) + } +} + +func TestMinioObjectRemover_ReturnsTheServersError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `AccessDeniedAccess Denied`) + })) + defer srv.Close() + + remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"}) + if err != nil { + t.Fatal(err) + } + if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "k"); err == nil { + t.Fatal("RemoveObject = nil, want the AccessDenied error") + } +} From e66b764734a505a85009922c36629a4d674512c6 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Sun, 8 Nov 2026 20:21:22 +0300 Subject: [PATCH 068/174] fix(backups): let an unfinished MariaDB run block a legacy dump delete A Backup taken before dumps were recorded finds its dump by window, and only finished neighbour runs were allowed to contest a candidate. A run uploads its dump and prunes older ones through maxRetention before its operator Backup turns Complete, so in that gap the pruned run's cleanup saw the newer run's dump as the only candidate and deleted it, leaving that Backup Ready with nothing to restore. The size probe and the restore keep ignoring unfinished runs, where a wrong guess costs a size or a refused restore. Deletion now treats a run that has not completed as one whose window is still open, so the dump is left behind with a Warning, as for any other unproven attribution. Holding the Backup would not help: once that run completes its window still covers the candidate, and the attribution stays ambiguous. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../backupcontroller/mariadb_artifact_size.go | 37 ++++++++----- .../mariadbstrategy_cleanup.go | 2 +- .../mariadbstrategy_cleanup_test.go | 52 +++++++++++++++++++ 3 files changed, 77 insertions(+), 14 deletions(-) diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index f9b9973ad..4e0525517 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -64,18 +64,19 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt if err != nil { return mariadbDump{}, err } - return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt) + return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt, false) } // findMariaDBDumpWith is findMariaDBDump reading the bucket through lister, // for a caller that reaches the bucket by another endpoint than the one -// mdbBackup names. -func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { +// mdbBackup names. withUnfinished lets runs that have not completed contest +// the dump too, as neighbourMariaDBRuns describes. +func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time, withUnfinished bool) (mariadbDump, error) { s3 := mdbBackup.Spec.Storage.S3 if s3 == nil { return mariadbDump{}, errors.New("the operator Backup does not store to S3") } - others, err := finishedMariaDBRuns(ctx, reader, mdbBackup) + others, err := neighbourMariaDBRuns(ctx, reader, mdbBackup, withUnfinished) if err != nil { return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) } @@ -136,6 +137,10 @@ type mariadbRun struct { lower, upper time.Time } +// mariadbUnfinishedRunEnd closes the window of a run that has not completed, +// which can still write its dump at any time. +var mariadbUnfinishedRunEnd = time.Date(9999, time.December, 31, 0, 0, 0, 0, time.UTC) + func newMariaDBRun(b *mariadbtypes.Backup, created, completed time.Time) mariadbRun { return mariadbRun{ name: b.Namespace + "/" + b.Name, @@ -161,8 +166,8 @@ func (r mariadbRun) placesOnEitherClock(d mariadbDump) bool { // dump in the window on one clock only may be this run's own, misplaced by // a skewed clock, which leaves any other candidate unproven. A neighbour's // dump is still taken if this run's own is missing from the listing, or -// misplaced on both clocks, and the neighbour's operator Backup is gone or -// has not completed. +// misplaced on both clocks, and the neighbour's operator Backup is gone or, +// unless others includes unfinished runs, has not completed. func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, others []mariadbRun) (mariadbDump, error) { var near []mariadbDump for _, obj := range objects { @@ -194,14 +199,17 @@ func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, oth return d, nil } -// finishedMariaDBRuns returns the runs of the other operator Backups that +// neighbourMariaDBRuns returns the runs of the other operator Backups that // write under the same bucket and prefix as b and have completed, whether -// their Job succeeded or failed. One still running is left out: its window -// has no end yet, and an operator Backup stuck short of Complete, as one -// whose Job pod never starts is, would otherwise contest every later run of -// the application for good. The endpoint is not compared, since one has -// several spellings, and a false neighbour only leaves a size unset. -func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup) ([]mariadbRun, error) { +// their Job succeeded or failed. One still running is left out unless +// withUnfinished is set: its window has no end yet, and an operator Backup +// stuck short of Complete, as one whose Job pod never starts is, would +// otherwise contest every later run of the application for good. That is +// the price of deleting a dump, since a run can upload its dump and prune +// older ones through maxRetention before it completes. The endpoint is not +// compared, since one has several spellings, and a false neighbour only +// keeps a dump unattributed. +func neighbourMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup, withUnfinished bool) ([]mariadbRun, error) { list := &mariadbtypes.BackupList{} if err := reader.List(ctx, list); err != nil { return nil, err @@ -218,6 +226,9 @@ func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbty } cond := apimeta.FindStatusCondition(other.Status.Conditions, mariadbtypes.ConditionTypeComplete) if cond == nil || cond.Status != metav1.ConditionTrue { + if withUnfinished { + runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, mariadbUnfinishedRunEnd)) + } continue } runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, cond.LastTransitionTime.Time)) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 483ded502..0a2fc1e34 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -117,7 +117,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac if err != nil { return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) } - dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time) + dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time, true) if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 745f7aef4..9b41f4701 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -385,6 +385,58 @@ func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpANeighbourCouldOwnIsReleased( assertReleasedUntouched(t, r, rec, bucket, res, err) } +// A run started before this one completed can upload its dump and prune +// this run's through maxRetention before its own operator Backup completes. +// Its dump is then the only one in this run's window, and only its open +// window says it may be its own. +func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpAnUnfinishedRunCouldOwnIsReleased(t *testing.T) { + started := cleanupCreated.Add(2 * time.Minute) + cases := map[string][]metav1.Condition{ + "running": {{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(started), + }}, + "not yet reconciled": nil, + } + for name, conditions := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, started.Add(90*time.Second), "gzip"), LastModified: started.Add(100 * time.Second)}, + }} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(started) + running.Status.Conditions = conditions + backup := cleanupBackup("") + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +// An unfinished run contests only what it could have written: a dump +// that predates it, on both clocks, is still this run's. +func TestCleanupMariaDBBackup_LegacyBackupDeletesItsDumpDespiteALaterUnfinishedRun(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + backup := cleanupBackup("") + r, _ := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } +} + func TestCleanupMariaDBBackup_LegacyListingFailureRetries(t *testing.T) { bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) From 2e5a3bd6a7bcbd85f77e1043f782951ce37411cd Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Mon, 9 Nov 2026 15:47:34 +0700 Subject: [PATCH 069/174] fix(dashboard): keep the tenant logs page off application URLs Every console path of one or two segments belongs to an application list or detail page, so /console/observability/logs took the Overview of an application named logs whose kind has the plural observability. Kubernetes names must start with an alphanumeric character, so the reserved "-" segment cannot collide with any application. Assisted-by: LLM Signed-off-by: ZverGuy --- .../src/routes/ConsolePage.routing.test.tsx | 18 ++++++++++++++++-- .../apps/console/src/routes/ConsolePage.tsx | 2 +- .../src/routes/sidebar-sections.test.tsx | 2 +- .../console/src/routes/sidebar-sections.tsx | 2 +- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx index cf6982823..8e71ff98a 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx @@ -71,12 +71,12 @@ describe("ConsolePage routing", () => { expect(screen.queryByText(/Cluster-scoped capacity/i)).toBeNull() }) - it("serves the tenant logs page under observability/", async () => { + it("serves the tenant logs page under the reserved -/ segment", async () => { renderWithK8sProvider( , - { client: makeClient(), initialRoute: "/observability/logs" }, + { client: makeClient(), initialRoute: "/-/observability/logs" }, ) expect(await screen.findByText(/^Every application in/)).toBeInTheDocument() @@ -94,4 +94,18 @@ describe("ConsolePage routing", () => { await waitFor(() => expect(client.list).toHaveBeenCalled()) expect(screen.queryByText(/^Every application in/)).toBeNull() }) + + // No ApplicationDefinition plural or application name can be "-", so only + // routes under it are safe from shadowing an application's own pages. + it("leaves /observability/logs to an application named logs of kind observability", async () => { + renderWithK8sProvider( + + + , + { client: makeClient(), initialRoute: "/observability/logs" }, + ) + + expect(await screen.findByText("Loading…")).toBeInTheDocument() + expect(screen.queryByText(/^Every application in/)).toBeNull() + }) }) diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx index 2b2aed3e4..c78ae619e 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx @@ -97,7 +97,7 @@ export function ConsolePage() { path="migration/vmimporttasks/:name" element={} /> - } /> + } /> } /> } /> } /> diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx index 1fcc3cde4..e678c7dd4 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx @@ -117,7 +117,7 @@ describe("useConsoleSidebarSections — admin areas moved out", () => { wrapper: makeWrapper(client), }) await waitFor(() => expect(result.current.length).toBeGreaterThan(0)) - expect(findItem(result.current, "Logs")?.to).toBe("/console/observability/logs") + expect(findItem(result.current, "Logs")?.to).toBe("/console/-/observability/logs") expect(result.current.find((s) => s.title === "Observability")?.items).toHaveLength(1) }) }) diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx index 0cf1ce089..51c9ea00a 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx @@ -121,7 +121,7 @@ export function useConsoleSidebarSections(): SidebarSection[] { const observabilitySection: SidebarSection = { title: "Observability", - items: [{ label: "Logs", to: "/console/observability/logs", icon: ScrollText }], + items: [{ label: "Logs", to: "/console/-/observability/logs", icon: ScrollText }], } return [...categorySections, backupsSection, migrationSection, observabilitySection] From e623530c479ad92afd9c8a707aa12cd645f0ae93 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Mon, 9 Nov 2026 10:29:20 +0100 Subject: [PATCH 070/174] fix(kubernetes): detach volumes when the infra PVC is missing Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Assisted-by: LLM --- .../images/kubevirt-csi-driver/controller.go | 3 +- .../kubevirt-csi-driver/controller_test.go | 77 +++++++++++++++++-- 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go index 9029c9ff3..b61121758 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go @@ -424,7 +424,8 @@ func (w *WrappedControllerService) ControllerUnpublishVolume(ctx context.Context pvc, err := w.infraClient.CoreV1().PersistentVolumeClaims(w.infraNamespace).Get(ctx, dvName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { - return &csi.ControllerUnpublishVolumeResponse{}, nil + // A failed PVC creation can still leave a hotplug entry in the VM. + return w.ControllerService.ControllerUnpublishVolume(ctx, req) } return nil, err } diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go index 46e2e1dce..73b8ffc7d 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go @@ -310,13 +310,14 @@ const ( testVolumeName = "pvc-0" ) -// teardownClient models the CAPI/CAPK teardown race: the VM is gone while its -// VMI still reports the hot-plugged volume. The VMI drops the volume once -// RemoveVolumeFromVMI is called, or when the garbage collector deletes a -// VMI whose controlling VM is gone. +// teardownClient keeps hotplug state until removal or garbage collection, +// including when the VMI outlives its VM during CAPI/CAPK teardown. type teardownClient struct { kubevirtclient.Client + vm *kubevirtv1.VirtualMachine vmiOwners []metav1.OwnerReference + detached bool + vmRemovals int vmiRemovals int lookups int } @@ -328,6 +329,9 @@ func (c *teardownClient) vmiGarbageCollected() bool { func (c *teardownClient) GetWorkloadManagingVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachine, error) { c.lookups++ + if c.vm != nil { + return c.vm, nil + } return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachines"}, testVMName) } @@ -336,14 +340,27 @@ func (c *teardownClient) GetVirtualMachine(context.Context, string, string) (*ku vmi := &kubevirtv1.VirtualMachineInstance{ ObjectMeta: metav1.ObjectMeta{Name: testVMName, Namespace: testInfraNamespace, OwnerReferences: c.vmiOwners}, } - if c.vmiRemovals == 0 { - vmi.Status.VolumeStatus = []kubevirtv1.VolumeStatus{{Name: testVolumeName, HotplugVolume: &kubevirtv1.HotplugVolumeStatus{}}} + if !c.detached && c.vmRemovals == 0 && c.vmiRemovals == 0 { + vmi.Status.VolumeStatus = []kubevirtv1.VolumeStatus{{Name: testVolumeName, Phase: kubevirtv1.VolumePending, Reason: "FailedPvcNotFound", HotplugVolume: &kubevirtv1.HotplugVolumeStatus{}}} } return vmi, nil } +func (c *teardownClient) EnsureVolumeRemovedVM(context.Context, string, string, string) (bool, error) { + return c.vm == nil || len(c.vm.Spec.Template.Spec.Volumes) == 0, nil +} + func (c *teardownClient) EnsureVolumeRemovedVMI(context.Context, string, string, string) (bool, error) { - return c.vmiRemovals > 0, nil + return c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0, nil +} + +func (c *teardownClient) RemoveVolumeFromVM(_ context.Context, namespace, name string, opts *kubevirtv1.RemoveVolumeOptions) error { + if namespace != testInfraNamespace || name != testVMName || opts.Name != testVolumeName { + return errors.New("unexpected hotplug removal target") + } + c.vmRemovals++ + c.vm.Spec.Template.Spec.Volumes = nil + return nil } func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *kubevirtv1.RemoveVolumeOptions) error { @@ -352,7 +369,7 @@ func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *k } func (c *teardownClient) EnsureVolumeRemoved(context.Context, string, string, string, time.Duration) error { - if c.vmiRemovals > 0 || c.vmiGarbageCollected() { + if c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0 || c.vmiGarbageCollected() { return nil } return errors.New("timed out waiting for the VMI to drop the volume") @@ -410,6 +427,50 @@ func TestControllerUnpublishVolumeHonoursVMIHotplugFallback(t *testing.T) { } } +func TestControllerUnpublishVolumeMissingPVC(t *testing.T) { + tests := []struct { + name string + vmGone bool + detached bool + wantVMCalls int + wantVMICalls int + }{ + {name: "pending hotplug volume", wantVMCalls: 1}, + {name: "already detached", detached: true}, + {name: "VM gone, VMI still has volume", vmGone: true, wantVMICalls: 1}, + {name: "VM gone, no attached volume", vmGone: true, detached: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := &teardownClient{detached: tt.detached} + if !tt.vmGone { + client.vm = &kubevirtv1.VirtualMachine{Spec: kubevirtv1.VirtualMachineSpec{Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{}}} + if !tt.detached { + client.vm.Spec.Template.Spec.Volumes = []kubevirtv1.Volume{{ + Name: testVolumeName, + VolumeSource: kubevirtv1.VolumeSource{DataVolume: &kubevirtv1.DataVolumeSource{Name: testVolumeName, Hotpluggable: true}}, + }} + } + } + w := &WrappedControllerService{ + ControllerService: service.NewControllerService(client, testInfraNamespace, nil, util.StorageClassEnforcement{AllowAll: true}, true), + infraClient: fake.NewClientset(), + infraNamespace: testInfraNamespace, + } + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: testVolumeName, NodeId: testInfraNamespace + "/" + testVMName} + if _, err := w.ControllerUnpublishVolume(context.Background(), req); err != nil { + t.Fatalf("ControllerUnpublishVolume() error = %v", err) + } + if client.vmRemovals != tt.wantVMCalls || client.vmiRemovals != tt.wantVMICalls { + t.Errorf("VM/VMI removevolume calls = %d/%d, want %d/%d", client.vmRemovals, client.vmiRemovals, tt.wantVMCalls, tt.wantVMICalls) + } + if client.vm != nil && len(client.vm.Spec.Template.Spec.Volumes) != 0 { + t.Error("hotplug volume remains in VM spec") + } + }) + } +} + func nfsCNP(owners ...string) *unstructured.Unstructured { refs := make([]interface{}, 0, len(owners)) for _, owner := range owners { From 5f6679e4cff70b6e37535920427004259af8adac Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 9 Nov 2026 10:52:28 +0100 Subject: [PATCH 071/174] fix(ingress-nginx): raise client keepalive_time above the 1h default ingress-nginx has no ConfigMap key for the client-side keepalive_time, so nginx applies its 1h default and closes every HTTP/2 connection that reaches that age with a GOAWAY. Clients that multiplex a long operation over a single connection see it fail: a kopia repository maintenance run by Velero against SeaweedFS lasted just over an hour and died while listing blobs, with "http2: server sent GOAWAY and closed the connection; ErrCode=NO_ERROR", because the listing is not retried. Set keepalive_time to 12h through http-snippet. Idle connections are still closed by keep-alive after 75s, so this only lengthens the life of connections that are actually in use. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../tests/client_keepalive_time_test.yaml | 26 +++++++++++++++++++ packages/system/ingress-nginx/values.yaml | 7 +++++ 2 files changed, 33 insertions(+) create mode 100644 packages/system/ingress-nginx/tests/client_keepalive_time_test.yaml diff --git a/packages/system/ingress-nginx/tests/client_keepalive_time_test.yaml b/packages/system/ingress-nginx/tests/client_keepalive_time_test.yaml new file mode 100644 index 000000000..bb492e57c --- /dev/null +++ b/packages/system/ingress-nginx/tests/client_keepalive_time_test.yaml @@ -0,0 +1,26 @@ +suite: ingress-nginx client keepalive_time +release: + name: ingress-nginx + namespace: cozy-ingress-nginx +templates: + - charts/ingress-nginx/templates/controller-configmap.yaml +tests: + - it: raises the client keepalive_time above the nginx 1h default + asserts: + - equal: + path: data.http-snippet + value: keepalive_time 12h; + - it: keeps the snippet when a tenant ingress sets its own config keys + # The tenant ingress HelmRelease passes controller.config for whitelist, + # Cloudflare and PROXY protocol; Helm merges maps, so those keys must not + # drop the default snippet. + set: + ingress-nginx.controller.config.use-proxy-protocol: "true" + ingress-nginx.controller.config.whitelist-source-range: 10.0.0.0/8 + asserts: + - equal: + path: data.http-snippet + value: keepalive_time 12h; + - equal: + path: data.use-proxy-protocol + value: "true" diff --git a/packages/system/ingress-nginx/values.yaml b/packages/system/ingress-nginx/values.yaml index fca145843..1cb7b58da 100644 --- a/packages/system/ingress-nginx/values.yaml +++ b/packages/system/ingress-nginx/values.yaml @@ -47,6 +47,13 @@ ingress-nginx: upstream-keepalive-timeout: "60" upstream-keepalive-connections: "320" ssl-session-tickets: "true" + # Client-side keepalive_time has no ConfigMap key, so nginx keeps its + # 1h default and sends GOAWAY to any HTTP/2 connection older than that. + # A long S3 operation (kopia maintenance behind Velero) multiplexes its + # requests over one connection, and the in-flight object listing fails + # instead of being retried. Idle connections still close after + # keep-alive (75s). + http-snippet: "keepalive_time 12h;" resources: limits: cpu: "1" From 0972690f0d31426f892e7d7e04ad48cf33d5e9c3 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Mon, 9 Nov 2026 13:52:38 +0300 Subject: [PATCH 072/174] fix(backups): contest a recorded MariaDB dump before deleting it The size probe records the dump it attributes to a Backup, and it lets only runs that had already completed contest the attribution. A run can upload its dump and prune an older run's through maxRetention before its own operator Backup completes, so the probe of the pruned run can record the newer run's dump, and deleting that Backup then deleted a dump that another Backup still restores from. Deletion now contests the recorded dump against every run on the same bucket and prefix as they stand at deletion time, the unfinished ones included, and leaves it behind with a Warning when any could have written it, as the unrecorded path already does. A run that never completes therefore keeps every later dump its lifetime covers, which leaks storage rather than losing a backup. The probe is unchanged here. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup.go | 53 ++++- .../mariadbstrategy_cleanup_test.go | 215 ++++++++++++++++++ 2 files changed, 264 insertions(+), 4 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 0a2fc1e34..94a7d10f4 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "slices" "strings" "time" @@ -37,7 +38,8 @@ var mariadbDumpCleanupTimeout = 10 * time.Second // cleanupMariaDBBackup deletes the dump a MariaDB Backup was taken as, then // the operator Backup that wrote it. Every run of an application writes // under one prefix, so the only key deleted is one proven to be this -// Backup's: the dump it records, or, for a Backup taken before dumps were +// Backup's: the dump it records, unless contestedMariaDBDump finds another +// run that could have written it, or, for a Backup taken before dumps were // recorded, the one findMariaDBDump attributes to its operator Backup. When // no dump is proven the Backup is released with a Warning and nothing is // deleted, the operator Backup included, since it is what any later @@ -112,11 +114,20 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac s3ctx, cancel := context.WithTimeout(ctx, mariadbDumpCleanupTimeout) defer cancel() store := r.objectStore() - if name == "" { - lister, err := store.lister(conn) + lister, err := store.lister(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + if name != "" { + other, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name) if err != nil { - return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + return r.retryMariaDBCleanup(ctx, backup, location+name, err) } + if other != "" { + return r.releaseMariaDBCleanup(ctx, backup, location+name, + fmt.Sprintf("the recorded dump also falls in the lifetime of k8s.mariadb.com/Backup %s, which may have written it; the operator Backup is kept", other)), nil + } + } else { dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time, true) if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { return r.releaseMariaDBCleanup(ctx, backup, location, @@ -151,6 +162,40 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac return ctrl.Result{}, nil } +// contestedMariaDBDump returns the name of another operator Backup on op's +// bucket and prefix whose run, finished or not, could have written the dump +// name, or "" when none could. The size probe that recorded name let only +// runs finished by then contest it, and a run can upload its dump and prune +// op's through maxRetention before it completes. A dump no longer in the +// bucket is contested by nobody: deleting it again removes nothing. +func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, error) { + ts, ok := parseMariaDBLogicalBackupFile(name) + if !ok { + return "", fmt.Errorf("%q is not the name of a mariadb-operator dump", name) + } + s3 := op.Spec.Storage.S3 + key := mariadbOperatorS3Prefix(s3.Prefix) + name + objects, err := lister.ListObjects(ctx, s3.Bucket, key, false) + if err != nil { + return "", fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) + } + i := slices.IndexFunc(objects, func(obj s3ObjectInfo) bool { return obj.Key == key }) + if i < 0 { + return "", nil + } + others, err := neighbourMariaDBRuns(ctx, reader, op, true) + if err != nil { + return "", fmt.Errorf("list operator Backups: %w", err) + } + d := mariadbDump{Key: key, Name: name, Timestamp: ts, LastModified: objects[i].LastModified} + for _, other := range others { + if other.placesOnEitherClock(d) { + return other.name, nil + } + } + return "", nil +} + // activeRestoreOf returns the name of a RestoreJob that restores from // backup and has not finished, or "". func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 9b41f4701..ddca34523 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -22,6 +22,7 @@ import ( strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" ) @@ -437,6 +438,220 @@ func TestCleanupMariaDBBackup_LegacyBackupDeletesItsDumpDespiteALaterUnfinishedR } } +// The size probe that records a dump lets only finished runs contest it, so +// a run still unfinished then, which uploaded its dump and pruned this +// run's through maxRetention, can have its dump recorded here. Deletion +// contests the recorded dump again, with every run as it stands now. +func TestCleanupMariaDBBackup_RecordedDumpARunUnfinishedAtTheProbeCouldOwnIsReleased(t *testing.T) { + started := cleanupCreated.Add(time.Minute) + theirs := dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip") + cases := map[string][]metav1.Condition{ + "running": {{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(started), + }}, + "not yet reconciled": nil, + "completed since": {mariadbBackupSucceeded(metav1.NewTime(cleanupCompleted.Add(5 * time.Minute)))}, + } + for name, conditions := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: started.Add(time.Minute)}}} + other := cleanupOperatorBackup(t) + other.Name = "op-other" + other.CreationTimestamp = metav1.NewTime(started) + other.Status.Conditions = conditions + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), other, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(strings.TrimPrefix(theirs, cleanupPrefix))) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +// Either clock placing the recorded dump in another run's lifetime is +// enough: the backup pod's clock can be skewed while the store's is not. +func TestCleanupMariaDBBackup_RecordedDumpAnotherRunPlacesOnOneClockIsReleased(t *testing.T) { + stamped := cleanupCreated.Add(20 * time.Second) + uploaded := cleanupCreated.Add(2 * time.Minute) + theirs := dumpKey(cleanupPrefix, stamped, "gzip") + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: uploaded}}} + other := cleanupOperatorBackup(t) + other.Name = "op-other" + other.CreationTimestamp = metav1.NewTime(uploaded.Add(30 * time.Second)) + other.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), other, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(strings.TrimPrefix(theirs, cleanupPrefix))) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +// The listing is by prefix, so a longer key sharing it is not the dump and +// contests nothing. +func TestCleanupMariaDBBackup_RecordedDumpIgnoresAKeyItIsOnlyAPrefixOf(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key + ".tmp", LastModified: cleanupCompleted.Add(2 * time.Hour)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } +} + +// A name the dump time cannot be read from leaves one clock out of the +// contest, so it is an error rather than a contest on half the evidence. +func TestContestedMariaDBDump_RejectsANameWithoutADumpTime(t *testing.T) { + op := cleanupOperatorBackup(t) + name := "backup.not-a-time.sql" + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + name, LastModified: cleanupCreated}}} + c := newMariaDBStrategyTestClient(t, op) + if _, err := contestedMariaDBDump(context.Background(), bucket, c, op, name); err == nil { + t.Fatal("err = nil, want the unreadable dump time reported") + } +} + +// A retry after the dump was deleted but the operator Backup was not finds +// nothing to contest, whichever run's lifetime the dump's name falls in. +func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T) { + bucket := &fakeS3Bucket{} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCreated) + running.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted") + } + if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want no ArtifactNotDeleted", events) + } +} + +// A run started after the recorded dump was uploaded cannot have written it. +func TestCleanupMariaDBBackup_DeletesItsRecordedDumpDespiteALaterUnfinishedRun(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } +} + +func TestCleanupMariaDBBackup_RecordedDumpListingFailureRetries(t *testing.T) { + bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +// Without the list of runs nothing is known about who else could have +// written the dump, so it is kept and the delete retried. +func TestCleanupMariaDBBackup_RecordedDumpWithUnreadableRunsRetries(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.BackupList); ok { + return errors.New("apiserver unavailable") + } + return c.List(ctx, list, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +// The whole sequence: run B uploads its dump and prunes this run's before +// B completes, this run's Backup is created and probed, then deleted. B's +// dump must survive whatever the probe recorded. +func TestMariaDBBackup_ProbeThenCleanupKeepsADumpAnUnfinishedRunUploaded(t *testing.T) { + apps := mariadbapp.GroupName + started := cleanupCreated.Add(time.Minute) + theirs := s3ObjectInfo{Key: dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip"), Size: 42, LastModified: started.Add(time.Minute)} + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "tenant/db" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bk"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "db", APIGroup: &apps}, + }, + } + other := func() *mariadbtypes.Backup { + b := cleanupOperatorBackup(t) + b.Name = "op-other" + b.CreationTimestamp = metav1.NewTime(started) + b.Status.Conditions = nil + return b + } + + prober := reconcilerWithLister(t, &fakeS3Lister{objects: []s3ObjectInfo{theirs}}, job, other(), s3CredsSecret("tenant", "creds")) + created, err := prober.createMariaDBBackupArtifact(context.Background(), job, resolved, cleanupOperatorBackup(t), rendered) + if err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + if got, want := created.Spec.DriverMetadata[mariadbDumpFileKey], strings.TrimPrefix(theirs.Key, cleanupPrefix); got != want { + t.Fatalf("probe recorded dump %q, want %q: this test covers the recorded path", got, want) + } + + backup := created.DeepCopy() + backup.ResourceVersion = "" + now := metav1.Now() + backup.DeletionTimestamp = &now + backup.Finalizers = []string{backupFinalizer} + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{theirs}} + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), other(), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + func TestCleanupMariaDBBackup_LegacyListingFailureRetries(t *testing.T) { bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) From a1e28cccddce7a76a57da0b95b0fe80cf5b276ff Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Mon, 9 Nov 2026 12:51:14 +0100 Subject: [PATCH 073/174] test(kubernetes): cover missing VMI during volume unpublish Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Assisted-by: LLM --- .../kubevirt-csi-driver/controller_test.go | 49 +++++++++++++------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go index 73b8ffc7d..c6029f17b 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go @@ -314,12 +314,15 @@ const ( // including when the VMI outlives its VM during CAPI/CAPK teardown. type teardownClient struct { kubevirtclient.Client - vm *kubevirtv1.VirtualMachine - vmiOwners []metav1.OwnerReference - detached bool - vmRemovals int - vmiRemovals int - lookups int + vm *kubevirtv1.VirtualMachine + vmiOwners []metav1.OwnerReference + detached bool + vmiGone bool + vmiGoneAfterCheck bool + vmiLookups int + vmRemovals int + vmiRemovals int + lookups int } func (c *teardownClient) vmiGarbageCollected() bool { @@ -337,6 +340,10 @@ func (c *teardownClient) GetWorkloadManagingVirtualMachine(context.Context, stri func (c *teardownClient) GetVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachineInstance, error) { c.lookups++ + c.vmiLookups++ + if c.vmiGone { + return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachineinstances"}, testVMName) + } vmi := &kubevirtv1.VirtualMachineInstance{ ObjectMeta: metav1.ObjectMeta{Name: testVMName, Namespace: testInfraNamespace, OwnerReferences: c.vmiOwners}, } @@ -351,7 +358,11 @@ func (c *teardownClient) EnsureVolumeRemovedVM(context.Context, string, string, } func (c *teardownClient) EnsureVolumeRemovedVMI(context.Context, string, string, string) (bool, error) { - return c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0, nil + removed := c.vmiGone || c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0 + if c.vmiGoneAfterCheck { + c.vmiGone = true + } + return removed, nil } func (c *teardownClient) RemoveVolumeFromVM(_ context.Context, namespace, name string, opts *kubevirtv1.RemoveVolumeOptions) error { @@ -369,7 +380,7 @@ func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *k } func (c *teardownClient) EnsureVolumeRemoved(context.Context, string, string, string, time.Duration) error { - if c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0 || c.vmiGarbageCollected() { + if c.vmiGone || c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0 || c.vmiGarbageCollected() { return nil } return errors.New("timed out waiting for the VMI to drop the volume") @@ -429,20 +440,25 @@ func TestControllerUnpublishVolumeHonoursVMIHotplugFallback(t *testing.T) { func TestControllerUnpublishVolumeMissingPVC(t *testing.T) { tests := []struct { - name string - vmGone bool - detached bool - wantVMCalls int - wantVMICalls int + name string + vmGone bool + detached bool + vmiGone bool + vmiGoneAfterCheck bool + wantVMILookups int + wantVMCalls int + wantVMICalls int }{ {name: "pending hotplug volume", wantVMCalls: 1}, {name: "already detached", detached: true}, - {name: "VM gone, VMI still has volume", vmGone: true, wantVMICalls: 1}, + {name: "VM gone, VMI still has volume", vmGone: true, wantVMICalls: 1, wantVMILookups: 1}, {name: "VM gone, no attached volume", vmGone: true, detached: true}, + {name: "VM and VMI gone", vmGone: true, vmiGone: true}, + {name: "VMI disappears after attachment check", vmGone: true, vmiGoneAfterCheck: true, wantVMILookups: 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - client := &teardownClient{detached: tt.detached} + client := &teardownClient{detached: tt.detached, vmiGone: tt.vmiGone, vmiGoneAfterCheck: tt.vmiGoneAfterCheck} if !tt.vmGone { client.vm = &kubevirtv1.VirtualMachine{Spec: kubevirtv1.VirtualMachineSpec{Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{}}} if !tt.detached { @@ -464,6 +480,9 @@ func TestControllerUnpublishVolumeMissingPVC(t *testing.T) { if client.vmRemovals != tt.wantVMCalls || client.vmiRemovals != tt.wantVMICalls { t.Errorf("VM/VMI removevolume calls = %d/%d, want %d/%d", client.vmRemovals, client.vmiRemovals, tt.wantVMCalls, tt.wantVMICalls) } + if client.vmiLookups != tt.wantVMILookups { + t.Errorf("GetVirtualMachine calls = %d, want %d", client.vmiLookups, tt.wantVMILookups) + } if client.vm != nil && len(client.vm.Spec.Template.Spec.Volumes) != 0 { t.Error("hotplug volume remains in VM spec") } From 74fbb34aca35beabe6c42a7c02f89fd3bc7a40e8 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Mon, 9 Nov 2026 18:47:08 +0300 Subject: [PATCH 074/174] docs(backups): say who deletes a MariaDB Backup The comment justified deleting a dump in a tenant's own bucket by the tenant asking for it, but tenant roles grant no delete on Backups: only an administrator or the namespace teardown deletes one, and the reader of this branch should know whose request the delete answers. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/backup_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index f817e696b..9d755059e 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -137,8 +137,8 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // backup run of the same prefix and a Plan never deletes a Backup, so // a dump on the platform bucket is deleted here or never. A dump in // a tenant's own bucket is deleted too: the Backup proves it owns - // it, and the tenant deleting the Backup is asking for exactly that - // (see cleanupMariaDBBackup). + // it, and tenants cannot delete Backups, so the delete comes from an + // administrator or from namespace teardown (see cleanupMariaDBBackup). return r.cleanupMariaDBBackup(ctx, backup) case strategyv1alpha1.MongoDBStrategyKind: // Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the From 4be80657beaf164067a0e08675f951452bd8b3dd Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 10 Nov 2026 00:41:10 +0700 Subject: [PATCH 075/174] test(dashboard): prove /observability/logs opens the application page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The list page shows the same "Loading…" text as the detail page, so the previous check could not tell which route matched. A missing instance makes only the detail page answer with its own name, and the GET it issues names the plural and the application. Assisted-by: LLM Signed-off-by: ZverGuy --- .../src/routes/ConsolePage.routing.test.tsx | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx index 8e71ff98a..c3b014f4d 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx @@ -98,14 +98,26 @@ describe("ConsolePage routing", () => { // No ApplicationDefinition plural or application name can be "-", so only // routes under it are safe from shadowing an application's own pages. it("leaves /observability/logs to an application named logs of kind observability", async () => { + const client = makeClient() + vi.mocked(client.list).mockImplementation(async (_g, _v, plural) => ({ + apiVersion: "v1", + kind: `${plural}List`, + metadata: {}, + items: + plural === "tenantnamespaces" + ? [{ apiVersion: "core.cozystack.io/v1alpha1", kind: "TenantNamespace", metadata: { name: "tenant-root" } }] + : [], + }) as K8sList) + const get = vi.spyOn(client, "get").mockRejectedValue(new Error("not found")) renderWithK8sProvider( , - { client: makeClient(), initialRoute: "/observability/logs" }, + { client, initialRoute: "/observability/logs" }, ) - expect(await screen.findByText("Loading…")).toBeInTheDocument() + expect(await screen.findByText(/^Application/)).toHaveTextContent("Application logs not found.") + expect(get).toHaveBeenCalledWith(expect.anything(), expect.anything(), "observability", "logs", expect.anything()) expect(screen.queryByText(/^Every application in/)).toBeNull() }) }) From 7eff63b028cf769fcd1237e9f7fdd3dc568588a9 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 10 Nov 2026 00:51:01 +0700 Subject: [PATCH 076/174] refactor(dashboard): drop the scope-keyed placeholder from the logs view The logs view remounts under a key per scope, so its queries start with no previous result to carry over and the scope check never had anything to refuse. The key is the one guard, and the tenant-switch tests fail without it. Assisted-by: LLM Signed-off-by: ZverGuy --- .../console/src/components/observability/LogsView.tsx | 5 +---- .../console/apps/console/src/lib/observability/proxy.ts | 9 --------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx index 8c05a8743..05535e12e 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -14,7 +14,6 @@ import { import { fieldValues, isScopeWide, - keepWithinScope, logsRoot, queryLogs, type ObservabilityScope, @@ -81,7 +80,7 @@ function placeholderFor({ follow, stopped, loading, failed, longestRange, retry return } -// Filter, pod and container belong to one scope; a new tenant or application starts clean. +// Filter, pod, container and the lines already read belong to one scope; a new tenant or application starts clean. export function LogsView({ scope }: { scope: ObservabilityScope }) { const root = logsRoot(scope) return @@ -107,7 +106,6 @@ function ScopedLogsView({ root }: { root: string }) { query: buildLogsQuery({ filter: "", pod: narrowTo }), ...windowFor(range), }, signal), - placeholderData: keepWithinScope(root), retry: false, }) const pods = useQuery(pickerQuery(POD_FIELD)) @@ -117,7 +115,6 @@ function ScopedLogsView({ root }: { root: string }) { queryKey: ["observability", "logs", root, query, range], queryFn: ({ signal }) => queryLogs(client, root, { query, limit: HISTORY_LIMIT, ...windowFor(range) }, signal), enabled: !follow, - placeholderData: keepWithinScope(root), retry: false, }) const tail = useLogTail(root, query, followRun, { cap: FOLLOW_CAP }) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts index d33052f70..c28bd12d4 100644 --- a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts @@ -20,15 +20,6 @@ export function logsRoot(scope: ObservabilityScope): string { return app ? `${app}/log/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/logs/` } -// The provider's default keeps a useQuery's previous result while its key -// changes; across scopes that would show one tenant's or application's data -// under another. (useQueries starts a new observer per key, so it never does.) -// Every observability query key carries its root at index 2. -export function keepWithinScope(root: string) { - return (previous: T | undefined, previousQuery?: { queryKey: readonly unknown[] }): T | undefined => - previousQuery?.queryKey[2] === root ? previous : undefined -} - export type ProxyErrorKind = | "bad-request" | "unauthorized" From 48f459e103f4b3a3bc6a79e3932160cee659c788 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 10 Nov 2026 00:54:51 +0700 Subject: [PATCH 077/174] fix(dashboard): say the logs are loading while an empty result stands in The previous result stays on screen while a new range or filter loads. When that result was empty, the query reports success, so the view said no lines matched the range it had not read yet. Assisted-by: LLM Signed-off-by: ZverGuy --- .../components/observability/LogsView.test.tsx | 17 +++++++++++++++++ .../src/components/observability/LogsView.tsx | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx index a0df67c68..47cbf1c22 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -93,6 +93,23 @@ describe("LogsView with the production query defaults", () => { expect(screen.getByText("ready to accept connections")).toBeInTheDocument() }) + + it("says it is loading, not that nothing matched, while a longer range replaces an empty one", async () => { + const { client } = clientWith((endpoint, params) => { + if (!endpoint.endsWith("query")) return defaultHandler(endpoint, params) + return Date.parse(params.get("start") ?? "") < Date.now() - 2 * 3600_000 + ? new Promise(() => {}) + : new Response("") + }) + renderWithProductionDefaults(client, "tenant-range-a") + const frame = () => screen.getByRole("region", { name: "Log lines" }) + expect(await within(frame()).findByText("No log lines for this range and filter.")).toBeInTheDocument() + + await userEvent.click(screen.getByRole("button", { name: "24h" })) + + await waitFor(() => expect(within(frame()).getByRole("status")).toHaveAttribute("aria-busy", "true")) + expect(within(frame()).queryByText("No log lines for this range and filter.")).toBeNull() + }) }) describe("LogsView", () => { diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx index 05535e12e..1563d90b5 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -168,7 +168,7 @@ function ScopedLogsView({ root }: { root: string }) { // Previous results stay as placeholder data while a new query loads, so only a new key resets the scroll pin. key={followRun ?? `history\n${query}\n${range}`} entries={entries} - placeholder={placeholderFor({ follow, stopped: tail.state === "stopped", loading: history.isLoading, failed: Boolean(error), longestRange: range === LONGEST_RANGE, retry: () => void history.refetch() })} + placeholder={placeholderFor({ follow, stopped: tail.state === "stopped", loading: history.isLoading || (history.isPlaceholderData && history.isFetching), failed: Boolean(error), longestRange: range === LONGEST_RANGE, retry: () => void history.refetch() })} /> ) From 051442b3187abbe0eb73e21924ee1eebd1b1e701 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 10 Nov 2026 00:58:29 +0700 Subject: [PATCH 078/174] fix(dashboard): mark previous log lines while a new query replaces them The previous result stays on screen while a new range or filter loads, with nothing saying those lines answer the old query. A status line now says so until the new result arrives. The empty case reads a placeholder as loading whether or not the request is in flight, so a paused query does not claim that nothing matched either. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observability/LogsView.test.tsx | 31 +++++++++++++++++++ .../src/components/observability/LogsView.tsx | 7 ++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx index 47cbf1c22..be8bff1a8 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -110,6 +110,22 @@ describe("LogsView with the production query defaults", () => { await waitFor(() => expect(within(frame()).getByRole("status")).toHaveAttribute("aria-busy", "true")) expect(within(frame()).queryByText("No log lines for this range and filter.")).toBeNull() }) + + it("marks the lines of the previous range as standing in while a longer range loads", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") && Date.parse(params.get("start") ?? "") < Date.now() - 2 * 3600_000 + ? new Promise(() => {}) + : defaultHandler(endpoint, params), + ) + renderWithProductionDefaults(client, "tenant-range-b") + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + expect(screen.queryByText(/previous lines/)).toBeNull() + + await userEvent.click(screen.getByRole("button", { name: "24h" })) + + expect(await screen.findByText(/previous lines/)).toHaveAttribute("aria-busy", "true") + expect(screen.getByText("ready to accept connections")).toBeInTheDocument() + }) }) describe("LogsView", () => { @@ -319,6 +335,21 @@ describe("LogsView", () => { expect(screen.queryByRole("button", { name: "Retry" })).toBeNull() }) + it("says it is loading while a retry runs", async () => { + let calls = 0 + const { client } = clientWith((endpoint, params) => { + if (!endpoint.endsWith("query")) return defaultHandler(endpoint, params) + return calls++ === 0 ? new Response("upstream down", { status: 502 }) : new Promise(() => {}) + }) + renderWithK8sProvider(, { client }) + + await userEvent.click(await screen.findByRole("button", { name: "Retry" })) + + const frame = screen.getByRole("region", { name: "Log lines" }) + await waitFor(() => expect(within(frame).getByRole("status")).toHaveAttribute("aria-busy", "true")) + expect(within(frame).queryByRole("button", { name: "Retry" })).toBeNull() + }) + it("opens a new history result at its newest line, not the old scroll position", async () => { const { client } = clientWith((endpoint, params) => endpoint.endsWith("query") diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx index 1563d90b5..a4376df47 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -163,12 +163,17 @@ function ScopedLogsView({ root }: { root: string }) { {` · newest ${FOLLOW_CAP} lines kept`}

)} + {!follow && history.isPlaceholderData && entries.length > 0 && ( +

+ Showing the previous lines until the new range or filter loads… +

+ )} {error ? : null} void history.refetch() })} + placeholder={placeholderFor({ follow, stopped: tail.state === "stopped", loading: history.isLoading || history.isPlaceholderData, failed: Boolean(error), longestRange: range === LONGEST_RANGE, retry: () => void history.refetch() })} /> ) From 7105364161b9b0dea4afaae49681647c32edf1ca Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 10 Nov 2026 01:02:17 +0700 Subject: [PATCH 079/174] test(dashboard): pin when the previous-lines note goes away The note must leave as soon as the new result lands and must not appear over live lines while following; neither was exercised, since the only test kept the longer range pending forever. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observability/LogsView.test.tsx | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx index be8bff1a8..faba85a4e 100644 --- a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -126,6 +126,45 @@ describe("LogsView with the production query defaults", () => { expect(await screen.findByText(/previous lines/)).toHaveAttribute("aria-busy", "true") expect(screen.getByText("ready to accept connections")).toBeInTheDocument() }) + + it("drops the standing-in note once the longer range answers", async () => { + let answer: (r: Response) => void = () => {} + const longer = new Promise((resolve) => { answer = resolve }) + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") && Date.parse(params.get("start") ?? "") < Date.now() - 2 * 3600_000 + ? longer + : defaultHandler(endpoint, params), + ) + renderWithProductionDefaults(client, "tenant-range-c") + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + await userEvent.click(screen.getByRole("button", { name: "24h" })) + expect(await screen.findByText(/previous lines/)).toBeInTheDocument() + + answer(new Response(`${line("2026-11-03T12:00:00Z", "checkpoint complete")}\n`)) + + expect(await screen.findByText("checkpoint complete")).toBeInTheDocument() + expect(screen.queryByText(/previous lines/)).toBeNull() + expect(screen.queryByText("ready to accept connections")).toBeNull() + }) + + it("shows no standing-in note while following", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("tail") + ? new Response(`${line("2026-11-04T10:00:02Z", "live line")}\n`) + : endpoint.endsWith("query") && params.get("query")?.includes("error") + ? new Promise(() => {}) + : defaultHandler(endpoint, params), + ) + renderWithProductionDefaults(client, "tenant-follow-a") + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + + await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}") + expect(await screen.findByText(/previous lines/)).toBeInTheDocument() + await userEvent.click(screen.getByRole("button", { name: "Follow" })) + + expect(await screen.findByText("live line")).toBeInTheDocument() + expect(screen.queryByText(/previous lines/)).toBeNull() + }) }) describe("LogsView", () => { From ecb5de6051b9ece64dacf12dfa0a1aa9b29634d8 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:57:50 +0700 Subject: [PATCH 080/174] feat(observability-proxy): map application plurals to their kinds Paths and RBAC name an application by its plural, while the lineage labels that scope its logs and series carry the kind. The mapping comes from ApplicationDefinitions, kept in one place so both application roots scope by the same kind. A plural declared for two kinds is refused rather than resolved by cache order. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appkinds.go | 97 ++++++++++++++++++++ internal/observabilityproxy/appkinds_test.go | 77 ++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 internal/observabilityproxy/appkinds.go create mode 100644 internal/observabilityproxy/appkinds_test.go diff --git a/internal/observabilityproxy/appkinds.go b/internal/observabilityproxy/appkinds.go new file mode 100644 index 000000000..5e545f390 --- /dev/null +++ b/internal/observabilityproxy/appkinds.go @@ -0,0 +1,97 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "errors" + "fmt" + "slices" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/cache" +) + +// ApplicationDefinitionsResource is where the plural of every application +// kind is declared (api/v1alpha1/applicationdefinitions_types.go). +var ApplicationDefinitionsResource = schema.GroupVersionResource{Group: "cozystack.io", Version: "v1alpha1", Resource: "applicationdefinitions"} + +// ErrKindsNotSynced means the ApplicationDefinitions cache has not been +// filled yet, so an unknown plural cannot be told from a missing one. +var ErrKindsNotSynced = errors.New("application definitions not synced") + +const pluralIndex = "plural" + +// KindLookup maps an application plural, as it appears in paths and RBAC, +// to the kind the lineage labels carry. An unknown plural is "", nil. +type KindLookup interface { + Kind(plural string) (string, error) +} + +// ApplicationPluralIndexers indexes ApplicationDefinitions by +// spec.application.plural; the informer feeding ApplicationKinds needs it. +func ApplicationPluralIndexers() cache.Indexers { + return cache.Indexers{pluralIndex: func(obj any) ([]string, error) { + u, ok := obj.(*unstructured.Unstructured) + if !ok { + return nil, nil + } + plural, _, _ := unstructured.NestedString(u.Object, "spec", "application", "plural") + if plural == "" { + return nil, nil + } + return []string{plural}, nil + }} +} + +// ApplicationKinds is the one place the proxy maps a plural to its kind; +// both application roots use it. +type ApplicationKinds struct { + indexer cache.Indexer + hasSynced func() bool +} + +func NewApplicationKinds(indexer cache.Indexer, hasSynced func() bool) *ApplicationKinds { + return &ApplicationKinds{indexer: indexer, hasSynced: hasSynced} +} + +func (k *ApplicationKinds) Kind(plural string) (string, error) { + if !k.hasSynced() { + return "", ErrKindsNotSynced + } + objs, err := k.indexer.ByIndex(pluralIndex, plural) + if err != nil { + return "", err + } + var kinds []string + for _, obj := range objs { + kind, _, _ := unstructured.NestedString(obj.(*unstructured.Unstructured).Object, "spec", "application", "kind") + if kind != "" && !slices.Contains(kinds, kind) { + kinds = append(kinds, kind) + } + } + switch len(kinds) { + case 0: + return "", nil + case 1: + return kinds[0], nil + } + // Picking one would scope the request to whichever definition the + // cache happens to list first. + slices.Sort(kinds) + return "", fmt.Errorf("plural %q is declared for kinds %v", plural, kinds) +} diff --git a/internal/observabilityproxy/appkinds_test.go b/internal/observabilityproxy/appkinds_test.go new file mode 100644 index 000000000..6eb8d60bc --- /dev/null +++ b/internal/observabilityproxy/appkinds_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "errors" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/tools/cache" +) + +func appDefinition(name, kind, plural string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "cozystack.io/v1alpha1", + "kind": "ApplicationDefinition", + "metadata": map[string]any{"name": name}, + "spec": map[string]any{"application": map[string]any{"kind": kind, "plural": plural, "singular": name}}, + }} +} + +func newTestKinds(t *testing.T, synced bool, defs ...*unstructured.Unstructured) *ApplicationKinds { + t.Helper() + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, ApplicationPluralIndexers()) + for _, d := range defs { + if err := indexer.Add(d); err != nil { + t.Fatal(err) + } + } + return NewApplicationKinds(indexer, func() bool { return synced }) +} + +func TestApplicationKinds(t *testing.T) { + kinds := newTestKinds(t, true, + appDefinition("postgres", "Postgres", "postgreses"), + appDefinition("vm-instance", "VMInstance", "vminstances"), + appDefinition("broken", "", "brokens"), + ) + for plural, want := range map[string]string{"postgreses": "Postgres", "vminstances": "VMInstance", "postgres": "", "brokens": "", "": ""} { + got, err := kinds.Kind(plural) + if err != nil || got != want { + t.Errorf("Kind(%q) = %q, %v; want %q", plural, got, err, want) + } + } +} + +func TestApplicationKindsNotSynced(t *testing.T) { + kinds := newTestKinds(t, false, appDefinition("postgres", "Postgres", "postgreses")) + if _, err := kinds.Kind("postgreses"); !errors.Is(err, ErrKindsNotSynced) { + t.Fatalf("Kind before sync: err = %v, want ErrKindsNotSynced", err) + } +} + +func TestApplicationKindsAmbiguous(t *testing.T) { + kinds := newTestKinds(t, true, appDefinition("a", "Postgres", "postgreses"), appDefinition("b", "Postgresql", "postgreses")) + if kind, err := kinds.Kind("postgreses"); err == nil { + t.Fatalf("two kinds for one plural resolved to %q", kind) + } + same := newTestKinds(t, true, appDefinition("a", "Postgres", "postgreses"), appDefinition("b", "Postgres", "postgreses")) + if kind, err := same.Kind("postgreses"); err != nil || kind != "Postgres" { + t.Fatalf("duplicate definitions of one kind: %q, %v", kind, err) + } +} From 2f600d5354f672b5e53d382c248d867624298dc2 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 05:59:44 +0700 Subject: [PATCH 081/174] feat(observability-proxy): serve the application logs root A viewer of one application gets that application's lines without writing a namespace query and filtering by hand, and a grant narrowed to one application by resourceNames is honoured on every logs endpoint, tail and the pickers included. The namespace filter stays and the application's identity is ANDed next to it as a second stream filter, so the result can only be narrower than the namespace root. The plural is mapped to its kind only after the caller is authorized, so an anonymous client cannot probe which applications exist, and the proxy is not ready until the ApplicationDefinitions cache is. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 20 ++- internal/observabilityproxy/server.go | 43 +++++-- .../observabilityproxy/server_app_test.go | 114 ++++++++++++++++++ .../observabilityproxy/server_metrics_test.go | 2 +- internal/observabilityproxy/server_test.go | 21 +++- .../observability-proxy/templates/rbac.yaml | 5 + .../observability-proxy/tests/rbac_test.yaml | 5 +- 7 files changed, 192 insertions(+), 18 deletions(-) create mode 100644 internal/observabilityproxy/server_app_test.go diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index a3687fe08..1de6e6e14 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -39,6 +39,8 @@ import ( "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" @@ -178,13 +180,24 @@ func run(o options) error { return fmt.Errorf("building client: %w", err) } + dynClient, err := dynamic.NewForConfig(restConfig) + if err != nil { + return fmt.Errorf("building dynamic client: %w", err) + } + factory := informers.NewSharedInformerFactory(client, 0) namespaces := factory.Core().V1().Namespaces() lister := namespaces.Lister() + appDefs := dynamicinformer.NewDynamicSharedInformerFactory(dynClient, 0). + ForResource(observabilityproxy.ApplicationDefinitionsResource).Informer() + if err := appDefs.AddIndexers(observabilityproxy.ApplicationPluralIndexers()); err != nil { + return fmt.Errorf("indexing application definitions: %w", err) + } factory.Start(ctx.Done()) + go appDefs.Run(ctx.Done()) var ready atomic.Bool go func() { - if cache.WaitForCacheSync(ctx.Done(), namespaces.Informer().HasSynced) { + if cache.WaitForCacheSync(ctx.Done(), namespaces.Informer().HasSynced, appDefs.HasSynced) { ready.Store(true) } }() @@ -198,13 +211,14 @@ func run(o options) error { Limiter: o.limiter, RequestTimeout: o.requestTimeout, TailIdleTimeout: o.tailIdleTimeout, - }, observabilityproxy.NewAuthorizer(client, o.authCacheTTL), observabilityproxy.NewStoreResolver(lister, namespaces.Informer().HasSynced, o.rootTenant), reg) + }, observabilityproxy.NewAuthorizer(client, o.authCacheTTL), observabilityproxy.NewStoreResolver(lister, namespaces.Informer().HasSynced, o.rootTenant), + observabilityproxy.NewApplicationKinds(appDefs.GetIndexer(), appDefs.HasSynced), reg) probes := http.NewServeMux() probes.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "ok") }) probes.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { if !ready.Load() { - http.Error(w, "namespace cache not synced", http.StatusServiceUnavailable) + http.Error(w, "namespace or application definition cache not synced", http.StatusServiceUnavailable) return } _, _ = io.WriteString(w, "ok") diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index eb540a271..617971bf2 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -73,13 +73,14 @@ type Server struct { cfg Config auth *Authorizer stores StoreLookup + kinds KindLookup client *http.Client limits *limiter now func() time.Time requests *prometheus.CounterVec } -func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus.Registerer) *Server { +func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, kinds KindLookup, reg prometheus.Registerer) *Server { if cfg.UpstreamURL == nil { domain := cfg.ClusterDomain cfg.UpstreamURL = func(store string) string { @@ -96,6 +97,7 @@ func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus. cfg: cfg, auth: auth, stores: stores, + kinds: kinds, // No client timeout: it would cut live tails. Non-tail requests are // bounded by a context deadline instead. client: &http.Client{Transport: http.DefaultTransport.(*http.Transport).Clone()}, @@ -172,12 +174,7 @@ func writeError(w http.ResponseWriter, e *HTTPError) { http.Error(w, e.Message, e.Code) } -// serve answers a root it has no handler for before authenticating, so an -// unknown path costs no review calls. func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTTPError { - if route.Kind != NamespaceLogs && route.Kind != NamespaceMetrics { - return errorf(http.StatusNotFound, "not found") - } user, herr := s.authorize(r, route) if herr != nil { return herr @@ -190,8 +187,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT w.Header().Set("Retry-After", strconv.FormatInt(int64(math.Ceil(wait.Seconds())), 10)) return errorf(http.StatusTooManyRequests, "request rate limit exceeded") } - if route.Kind == NamespaceMetrics { + var kind string + if route.Kind == ApplicationLog || route.Kind == ApplicationMetrics { + if kind, herr = s.applicationKind(route.Plural); herr != nil { + return herr + } + } + switch route.Kind { + case NamespaceMetrics: return s.serveMetrics(w, r, route, user) + case ApplicationMetrics: + return errorf(http.StatusNotFound, "not found") } endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/") if !ok { @@ -214,6 +220,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT if herr != nil { return herr } + if kind != "" { + params.Add("extra_stream_filters", applicationStreamFilter(kind, route.Name)) + } store, err := s.stores.Resolve(route.Namespace) if err != nil { @@ -322,6 +331,26 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout return nil } +func (s *Server) applicationKind(plural string) (string, *HTTPError) { + kind, err := s.kinds.Kind(plural) + if err != nil { + slog.Error("mapping application plural to kind", "plural", plural, "error", err) + return "", errorf(http.StatusServiceUnavailable, "cannot resolve the application kind") + } + if kind == "" { + return "", errorf(http.StatusNotFound, "unknown application resource %q", plural) + } + return kind, nil +} + +// applicationStreamFilter selects the streams Fluent Bit labels with the +// pod's lineage. vlselect ANDs each extra_stream_filters value into the +// query and every subquery (Query.AddExtraFilters in lib/logstorage), so +// it narrows the namespace filter sent next to it. +func applicationStreamFilter(kind, name string) string { + return "{apps_cozystack_io_application_kind=" + strconv.Quote(kind) + ",apps_cozystack_io_application_name=" + strconv.Quote(name) + "}" +} + // post sends params as a form with only the given headers, so nothing the // client sent reaches the store, and tells an unreachable store (502) from // one that missed its deadline or, for a tail, its idle timeout (504). diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go new file mode 100644 index 000000000..0a76ab9a5 --- /dev/null +++ b/internal/observabilityproxy/server_app_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "io" + "net/http" + "net/url" + "slices" + "testing" + + authorizationv1 "k8s.io/api/authorization/v1" +) + +const appRoot = "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/" + +// appPolicy lets alice read every postgres in tenant-a, as the view role's +// apps.cozystack.io wildcard does, and bob only "a", as a grant narrowed +// with resourceNames does. +func appPolicy(user string, a *authorizationv1.ResourceAttributes) bool { + if a.Namespace != "tenant-a" || a.Verb != "get" || a.Group != "apps.cozystack.io" || a.Resource != "postgreses" || + (a.Subresource != "log" && a.Subresource != "metrics") { + return false + } + return user == "alice" || a.Name == "a" +} + +func TestServerApplicationLogsScope(t *testing.T) { + h := newPolicyHarness(t, Config{}, nil, appPolicy, nil) + hostile := url.Values{ + "query": {"error"}, + "field": {"level"}, + "extra_filters": {`{kubernetes_namespace_name="tenant-b"}`}, + "extra_stream_filters": {`{apps_cozystack_io_application_name=~".*"}`}, + "AccountID": {"9"}, + } + headers := map[string]string{"AccountID": "9", "ProjectID": "9"} + endpoints := []string{"query", "hits", "stats_query_range", "tail", "field_values", "stream_field_values", "facets"} + for _, ep := range endpoints { + resp := h.do(http.MethodPost, appRoot+"db.main/log/select/logsql/"+ep+"?extra_stream_filters=x", "good", hostile, headers) + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("%s answered %d", ep, resp.StatusCode) + } + } + want := authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "log", Name: "db.main", Namespace: "tenant-a"} + if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want { + t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want) + } + reqs := h.requests() + if len(reqs) != len(endpoints) { + t.Fatalf("upstream saw %d requests, want %d", len(reqs), len(endpoints)) + } + // Two values: vlselect ANDs every extra_stream_filters value into the + // query and each of its subqueries. + wantFilters := []string{wantScope, `{apps_cozystack_io_application_kind="Postgres",apps_cozystack_io_application_name="db.main"}`} + for i, r := range reqs { + if r.path != "/select/logsql/"+endpoints[i] { + t.Errorf("upstream path = %s, want %s", r.path, endpoints[i]) + } + if got := r.form["extra_stream_filters"]; !slices.Equal(got, wantFilters) { + t.Errorf("%s: extra_stream_filters = %q, want %q", endpoints[i], got, wantFilters) + } + if r.form.Has("extra_filters") || r.form.Has("AccountID") || r.headers.Get("AccountID") != "0" || r.headers.Get("ProjectID") != "0" { + t.Errorf("%s: client scope reached the store: %v %v", endpoints[i], r.form, r.headers) + } + } +} + +func TestServerApplicationKinds(t *testing.T) { + tests := []struct { + name string + kinds KindLookup + path string + want int + }{ + {"unknown plural logs", nil, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/mysqls/a/log/select/logsql/query", http.StatusNotFound}, + {"unknown plural metrics", nil, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/mysqls/a/metrics/api/v1/query?query=up", http.StatusNotFound}, + {"not synced logs", newTestKinds(t, false), appRoot + "a/log/select/logsql/query", http.StatusServiceUnavailable}, + {"not synced metrics", newTestKinds(t, false), appRoot + "a/metrics/api/v1/query?query=up", http.StatusServiceUnavailable}, + {"ambiguous plural", newTestKinds(t, true, appDefinition("x", "Postgres", "postgreses"), appDefinition("y", "Pg", "postgreses")), appRoot + "a/log/select/logsql/query", http.StatusServiceUnavailable}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := newPolicyHarness(t, Config{}, nil, func(string, *authorizationv1.ResourceAttributes) bool { return true }, tt.kinds) + if got := h.status(http.MethodGet, tt.path, "good"); got != tt.want { + t.Fatalf("status = %d, want %d", got, tt.want) + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests", n) + } + }) + } + // The kind is looked up only for an authorized caller. + h := newPolicyHarness(t, Config{}, nil, appPolicy, newTestKinds(t, false)) + if got := h.status(http.MethodGet, appRoot+"b/log/select/logsql/query", "bob-token"); got != http.StatusForbidden { + t.Fatalf("denied caller answered %d, want 403", got) + } +} diff --git a/internal/observabilityproxy/server_metrics_test.go b/internal/observabilityproxy/server_metrics_test.go index 1ef5e761e..1a3d1de10 100644 --- a/internal/observabilityproxy/server_metrics_test.go +++ b/internal/observabilityproxy/server_metrics_test.go @@ -203,7 +203,7 @@ func TestServerMetricsConcurrency(t *testing.T) { } func TestDefaultMetricsUpstreamURL(t *testing.T) { - s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil) + s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil, nil) if got, want := s.cfg.Metrics.UpstreamURL("shortterm"), "http://vmselect-shortterm.tenant-root.svc.cozy.local:8481/select/0/prometheus"; got != want { t.Fatalf("upstream = %s, want %s", got, want) } diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index 2670d0e3e..b541a18e1 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -55,10 +55,19 @@ func (m mapStores) Resolve(ns string) (string, error) { return m[ns], nil } // (alice) and "bob-token" (bob), both allowed list logs and metrics in // tenant-a only, and to an upstream served by handler. func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { - h := &harness{t: t} - h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, func(_ string, a *authorizationv1.ResourceAttributes) bool { + return newPolicyHarness(t, cfg, handler, func(_ string, a *authorizationv1.ResourceAttributes) bool { return a.Namespace == "tenant-a" && (a.Resource == "logs" || a.Resource == "metrics") - }) + }, nil) +} + +// newPolicyHarness is newHarness with the given SubjectAccessReview policy +// and plural-to-kind mapping; nil kinds knows only postgreses. +func newPolicyHarness(t *testing.T, cfg Config, handler http.HandlerFunc, allow func(string, *authorizationv1.ResourceAttributes) bool, kinds KindLookup) *harness { + h := &harness{t: t} + h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, allow) + if kinds == nil { + kinds = newTestKinds(t, true, appDefinition("postgres", "Postgres", "postgreses")) + } h.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { t.Errorf("upstream ParseForm: %v", err) @@ -90,7 +99,7 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { if cfg.Logs == (LogsLimits{}) { cfg.Logs = testLimits() } - h.server = NewServer(cfg, NewAuthorizer(h.kube.client, 10*time.Second), mapStores{"tenant-a": "tenant-a-store"}, nil) + h.server = NewServer(cfg, NewAuthorizer(h.kube.client, 10*time.Second), mapStores{"tenant-a": "tenant-a-store"}, kinds, nil) h.server.now = func() time.Time { return testNow } h.proxy = httptest.NewServer(h.server) t.Cleanup(h.proxy.Close) @@ -154,8 +163,8 @@ func TestServerStatusCodes(t *testing.T) { {"unknown metrics endpoint", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/status/tsdb", "good", http.StatusNotFound}, {"metrics in another namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/metrics/api/v1/query?query=up", "good", http.StatusForbidden}, {"metrics without token", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query?query=up", "", http.StatusUnauthorized}, - {"application log not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusNotFound}, - {"application metrics not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusNotFound}, + {"application log without a grant", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusForbidden}, + {"application metrics without a grant", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusForbidden}, {"tenant_ids not proxied", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/tenant_ids", "good", http.StatusNotFound}, {"unknown logsql endpoint", http.MethodGet, logsRoot + "stats_query", "good", http.StatusNotFound}, {"logs root itself", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/", "good", http.StatusNotFound}, diff --git a/packages/system/observability-proxy/templates/rbac.yaml b/packages/system/observability-proxy/templates/rbac.yaml index 3c5392903..f3f67127c 100644 --- a/packages/system/observability-proxy/templates/rbac.yaml +++ b/packages/system/observability-proxy/templates/rbac.yaml @@ -15,6 +15,11 @@ rules: - apiGroups: [""] resources: ["namespaces"] verbs: ["list", "watch"] +# Application roots map the plural in the path to the kind the lineage +# labels carry, from an informer cache as well. +- apiGroups: ["cozystack.io"] + resources: ["applicationdefinitions"] + verbs: ["list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/packages/system/observability-proxy/tests/rbac_test.yaml b/packages/system/observability-proxy/tests/rbac_test.yaml index ef96c65d7..5382c2ab6 100644 --- a/packages/system/observability-proxy/tests/rbac_test.yaml +++ b/packages/system/observability-proxy/tests/rbac_test.yaml @@ -12,7 +12,7 @@ release: namespace: cozy-observability-proxy tests: - - it: lets the proxy review tokens and access, and list and watch namespaces + - it: lets the proxy review tokens and access, and list and watch namespaces and application definitions template: templates/rbac.yaml documentSelector: path: kind @@ -30,6 +30,9 @@ tests: - apiGroups: [""] resources: ["namespaces"] verbs: ["list", "watch"] + - apiGroups: ["cozystack.io"] + resources: ["applicationdefinitions"] + verbs: ["list", "watch"] - it: binds the proxy role to the proxy ServiceAccount in the release namespace template: templates/rbac.yaml From dcdf0adab3171c0139766f0999cfb975d0ff97c1 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 06:03:40 +0700 Subject: [PATCH 082/174] feat(observability-proxy): serve the application metrics root Graphs of one application need its series without the caller knowing which pods and volumes it ran on. Exporter series carry the identity the scrape class stamps, but cAdvisor and kubelet volume series do not, so the scope ORs three branches: the identity, and the application's pods and PVCs as kube-state-metrics labelled them over the window. Each branch also carries the namespace, so the scope is never wider than the namespace root. The lookup reaches back by the query's own lookback, bounded by a flag, so pods replaced inside an expression's range still count. A failed lookup only drops its branch. A list over its cap answers 422 instead of being truncated, which would hide series without saying so. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 5 + cmd/observability-proxy/main_test.go | 10 +- internal/observabilityproxy/appscope.go | 158 ++++++++++++++++ internal/observabilityproxy/appscope_test.go | 116 ++++++++++++ internal/observabilityproxy/metricsparams.go | 20 ++- internal/observabilityproxy/server.go | 18 +- .../observabilityproxy/server_app_test.go | 170 ++++++++++++++++++ internal/observabilityproxy/server_test.go | 3 + 8 files changed, 492 insertions(+), 8 deletions(-) create mode 100644 internal/observabilityproxy/appscope.go create mode 100644 internal/observabilityproxy/appscope_test.go diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 1de6e6e14..4a99aa4cc 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -94,6 +94,9 @@ func parseFlags(args []string) (options, error) { fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use") fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return") fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") + fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs at most") + fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") + fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables") fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user") @@ -134,6 +137,8 @@ func parseFlags(args []string) (options, error) { return o, errors.New("--metrics-shortterm-retention, --metrics-max-range and --metrics-max-timeout must be positive") case o.metrics.Limits.MaxPoints < 2: return o, errors.New("--metrics-max-points must be at least 2") + case o.metrics.App.Lookback < 0 || o.metrics.App.MaxPods < 1 || o.metrics.App.MaxPVCs < 1: + return o, errors.New("--metrics-app-lookback must not be negative and --metrics-app-max-pods and --metrics-app-max-pvcs must be positive") case o.logs.MaxTailResume < 0: return o, errors.New("--logs-max-tail-resume must not be negative") // The proxy's deadline starts before the request reaches vmselect, so diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go index bf7ac396b..6b5c60d7d 100644 --- a/cmd/observability-proxy/main_test.go +++ b/cmd/observability-proxy/main_test.go @@ -40,6 +40,9 @@ func TestParseFlagsDefaults(t *testing.T) { if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second { t.Errorf("metrics limit defaults changed: %+v", m.Limits) } + if m.App.Lookback != 24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 { + t.Errorf("application scope defaults changed: %+v", m.App) + } if m.RootTenant != o.rootTenant { t.Errorf("metrics root tenant = %q, want --root-tenant", m.RootTenant) } @@ -63,13 +66,15 @@ func TestParseFlagsMetricsOverrides(t *testing.T) { o, err := parseFlags([]string{ "--root-tenant=tenant-x", "--metrics-shortterm-store=fast", "--metrics-longterm-store=slow", "--metrics-shortterm-retention=24h", "--metrics-max-range=168h", "--metrics-max-points=500", "--metrics-max-timeout=10s", + "--metrics-app-lookback=0s", "--metrics-app-max-pods=7", "--metrics-app-max-pvcs=3", }) if err != nil { t.Fatal(err) } m := o.metrics if m.RootTenant != "tenant-x" || m.ShortTermStore != "fast" || m.LongTermStore != "slow" || m.ShortTermRetention != 24*time.Hour || - m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second { + m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second || + m.App.Lookback != 0 || m.App.MaxPods != 7 || m.App.MaxPVCs != 3 { t.Errorf("overrides not applied: %+v", m) } } @@ -101,6 +106,9 @@ func TestParseFlagsInvalid(t *testing.T) { {"--metrics-max-range=0s"}, {"--metrics-max-points=1"}, {"--metrics-max-timeout=0s"}, + {"--metrics-app-lookback=-1h"}, + {"--metrics-app-max-pods=0"}, + {"--metrics-app-max-pvcs=0"}, {"--unknown"}, // The proxy would cut the request while vmselect is still within // the timeout it was given. diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go new file mode 100644 index 000000000..b969cc502 --- /dev/null +++ b/internal/observabilityproxy/appscope.go @@ -0,0 +1,158 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +// AppScopeLimits bound the pod and PVC lookup of the application metrics +// root. +type AppScopeLimits struct { + // Lookback is how far before the window the lookup reaches at most, so + // pods gone before the window but inside an expression's range count. + Lookback time.Duration + MaxPods int + MaxPVCs int +} + +// maxLookupBytes bounds a lookup answer; the caps keep a valid one far +// below it. +const maxLookupBytes = 8 << 20 + +// The lineage labels as the scrape class writes them on series, and as +// kube-state-metrics writes them on kube_pod_labels and +// kube_persistentvolumeclaim_labels. +const ( + seriesKindLabel = "apps_cozystack_io_application_kind" + seriesNameLabel = "apps_cozystack_io_application_name" + ksmKindLabel = "label_" + seriesKindLabel + ksmNameLabel = "label_" + seriesNameLabel +) + +func identitySelector(ns, kindLabel, nameLabel, kind, name string) string { + return "{namespace=" + strconv.Quote(ns) + "," + kindLabel + "=" + strconv.Quote(kind) + "," + nameLabel + "=" + strconv.Quote(name) + "}" +} + +// nameBranch matches label against any of names within ns. With no names +// it is "": a branch of pod=~"^()$" would match every series without a pod +// label. +func nameBranch(ns, label string, names []string) string { + if len(names) == 0 { + return "" + } + quoted := make([]string, len(names)) + for i, n := range names { + quoted[i] = regexp.QuoteMeta(n) + } + return "{namespace=" + strconv.Quote(ns) + "," + label + "=~" + strconv.Quote("^("+strings.Join(quoted, "|")+")$") + "}" +} + +// applicationSelectors is the application scope: series stamped with the +// application's identity, plus series of its pods and PVCs for those not +// stamped (cAdvisor, kubelet volume stats). Every branch carries the +// namespace, so their OR stays inside it. +// +// Within the namespace, both the identity and the pod and PVC lists come +// from lineage labels on pods and PVCs. They separate applications for a +// grant narrowed by resourceNames only while whoever creates a pod or PVC +// cannot choose those labels; the lineage webhook keeps them as set on an +// object it finds no owner for, and the logs scope rests on the same. +func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { + out := []string{identitySelector(ns, seriesKindLabel, seriesNameLabel, kind, name)} + for _, b := range []string{nameBranch(ns, "pod", pods), nameBranch(ns, "persistentvolumeclaim", pvcs)} { + if b != "" { + out = append(out, b) + } + } + return out +} + +// applicationScope looks the application's pods and PVCs up in store over +// the request window, reaching back by the query's own lookback up to the +// configured limit. A failed lookup drops its branch, which only narrows +// the answer; a list over its cap is refused, because truncating it would +// silently drop series. +func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { + lim := s.cfg.Metrics.App + start := req.from.Add(-lim.Lookback) + if req.start.After(start) { + start = req.start + } + var lists [2][]string + for i, l := range []struct { + label, metric string + max int + }{{"pod", "kube_pod_labels", lim.MaxPods}, {"persistentvolumeclaim", "kube_persistentvolumeclaim_labels", lim.MaxPVCs}} { + match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name) + names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.to, l.max) + if err != nil { + slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) + continue + } + if len(names) > l.max { + return nil, errorf(http.StatusUnprocessableEntity, "the application has more than %d %ss in this window; use a shorter range", l.max, l.label) + } + lists[i] = names + } + return applicationSelectors(ns, kind, name, lists[0], lists[1]), nil +} + +// lookupNames asks for at most max+1 values, enough to tell a list over +// the cap without reading all of it. +func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, match, ns string, start, end time.Time, max int) ([]string, error) { + params := url.Values{} + params.Set("match[]", match) + params.Set("extra_filters[]", namespaceSelector(ns)) + params.Set("start", formatTime(start)) + params.Set("end", formatTime(end)) + params.Set("limit", strconv.Itoa(max+1)) + resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/api/v1/label/"+label+"/values", params, "metrics store", nil) + if herr != nil { + return nil, herr + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status) + } + var body struct { + Status string `json:"status"` + Data []string `json:"data"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, maxLookupBytes)).Decode(&body); err != nil { + return nil, err + } + if body.Status != "success" { + return nil, errorf(http.StatusBadGateway, "metrics store answered status %q", body.Status) + } + names := body.Data[:0] + for _, n := range body.Data { + if n != "" { + names = append(names, n) + } + } + return names, nil +} diff --git a/internal/observabilityproxy/appscope_test.go b/internal/observabilityproxy/appscope_test.go new file mode 100644 index 000000000..af028b867 --- /dev/null +++ b/internal/observabilityproxy/appscope_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "net/url" + "regexp" + "slices" + "strconv" + "testing" + "time" +) + +const wantIdentityBranch = `{namespace="tenant-a",apps_cozystack_io_application_kind="Postgres",apps_cozystack_io_application_name="db.main"}` + +func TestApplicationSelectors(t *testing.T) { + tests := []struct { + name string + pods, pvcs []string + want []string + }{ + {"identity only", nil, nil, []string{wantIdentityBranch}}, + {"pods", []string{"db-main-1", "db.main-2"}, nil, []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(db-main-1|db\\.main-2)$"}`, + }}, + {"pvcs", nil, []string{"data-db-main-1"}, []string{ + wantIdentityBranch, + `{namespace="tenant-a",persistentvolumeclaim=~"^(data-db-main-1)$"}`, + }}, + {"all three", []string{"p"}, []string{"v"}, []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(p)$"}`, + `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := applicationSelectors("tenant-a", "Postgres", "db.main", tt.pods, tt.pvcs); !slices.Equal(got, tt.want) { + t.Fatalf("got %q\nwant %q", got, tt.want) + } + }) + } +} + +// DNS-1123 keeps most metacharacters out of names, but the names come +// back from the store, so the encoder must not rely on that. +func TestNameBranchEscapes(t *testing.T) { + names := []string{`a.b`, `x|.*`, `)|(.*`, `q"} or {namespace=~".+`, `back\slash`, `$^+?[]{}`} + sel := nameBranch("tenant-a", "pod", names) + m := regexp.MustCompile(`^\{namespace="tenant-a",pod=~(".*")\}$`).FindStringSubmatch(sel) + if m == nil { + t.Fatalf("selector %s is not one namespace and one pod matcher", sel) + } + pattern, err := strconv.Unquote(m[1]) + if err != nil { + t.Fatalf("pod matcher %s does not unquote: %v", m[1], err) + } + re := regexp.MustCompile(pattern) + for _, n := range names { + if !re.MatchString(n) { + t.Errorf("%q does not match its own branch", n) + } + } + for _, other := range []string{"axb", "x", "anything", "tenant-b", "", `back\\slash`} { + if re.MatchString(other) { + t.Errorf("branch matches %q", other) + } + } + if nameBranch("tenant-a", "pod", nil) != "" { + t.Error("an empty list produced a branch, which would match series without the label") + } +} + +func TestMetricsRequestScope(t *testing.T) { + branches := []string{"{a}", "{b}"} + labels := buildMetrics(t, "api/v1/labels", url.Values{}) + labels.setScope(branches) + if got := labels.params["extra_filters[]"]; !slices.Equal(got, branches) { + t.Errorf("extra_filters[] = %q", got) + } + if got := labels.params["match[]"]; !slices.Equal(got, branches) { + t.Errorf("default match[] = %q, want the branches", got) + } + client := buildMetrics(t, "api/v1/labels", url.Values{"match[]": {"up"}}) + client.setScope(branches) + if got := client.params["match[]"]; !slices.Equal(got, []string{"up"}) { + t.Errorf("client match[] replaced: %q", got) + } + query := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}) + query.setScope(branches) + if query.params.Has("match[]") || !slices.Equal(query.params["extra_filters[]"], branches) { + t.Errorf("query params = %v", query.params) + } + if !query.from.Equal(testNow) || !query.to.Equal(testNow) { + t.Errorf("instant window = [%s, %s], want now", query.from, query.to) + } + rng := buildMetrics(t, "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}) + if !rng.from.Equal(testNow.Add(-time.Hour)) || !rng.to.Equal(testNow) { + t.Errorf("range window = [%s, %s]", rng.from, rng.to) + } +} diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index 8beb347d0..a0bb42f0d 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -46,6 +46,7 @@ type MetricsConfig struct { LongTermStore string ShortTermRetention time.Duration Limits MetricsLimits + App AppScopeLimits // UpstreamURL maps a store name to its Prometheus API base URL, without // a trailing slash; nil means the in-cluster vmselect Service. UpstreamURL func(store string) string @@ -77,6 +78,20 @@ type metricsRequest struct { // start is the earliest instant the request reads, for the store choice; // zero when that is unknown, which picks the long-term store. start time.Time + // from and to are the window the query is evaluated over, without the + // lookback of its expression. + from, to time.Time + // scopedMatch is set when match[] holds the scope, not client matchers. + scopedMatch bool +} + +// setScope replaces the namespace scope with the given selectors, which +// vmselect ORs. +func (req *metricsRequest) setScope(selectors []string) { + req.params["extra_filters[]"] = selectors + if req.scopedMatch { + req.params["match[]"] = selectors + } } // namespaceSelector quotes ns the way MetricsQL unquotes a label value @@ -127,6 +142,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti out.Set("step", formatSeconds(step)) } req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step) + req.from, req.to = at, at return req, nil case "api/v1/query_range": @@ -150,6 +166,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints) } req.start = queryStart(out.Get("query"), start, step) + req.from, req.to = start, end out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) out.Set("step", formatSeconds(step)) @@ -172,7 +189,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti if herr != nil { return req, herr } - req.start = start + req.start, req.from, req.to = start, start, end out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) @@ -194,6 +211,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti // extra_filters[] on the label APIs unless a match[] is present // (getCommonParamsInternal), so the scope is repeated here. out.Set("match[]", namespaceSelector(namespace)) + req.scopedMatch = true } return req, nil } diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index 617971bf2..fd74491b5 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -193,11 +193,8 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT return herr } } - switch route.Kind { - case NamespaceMetrics: - return s.serveMetrics(w, r, route, user) - case ApplicationMetrics: - return errorf(http.StatusNotFound, "not found") + if route.Kind == NamespaceMetrics || route.Kind == ApplicationMetrics { + return s.serveMetrics(w, r, route, user, kind) } endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/") if !ok { @@ -288,7 +285,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT return nil } -func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user string) *HTTPError { +// serveMetrics scopes to the namespace, or with kind set to the application +// route.Name of that kind within it. +func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user, kind string) *HTTPError { release, ok := s.limits.acquire(user) if !ok { return tooManyConcurrent(w) @@ -310,6 +309,13 @@ func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Rout defer cancel() } store := s.cfg.Metrics.store(req.start, now) + if kind != "" { + selectors, herr := s.applicationScope(ctx, r, store, req, route.Namespace, kind, route.Name) + if herr != nil { + return herr + } + req.setScope(selectors) + } resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/"+req.path, req.params, "metrics store", nil) if herr != nil { return herr diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 0a76ab9a5..14807637e 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -17,11 +17,14 @@ limitations under the License. package observabilityproxy import ( + "encoding/json" "io" "net/http" "net/url" "slices" + "strings" "testing" + "time" authorizationv1 "k8s.io/api/authorization/v1" ) @@ -112,3 +115,170 @@ func TestServerApplicationKinds(t *testing.T) { t.Fatalf("denied caller answered %d, want 403", got) } } + +const wantPodLookup = `kube_pod_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}` + +// appMetricsUpstream answers the pod and PVC lookups with the given names, +// or with status when it is not 200, and every other path like vmselect. +func appMetricsUpstream(pods, pvcs []string, podStatus int) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + names := map[string][]string{"/api/v1/label/pod/values": pods, "/api/v1/label/persistentvolumeclaim/values": pvcs} + list, lookup := names[r.URL.Path] + if !lookup { + promUpstream(w, r) + return + } + if r.URL.Path == "/api/v1/label/pod/values" && podStatus != http.StatusOK { + http.Error(w, "unavailable", podStatus) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"status": "success", "data": list}) + } +} + +func TestServerApplicationMetricsScope(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"db-main-0", "db.main-1"}, []string{"data-db-main-0"}, http.StatusOK), appPolicy, nil) + hostile := url.Values{ + "query": {"sum(rate(container_cpu_usage_seconds_total[5m]))"}, + "start": {unix(testNow.Add(-time.Hour))}, + "end": {unix(testNow)}, + "step": {"60"}, + "extra_filters": {`{namespace=~".+"}`}, + "extra_filters[]": {`{namespace="tenant-a"}`}, + "extra_label": {"apps_cozystack_io_application_name=b"}, + } + resp := h.do(http.MethodPost, appRoot+"db.main/metrics/api/v1/query_range", "good", hostile, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("answered %d %q", resp.StatusCode, body) + } + want := authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "metrics", Name: "db.main", Namespace: "tenant-a"} + if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want { + t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want) + } + reqs := h.requests() + if len(reqs) != 3 { + t.Fatalf("upstream saw %d requests, want two lookups and the query", len(reqs)) + } + lookups := map[string]string{ + "/api/v1/label/pod/values": wantPodLookup, + "/api/v1/label/persistentvolumeclaim/values": `kube_persistentvolumeclaim_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}`, + } + for _, r := range reqs[:2] { + if got := r.form["match[]"]; len(got) != 1 || got[0] != lookups[r.path] { + t.Errorf("%s match[] = %q", r.path, got) + } + if got := r.form["extra_filters[]"]; !slices.Equal(got, []string{wantMetricsScope}) { + t.Errorf("%s extra_filters[] = %q", r.path, got) + } + start, herr := parseMetricsTime("start", r.form.Get("start")) + if herr != nil || start.After(testNow.Add(-time.Hour)) || start.Before(testNow.Add(-25*time.Hour)) { + t.Errorf("%s start = %s, want within the lookback before the window", r.path, r.form.Get("start")) + } + if r.form.Get("end") != formatTime(testNow) || r.form.Get("limit") != "101" { + t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit")) + } + } + main := reqs[2] + wantBranches := []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(db-main-0|db\\.main-1)$"}`, + `{namespace="tenant-a",persistentvolumeclaim=~"^(data-db-main-0)$"}`, + } + if main.path != "/api/v1/query_range" || !slices.Equal(main.form["extra_filters[]"], wantBranches) { + t.Fatalf("query %s extra_filters[] = %q, want %q", main.path, main.form["extra_filters[]"], wantBranches) + } + if main.form.Has("extra_filters") || main.form.Has("extra_label") { + t.Fatalf("client scope reached the store: %v", main.form) + } + if !slices.Equal(h.stores, []string{"shortterm", "shortterm", "shortterm"}) { + t.Fatalf("stores = %q, want the lookups on the query's store", h.stores) + } +} + +func TestServerApplicationMetricsBranches(t *testing.T) { + podBranch := `{namespace="tenant-a",pod=~"^(p)$"}` + tests := []struct { + name string + pods, pvcs []string + podStatus int + cfg Config + path string + wantCode int + want []string + }{ + {name: "no pods or pvcs", path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, + {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusOK, + want: []string{wantIdentityBranch, `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`}}, + {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, + path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, + {name: "pvcs over the cap", pods: []string{"p"}, pvcs: []string{"v", "w"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, + path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, + {name: "labels get the branches as match[]", pods: []string{"p"}, path: "api/v1/labels", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, + {name: "series needs match[]", path: "api/v1/series", wantCode: http.StatusBadRequest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.podStatus == 0 { + tt.podStatus = http.StatusOK + } + h := newPolicyHarness(t, tt.cfg, appMetricsUpstream(tt.pods, tt.pvcs, tt.podStatus), appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tt.path, "good"); got != tt.wantCode { + t.Fatalf("status = %d, want %d", got, tt.wantCode) + } + var main *upstreamRequest + for _, r := range h.requests() { + if !strings.HasPrefix(r.path, "/api/v1/label/pod/") && !strings.HasPrefix(r.path, "/api/v1/label/persistentvolumeclaim/") { + main = &r + } + } + if tt.want == nil { + if main != nil { + t.Fatalf("query reached the store: %s %v", main.path, main.form) + } + return + } + if main == nil || !slices.Equal(main.form["extra_filters[]"], tt.want) { + t.Fatalf("extra_filters[] = %v, want %q", main, tt.want) + } + if strings.HasSuffix(tt.path, "labels") && !slices.Equal(main.form["match[]"], tt.want) { + t.Fatalf("match[] = %q, want the branches", main.form["match[]"]) + } + }) + } +} + +// A grant narrowed to application "a" by resourceNames must not reach "b" +// through any endpoint of either root. +func TestServerApplicationNarrowedGrant(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"a-0"}, nil, http.StatusOK), appPolicy, nil) + paths := []string{ + "metrics/api/v1/query?query=up", + "metrics/api/v1/query_range?query=up&start=" + unix(testNow.Add(-time.Hour)) + "&end=" + unix(testNow) + "&step=60", + "metrics/api/v1/series?match[]=up", + "metrics/api/v1/labels", + "metrics/api/v1/label/pod/values", + "log/select/logsql/query", + "log/select/logsql/hits", + "log/select/logsql/stats_query_range?query=" + url.QueryEscape("* | stats count()"), + "log/select/logsql/tail", + "log/select/logsql/field_values?field=level", + "log/select/logsql/stream_field_values?field=level", + "log/select/logsql/facets", + } + for _, p := range paths { + if got := h.status(http.MethodGet, appRoot+"b/"+p, "bob-token"); got != http.StatusForbidden { + t.Errorf("b/%s answered %d, want 403", p, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for a denied application", n) + } + for _, p := range []string{"metrics/api/v1/labels", "log/select/logsql/query"} { + if got := h.status(http.MethodGet, appRoot+"a/"+p, "bob-token"); got != http.StatusOK { + t.Errorf("a/%s answered %d, want 200", p, got) + } + } +} diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index b541a18e1..168c402fe 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -96,6 +96,9 @@ func newPolicyHarness(t *testing.T, cfg Config, handler http.HandlerFunc, allow if cfg.Metrics.Limits == (MetricsLimits{}) { cfg.Metrics.Limits = testMetricsLimits() } + if cfg.Metrics.App == (AppScopeLimits{}) { + cfg.Metrics.App = AppScopeLimits{Lookback: 24 * time.Hour, MaxPods: 100, MaxPVCs: 100} + } if cfg.Logs == (LogsLimits{}) { cfg.Logs = testLimits() } From e6697ea4f176d1b059a40bab9bb87b31340d4e08 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 06:11:18 +0700 Subject: [PATCH 083/174] test(observability-proxy): cover malformed pod lookups and the lookup window An empty name in a lookup answer would turn the pod branch into a pattern that matches every series without a pod label, widening the application scope to the namespace; nothing pinned the guard against it. Malformed answers must only drop their branch, and the lookup must start at the query's own lookback, or at the configured limit when that lookback is unknown. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appkinds.go | 6 ++- internal/observabilityproxy/appscope.go | 6 +-- .../observabilityproxy/server_app_test.go | 39 +++++++++++++++++-- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/internal/observabilityproxy/appkinds.go b/internal/observabilityproxy/appkinds.go index 5e545f390..d58368a13 100644 --- a/internal/observabilityproxy/appkinds.go +++ b/internal/observabilityproxy/appkinds.go @@ -79,7 +79,11 @@ func (k *ApplicationKinds) Kind(plural string) (string, error) { } var kinds []string for _, obj := range objs { - kind, _, _ := unstructured.NestedString(obj.(*unstructured.Unstructured).Object, "spec", "application", "kind") + u, ok := obj.(*unstructured.Unstructured) + if !ok { + continue + } + kind, _, _ := unstructured.NestedString(u.Object, "spec", "application", "kind") if kind != "" && !slices.Contains(kinds, kind) { kinds = append(kinds, kind) } diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index b969cc502..3b192d056 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -121,15 +121,15 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st return applicationSelectors(ns, kind, name, lists[0], lists[1]), nil } -// lookupNames asks for at most max+1 values, enough to tell a list over +// lookupNames asks for at most limit+1 values, enough to tell a list over // the cap without reading all of it. -func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, match, ns string, start, end time.Time, max int) ([]string, error) { +func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, match, ns string, start, end time.Time, limit int) ([]string, error) { params := url.Values{} params.Set("match[]", match) params.Set("extra_filters[]", namespaceSelector(ns)) params.Set("start", formatTime(start)) params.Set("end", formatTime(end)) - params.Set("limit", strconv.Itoa(max+1)) + params.Set("limit", strconv.Itoa(limit+1)) resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/api/v1/label/"+label+"/values", params, "metrics store", nil) if herr != nil { return nil, herr diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 14807637e..25d8f2a7f 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -173,9 +173,9 @@ func TestServerApplicationMetricsScope(t *testing.T) { if got := r.form["extra_filters[]"]; !slices.Equal(got, []string{wantMetricsScope}) { t.Errorf("%s extra_filters[] = %q", r.path, got) } - start, herr := parseMetricsTime("start", r.form.Get("start")) - if herr != nil || start.After(testNow.Add(-time.Hour)) || start.Before(testNow.Add(-25*time.Hour)) { - t.Errorf("%s start = %s, want within the lookback before the window", r.path, r.form.Get("start")) + // The query's own lookback is known and shorter than the 24h cap. + if want := formatTime(buildMetrics(t, "api/v1/query_range", hostile).start); r.form.Get("start") != want { + t.Errorf("%s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } if r.form.Get("end") != formatTime(testNow) || r.form.Get("limit") != "101" { t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit")) @@ -204,6 +204,7 @@ func TestServerApplicationMetricsBranches(t *testing.T) { name string pods, pvcs []string podStatus int + podBody string cfg Config path string wantCode int @@ -218,13 +219,29 @@ func TestServerApplicationMetricsBranches(t *testing.T) { path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "labels get the branches as match[]", pods: []string{"p"}, path: "api/v1/labels", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, {name: "series needs match[]", path: "api/v1/series", wantCode: http.StatusBadRequest}, + // An empty name would make pod=~"^(|p)$", which matches every + // series without a pod label. + {name: "empty pod name dropped", pods: []string{"", "p"}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, + {name: "only empty pod names", pods: []string{""}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, + {name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, + {name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if tt.podStatus == 0 { tt.podStatus = http.StatusOK } - h := newPolicyHarness(t, tt.cfg, appMetricsUpstream(tt.pods, tt.pvcs, tt.podStatus), appPolicy, nil) + upstream := appMetricsUpstream(tt.pods, tt.pvcs, tt.podStatus) + if tt.podBody != "" { + upstream = func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/label/pod/values" { + _, _ = io.WriteString(w, tt.podBody) + return + } + appMetricsUpstream(nil, nil, http.StatusOK)(w, r) + } + } + h := newPolicyHarness(t, tt.cfg, upstream, appPolicy, nil) if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tt.path, "good"); got != tt.wantCode { t.Fatalf("status = %d, want %d", got, tt.wantCode) } @@ -282,3 +299,17 @@ func TestServerApplicationNarrowedGrant(t *testing.T) { } } } + +// An expression whose lookback cannot be bounded looks pods up from the +// configured lookback before the window. +func TestServerApplicationLookupUnknownLookback(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + q := url.Values{"query": {"up @ end()"}, "time": {unix(testNow)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + r := h.requests()[0] + if want := formatTime(testNow.Add(-24 * time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { + t.Fatalf("lookup %s start = %s, want %s", r.path, r.form.Get("start"), want) + } +} From dd7116ea451d11e782852a545dee03b5e83c81e4 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Wed, 4 Nov 2026 06:14:16 +0700 Subject: [PATCH 084/174] fix(observability-proxy): check the lookup cap before dropping empty names The store cuts a lookup at the cap plus one value. Dropping empty values first could bring such a cut list back under the cap and serve it as complete, hiding the application's other pods without the 422 that asks for a shorter range. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 5 +++++ internal/observabilityproxy/server_app_test.go | 2 ++ 2 files changed, 7 insertions(+) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 3b192d056..e259831da 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -148,6 +148,11 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, if body.Status != "success" { return nil, errorf(http.StatusBadGateway, "metrics store answered status %q", body.Status) } + if len(body.Data) > limit { + // Over the cap before dropping empty values: the store cut the + // list at limit+1, so a shorter filtered list could hide names. + return body.Data, nil + } names := body.Data[:0] for _, n := range body.Data { if n != "" { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 25d8f2a7f..6d353d00f 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -215,6 +215,8 @@ func TestServerApplicationMetricsBranches(t *testing.T) { want: []string{wantIdentityBranch, `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`}}, {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, + {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, + path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "pvcs over the cap", pods: []string{"p"}, pvcs: []string{"v", "w"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "labels get the branches as match[]", pods: []string{"p"}, path: "api/v1/labels", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, From c4593d4c3edcb4b999be72f6bace5dc707b8af50 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:34:41 +0700 Subject: [PATCH 085/174] fix(observability-proxy): require namespace access on application roots The application roots asked only for the application subresource. Their pod and PVC branches select by namespace and name across the whole lookup window, so a caller granted only application "a" received the older series of application "b" whenever "b" had used one of the same pod or PVC names. The lineage labels the scope rests on can also be set by whoever creates an unowned pod. Until the scope carries application identity on every branch, the application roots also require read access to the namespace root, the same review the namespace root asks. Every tenant role that reads applications already holds it, so no current grant loses access; a grant narrowed by resourceNames alone is refused instead of trusted. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 11 ++++---- internal/observabilityproxy/route.go | 15 ++++++++++ internal/observabilityproxy/route_test.go | 24 ++++++++++++++++ internal/observabilityproxy/server.go | 16 ++++++----- .../observabilityproxy/server_app_test.go | 28 +++++++++++++------ internal/observabilityproxy/server_test.go | 2 +- 6 files changed, 75 insertions(+), 21 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index e259831da..55b51afe6 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -76,11 +76,12 @@ func nameBranch(ns, label string, names []string) string { // stamped (cAdvisor, kubelet volume stats). Every branch carries the // namespace, so their OR stays inside it. // -// Within the namespace, both the identity and the pod and PVC lists come -// from lineage labels on pods and PVCs. They separate applications for a -// grant narrowed by resourceNames only while whoever creates a pod or PVC -// cannot choose those labels; the lineage webhook keeps them as set on an -// object it finds no owner for, and the logs scope rests on the same. +// Within the namespace this narrows, it does not isolate: the identity and +// the pod and PVC lists come from lineage labels that the lineage webhook +// keeps as set on an object it finds no owner for, and the pod and PVC +// branches match by name, so a name another application held within the +// lookup window pulls that application's series in. Route.Reviews makes +// the application roots require read access to the namespace root too. func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { out := []string{identitySelector(ns, seriesKindLabel, seriesNameLabel, kind, name)} for _, b := range []string{nameBranch(ns, "pod", pods), nameBranch(ns, "persistentvolumeclaim", pvcs)} { diff --git a/internal/observabilityproxy/route.go b/internal/observabilityproxy/route.go index 56ee9e496..1baff4578 100644 --- a/internal/observabilityproxy/route.go +++ b/internal/observabilityproxy/route.go @@ -133,6 +133,21 @@ func ParseRoute(escapedPath string) (Route, *HTTPError) { // ResourceAttributes is the single SubjectAccessReview a route requires. It // depends on the path only, so a POST query authorizes exactly like a GET. +// Reviews lists every access a request on the route needs. An application +// root also needs read access to its namespace root: its pod and PVC +// branches match by name across the window, so a name another application +// held earlier reaches that application's series, and a grant narrowed by +// resourceNames would read more than it names. +func (r Route) Reviews() []authorizationv1.ResourceAttributes { + switch r.Kind { + case ApplicationLog: + return []authorizationv1.ResourceAttributes{Route{Kind: NamespaceLogs, Namespace: r.Namespace}.ResourceAttributes(), r.ResourceAttributes()} + case ApplicationMetrics: + return []authorizationv1.ResourceAttributes{Route{Kind: NamespaceMetrics, Namespace: r.Namespace}.ResourceAttributes(), r.ResourceAttributes()} + } + return []authorizationv1.ResourceAttributes{r.ResourceAttributes()} +} + func (r Route) ResourceAttributes() authorizationv1.ResourceAttributes { switch r.Kind { case NamespaceLogs: diff --git a/internal/observabilityproxy/route_test.go b/internal/observabilityproxy/route_test.go index f34311e0f..a8e85b377 100644 --- a/internal/observabilityproxy/route_test.go +++ b/internal/observabilityproxy/route_test.go @@ -18,6 +18,7 @@ package observabilityproxy import ( "net/http" + "slices" "testing" authorizationv1 "k8s.io/api/authorization/v1" @@ -130,3 +131,26 @@ func TestRouteResourceAttributes(t *testing.T) { } } } + +func TestRouteReviews(t *testing.T) { + nsLogs := authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "logs", Namespace: "a"} + nsMetrics := authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "metrics", Namespace: "a"} + tests := []struct { + route Route + want []authorizationv1.ResourceAttributes + }{ + {Route{Kind: NamespaceLogs, Namespace: "a"}, []authorizationv1.ResourceAttributes{nsLogs}}, + {Route{Kind: NamespaceMetrics, Namespace: "a"}, []authorizationv1.ResourceAttributes{nsMetrics}}, + {Route{Kind: ApplicationLog, Namespace: "a", Plural: "postgreses", Name: "db"}, []authorizationv1.ResourceAttributes{ + nsLogs, {Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "log", Name: "db", Namespace: "a"}, + }}, + {Route{Kind: ApplicationMetrics, Namespace: "a", Plural: "postgreses", Name: "db"}, []authorizationv1.ResourceAttributes{ + nsMetrics, {Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "metrics", Name: "db", Namespace: "a"}, + }}, + } + for _, tt := range tests { + if got := tt.route.Reviews(); !slices.Equal(got, tt.want) { + t.Errorf("%v.Reviews() = %+v, want %+v", tt.route.Kind, got, tt.want) + } + } +} diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index fd74491b5..cf873af72 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -451,13 +451,15 @@ func (s *Server) authorize(r *http.Request, route Route) (string, *HTTPError) { slog.Error("token review failed", "error", err) return "", errorf(http.StatusServiceUnavailable, "authentication unavailable") } - allowed, err := s.auth.Authorize(ctx, user, route.ResourceAttributes()) - if err != nil { - slog.Error("subject access review failed", "error", err) - return "", errorf(http.StatusServiceUnavailable, "authorization unavailable") - } - if !allowed { - return "", errorf(http.StatusForbidden, "forbidden") + for _, attrs := range route.Reviews() { + allowed, err := s.auth.Authorize(ctx, user, attrs) + if err != nil { + slog.Error("subject access review failed", "error", err) + return "", errorf(http.StatusServiceUnavailable, "authorization unavailable") + } + if !allowed { + return "", errorf(http.StatusForbidden, "forbidden") + } } return user.Username, nil } diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 6d353d00f..a767e7f9e 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -31,11 +31,18 @@ import ( const appRoot = "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/" -// appPolicy lets alice read every postgres in tenant-a, as the view role's -// apps.cozystack.io wildcard does, and bob only "a", as a grant narrowed -// with resourceNames does. +// appPolicy lets alice read every postgres in tenant-a and the namespace +// roots, as the view role does. bob and carol may read only postgres "a", +// as a grant narrowed with resourceNames does; carol may also read the +// namespace roots. func appPolicy(user string, a *authorizationv1.ResourceAttributes) bool { - if a.Namespace != "tenant-a" || a.Verb != "get" || a.Group != "apps.cozystack.io" || a.Resource != "postgreses" || + if a.Namespace != "tenant-a" { + return false + } + if a.Verb == "list" && a.Group == "cozystack.io" && (a.Resource == "logs" || a.Resource == "metrics") && a.Name == "" && a.Subresource == "" { + return user == "alice" || user == "carol" + } + if a.Verb != "get" || a.Group != "apps.cozystack.io" || a.Resource != "postgreses" || (a.Subresource != "log" && a.Subresource != "metrics") { return false } @@ -270,7 +277,10 @@ func TestServerApplicationMetricsBranches(t *testing.T) { } // A grant narrowed to application "a" by resourceNames must not reach "b" -// through any endpoint of either root. +// through any endpoint of either root. Without read access to the +// namespace roots it does not reach "a" either: the pod and PVC branches +// match by name, and a name another application used earlier in the +// window would pull its series in. func TestServerApplicationNarrowedGrant(t *testing.T) { h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"a-0"}, nil, http.StatusOK), appPolicy, nil) paths := []string{ @@ -288,15 +298,17 @@ func TestServerApplicationNarrowedGrant(t *testing.T) { "log/select/logsql/facets", } for _, p := range paths { - if got := h.status(http.MethodGet, appRoot+"b/"+p, "bob-token"); got != http.StatusForbidden { - t.Errorf("b/%s answered %d, want 403", p, got) + for _, denied := range []struct{ app, token string }{{"b", "bob-token"}, {"b", "carol-token"}, {"a", "bob-token"}} { + if got := h.status(http.MethodGet, appRoot+denied.app+"/"+p, denied.token); got != http.StatusForbidden { + t.Errorf("%s on %s/%s answered %d, want 403", denied.token, denied.app, p, got) + } } } if n := len(h.requests()); n != 0 { t.Fatalf("upstream saw %d requests for a denied application", n) } for _, p := range []string{"metrics/api/v1/labels", "log/select/logsql/query"} { - if got := h.status(http.MethodGet, appRoot+"a/"+p, "bob-token"); got != http.StatusOK { + if got := h.status(http.MethodGet, appRoot+"a/"+p, "carol-token"); got != http.StatusOK { t.Errorf("a/%s answered %d, want 200", p, got) } } diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index 168c402fe..ce2ba669b 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -64,7 +64,7 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { // and plural-to-kind mapping; nil kinds knows only postgreses. func newPolicyHarness(t *testing.T, cfg Config, handler http.HandlerFunc, allow func(string, *authorizationv1.ResourceAttributes) bool, kinds KindLookup) *harness { h := &harness{t: t} - h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, allow) + h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob", "carol-token": "carol"}, allow) if kinds == nil { kinds = newTestKinds(t, true, appDefinition("postgres", "Postgres", "postgreses")) } From 07f4f187cef1a96e6f39247eeb2bf8ae7ff2fd79 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:36:03 +0700 Subject: [PATCH 086/174] fix(observability-proxy): cover forward-shifted samples in the app lookup The pod and PVC lookup of the application metrics root ended at the evaluation time, but a negative offset reads samples after it. An instant query evaluated two hours ago with "offset -1h" reads samples from one hour ago, so a pod created in between was missing from the scope and its series were dropped while the request succeeded. The lookup now ends at the latest instant the query reads, its window end plus the sum of its negative offsets. A query whose latest sample cannot be bounded, an @ modifier other than start() or end() or an offset that is not a plain duration, is refused with 422 on the application root; the namespace root has no lookup and still serves it. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 12 ++-- .../observabilityproxy/metricslookback.go | 64 +++++++++++++++---- internal/observabilityproxy/metricsparams.go | 11 +++- .../observabilityproxy/metricsparams_test.go | 37 +++++++++++ .../observabilityproxy/server_app_test.go | 36 +++++++++++ 5 files changed, 142 insertions(+), 18 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 55b51afe6..88acb4ba4 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -94,10 +94,14 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // applicationScope looks the application's pods and PVCs up in store over // the request window, reaching back by the query's own lookback up to the -// configured limit. A failed lookup drops its branch, which only narrows -// the answer; a list over its cap is refused, because truncating it would -// silently drop series. +// configured limit and forward to the latest sample a negative offset +// reads. A failed lookup drops its branch, which only narrows the answer; +// a list over its cap, or a query whose latest sample is unknown, is +// refused, because the answer would silently miss series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { + if req.last.IsZero() { + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root") + } lim := s.cfg.Metrics.App start := req.from.Add(-lim.Lookback) if req.start.After(start) { @@ -109,7 +113,7 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st max int }{{"pod", "kube_pod_labels", lim.MaxPods}, {"persistentvolumeclaim", "kube_persistentvolumeclaim_labels", lim.MaxPVCs}} { match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name) - names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.to, l.max) + names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max) if err != nil { slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) continue diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index e55d189a5..442e9c8d9 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -113,10 +113,55 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { sum += d } lower := strings.ToLower(query) + back, _, ok := scanOffsets(lower) + if !ok { + return 0, false + } + sum += back + if hasKeyword(lower, "with") { + return 0, false + } + n := float64(subqueries + strings.Count(query, "(")) + sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval) + if sum > float64(maxMetricsDuration) { + return 0, false + } + return time.Duration(sum), true +} + +// queryLookahead is an upper bound on how far after its evaluation window +// a query reads, from the same lexical scan. Only a negative offset reads +// forward; the negative offsets are summed, which covers nesting. An @ +// modifier pins samples to any instant unless it names start() or end(), +// which lie in the window, and WITH can apply one offset several times; +// either makes the bound unknown. +func queryLookahead(query string) (time.Duration, bool) { + lower := strings.ToLower(query) + for rest := lower; ; { + at := strings.IndexByte(rest, '@') + if at < 0 { + break + } + rest = strings.TrimLeft(rest[at+1:], " \t\n\r") + if !strings.HasPrefix(rest, "start()") && !strings.HasPrefix(rest, "end()") { + return 0, false + } + } + _, ahead, ok := scanOffsets(lower) + if !ok || ahead > 0 && hasKeyword(lower, "with") || ahead > float64(maxMetricsDuration) { + return 0, false + } + return time.Duration(ahead), true +} + +// scanOffsets sums every offset of the lowercased query, the positive ones +// into back and the negative ones into ahead, in nanoseconds; ok is false +// when one is not a plain duration. +func scanOffsets(lower string) (back, ahead float64, ok bool) { for i := 0; ; { k := strings.Index(lower[i:], "offset") if k < 0 { - break + return back, ahead, true } k += i i = k + len("offset") @@ -134,21 +179,14 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { } d, ok := parseQueryDuration(arg[:end]) if !ok { - return 0, false + return 0, 0, false } - if !negative { - sum += d + if negative { + ahead += d + } else { + back += d } } - if hasKeyword(lower, "with") { - return 0, false - } - n := float64(subqueries + strings.Count(query, "(")) - sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval) - if sum > float64(maxMetricsDuration) { - return 0, false - } - return time.Duration(sum), true } // isKeyword reports whether lower[start:end] stands on its own rather than diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index a0bb42f0d..b55f09318 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -81,6 +81,9 @@ type metricsRequest struct { // from and to are the window the query is evaluated over, without the // lookback of its expression. from, to time.Time + // last is the latest instant the request reads, past to by a negative + // offset; zero when that is unknown. + last time.Time // scopedMatch is set when match[] holds the scope, not client matchers. scopedMatch bool } @@ -143,6 +146,9 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step) req.from, req.to = at, at + if ahead, ok := queryLookahead(out.Get("query")); ok { + req.last = at.Add(ahead) + } return req, nil case "api/v1/query_range": @@ -167,6 +173,9 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } req.start = queryStart(out.Get("query"), start, step) req.from, req.to = start, end + if ahead, ok := queryLookahead(out.Get("query")); ok { + req.last = end.Add(ahead) + } out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) out.Set("step", formatSeconds(step)) @@ -189,7 +198,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti if herr != nil { return req, herr } - req.start, req.from, req.to = start, start, end + req.start, req.from, req.to, req.last = start, start, end, end out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index a4a435567..5c39fdfff 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -429,3 +429,40 @@ func TestBuildMetricsParamsPointsAsVmselectCounts(t *testing.T) { t.Fatalf("11001 points answered %d, want 422", got) } } + +func TestQueryLookahead(t *testing.T) { + tests := []struct { + query string + want time.Duration + }{ + {`up`, 0}, + {`x offset 1h`, 0}, + {`x offset -1h`, time.Hour}, + {`x OFFSET - 90m`, 90 * time.Minute}, + {`rate(x[5m] offset -30m) offset -1h`, 90 * time.Minute}, + {`x offset -1h + y offset 2h`, time.Hour}, + {`x @ end()`, 0}, + {`x @ start() offset -1h`, time.Hour}, + {`node_offset_seconds - 1`, 0}, + } + for _, tt := range tests { + t.Run(tt.query, func(t *testing.T) { + got, ok := queryLookahead(tt.query) + if !ok || got != tt.want { + t.Fatalf("queryLookahead = %s, %v; want %s", got, ok, tt.want) + } + }) + } + for _, q := range []string{ + `x @ 1700000000`, + `x @ (end() - 1h)`, + `x{a="a@b"}`, + `x offset -5i`, + `x offset -$__interval`, + `with (f(a) = a offset -1h) f(f(x))`, + } { + if got, ok := queryLookahead(q); ok { + t.Errorf("queryLookahead(%q) = %s, want unknown", q, got) + } + } +} diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index a767e7f9e..c7e7a67ef 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -327,3 +327,39 @@ func TestServerApplicationLookupUnknownLookback(t *testing.T) { t.Fatalf("lookup %s start = %s, want %s", r.path, r.form.Get("start"), want) } } + +// TestServerApplicationLookupCoversNegativeOffset evaluates two hours ago +// a selector shifted an hour forward: the pod lookup must reach the time +// of the samples, or a pod created in between is left out. +func TestServerApplicationLookupCoversNegativeOffset(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + at := testNow.Add(-2 * time.Hour) + q := url.Values{"query": {"container_cpu_usage_seconds_total offset -1h"}, "time": {unix(at)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + r := h.requests()[0] + if want := formatTime(at.Add(time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("end") != want { + t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want) + } +} + +// TestServerApplicationRefusesUnboundedSampleTime refuses a query whose +// samples may lie at a time the lookup cannot be sized for, rather than +// answer with series silently left out. +func TestServerApplicationRefusesUnboundedSampleTime(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{"up @ 1700000000", "up offset -5i"} { + q := url.Values{"query": {query}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for a refused query", n) + } + // The namespace root has no lookup to size and still serves it. + if got := h.status(http.MethodGet, metricsRoot+"api/v1/query?"+url.Values{"query": {"up @ 1700000000"}}.Encode(), "good"); got != http.StatusOK { + t.Fatalf("namespace root answered %d", got) + } +} From 97a55a167bf786809a88d50d65da2f2a2225f55d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:44:23 +0700 Subject: [PATCH 087/174] docs(observability-proxy): put the route review comments on their functions Reviews was inserted between ResourceAttributes and its doc comment, so go doc showed the ResourceAttributes text on Reviews, and that text still called it the single review a route needs, which application roots no longer are. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/route.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/route.go b/internal/observabilityproxy/route.go index 1baff4578..49a7c1770 100644 --- a/internal/observabilityproxy/route.go +++ b/internal/observabilityproxy/route.go @@ -131,8 +131,6 @@ func ParseRoute(escapedPath string) (Route, *HTTPError) { return r, nil } -// ResourceAttributes is the single SubjectAccessReview a route requires. It -// depends on the path only, so a POST query authorizes exactly like a GET. // Reviews lists every access a request on the route needs. An application // root also needs read access to its namespace root: its pod and PVC // branches match by name across the window, so a name another application @@ -148,6 +146,9 @@ func (r Route) Reviews() []authorizationv1.ResourceAttributes { return []authorizationv1.ResourceAttributes{r.ResourceAttributes()} } +// ResourceAttributes is the SubjectAccessReview of the route's own +// resource. It depends on the path only, so a POST query authorizes +// exactly like a GET. func (r Route) ResourceAttributes() authorizationv1.ResourceAttributes { switch r.Kind { case NamespaceLogs: From 1a3517fa81d6ca87f4387ce4e50e831d981d60e7 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:55:50 +0700 Subject: [PATCH 088/174] fix(observability-proxy): fail an application query whose lookup failed When the pod or PVC lookup failed, its branch was dropped and the query still answered 200, without the cAdvisor and volume series of the application, so a store error showed up as an empty graph. That is the same silently incomplete answer that a lookup over its cap is refused for. A failed lookup now fails the request: 504 when the store missed the deadline, 502 otherwise. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 13 ++++++++----- internal/observabilityproxy/server_app_test.go | 9 +++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 88acb4ba4..ff1955dc9 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -95,9 +95,9 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // applicationScope looks the application's pods and PVCs up in store over // the request window, reaching back by the query's own lookback up to the // configured limit and forward to the latest sample a negative offset -// reads. A failed lookup drops its branch, which only narrows the answer; -// a list over its cap, or a query whose latest sample is unknown, is -// refused, because the answer would silently miss series. +// reads. A failed lookup, a list over its cap, or a query whose latest +// sample is unknown fails the request, because a successful answer would +// silently miss the application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root") @@ -115,8 +115,11 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name) names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max) if err != nil { - slog.Warn("application lookup failed; its branch is left out", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) - continue + slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) + if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusGatewayTimeout { + return nil, herr + } + return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label) } if len(names) > l.max { return nil, errorf(http.StatusUnprocessableEntity, "the application has more than %d %ss in this window; use a shorter range", l.max, l.label) diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index c7e7a67ef..268ea7b8f 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -218,8 +218,9 @@ func TestServerApplicationMetricsBranches(t *testing.T) { want []string }{ {name: "no pods or pvcs", path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, - {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusOK, - want: []string{wantIdentityBranch, `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`}}, + // A failed lookup would leave the application's own series out of + // a successful answer, so it fails the request instead. + {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, @@ -232,8 +233,8 @@ func TestServerApplicationMetricsBranches(t *testing.T) { // series without a pod label. {name: "empty pod name dropped", pods: []string{"", "p"}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, {name: "only empty pod names", pods: []string{""}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, - {name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, - {name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, + {name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, + {name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 66e2b99f0981cd440a6d9b538c29f51fdaacadf0 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 00:56:32 +0700 Subject: [PATCH 089/174] fix(observability-proxy): reach the aligned end in the app lookup vmselect rounds a range query's end up to a whole step before evaluating it, so the last point can read samples up to one step past the end the client sent. The pod and PVC lookup stopped at that end and could miss a pod created within the last step. The lookup end of a range query now adds one step. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/metricsparams.go | 3 ++- internal/observabilityproxy/server_app_test.go | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index b55f09318..721496fe3 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -173,8 +173,9 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } req.start = queryStart(out.Get("query"), start, step) req.from, req.to = start, end + // vmselect rounds end up to a whole step (AdjustStartEnd). if ahead, ok := queryLookahead(out.Get("query")); ok { - req.last = end.Add(ahead) + req.last = end.Add(step + ahead) } out.Set("start", formatTime(start)) out.Set("end", formatTime(end)) diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 268ea7b8f..395deb18b 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -184,7 +184,8 @@ func TestServerApplicationMetricsScope(t *testing.T) { if want := formatTime(buildMetrics(t, "api/v1/query_range", hostile).start); r.form.Get("start") != want { t.Errorf("%s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } - if r.form.Get("end") != formatTime(testNow) || r.form.Get("limit") != "101" { + // One step past end, which vmselect rounds up to a whole step. + if r.form.Get("end") != formatTime(testNow.Add(time.Minute)) || r.form.Get("limit") != "101" { t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit")) } } @@ -364,3 +365,18 @@ func TestServerApplicationRefusesUnboundedSampleTime(t *testing.T) { t.Fatalf("namespace root answered %d", got) } } + +// TestServerApplicationLookupCoversAlignedEnd: vmselect rounds a range +// query's end up to a whole step, so the lookup reaches one step past it. +func TestServerApplicationLookupCoversAlignedEnd(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + end := testNow.Add(-90 * time.Minute) + q := url.Values{"query": {"up"}, "start": {unix(end.Add(-6 * time.Hour))}, "end": {unix(end)}, "step": {"3600"}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query_range?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + r := h.requests()[0] + if want := formatTime(end.Add(time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("end") != want { + t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want) + } +} From bc823b704e5994301f62d6add1e28971768e7ff9 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:01:57 +0700 Subject: [PATCH 090/174] fix(observability-proxy): answer 504 to a lookup that misses the deadline Only a deadline hit before the lookup's response headers was reported as 504. When the store sent its headers and then stalled, the deadline expired while the body was decoded, and the decode error came back as a generic 502, so a slow store looked like a broken one. The request deadline is now checked whichever step of the lookup it cut, and both cases are covered by a test. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 7 +++-- .../observabilityproxy/server_app_test.go | 28 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index ff1955dc9..80d12cbe2 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -19,6 +19,7 @@ package observabilityproxy import ( "context" "encoding/json" + "errors" "io" "log/slog" "net/http" @@ -116,8 +117,10 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max) if err != nil { slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) - if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusGatewayTimeout { - return nil, herr + // The deadline can also expire while the answer is read, after + // post has handed it over. + if errors.Is(context.Cause(ctx), context.DeadlineExceeded) { + return nil, errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", l.label) } return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label) } diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 395deb18b..53216e575 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -380,3 +380,31 @@ func TestServerApplicationLookupCoversAlignedEnd(t *testing.T) { t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want) } } + +// TestServerApplicationLookupDeadline: a lookup that misses the request +// deadline answers 504, whether the store sent nothing or stalled after +// its headers, and the query itself is not sent. +func TestServerApplicationLookupDeadline(t *testing.T) { + for _, headers := range []bool{false, true} { + h := newPolicyHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/label/pod/values" { + appMetricsUpstream(nil, nil, http.StatusOK)(w, r) + return + } + if headers { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"status":"success","data":[`) + w.(http.Flusher).Flush() + } + <-r.Context().Done() + }, appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusGatewayTimeout { + t.Fatalf("headers sent %v: status = %d, want 504", headers, got) + } + for _, r := range h.requests() { + if !strings.HasPrefix(r.path, "/api/v1/label/") { + t.Fatalf("headers sent %v: the query reached the store after a failed lookup", headers) + } + } + } +} From 84665a42a798d935a1b078d6fbdd3c720c62bffc Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:28:12 +0700 Subject: [PATCH 091/174] fix(observability-proxy): refuse app queries that read past the lookup The pod and PVC lookup never started earlier than --metrics-app-lookback before the window. A query that read further back, such as rate(x[7d]) with the 24h default, or whose lookback is unknown (an @ modifier, WITH, a step-multiple window), answered 200 without the series of pods that existed only before the lookup start. Such queries now answer 422 on the application root, matching how a query whose latest sample cannot be covered is handled; the namespace root has no lookup and still serves them. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 2 +- internal/observabilityproxy/appscope.go | 17 ++++++----- .../observabilityproxy/server_app_test.go | 29 ++++++++++++++----- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 4a99aa4cc..93f4c4553 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -94,7 +94,7 @@ func parseFlags(args []string) (options, error) { fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use") fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return") fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") - fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs at most") + fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs; a query reading further back answers 422") fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 80d12cbe2..835d29b84 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -94,20 +94,21 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { } // applicationScope looks the application's pods and PVCs up in store over -// the request window, reaching back by the query's own lookback up to the -// configured limit and forward to the latest sample a negative offset -// reads. A failed lookup, a list over its cap, or a query whose latest -// sample is unknown fails the request, because a successful answer would -// silently miss the application's series. +// the samples the request reads: back by the query's own lookback, and +// forward to the latest sample a negative offset reads. A query that reads +// further back than the configured lookback, or whose earliest or latest +// sample is unknown, a failed lookup, and a list over its cap all fail the +// request, because a successful answer would silently miss the +// application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root") } lim := s.cfg.Metrics.App - start := req.from.Add(-lim.Lookback) - if req.start.After(start) { - start = req.start + if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { + return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback) } + start := req.start var lists [2][]string for i, l := range []struct { label, metric string diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 53216e575..d3dcb683a 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -316,17 +316,30 @@ func TestServerApplicationNarrowedGrant(t *testing.T) { } } -// An expression whose lookback cannot be bounded looks pods up from the -// configured lookback before the window. -func TestServerApplicationLookupUnknownLookback(t *testing.T) { - h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) - q := url.Values{"query": {"up @ end()"}, "time": {unix(testNow)}} +// TestServerApplicationLookbackBeyondCap: the lookup reaches back at most +// --metrics-app-lookback, so a query that reads further back, or whose +// lookback is unknown, would miss pods that existed only earlier; it is +// refused rather than answered without their series. +func TestServerApplicationLookbackBeyondCap(t *testing.T) { + cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 100, MaxPVCs: 100}}} + h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{"rate(x[7d])", "x[30d] @ end()", "rate(x[5i])"} { + q := url.Values{"query": {query}, "time": {unix(testNow)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for refused queries", n) + } + q := url.Values{"query": {"rate(x[5m])"}, "time": {unix(testNow)}} if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { - t.Fatalf("status = %d", got) + t.Fatalf("rate(x[5m]) answered %d, want 200", got) } r := h.requests()[0] - if want := formatTime(testNow.Add(-24 * time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { - t.Fatalf("lookup %s start = %s, want %s", r.path, r.form.Get("start"), want) + want := formatTime(buildMetrics(t, "api/v1/query", q).start) + if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { + t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } } From 84d5b4ff0aa6e5056d4ea244eb5ba18baff6a0b8 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:28:34 +0700 Subject: [PATCH 092/174] fix(observability-proxy): require a positive application lookback Every query reads some time before its window, at least the implicit rollup window, so with the lookback refusal a zero --metrics-app-lookback would turn every application metrics query into a 422. It is now refused at startup instead. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 4 ++-- cmd/observability-proxy/main_test.go | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 93f4c4553..7fe145229 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -137,8 +137,8 @@ func parseFlags(args []string) (options, error) { return o, errors.New("--metrics-shortterm-retention, --metrics-max-range and --metrics-max-timeout must be positive") case o.metrics.Limits.MaxPoints < 2: return o, errors.New("--metrics-max-points must be at least 2") - case o.metrics.App.Lookback < 0 || o.metrics.App.MaxPods < 1 || o.metrics.App.MaxPVCs < 1: - return o, errors.New("--metrics-app-lookback must not be negative and --metrics-app-max-pods and --metrics-app-max-pvcs must be positive") + case o.metrics.App.Lookback <= 0 || o.metrics.App.MaxPods < 1 || o.metrics.App.MaxPVCs < 1: + return o, errors.New("--metrics-app-lookback, --metrics-app-max-pods and --metrics-app-max-pvcs must be positive") case o.logs.MaxTailResume < 0: return o, errors.New("--logs-max-tail-resume must not be negative") // The proxy's deadline starts before the request reaches vmselect, so diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go index 6b5c60d7d..bf8e032c2 100644 --- a/cmd/observability-proxy/main_test.go +++ b/cmd/observability-proxy/main_test.go @@ -66,7 +66,7 @@ func TestParseFlagsMetricsOverrides(t *testing.T) { o, err := parseFlags([]string{ "--root-tenant=tenant-x", "--metrics-shortterm-store=fast", "--metrics-longterm-store=slow", "--metrics-shortterm-retention=24h", "--metrics-max-range=168h", "--metrics-max-points=500", "--metrics-max-timeout=10s", - "--metrics-app-lookback=0s", "--metrics-app-max-pods=7", "--metrics-app-max-pvcs=3", + "--metrics-app-lookback=2h", "--metrics-app-max-pods=7", "--metrics-app-max-pvcs=3", }) if err != nil { t.Fatal(err) @@ -74,7 +74,7 @@ func TestParseFlagsMetricsOverrides(t *testing.T) { m := o.metrics if m.RootTenant != "tenant-x" || m.ShortTermStore != "fast" || m.LongTermStore != "slow" || m.ShortTermRetention != 24*time.Hour || m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second || - m.App.Lookback != 0 || m.App.MaxPods != 7 || m.App.MaxPVCs != 3 { + m.App.Lookback != 2*time.Hour || m.App.MaxPods != 7 || m.App.MaxPVCs != 3 { t.Errorf("overrides not applied: %+v", m) } } @@ -107,6 +107,9 @@ func TestParseFlagsInvalid(t *testing.T) { {"--metrics-max-points=1"}, {"--metrics-max-timeout=0s"}, {"--metrics-app-lookback=-1h"}, + // Every query reads some time before its window, so a zero lookback + // would refuse every application metrics query. + {"--metrics-app-lookback=0s"}, {"--metrics-app-max-pods=0"}, {"--metrics-app-max-pvcs=0"}, {"--unknown"}, From 1a3e1827ef2104e7f0ae881bd03f284442e4a5d5 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:29:13 +0700 Subject: [PATCH 093/174] fix(observability-proxy): pass a store 422 from the app lookup through vmselect answers 422 when a label lookup exceeds its own series limits. The application root turned that into a 502, which reads as a broken store, while the client could have fixed it by narrowing the window. The 422 and vmselect's message now reach the client. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 9 +++++++++ internal/observabilityproxy/server_app_test.go | 3 +++ 2 files changed, 12 insertions(+) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 835d29b84..eec1f0eb6 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -120,6 +120,9 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) // The deadline can also expire while the answer is read, after // post has handed it over. + if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusUnprocessableEntity { + return nil, herr + } if errors.Is(context.Cause(ctx), context.DeadlineExceeded) { return nil, errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", l.label) } @@ -147,6 +150,12 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, return nil, herr } defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusUnprocessableEntity { + // vmselect's refusal of a lookup over its own limits; the client + // can act on it by narrowing the window. + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<10)) + return nil, errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, strings.TrimSpace(string(msg))) + } if resp.StatusCode != http.StatusOK { return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status) } diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index d3dcb683a..d1d691b00 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -222,6 +222,9 @@ func TestServerApplicationMetricsBranches(t *testing.T) { // A failed lookup would leave the application's own series out of // a successful answer, so it fails the request instead. {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, + // vmselect refuses a lookup over its own series limits with 422; + // the client can act on that by narrowing the window. + {name: "pod lookup over the store's limits", pods: []string{"p"}, podStatus: http.StatusUnprocessableEntity, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, From 86dddb447a81f584f9e017cb4fd9223c9e63137b Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:29:14 +0700 Subject: [PATCH 094/174] docs(observability-proxy): note the store calls behind one app request The per-user rate counts client requests, but an application metrics request makes three store calls, so an operator sizing the rate against store capacity needs to know the multiplier. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 7fe145229..a3fa9b259 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -98,6 +98,9 @@ func parseFlags(args []string) (options, error) { fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") + // Counted per client request: an application metrics request costs the + // store three calls (the pod and PVC lookups and the query) for one + // token, so the store load per user can be three times this rate. fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables") fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user") fs.IntVar(&o.limiter.MaxTailsPerUser, "max-tails-per-user", 5, "open tails per user; 0 disables") From 85fb98dd2cd5029adca903da074ab156877abb2a Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:36:52 +0700 Subject: [PATCH 095/174] fix(observability-proxy): stop refusing ordinary app queries on lookback With the lookback refusal, the 24h default turned common application queries into 422: a day's increase, and month-long panels at the step a dashboard picks for them. The lookback estimate is a loose upper bound, made for the store choice, and counts every "(" as an implicit subquery, so it far exceeds what such queries read. The lookback is only the refusal threshold, since the lookup itself spans the window the query reads, so its default now matches --metrics-max-range. The chart passes it from values, so an operator can tune it. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 6 ++++- cmd/observability-proxy/main_test.go | 2 +- .../observabilityproxy/server_app_test.go | 25 +++++++++++++++++++ .../templates/deployment.yaml | 1 + .../tests/deployment_test.yaml | 7 ++++++ .../system/observability-proxy/values.yaml | 3 +++ 6 files changed, 42 insertions(+), 2 deletions(-) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index a3fa9b259..487b6d04d 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -94,7 +94,11 @@ func parseFlags(args []string) (options, error) { fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use") fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return") fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") - fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 24*time.Hour, "how far before the window the application root looks up the application's pods and PVCs; a query reading further back answers 422") + // Only the refusal threshold: the lookup itself spans the window the + // query reads, and that estimate is a loose upper bound (every "(" + // counts as an implicit subquery), so the default matches + // --metrics-max-range to serve month-long panels at a dashboard step. + fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 30*24*time.Hour, "how far before the window an application metrics query may read; one reading further back, or by an unknown amount, answers 422") fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") diff --git a/cmd/observability-proxy/main_test.go b/cmd/observability-proxy/main_test.go index bf8e032c2..616d25c51 100644 --- a/cmd/observability-proxy/main_test.go +++ b/cmd/observability-proxy/main_test.go @@ -40,7 +40,7 @@ func TestParseFlagsDefaults(t *testing.T) { if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second { t.Errorf("metrics limit defaults changed: %+v", m.Limits) } - if m.App.Lookback != 24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 { + if m.App.Lookback != 30*24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 { t.Errorf("application scope defaults changed: %+v", m.App) } if m.RootTenant != o.rootTenant { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index d1d691b00..0b894b8fc 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -424,3 +424,28 @@ func TestServerApplicationLookupDeadline(t *testing.T) { } } } + +// TestServerApplicationDefaultLookbackServesLongPanels: the lookback is +// only the refusal threshold, and the lookback estimate is a loose upper +// bound, so the default must serve a day's increase and a month-long +// panel at the step a dashboard picks for it. +func TestServerApplicationDefaultLookbackServesLongPanels(t *testing.T) { + cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: 30 * 24 * time.Hour, MaxPods: 100, MaxPVCs: 100}}} + h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + month := url.Values{"start": {unix(testNow.Add(-29 * 24 * time.Hour))}, "end": {unix(testNow)}} + for _, tc := range []struct{ path, query, step string }{ + {"api/v1/query", "increase(x[24h])", ""}, + {"api/v1/query_range", "rate(x[5m])", "21600"}, + {"api/v1/query_range", "histogram_quantile(0.99, sum by (le) (rate(x[5m])))", "7200"}, + } { + q := url.Values{"query": {tc.query}} + if tc.step != "" { + q.Set("step", tc.step) + q.Set("start", month.Get("start")) + q.Set("end", month.Get("end")) + } + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tc.path+"?"+q.Encode(), "good"); got != http.StatusOK { + t.Errorf("%s step %s answered %d, want 200", tc.query, tc.step, got) + } + } +} diff --git a/packages/system/observability-proxy/templates/deployment.yaml b/packages/system/observability-proxy/templates/deployment.yaml index d49de140b..5cb32833f 100644 --- a/packages/system/observability-proxy/templates/deployment.yaml +++ b/packages/system/observability-proxy/templates/deployment.yaml @@ -41,6 +41,7 @@ spec: - --metrics-shortterm-store={{ .Values.observabilityProxy.metrics.shorttermStore }} - --metrics-longterm-store={{ .Values.observabilityProxy.metrics.longtermStore }} - --metrics-shortterm-retention={{ .Values.observabilityProxy.metrics.shorttermRetention }} + - --metrics-app-lookback={{ .Values.observabilityProxy.metrics.appLookback }} ports: - name: http containerPort: 8080 diff --git a/packages/system/observability-proxy/tests/deployment_test.yaml b/packages/system/observability-proxy/tests/deployment_test.yaml index a5e948920..0f0104660 100644 --- a/packages/system/observability-proxy/tests/deployment_test.yaml +++ b/packages/system/observability-proxy/tests/deployment_test.yaml @@ -51,6 +51,9 @@ tests: - contains: path: spec.template.spec.containers[0].args content: --metrics-shortterm-retention=72h + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-app-lookback=720h - it: follows renamed metrics stores set: @@ -58,6 +61,7 @@ tests: shorttermStore: fast longtermStore: slow shorttermRetention: 24h + appLookback: 48h asserts: - contains: path: spec.template.spec.containers[0].args @@ -68,6 +72,9 @@ tests: - contains: path: spec.template.spec.containers[0].args content: --metrics-shortterm-retention=24h + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-app-lookback=48h - it: falls back to cozy.local when the platform passes no cluster domain asserts: diff --git a/packages/system/observability-proxy/values.yaml b/packages/system/observability-proxy/values.yaml index 67334a237..3e3ac70fe 100644 --- a/packages/system/observability-proxy/values.yaml +++ b/packages/system/observability-proxy/values.yaml @@ -14,6 +14,9 @@ observabilityProxy: shorttermStore: shortterm longtermStore: longterm shorttermRetention: 72h + # How far before its window an application metrics query may read; + # one reading further back answers 422 on the application root. + appLookback: 720h resources: requests: cpu: 10m From caa4e8d0e3d5ab9f05fce28c96e0c54e4aed35b1 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:37:59 +0700 Subject: [PATCH 096/174] fix(observability-proxy): keep only the series-limit 422 of a lookup vmselect answers 422 to every error that carries no status of its own, its query timeout and storage failures included, so passing every lookup 422 through told the client to narrow a window when the store was failing, and bypassed the 504 for a missed deadline. Only vmselect's series-limit message stays a 422, with the message taken from its JSON answer; its timeout becomes 504 and anything else 502. The request deadline is checked before the store's status. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 35 ++++++++++++---- .../observabilityproxy/server_app_test.go | 41 +++++++++++++++++-- 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index eec1f0eb6..796280eb6 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -120,12 +120,12 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) // The deadline can also expire while the answer is read, after // post has handed it over. - if herr, ok := err.(*HTTPError); ok && herr.Code == http.StatusUnprocessableEntity { - return nil, herr - } if errors.Is(context.Cause(ctx), context.DeadlineExceeded) { return nil, errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", l.label) } + if herr, ok := err.(*HTTPError); ok && (herr.Code == http.StatusUnprocessableEntity || herr.Code == http.StatusGatewayTimeout) { + return nil, herr + } return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label) } if len(names) > l.max { @@ -151,10 +151,7 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, } defer func() { _ = resp.Body.Close() }() if resp.StatusCode == http.StatusUnprocessableEntity { - // vmselect's refusal of a lookup over its own limits; the client - // can act on it by narrowing the window. - msg, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<10)) - return nil, errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, strings.TrimSpace(string(msg))) + return nil, lookupRefusal(label, resp.Body) } if resp.StatusCode != http.StatusOK { return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status) @@ -182,3 +179,27 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, } return names, nil } + +// seriesLimitError is the message vmstorage gives when a search matches +// more series than the -search.max* flags allow (lib/storage/index_db.go); +// cluster vmselect wraps it in its own context, hence a substring match. +const seriesLimitError = "the number of matching timeseries exceeds" + +// lookupRefusal reads a vmselect 422. vmselect answers 422 to any error +// without a status of its own (httpserver.SendPrometheusError), storage +// failures and its own timeout included, so only the series limit, which +// a narrower window fixes, stays a 422 for the client. +func lookupRefusal(label string, body io.Reader) *HTTPError { + var answer struct { + Error string `json:"error"` + } + _ = json.NewDecoder(io.LimitReader(body, 64<<10)).Decode(&answer) + switch { + case strings.Contains(answer.Error, seriesLimitError): + return errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, answer.Error) + case strings.Contains(answer.Error, "timeout exceeded"): + return errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", label) + default: + return errorf(http.StatusBadGateway, "the application's %s lookup failed: %s", label, answer.Error) + } +} diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 0b894b8fc..2cd2a0eda 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -222,9 +222,6 @@ func TestServerApplicationMetricsBranches(t *testing.T) { // A failed lookup would leave the application's own series out of // a successful answer, so it fails the request instead. {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, - // vmselect refuses a lookup over its own series limits with 422; - // the client can act on that by narrowing the window. - {name: "pod lookup over the store's limits", pods: []string{"p"}, podStatus: http.StatusUnprocessableEntity, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, @@ -449,3 +446,41 @@ func TestServerApplicationDefaultLookbackServesLongPanels(t *testing.T) { } } } + +// TestServerApplicationLookupStoreErrors: vmselect answers 422 to any error +// without a status of its own (SendPrometheusError), so only its series +// limit, which a narrower window fixes, reaches the client as 422. +func TestServerApplicationLookupStoreErrors(t *testing.T) { + for _, tc := range []struct { + label, message string + want int + }{ + {"pod", "the number of matching timeseries exceeds 300000; either narrow down the search or increase -search.max* command-line flag values", http.StatusUnprocessableEntity}, + {"persistentvolumeclaim", "the number of matching timeseries exceeds 300000; either narrow down the search", http.StatusUnprocessableEntity}, + {"pod", "cannot fetch label values from vmstorage nodes: the number of matching timeseries exceeds 300000", http.StatusUnprocessableEntity}, + {"pod", "timeout exceeded before starting the query processing: 30s", http.StatusGatewayTimeout}, + {"pod", "error when searching for label values: cannot obtain data from vmstorage", http.StatusBadGateway}, + } { + h := newPolicyHarness(t, Config{}, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/label/"+tc.label+"/values" { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnprocessableEntity) + _ = json.NewEncoder(w).Encode(map[string]string{"status": "error", "errorType": "422", "error": tc.message}) + return + } + appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r) + }, appPolicy, nil) + resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good", nil, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != tc.want { + t.Errorf("%s lookup %q answered %d, want %d", tc.label, tc.message, resp.StatusCode, tc.want) + } + if tc.want == http.StatusUnprocessableEntity && !strings.Contains(string(body), "the number of matching timeseries exceeds 300000") { + t.Errorf("the store's limit message did not reach the client: %q", body) + } + if strings.Contains(string(body), `"errorType"`) { + t.Errorf("the store's raw JSON reached the client: %q", body) + } + } +} From 336961d7cd6dc7293caebd670c96b459b4ade6fc Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:38:10 +0700 Subject: [PATCH 097/174] docs(observability-proxy): put the store-call multiplier in the flag help The note that an application metrics request makes three store calls sat in a source comment, where an operator reading --help never sees it. It is now part of the --rate-limit-per-user help text. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index 487b6d04d..f5676b6d5 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -102,10 +102,7 @@ func parseFlags(args []string) (options, error) { fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables") - // Counted per client request: an application metrics request costs the - // store three calls (the pod and PVC lookups and the query) for one - // token, so the store load per user can be three times this rate. - fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables") + fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained client requests per second per user; an application metrics request makes three store calls (pod lookup, PVC lookup, query); 0 disables") fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user") fs.IntVar(&o.limiter.MaxTailsPerUser, "max-tails-per-user", 5, "open tails per user; 0 disables") fs.IntVar(&o.limiter.MaxTails, "max-tails", 200, "open tails in total; 0 disables") From c9c02b2d52e31f70e2a28a2d24c2929c8d042d74 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:45:13 +0700 Subject: [PATCH 098/174] test(observability-proxy): cover a store 504 on the app lookup The lookup now passes a 504 from the store through, where it used to answer 502, but only the 504 read from a 422 body was tested. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/server_app_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 2cd2a0eda..dcbf1e2a2 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -222,6 +222,7 @@ func TestServerApplicationMetricsBranches(t *testing.T) { // A failed lookup would leave the application's own series out of // a successful answer, so it fails the request instead. {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, + {name: "pod lookup times out at the store", pods: []string{"p"}, podStatus: http.StatusGatewayTimeout, path: "api/v1/query?query=up", wantCode: http.StatusGatewayTimeout}, {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, From 501f0d98f7c82e02169c33219d2772be49c137fa Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 01:45:13 +0700 Subject: [PATCH 099/174] docs(observability-proxy): drop the claimed link between two flag defaults The comment said the application lookback default matches --metrics-max-range, but nothing ties the two: changing the range flag leaves the lookback where it is. It now gives only the reason for the length. Assisted-by: LLM Signed-off-by: ZverGuy --- cmd/observability-proxy/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/observability-proxy/main.go b/cmd/observability-proxy/main.go index f5676b6d5..66cc614f0 100644 --- a/cmd/observability-proxy/main.go +++ b/cmd/observability-proxy/main.go @@ -96,8 +96,8 @@ func parseFlags(args []string) (options, error) { fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set") // Only the refusal threshold: the lookup itself spans the window the // query reads, and that estimate is a loose upper bound (every "(" - // counts as an implicit subquery), so the default matches - // --metrics-max-range to serve month-long panels at a dashboard step. + // counts as an implicit subquery), so the default is long enough to + // serve month-long panels at a dashboard step. fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 30*24*time.Hour, "how far before the window an application metrics query may read; one reading further back, or by an unknown amount, answers 422") fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422") fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422") From 0bb98ced05bfeca07e15edf31c4470d05ae3ecfc Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 02:24:13 +0700 Subject: [PATCH 100/174] fix(observability-proxy): say that every @ is refused on app roots The lookahead scan let @ start() and @ end() through and the 422 named them as allowed, but the lookback scan cannot bound any @, so every @ query was refused anyway, with a message pointing at forms that would not pass. Both scans now treat any @ as unknown and the message says so. The lookback-cap test now exercises the cap with a window just past it rather than an @ query. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 2 +- .../observabilityproxy/metricslookback.go | 20 +++++----------- .../observabilityproxy/metricsparams_test.go | 6 +++-- .../observabilityproxy/server_app_test.go | 24 ++++++++++++++++++- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 796280eb6..e41dd1330 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -102,7 +102,7 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { - return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ other than start() or end(), or an offset that is not a plain duration); use the namespace root") + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ modifier, or an offset that is not a plain duration); use the namespace root") } lim := s.cfg.Metrics.App if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 442e9c8d9..e9e7cb0e9 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -131,22 +131,14 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { // queryLookahead is an upper bound on how far after its evaluation window // a query reads, from the same lexical scan. Only a negative offset reads -// forward; the negative offsets are summed, which covers nesting. An @ -// modifier pins samples to any instant unless it names start() or end(), -// which lie in the window, and WITH can apply one offset several times; -// either makes the bound unknown. +// forward; the negative offsets are summed, which covers nesting. Any @ +// makes the bound unknown, as it does for queryLookback, and so does WITH, +// which can apply one offset several times. func queryLookahead(query string) (time.Duration, bool) { - lower := strings.ToLower(query) - for rest := lower; ; { - at := strings.IndexByte(rest, '@') - if at < 0 { - break - } - rest = strings.TrimLeft(rest[at+1:], " \t\n\r") - if !strings.HasPrefix(rest, "start()") && !strings.HasPrefix(rest, "end()") { - return 0, false - } + if strings.Contains(query, "@") { + return 0, false } + lower := strings.ToLower(query) _, ahead, ok := scanOffsets(lower) if !ok || ahead > 0 && hasKeyword(lower, "with") || ahead > float64(maxMetricsDuration) { return 0, false diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 5c39fdfff..5e0d548b7 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -441,8 +441,6 @@ func TestQueryLookahead(t *testing.T) { {`x OFFSET - 90m`, 90 * time.Minute}, {`rate(x[5m] offset -30m) offset -1h`, 90 * time.Minute}, {`x offset -1h + y offset 2h`, time.Hour}, - {`x @ end()`, 0}, - {`x @ start() offset -1h`, time.Hour}, {`node_offset_seconds - 1`, 0}, } for _, tt := range tests { @@ -455,6 +453,10 @@ func TestQueryLookahead(t *testing.T) { } for _, q := range []string{ `x @ 1700000000`, + // The lookback scan cannot bound any @, so the application root + // refuses every one; the lookahead does not tell them apart. + `x @ end()`, + `x @ start() offset -1h`, `x @ (end() - 1h)`, `x{a="a@b"}`, `x offset -5i`, diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index dcbf1e2a2..8ae395d45 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -324,7 +324,9 @@ func TestServerApplicationNarrowedGrant(t *testing.T) { func TestServerApplicationLookbackBeyondCap(t *testing.T) { cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 100, MaxPVCs: 100}}} h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) - for _, query := range []string{"rate(x[7d])", "x[30d] @ end()", "rate(x[5i])"} { + // max_over_time(x[2h]) reads just past the 1h cap; rate(x[5i]) has an + // unknown lookback. + for _, query := range []string{"rate(x[7d])", "max_over_time(x[2h])", "rate(x[5i])"} { q := url.Values{"query": {query}, "time": {unix(testNow)}} if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity { t.Errorf("%s answered %d, want 422", query, got) @@ -485,3 +487,23 @@ func TestServerApplicationLookupStoreErrors(t *testing.T) { } } } + +// TestServerApplicationRefusesEveryAt: no @ form can be bounded, and the +// answer says so rather than naming forms that would pass. +func TestServerApplicationRefusesEveryAt(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{"up @ end()", "up @ start()", "up @ 1700000000"} { + resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+url.Values{"query": {query}}.Encode(), "good", nil, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, resp.StatusCode) + } + if strings.Contains(string(body), "start() or end()") { + t.Errorf("%s: the refusal names @ forms as allowed: %q", query, body) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for refused queries", n) + } +} From 41e61cb288b5d5a0c2cd250c5effb67601927bfb Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 02:24:54 +0700 Subject: [PATCH 101/174] fix(observability-proxy): keep store error text out of lookup answers A store error other than the series limit was built into the lookup's 502 message, and that text can name vmstorage nodes and addresses. The handler happened to replace the message before it reached the client, but nothing held it there. The text now goes to the log only. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 4 +++- internal/observabilityproxy/appscope_test.go | 13 +++++++++++++ internal/observabilityproxy/server_app_test.go | 5 +++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index e41dd1330..561a92ab8 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -200,6 +200,8 @@ func lookupRefusal(label string, body io.Reader) *HTTPError { case strings.Contains(answer.Error, "timeout exceeded"): return errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", label) default: - return errorf(http.StatusBadGateway, "the application's %s lookup failed: %s", label, answer.Error) + // The text may name vmstorage nodes, so it stays in the log. + slog.Warn("application lookup refused by the store", "label", label, "error", answer.Error) + return errorf(http.StatusBadGateway, "the application's %s lookup failed", label) } } diff --git a/internal/observabilityproxy/appscope_test.go b/internal/observabilityproxy/appscope_test.go index af028b867..c4b8e5d69 100644 --- a/internal/observabilityproxy/appscope_test.go +++ b/internal/observabilityproxy/appscope_test.go @@ -17,10 +17,12 @@ limitations under the License. package observabilityproxy import ( + "net/http" "net/url" "regexp" "slices" "strconv" + "strings" "testing" "time" ) @@ -114,3 +116,14 @@ func TestMetricsRequestScope(t *testing.T) { t.Errorf("range window = [%s, %s]", rng.from, rng.to) } } + +// TestLookupRefusalKeepsStoreErrorsInternal: a store error other than the +// series limit may name vmstorage nodes; it goes to the log, not into a +// message that could reach the client. +func TestLookupRefusalKeepsStoreErrorsInternal(t *testing.T) { + body := strings.NewReader(`{"status":"error","error":"cannot fetch from vmstorage-longterm-0.vmstorage:8401: connection refused"}`) + herr := lookupRefusal("pod", body) + if herr.Code != http.StatusBadGateway || strings.Contains(herr.Message, "vmstorage") { + t.Fatalf("lookupRefusal = %d %q, want 502 without the store's text", herr.Code, herr.Message) + } +} diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 8ae395d45..9a56e9e18 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -485,6 +485,11 @@ func TestServerApplicationLookupStoreErrors(t *testing.T) { if strings.Contains(string(body), `"errorType"`) { t.Errorf("the store's raw JSON reached the client: %q", body) } + // Other store errors may name vmstorage nodes; only the limit + // message, which the client can act on, is passed on. + if tc.want != http.StatusUnprocessableEntity && strings.Contains(string(body), "vmstorage") { + t.Errorf("the store's error text reached the client: %q", body) + } } } From 5fc9e5a0b9fa83f925b62e68783e047b6106869c Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 02:33:22 +0700 Subject: [PATCH 102/174] fix(observability-proxy): pass on only the limit text of a lookup 422 The series-limit refusal of a lookup reached the client with all of cluster vmselect's wrapping, and that context can name vmstorage nodes, the leak the generic 502 had just closed for other errors. The client now gets the message from the limit on. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 5 ++++- internal/observabilityproxy/server_app_test.go | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 561a92ab8..526bd0b52 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -196,7 +196,10 @@ func lookupRefusal(label string, body io.Reader) *HTTPError { _ = json.NewDecoder(io.LimitReader(body, 64<<10)).Decode(&answer) switch { case strings.Contains(answer.Error, seriesLimitError): - return errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, answer.Error) + // From the limit on: what precedes it is cluster vmselect's context, + // which can name vmstorage nodes. + limit := answer.Error[strings.Index(answer.Error, seriesLimitError):] + return errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, limit) case strings.Contains(answer.Error, "timeout exceeded"): return errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", label) default: diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 9a56e9e18..52cfaa255 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -487,7 +487,7 @@ func TestServerApplicationLookupStoreErrors(t *testing.T) { } // Other store errors may name vmstorage nodes; only the limit // message, which the client can act on, is passed on. - if tc.want != http.StatusUnprocessableEntity && strings.Contains(string(body), "vmstorage") { + if strings.Contains(string(body), "vmstorage") { t.Errorf("the store's error text reached the client: %q", body) } } From 486f13dce690507a12319a63166b1ac85b0ba398 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 02:33:22 +0700 Subject: [PATCH 103/174] docs(observability-proxy): name every query form the app 422 refuses The refusal named only the @ modifier and non-literal offsets, but the same answer covers an @ inside a label value and WITH with a negative offset, so a client could not tell why such a query was refused. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 526bd0b52..6c3a9fb4f 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -102,7 +102,7 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { - return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ modifier, or an offset that is not a plain duration); use the namespace root") + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, WITH with a negative offset, or an offset that is not a plain duration); use the namespace root") } lim := s.cfg.Metrics.App if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { From 5418d6b40e864022c32914e57f6b2084e6ccdc81 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:05:54 +0700 Subject: [PATCH 104/174] fix(observability-proxy): refuse compound offsets on the app root The offset scan read a duration up to its first sign and ignored the rest. metricsql lexes 1h-30m as one duration and DurationValue makes every part after the first negative one subtract, so offset -1h-30m shifts samples 90 minutes forward while the scan covered 60, and offset 1h-2h shifts them an hour forward while the scan counted it as looking back. The pod and PVC lookup then missed pods created in the gap and the query answered 200 without them. A duration followed directly by a minus now makes the scan unknown, which answers 422 on the application root. A plus or a spaced sign is an operator after the offset in metricsql, not part of it, and is still read. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observabilityproxy/metricslookback.go | 6 +++++ .../observabilityproxy/metricsparams_test.go | 13 +++++++++++ .../observabilityproxy/server_app_test.go | 22 +++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index e9e7cb0e9..886d9e43b 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -169,6 +169,12 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { if end < 0 { end = len(arg) } + // metricsql lexes 1h-30m as one duration whose parts after the + // first negative one all subtract (DurationValue), so -1h-30m is + // -90m and 1h-2h is -1h; the scan does not read such compounds. + if end < len(arg) && arg[end] == '-' { + return 0, 0, false + } d, ok := parseQueryDuration(arg[:end]) if !ok { return 0, 0, false diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 5e0d548b7..29802a220 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -442,6 +442,12 @@ func TestQueryLookahead(t *testing.T) { {`rate(x[5m] offset -30m) offset -1h`, 90 * time.Minute}, {`x offset -1h + y offset 2h`, time.Hour}, {`node_offset_seconds - 1`, 0}, + // metricsql lexes a duration as one token, so a sign with spaces, + // or a plus, is an operator after the offset, not part of it. + {`x offset 1h+30m`, 0}, + {`x offset 1h - 30m`, 0}, + {`x offset -1h + 30m`, time.Hour}, + {`x offset -1h30m`, 90 * time.Minute}, } for _, tt := range tests { t.Run(tt.query, func(t *testing.T) { @@ -460,6 +466,13 @@ func TestQueryLookahead(t *testing.T) { `x @ (end() - 1h)`, `x{a="a@b"}`, `x offset -5i`, + // Compound durations with a negative part (metricsql DurationValue): + // -1h-30m is -90m and 1h-2h is -1h, neither of which the scan reads. + `x offset -1h-30m`, + `x offset - 1h-30m`, + `x offset 1h-2h`, + `x offset 1h-30m`, + `rate(x[5m] offset 1h-30m)`, `x offset -$__interval`, `with (f(a) = a offset -1h) f(f(x))`, } { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 52cfaa255..e4f0ca34f 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -512,3 +512,25 @@ func TestServerApplicationRefusesEveryAt(t *testing.T) { t.Fatalf("upstream saw %d requests for refused queries", n) } } + +// TestServerApplicationRefusesForwardReads: queries that read past the +// evaluation time by an amount the scan cannot bound answer 422 and send +// nothing upstream, rather than answer 200 without pods created in that +// time. +func TestServerApplicationRefusesForwardReads(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{ + "container_cpu_usage_seconds_total offset -1h-30m", + "container_cpu_usage_seconds_total offset 1h-2h", + } { + q := url.Values{"query": {query}, "time": {unix(testNow.Add(-2 * time.Hour))}} + resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good", nil, nil) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, resp.StatusCode) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for refused queries", n) + } +} From cb35399ee70f49399c467bb2a4be9117b923255d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:06:04 +0700 Subject: [PATCH 105/174] fix(observability-proxy): refuse rollup_candlestick on the app root vmselect evaluates rollup_candlestick one step ahead of the evaluation time, with no offset in the query: evalRollupFuncWithoutAt adds the step in effect to the window's start and end (app/vmselect/promql/ eval.go in v1.136.0-cluster, unchanged in v1.139.0). The lookahead scan returned zero for it, so the lookup stopped at the evaluation time and pods created during that step were left out of a 200. The step in effect is the subquery's when the call is nested, so rather than size it the application root answers 422. No other function reads ahead: the subquery evaluation past its end in evalRollupFuncWithSubquery is outside the outer rollup's window. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 2 +- internal/observabilityproxy/metricslookback.go | 8 +++++++- internal/observabilityproxy/metricsparams_test.go | 5 +++++ internal/observabilityproxy/server_app_test.go | 1 + 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 6c3a9fb4f..9911abeeb 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -102,7 +102,7 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { - return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, WITH with a negative offset, or an offset that is not a plain duration); use the namespace root") + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, rollup_candlestick, WITH with a negative offset, or an offset that is not a plain duration); use the namespace root") } lim := s.cfg.Metrics.App if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 886d9e43b..379a42be0 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -133,12 +133,18 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { // a query reads, from the same lexical scan. Only a negative offset reads // forward; the negative offsets are summed, which covers nesting. Any @ // makes the bound unknown, as it does for queryLookback, and so does WITH, -// which can apply one offset several times. +// which can apply one offset several times. vmselect also evaluates +// rollup_candlestick one step ahead of the evaluation time +// (evalRollupFuncWithoutAt), with the step of whatever subquery it sits +// in, so it makes the bound unknown too. func queryLookahead(query string) (time.Duration, bool) { if strings.Contains(query, "@") { return 0, false } lower := strings.ToLower(query) + if hasKeyword(lower, "rollup_candlestick") { + return 0, false + } _, ahead, ok := scanOffsets(lower) if !ok || ahead > 0 && hasKeyword(lower, "with") || ahead > float64(maxMetricsDuration) { return 0, false diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 29802a220..d320333fd 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -473,6 +473,11 @@ func TestQueryLookahead(t *testing.T) { `x offset 1h-2h`, `x offset 1h-30m`, `rate(x[5m] offset 1h-30m)`, + // vmselect evaluates rollup_candlestick one step ahead + // (evalRollupFuncWithoutAt), whatever the offset. + `rollup_candlestick(x)`, + `ROLLUP_CANDLESTICK(x[5m])`, + `sum(rollup_candlestick(x[1h:5m]))`, `x offset -$__interval`, `with (f(a) = a offset -1h) f(f(x))`, } { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index e4f0ca34f..3ede9f239 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -522,6 +522,7 @@ func TestServerApplicationRefusesForwardReads(t *testing.T) { for _, query := range []string{ "container_cpu_usage_seconds_total offset -1h-30m", "container_cpu_usage_seconds_total offset 1h-2h", + "rollup_candlestick(container_cpu_usage_seconds_total)", } { q := url.Values{"query": {query}, "time": {unix(testNow.Add(-2 * time.Hour))}} resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good", nil, nil) From 84f19e14d88b7a6567ad6b98f67120128e71465b Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:11:42 +0700 Subject: [PATCH 106/174] fix(observability-proxy): reach the subquery steps in the app lookup vmselect evaluates a subquery up to its end plus one subquery step, rounded up to the step (evalRollupFuncWithSubquery in app/vmselect/promql/eval.go), and range_* transforms (setLastValues) and keep_next_value carry those later values back onto earlier points. A series that starts after the evaluation time, such as a pod created within that reach, can therefore appear in the answer, which the earlier claim that only rollup_candlestick reads ahead missed. The lookup stopped at the evaluation time and left such a pod out of a 200. The lookahead now adds two steps per subquery level, counting every "(" as a possible implicit subquery and using the largest step seen, as the lookback already does. Overreaching only widens the lookup over the application's own pods. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observabilityproxy/metricslookback.go | 52 +++++++++++++++---- internal/observabilityproxy/metricsparams.go | 4 +- .../observabilityproxy/metricsparams_test.go | 22 ++++++-- .../observabilityproxy/server_app_test.go | 22 +++++++- 4 files changed, 84 insertions(+), 16 deletions(-) diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 379a42be0..ffff274a2 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -130,14 +130,19 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { } // queryLookahead is an upper bound on how far after its evaluation window -// a query reads, from the same lexical scan. Only a negative offset reads -// forward; the negative offsets are summed, which covers nesting. Any @ -// makes the bound unknown, as it does for queryLookback, and so does WITH, -// which can apply one offset several times. vmselect also evaluates -// rollup_candlestick one step ahead of the evaluation time -// (evalRollupFuncWithoutAt), with the step of whatever subquery it sits -// in, so it makes the bound unknown too. -func queryLookahead(query string) (time.Duration, bool) { +// a query reads, from the same lexical scan, with step the query's step. +// A negative offset reads forward; the negative offsets are summed, which +// covers nesting. vmselect also evaluates a subquery up to its end plus +// its step, rounded up to the step (evalRollupFuncWithSubquery), and +// range_* transforms and keep_next_value carry those later values back +// onto earlier points, so each subquery level adds two steps; as for the +// lookback, every "(" counts as a possible implicit subquery and each +// term uses the largest step seen. Any @ makes the bound unknown, as it +// does for queryLookback, and so does WITH, which can apply one offset +// several times. vmselect also evaluates rollup_candlestick one step ahead +// of the evaluation time (evalRollupFuncWithoutAt), so it makes the bound +// unknown too. +func queryLookahead(query string, step time.Duration) (time.Duration, bool) { if strings.Contains(query, "@") { return 0, false } @@ -146,7 +151,36 @@ func queryLookahead(query string) (time.Duration, bool) { return 0, false } _, ahead, ok := scanOffsets(lower) - if !ok || ahead > 0 && hasKeyword(lower, "with") || ahead > float64(maxMetricsDuration) { + if !ok || ahead > 0 && hasKeyword(lower, "with") { + return 0, false + } + subqueries := 0 + maxStep := float64(step) + for rest := query; ; { + open := strings.IndexByte(rest, '[') + if open < 0 { + break + } + closing := strings.IndexByte(rest[open:], ']') + if closing < 0 { + return 0, false + } + _, subStep, isSubquery := strings.Cut(rest[open+1:open+closing], ":") + rest = rest[open+closing+1:] + if !isSubquery { + continue + } + subqueries++ + if strings.TrimSpace(subStep) != "" { + d, ok := parseQueryDuration(subStep) + if !ok { + return 0, false + } + maxStep = max(maxStep, d) + } + } + ahead += 2 * float64(subqueries+strings.Count(query, "(")) * maxStep + if ahead > float64(maxMetricsDuration) { return 0, false } return time.Duration(ahead), true diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index 721496fe3..63be3c497 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -146,7 +146,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step) req.from, req.to = at, at - if ahead, ok := queryLookahead(out.Get("query")); ok { + if ahead, ok := queryLookahead(out.Get("query"), step); ok { req.last = at.Add(ahead) } return req, nil @@ -174,7 +174,7 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti req.start = queryStart(out.Get("query"), start, step) req.from, req.to = start, end // vmselect rounds end up to a whole step (AdjustStartEnd). - if ahead, ok := queryLookahead(out.Get("query")); ok { + if ahead, ok := queryLookahead(out.Get("query"), step); ok { req.last = end.Add(step + ahead) } out.Set("start", formatTime(start)) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index d320333fd..95ee047d8 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -439,7 +439,9 @@ func TestQueryLookahead(t *testing.T) { {`x offset 1h`, 0}, {`x offset -1h`, time.Hour}, {`x OFFSET - 90m`, 90 * time.Minute}, - {`rate(x[5m] offset -30m) offset -1h`, 90 * time.Minute}, + // Each "(" may be an implicit subquery, which vmselect evaluates + // up to two steps past its end; here the step is 1m. + {`rate(x[5m] offset -30m) offset -1h`, 92 * time.Minute}, {`x offset -1h + y offset 2h`, time.Hour}, {`node_offset_seconds - 1`, 0}, // metricsql lexes a duration as one token, so a sign with spaces, @@ -451,7 +453,7 @@ func TestQueryLookahead(t *testing.T) { } for _, tt := range tests { t.Run(tt.query, func(t *testing.T) { - got, ok := queryLookahead(tt.query) + got, ok := queryLookahead(tt.query, time.Minute) if !ok || got != tt.want { t.Fatalf("queryLookahead = %s, %v; want %s", got, ok, tt.want) } @@ -481,8 +483,22 @@ func TestQueryLookahead(t *testing.T) { `x offset -$__interval`, `with (f(a) = a offset -1h) f(f(x))`, } { - if got, ok := queryLookahead(q); ok { + if got, ok := queryLookahead(q, time.Minute); ok { t.Errorf("queryLookahead(%q) = %s, want unknown", q, got) } } } + +// TestQueryLookaheadSubquery: vmselect evaluates a subquery up to its end +// plus one subquery step, rounded up to the step (evalRollupFuncWithSubquery), +// and range_last and the other range_* transforms copy the last of those +// values back onto earlier points, so a series that starts after the +// evaluation time can reach the answer. +func TestQueryLookaheadSubquery(t *testing.T) { + for _, q := range []string{`max_over_time(range_last(x)[1h:1h])`, `max_over_time(keep_next_value(x)[1h:1h])`} { + got, ok := queryLookahead(q, time.Minute) + if !ok || got < 2*time.Hour { + t.Errorf("queryLookahead(%q) = %s, %v; want at least two subquery steps", q, got, ok) + } + } +} diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 3ede9f239..2d0d9b56e 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -184,8 +184,9 @@ func TestServerApplicationMetricsScope(t *testing.T) { if want := formatTime(buildMetrics(t, "api/v1/query_range", hostile).start); r.form.Get("start") != want { t.Errorf("%s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } - // One step past end, which vmselect rounds up to a whole step. - if r.form.Get("end") != formatTime(testNow.Add(time.Minute)) || r.form.Get("limit") != "101" { + // One step past end, which vmselect rounds up to a whole step, and + // two steps for each "(" that may be an implicit subquery. + if r.form.Get("end") != formatTime(testNow.Add(5*time.Minute)) || r.form.Get("limit") != "101" { t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit")) } } @@ -535,3 +536,20 @@ func TestServerApplicationRefusesForwardReads(t *testing.T) { t.Fatalf("upstream saw %d requests for refused queries", n) } } + +// TestServerApplicationLookupCoversSubquery: a pod created during the two +// subquery steps past the evaluation time can show up through range_last, +// so the lookup reaches that far. +func TestServerApplicationLookupCoversSubquery(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + at := testNow.Add(-6 * time.Hour) + q := url.Values{"query": {"max_over_time(range_last(container_cpu_usage_seconds_total)[1h:1h])"}, "time": {unix(at)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + r := h.requests()[0] + end, err := time.Parse(time.RFC3339, r.form.Get("end")) + if err != nil || r.path != "/api/v1/label/pod/values" || end.Before(at.Add(2*time.Hour)) { + t.Fatalf("lookup %s end = %s, want at least %s", r.path, r.form.Get("end"), formatTime(at.Add(2*time.Hour))) + } +} From 56fc64575a4bcc58d5cf5d66732dcdb190040188 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:11:54 +0700 Subject: [PATCH 107/174] fix(observability-proxy): read a minus after an offset as an operator The compound-offset refusal took any minus right after an offset's duration for a compound, so x offset 1h-1 or x offset 5m-y answered 422. metricsql continues a duration token only with a minus followed by a duration part (scanSingleDuration), so only that form is refused now; anything else is the subtraction it is. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/metricslookback.go | 7 ++++++- internal/observabilityproxy/metricsparams_test.go | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index ffff274a2..16f1f4cff 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -49,6 +49,10 @@ var ( querySeconds = regexp.MustCompile(`^\d+(?:\.\d+)?$`) ) +// negativeDurationPart is what metricsql's scanSingleDuration accepts +// after a minus inside one duration token. +var negativeDurationPart = regexp.MustCompile(`^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d|w|y|i)|\$__interval)`) + var queryDurationUnits = map[string]time.Duration{ "ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour, "d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour, @@ -212,7 +216,8 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { // metricsql lexes 1h-30m as one duration whose parts after the // first negative one all subtract (DurationValue), so -1h-30m is // -90m and 1h-2h is -1h; the scan does not read such compounds. - if end < len(arg) && arg[end] == '-' { + // A minus not followed by a duration part is an operator. + if end < len(arg) && arg[end] == '-' && negativeDurationPart.MatchString(arg[end+1:]) { return 0, 0, false } d, ok := parseQueryDuration(arg[:end]) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 95ee047d8..26db536cf 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -450,6 +450,9 @@ func TestQueryLookahead(t *testing.T) { {`x offset 1h - 30m`, 0}, {`x offset -1h + 30m`, time.Hour}, {`x offset -1h30m`, 90 * time.Minute}, + // A minus not followed by a duration is an operator. + {`x offset 1h-1`, 0}, + {`x offset 5m-y`, 0}, } for _, tt := range tests { t.Run(tt.query, func(t *testing.T) { From b876368a3fd057debcb8147fa96a1cb00d594522 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:16:24 +0700 Subject: [PATCH 108/174] fix(observability-proxy): refuse any minus right after an offset again Telling a compound duration from a subtraction by a duration-part pattern missed forms metricsql accepts after the minus: offset 5m-1.h is 55 minutes forward and offset 5m-.5h is 25, yet both read as five minutes back, and the lookup silently missed pods created in between. A pattern that followed scanSingleDuration exactly would still be a second copy of the grammar to keep in step, so any minus right after an offset's duration is refused again; offset 1h-1 and offset 5m-y now answer 422 as well, and metricsql cannot parse the latter anyway. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/metricslookback.go | 10 ++++------ internal/observabilityproxy/metricsparams_test.go | 10 +++++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 16f1f4cff..9f01936b7 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -49,10 +49,6 @@ var ( querySeconds = regexp.MustCompile(`^\d+(?:\.\d+)?$`) ) -// negativeDurationPart is what metricsql's scanSingleDuration accepts -// after a minus inside one duration token. -var negativeDurationPart = regexp.MustCompile(`^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d|w|y|i)|\$__interval)`) - var queryDurationUnits = map[string]time.Duration{ "ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour, "d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour, @@ -216,8 +212,10 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { // metricsql lexes 1h-30m as one duration whose parts after the // first negative one all subtract (DurationValue), so -1h-30m is // -90m and 1h-2h is -1h; the scan does not read such compounds. - // A minus not followed by a duration part is an operator. - if end < len(arg) && arg[end] == '-' && negativeDurationPart.MatchString(arg[end+1:]) { + // metricsql continues a duration after a minus in more forms than + // the scan can tell from a subtraction (-1.h and -.5h among them), + // so any minus right after the duration is refused. + if end < len(arg) && arg[end] == '-' { return 0, 0, false } d, ok := parseQueryDuration(arg[:end]) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 26db536cf..15067ed6c 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -450,9 +450,6 @@ func TestQueryLookahead(t *testing.T) { {`x offset 1h - 30m`, 0}, {`x offset -1h + 30m`, time.Hour}, {`x offset -1h30m`, 90 * time.Minute}, - // A minus not followed by a duration is an operator. - {`x offset 1h-1`, 0}, - {`x offset 5m-y`, 0}, } for _, tt := range tests { t.Run(tt.query, func(t *testing.T) { @@ -477,6 +474,12 @@ func TestQueryLookahead(t *testing.T) { `x offset - 1h-30m`, `x offset 1h-2h`, `x offset 1h-30m`, + // metricsql continues a duration with -1.h or -.5h too, so any minus + // right after the duration is refused rather than told apart. + `x offset 5m-1.h`, + `x offset 5m-.5h`, + `x offset 1h-1`, + `x offset 5m-y`, `rate(x[5m] offset 1h-30m)`, // vmselect evaluates rollup_candlestick one step ahead // (evalRollupFuncWithoutAt), whatever the offset. @@ -498,6 +501,7 @@ func TestQueryLookahead(t *testing.T) { // values back onto earlier points, so a series that starts after the // evaluation time can reach the answer. func TestQueryLookaheadSubquery(t *testing.T) { + for _, q := range []string{`max_over_time(range_last(x)[1h:1h])`, `max_over_time(keep_next_value(x)[1h:1h])`} { got, ok := queryLookahead(q, time.Minute) if !ok || got < 2*time.Hour { From d164790946a4b3a557c1bc1204ca6b4f8b78e57c Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:16:31 +0700 Subject: [PATCH 109/174] test(observability-proxy): pin the subquery lookahead exactly The subquery sizing was tested only as "at least two hours", so which step drives it, the fallback to the query step for an empty subquery step, and its refusals (an unreadable step, an unclosed bracket, a sum past the duration range) were not held by any test. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observabilityproxy/metricsparams_test.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 15067ed6c..75b88d7f3 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -480,6 +480,12 @@ func TestQueryLookahead(t *testing.T) { `x offset 5m-.5h`, `x offset 1h-1`, `x offset 5m-y`, + // A subquery step the scan cannot read, an unclosed bracket, and a + // sum past the duration range. + `rate(x[1h:5i])`, + `rate(x[1h:$__interval])`, + `rate(x[1h`, + `max_over_time(x[1h:100y])`, `rate(x[5m] offset 1h-30m)`, // vmselect evaluates rollup_candlestick one step ahead // (evalRollupFuncWithoutAt), whatever the offset. @@ -501,7 +507,16 @@ func TestQueryLookahead(t *testing.T) { // values back onto earlier points, so a series that starts after the // evaluation time can reach the answer. func TestQueryLookaheadSubquery(t *testing.T) { - + // One explicit subquery and one "(": two levels of two steps, at the + // subquery's step, or at the query's step when the subquery has none. + for q, want := range map[string]time.Duration{ + `max_over_time(x[1h:1h])`: 4 * time.Hour, + `max_over_time(x[1h:])`: 4 * time.Minute, + } { + if got, ok := queryLookahead(q, time.Minute); !ok || got != want { + t.Errorf("queryLookahead(%q) = %s, %v; want %s", q, got, ok, want) + } + } for _, q := range []string{`max_over_time(range_last(x)[1h:1h])`, `max_over_time(keep_next_value(x)[1h:1h])`} { got, ok := queryLookahead(q, time.Minute) if !ok || got < 2*time.Hour { From 054fac6ab0fa4c3e618080744d168c3a57f4bcc9 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:16:59 +0700 Subject: [PATCH 110/174] docs(observability-proxy): name subquery steps in the app lookup texts Since the lookup reaches subquery steps past the evaluation time, the applicationScope comment still described only negative offsets, and a query refused for a subquery step the scan cannot read got a 422 that listed every other cause but that one. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 5 +++-- internal/observabilityproxy/server_app_test.go | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 9911abeeb..4918c9588 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -95,14 +95,15 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // applicationScope looks the application's pods and PVCs up in store over // the samples the request reads: back by the query's own lookback, and -// forward to the latest sample a negative offset reads. A query that reads +// forward to the latest sample a negative offset or a subquery's +// evaluation past its end reads. A query that reads // further back than the configured lookback, or whose earliest or latest // sample is unknown, a failed lookup, and a list over its cap all fail the // request, because a successful answer would silently miss the // application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { - return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, rollup_candlestick, WITH with a negative offset, or an offset that is not a plain duration); use the namespace root") + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, rollup_candlestick, WITH with a negative offset, an offset that is not a plain duration, or a subquery step that is not one); use the namespace root") } lim := s.cfg.Metrics.App if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 2d0d9b56e..c451d35af 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -553,3 +553,15 @@ func TestServerApplicationLookupCoversSubquery(t *testing.T) { t.Fatalf("lookup %s end = %s, want at least %s", r.path, r.form.Get("end"), formatTime(at.Add(2*time.Hour))) } } + +// TestServerApplicationNamesUnreadableSubqueryStep: the refusal names the +// cause when a subquery step is what the scan cannot read. +func TestServerApplicationNamesUnreadableSubqueryStep(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+url.Values{"query": {"max_over_time(x[1h:5i])"}}.Encode(), "good", nil, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnprocessableEntity || !strings.Contains(string(body), "subquery step") { + t.Fatalf("answered %d %q, want 422 naming the subquery step", resp.StatusCode, body) + } +} From 8e491d15ddeb9dad0b82a9b5940342bcf2d666c5 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:21:16 +0700 Subject: [PATCH 111/174] fix(observability-proxy): accept only a token end after an offset metricsql also reads a trailing $__interval as part of an offset's duration: offset -1h$__interval is -(1h + step). The scan stopped at the "$" and read one hour, so the lookup missed pods created in the last step and the query answered 200 without them. Refusing a minus alone left every other continuation open. Only what certainly ends the duration token is now accepted after it: whitespace, a closing bracket or separator, or a binary operator other than minus; anything else answers 422 on the application root. The grammar comment no longer claims upper-case offset units are refused, as offsets are scanned lower-cased like metricsql reads them. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observabilityproxy/metricslookback.go | 19 +++++++++++++------ .../observabilityproxy/metricsparams_test.go | 4 ++++ .../observabilityproxy/server_app_test.go | 1 + 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 9f01936b7..55e78a0c2 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -41,14 +41,20 @@ const latencyOffset = 30 * time.Second // queryDurationPart is the subset of the MetricsQL duration grammar // (scanSingleDuration in metricsql) the scan understands. Step multiples -// (5i, $__interval), negative parts and upper-case units are left out on -// purpose: anything else makes the bound unknown. +// (5i, $__interval) and negative parts are left out on purpose: anything +// else makes the bound unknown. Offsets are scanned lower-cased, as +// metricsql reads units in any case; windows and steps in brackets are +// not, so an upper-case unit there is unknown. var ( queryDurationPart = regexp.MustCompile(`(\d+(?:\.\d+)?)(ms|s|m|h|d|w|y)`) queryDuration = regexp.MustCompile(`^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d|w|y))+$`) querySeconds = regexp.MustCompile(`^\d+(?:\.\d+)?$`) ) +// offsetTerminators may follow an offset's duration: whitespace, a +// closing bracket or separator, or a binary operator other than minus. +const offsetTerminators = " \t\n\r)]},+*/^%<>=!" + var queryDurationUnits = map[string]time.Duration{ "ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour, "d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour, @@ -212,10 +218,11 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { // metricsql lexes 1h-30m as one duration whose parts after the // first negative one all subtract (DurationValue), so -1h-30m is // -90m and 1h-2h is -1h; the scan does not read such compounds. - // metricsql continues a duration after a minus in more forms than - // the scan can tell from a subtraction (-1.h and -.5h among them), - // so any minus right after the duration is refused. - if end < len(arg) && arg[end] == '-' { + // metricsql continues a duration token in more forms than the scan + // reads: after a minus (-30m, -1.h, -.5h) and with a trailing + // $__interval, all summed by DurationValue. Only what certainly + // ends the token is accepted after the duration. + if end < len(arg) && !strings.ContainsRune(offsetTerminators, rune(arg[end])) { return 0, 0, false } d, ok := parseQueryDuration(arg[:end]) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 75b88d7f3..0fbbcd7a9 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -480,6 +480,10 @@ func TestQueryLookahead(t *testing.T) { `x offset 5m-.5h`, `x offset 1h-1`, `x offset 5m-y`, + // metricsql also reads a trailing $__interval as part of the + // duration: -1h$__interval is -(1h + step). + `x offset -1h$__interval`, + `rate(x[5m]) offset -1h$__interval`, // A subquery step the scan cannot read, an unclosed bracket, and a // sum past the duration range. `rate(x[1h:5i])`, diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index c451d35af..8b3a4e587 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -523,6 +523,7 @@ func TestServerApplicationRefusesForwardReads(t *testing.T) { for _, query := range []string{ "container_cpu_usage_seconds_total offset -1h-30m", "container_cpu_usage_seconds_total offset 1h-2h", + "container_cpu_usage_seconds_total offset -1h$__interval", "rollup_candlestick(container_cpu_usage_seconds_total)", } { q := url.Values{"query": {query}, "time": {unix(testNow.Add(-2 * time.Hour))}} From 3198de893ad0869995111f8879a62a3155e14225 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:37:04 +0700 Subject: [PATCH 112/174] fix(observability-proxy): refuse escapes outside strings on the app root metricsql unescapes identifiers (unescapeIdent in parser.go), so rollup\_candlestick(x + y) is rollup_candlestick to vmselect while the lexical scan saw another name: the lookup ended ten minutes past the evaluation time and vmselect read nearly thirteen. Any escaped keyword the scan looks for, offset included, had the same hole. A backslash outside a string literal now makes the scan unknown, which answers 422 on the application root; escapes inside label values are unaffected. With rollup_candlestick refused, two steps per "(" stay enough: a subquery evaluates at most one step past its end plus the alignment of that end to the step (evalRollupFuncWithSubquery), and the one step a candlestick over a non-selector would add on top cannot occur. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 2 +- .../observabilityproxy/metricslookback.go | 32 ++++++++++++++++--- .../observabilityproxy/metricsparams_test.go | 10 ++++++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 4918c9588..2f0f23751 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -103,7 +103,7 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { - return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, rollup_candlestick, WITH with a negative offset, an offset that is not a plain duration, or a subquery step that is not one); use the namespace root") + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, a backslash outside a string, rollup_candlestick, WITH with a negative offset, an offset that is not a plain duration, or a subquery step that is not one); use the namespace root") } lim := s.cfg.Metrics.App if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 55e78a0c2..7f936307f 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -145,11 +145,12 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { // lookback, every "(" counts as a possible implicit subquery and each // term uses the largest step seen. Any @ makes the bound unknown, as it // does for queryLookback, and so does WITH, which can apply one offset -// several times. vmselect also evaluates rollup_candlestick one step ahead -// of the evaluation time (evalRollupFuncWithoutAt), so it makes the bound -// unknown too. +// several times, and a backslash outside a string, which can spell an +// identifier the scan looks for differently. vmselect also evaluates +// rollup_candlestick one step ahead of the evaluation time +// (evalRollupFuncWithoutAt), so it makes the bound unknown too. func queryLookahead(query string, step time.Duration) (time.Duration, bool) { - if strings.Contains(query, "@") { + if strings.Contains(query, "@") || escapeOutsideString(query) { return 0, false } lower := strings.ToLower(query) @@ -237,6 +238,29 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { } } +// escapeOutsideString reports a backslash outside a string literal. +// metricsql unescapes identifiers (unescapeIdent), so rollup\_candlestick +// or offs\et name what the lexical scan looks for without its spelling. +// Inside double or single quotes a backslash escapes the next character; +// inside backquotes it is literal. +func escapeOutsideString(query string) bool { + var quote byte + for i := 0; i < len(query); i++ { + c := query[i] + switch { + case quote == 0 && c == '\\': + return true + case quote == 0 && (c == '"' || c == '\'' || c == '`'): + quote = c + case quote != 0 && quote != '`' && c == '\\': + i++ + case quote != 0 && c == quote: + quote = 0 + } + } + return false +} + // isKeyword reports whether lower[start:end] stands on its own rather than // inside an identifier. func isKeyword(lower string, start, end int) bool { diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 0fbbcd7a9..cf452a74f 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -444,6 +444,9 @@ func TestQueryLookahead(t *testing.T) { {`rate(x[5m] offset -30m) offset -1h`, 92 * time.Minute}, {`x offset -1h + y offset 2h`, time.Hour}, {`node_offset_seconds - 1`, 0}, + // An escape inside a string literal is only part of a label value. + {`x{a=~"a\\.b"}`, 0}, + {`x{a="\"rollup\\_candlestick\""}`, 0}, // metricsql lexes a duration as one token, so a sign with spaces, // or a plus, is an operator after the offset, not part of it. {`x offset 1h+30m`, 0}, @@ -491,6 +494,13 @@ func TestQueryLookahead(t *testing.T) { `rate(x[1h`, `max_over_time(x[1h:100y])`, `rate(x[5m] offset 1h-30m)`, + // metricsql unescapes identifiers (unescapeIdent), so an escaped + // rollup_candlestick, or any escape outside a string, could hide + // what the scan looks for. + `rollup\_candlestick(x + y)`, + `rollup_candle\stick(x)`, + `rollup\x5fcandlestick(x)`, + `x offs\et -1h`, // vmselect evaluates rollup_candlestick one step ahead // (evalRollupFuncWithoutAt), whatever the offset. `rollup_candlestick(x)`, From a54760fd9cc145b79c6c901011fe5cb6857a530a Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:37:53 +0700 Subject: [PATCH 113/174] fix(observability-proxy): start the instant app lookup a latency early vmselect evaluates an instant query whose time is within 30s of its own clock at that clock minus 30s (queryOffset in prometheus.go), not at time minus 30s. For a time slightly in the future the evaluation starts earlier than the proxy assumed: time=now+29s with a 1s step read from six minutes before, while the lookup started at five minutes 32 seconds, and pods present only in that gap were missing from a 200. The evaluation instant is always within (time-60s, time], so the lookup of an instant query now starts one more latency offset early. That needs no assumption about the proxy's clock matching vmselect's. The store choice on the namespace root is left as it is. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 8 +++++++ .../observabilityproxy/server_app_test.go | 24 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 2f0f23751..0fe8b05e9 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -110,6 +110,14 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback) } start := req.start + // vmselect evaluates an instant query whose time is within 30s of its + // own clock at that clock minus 30s (queryOffset in prometheus.go), so + // the evaluation instant lies anywhere in (time-60s, time], while + // req.start assumes time-30s. One more latency offset covers it without + // trusting the proxy's clock to match vmselect's. + if req.path == "api/v1/query" { + start = start.Add(-latencyOffset) + } var lists [2][]string for i, l := range []struct { label, metric string diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 8b3a4e587..39b6ac60f 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -341,7 +341,8 @@ func TestServerApplicationLookbackBeyondCap(t *testing.T) { t.Fatalf("rate(x[5m]) answered %d, want 200", got) } r := h.requests()[0] - want := formatTime(buildMetrics(t, "api/v1/query", q).start) + // An instant query may be evaluated up to one more latency offset early. + want := formatTime(buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset)) if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } @@ -566,3 +567,24 @@ func TestServerApplicationNamesUnreadableSubqueryStep(t *testing.T) { t.Fatalf("answered %d %q, want 422 naming the subquery step", resp.StatusCode, body) } } + +// TestServerApplicationLookupCoversLatencyOffset: vmselect evaluates an +// instant query whose time is within 30s of its own clock at its clock +// minus 30s (prometheus.go, queryOffset), so for a time slightly in the +// future it reads from earlier than time minus 30s. The lookup starts +// early enough for any evaluation instant in (time-60s, time]. +func TestServerApplicationLookupCoversLatencyOffset(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + at := testNow.Add(29 * time.Second) + q := url.Values{"query": {"container_cpu_usage_seconds_total"}, "time": {at.Format(time.RFC3339)}, "step": {"1s"}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + // The earliest read of the query evaluated at the earliest instant. + want := buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset) + r := h.requests()[0] + start, err := time.Parse(time.RFC3339, r.form.Get("start")) + if err != nil || start.After(want) { + t.Fatalf("lookup start = %s, want at most %s", r.form.Get("start"), formatTime(want)) + } +} From cf45a23bf3c79772928e606176133857ce800bd1 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:44:21 +0700 Subject: [PATCH 114/174] fix(observability-proxy): refuse comments outside strings on the app root The escape check tracked string literals but not comments: in metricsql a "#" outside a string comments out the rest of the line, so a quote inside a comment made the check treat the next line's code as string content, and rollup\_candlestick there was accepted again. The bracket and offset scans do not understand comments either. A "#" outside a string now makes the lookahead unknown as well, which answers 422 on the application root, rather than teaching every scan about comments; API clients do not send them. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 2 +- .../observabilityproxy/metricslookback.go | 23 +++++++++++-------- .../observabilityproxy/metricsparams_test.go | 5 ++++ 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 0fe8b05e9..53f8d5582 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -103,7 +103,7 @@ func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { // application's series. func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { if req.last.IsZero() { - return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, a backslash outside a string, rollup_candlestick, WITH with a negative offset, an offset that is not a plain duration, or a subquery step that is not one); use the namespace root") + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, a backslash or a comment outside a string, rollup_candlestick, WITH with a negative offset, an offset that is not a plain duration, or a subquery step that is not one); use the namespace root") } lim := s.cfg.Metrics.App if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index 7f936307f..b478f582b 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -145,12 +145,12 @@ func queryLookback(query string, step time.Duration) (time.Duration, bool) { // lookback, every "(" counts as a possible implicit subquery and each // term uses the largest step seen. Any @ makes the bound unknown, as it // does for queryLookback, and so does WITH, which can apply one offset -// several times, and a backslash outside a string, which can spell an -// identifier the scan looks for differently. vmselect also evaluates +// several times, and a backslash or a comment outside a string, which can +// hide what the scan looks for. vmselect also evaluates // rollup_candlestick one step ahead of the evaluation time // (evalRollupFuncWithoutAt), so it makes the bound unknown too. func queryLookahead(query string, step time.Duration) (time.Duration, bool) { - if strings.Contains(query, "@") || escapeOutsideString(query) { + if strings.Contains(query, "@") || unscannableOutsideString(query) { return 0, false } lower := strings.ToLower(query) @@ -238,17 +238,20 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { } } -// escapeOutsideString reports a backslash outside a string literal. -// metricsql unescapes identifiers (unescapeIdent), so rollup\_candlestick -// or offs\et name what the lexical scan looks for without its spelling. -// Inside double or single quotes a backslash escapes the next character; -// inside backquotes it is literal. -func escapeOutsideString(query string) bool { +// unscannableOutsideString reports a backslash or a "#" outside a string +// literal. metricsql unescapes identifiers (unescapeIdent), so +// rollup\_candlestick or offs\et name what the lexical scan looks for +// without its spelling; "#" starts a comment to the end of the line, and +// quotes or brackets inside one would mislead the string tracking here and +// the bracket and offset scans. Inside double or single quotes a backslash +// escapes the next character; inside backquotes it is literal, and a +// backquote escaped there makes metricsql refuse the string anyway. +func unscannableOutsideString(query string) bool { var quote byte for i := 0; i < len(query); i++ { c := query[i] switch { - case quote == 0 && c == '\\': + case quote == 0 && (c == '\\' || c == '#'): return true case quote == 0 && (c == '"' || c == '\'' || c == '`'): quote = c diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index cf452a74f..e201a6fa8 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -501,6 +501,11 @@ func TestQueryLookahead(t *testing.T) { `rollup_candle\stick(x)`, `rollup\x5fcandlestick(x)`, `x offs\et -1h`, + // A comment runs to the end of the line, so quotes in it would + // confuse the string tracking and brackets in it the scans. + "x # '\n+ rollup\\_candlestick(y) # '", + "x # \"\n+ rollup\\_candlestick(y) # \"", + "rate(x[5m]) # comment", // vmselect evaluates rollup_candlestick one step ahead // (evalRollupFuncWithoutAt), whatever the offset. `rollup_candlestick(x)`, From ac5a3c88c418b60216b0c6b93441f26b4229a98d Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 06:45:21 +0700 Subject: [PATCH 115/174] fix(observability-proxy): put the latency window in the instant start The second latency offset was added to the application lookup only, so the other readers of the instant query's earliest read still assumed time minus 30s: the store choice could send a query to the short-term store up to 30 seconds too late near its retention edge, the lookback cap compared against a start the lookup no longer used, and the field no longer meant the earliest instant read. The earliest read of an instant query now starts from time minus two latency offsets, the earliest instant vmselect can evaluate it at, and the lookup uses it as is. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 8 -------- internal/observabilityproxy/metricsparams.go | 6 +++++- .../observabilityproxy/metricsparams_test.go | 18 +++++++++++++++++- internal/observabilityproxy/server_app_test.go | 6 +++--- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index 53f8d5582..e6df8ba02 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -110,14 +110,6 @@ func (s *Server) applicationScope(ctx context.Context, r *http.Request, store st return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback) } start := req.start - // vmselect evaluates an instant query whose time is within 30s of its - // own clock at that clock minus 30s (queryOffset in prometheus.go), so - // the evaluation instant lies anywhere in (time-60s, time], while - // req.start assumes time-30s. One more latency offset covers it without - // trusting the proxy's clock to match vmselect's. - if req.path == "api/v1/query" { - start = start.Add(-latencyOffset) - } var lists [2][]string for i, l := range []struct { label, metric string diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index 63be3c497..f02620b3c 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -144,7 +144,11 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti } out.Set("step", formatSeconds(step)) } - req.start = queryStart(out.Get("query"), at.Add(-latencyOffset), step) + // vmselect evaluates an instant query whose time is within the + // latency offset of its own clock at that clock minus the offset + // (queryOffset in prometheus.go), so the evaluation instant lies in + // (time-2*offset, time]; the earlier end needs no clock to agree. + req.start = queryStart(out.Get("query"), at.Add(-2*latencyOffset), step) req.from, req.to = at, at if ahead, ok := queryLookahead(out.Get("query"), step); ok { req.last = at.Add(ahead) diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index e201a6fa8..34d36f7e5 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -227,7 +227,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) { if req.params.Get("end") != "2026-11-04T12:00:00.000Z" || req.params.Get("start") != "2026-11-04T11:55:00.000Z" { t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end")) } - if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-30*time.Second)) { + if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-2*latencyOffset)) { t.Errorf("instant query without time: params %v, start %v", q.params, q.start) } } @@ -543,3 +543,19 @@ func TestQueryLookaheadSubquery(t *testing.T) { } } } + +// TestBuildMetricsParamsInstantStartCoversLatencyOffset: vmselect evaluates +// an instant query whose time is within 30s of its clock at that clock +// minus 30s, anywhere in (time-60s, time], so the earliest read the store +// choice and the lookup rely on starts from time-60s. +func TestBuildMetricsParamsInstantStartCoversLatencyOffset(t *testing.T) { + at := testNow.Add(29 * time.Second) + req := buildMetrics(t, "api/v1/query", url.Values{"query": {"x"}, "time": {at.Format(time.RFC3339)}, "step": {"1s"}}) + lookback, ok := queryLookback("x", time.Second) + if !ok { + t.Fatal("lookback of x unknown") + } + if want := at.Add(-2*latencyOffset - lookback); req.start.After(want) { + t.Fatalf("start = %s, want at most %s", req.start, want) + } +} diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 39b6ac60f..e5c0d3710 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -341,8 +341,7 @@ func TestServerApplicationLookbackBeyondCap(t *testing.T) { t.Fatalf("rate(x[5m]) answered %d, want 200", got) } r := h.requests()[0] - // An instant query may be evaluated up to one more latency offset early. - want := formatTime(buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset)) + want := formatTime(buildMetrics(t, "api/v1/query", q).start) if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) } @@ -581,7 +580,8 @@ func TestServerApplicationLookupCoversLatencyOffset(t *testing.T) { t.Fatalf("status = %d", got) } // The earliest read of the query evaluated at the earliest instant. - want := buildMetrics(t, "api/v1/query", q).start.Add(-latencyOffset) + lookback, _ := queryLookback("container_cpu_usage_seconds_total", time.Second) + want := at.Add(-2*latencyOffset - lookback) r := h.requests()[0] start, err := time.Parse(time.RFC3339, r.form.Get("start")) if err != nil || start.After(want) { From 79ea8561de6f419d52830d3822a331d04c77e7c3 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 07:00:58 +0700 Subject: [PATCH 116/174] fix(observability-proxy): end a backquoted string only at an unescaped quote The string tracking treated a backslash inside backquotes as literal, but metricsql's scanString lets a backslash escape the closing quote in all three quote kinds, and a quoted label name keeps that escape ({`\``=""} parses). The proxy ended such a string early, read the rest of the query as string content, and accepted an escaped rollup_candlestick after it with the lookup ending too soon. Inside any quote a backslash now skips the next character, as scanString does, and the comment no longer claims metricsql refuses an escaped backquote. Assisted-by: LLM Signed-off-by: ZverGuy --- .../observabilityproxy/metricslookback.go | 9 +++++---- .../observabilityproxy/metricsparams_test.go | 5 +++++ .../observabilityproxy/server_app_test.go | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go index b478f582b..3571afc4c 100644 --- a/internal/observabilityproxy/metricslookback.go +++ b/internal/observabilityproxy/metricslookback.go @@ -243,9 +243,10 @@ func scanOffsets(lower string) (back, ahead float64, ok bool) { // rollup\_candlestick or offs\et name what the lexical scan looks for // without its spelling; "#" starts a comment to the end of the line, and // quotes or brackets inside one would mislead the string tracking here and -// the bracket and offset scans. Inside double or single quotes a backslash -// escapes the next character; inside backquotes it is literal, and a -// backquote escaped there makes metricsql refuse the string anyway. +// the bracket and offset scans. Inside any of the three quotes a backslash +// skips the next character, as metricsql's scanString does: an escaped +// closing quote does not end the string, backquotes included (a quoted +// label name keeps such an escape). func unscannableOutsideString(query string) bool { var quote byte for i := 0; i < len(query); i++ { @@ -255,7 +256,7 @@ func unscannableOutsideString(query string) bool { return true case quote == 0 && (c == '"' || c == '\'' || c == '`'): quote = c - case quote != 0 && quote != '`' && c == '\\': + case quote != 0 && c == '\\': i++ case quote != 0 && c == quote: quote = 0 diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 34d36f7e5..2cc22a4ae 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -506,6 +506,11 @@ func TestQueryLookahead(t *testing.T) { "x # '\n+ rollup\\_candlestick(y) # '", "x # \"\n+ rollup\\_candlestick(y) # \"", "rate(x[5m]) # comment", + // metricsql lets a backslash escape the closing quote in backquotes + // too (scanString), and quoted label names keep such escapes, so + // a backquoted string must not end at an escaped backquote. + "container_cpu_usage_seconds_total{`\\``=\"\"} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)", + "{`\\`a`} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)", // vmselect evaluates rollup_candlestick one step ahead // (evalRollupFuncWithoutAt), whatever the offset. `rollup_candlestick(x)`, diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index e5c0d3710..ade81ca04 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -588,3 +588,22 @@ func TestServerApplicationLookupCoversLatencyOffset(t *testing.T) { t.Fatalf("lookup start = %s, want at most %s", r.form.Get("start"), formatTime(want)) } } + +// TestServerApplicationRefusesBackquoteEscapeTail: an escaped backquote +// inside a quoted label name does not end the string in metricsql, so the +// escaped rollup_candlestick after it must still be refused. +func TestServerApplicationRefusesBackquoteEscapeTail(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{ + "container_cpu_usage_seconds_total{`\\``=\"\"} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)", + "{`\\`a`} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)", + } { + q := url.Values{"query": {query}, "time": {unix(testNow.Add(-6 * time.Hour))}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for refused queries", n) + } +} From e8e0cede61222e884441b7203094c18c13133736 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 18:09:46 +0700 Subject: [PATCH 117/174] fix(observability-proxy): forward the default instant query time An instant query without time had its application lookup sized at the proxy's clock, but the time was not forwarded, so vmselect evaluated at its own later clock. With lookups taking most of the request deadline, the evaluation lands well past the end of the lookup and leaves out pods first sampled in between. Forward the instant the lookup was sized for. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/metricsparams.go | 5 ++++- .../observabilityproxy/metricsparams_test.go | 2 +- .../observabilityproxy/server_app_test.go | 21 +++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go index f02620b3c..c40bc058f 100644 --- a/internal/observabilityproxy/metricsparams.go +++ b/internal/observabilityproxy/metricsparams.go @@ -134,8 +134,11 @@ func BuildMetricsParams(endpoint string, in url.Values, namespace string, now ti return req, herr } at = t - out.Set("time", formatTime(t)) } + // The default is forwarded too: the application lookup is sized + // for at, and vmselect's own clock reads later, by as long as the + // lookups took. + out.Set("time", formatTime(at)) step := instantDefaultStep if v := in.Get("step"); v != "" { var herr *HTTPError diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go index 2cc22a4ae..ba14a9781 100644 --- a/internal/observabilityproxy/metricsparams_test.go +++ b/internal/observabilityproxy/metricsparams_test.go @@ -227,7 +227,7 @@ func TestBuildMetricsParamsCanonicalForms(t *testing.T) { if req.params.Get("end") != "2026-11-04T12:00:00.000Z" || req.params.Get("start") != "2026-11-04T11:55:00.000Z" { t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end")) } - if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Has("time") || !q.start.Equal(testNow.Add(-15*time.Minute-2*latencyOffset)) { + if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Get("time") != "2026-11-04T12:00:00.000Z" || !q.start.Equal(testNow.Add(-15*time.Minute-2*latencyOffset)) { t.Errorf("instant query without time: params %v, start %v", q.params, q.start) } } diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index ade81ca04..7afaa3618 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -363,6 +363,27 @@ func TestServerApplicationLookupCoversNegativeOffset(t *testing.T) { } } +// TestServerApplicationForwardsDefaultTime: without time the lookup is +// sized at the proxy's clock, so that instant is forwarded. Otherwise +// vmselect evaluates at its own later clock, past the end of the lookup by +// however long the lookups took, and leaves out pods first sampled since. +func TestServerApplicationForwardsDefaultTime(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + q := url.Values{"query": {"container_cpu_usage_seconds_total"}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + reqs := h.requests() + lookup, final := reqs[0], reqs[len(reqs)-1] + want := formatTime(testNow) + if lookup.path != "/api/v1/label/pod/values" || lookup.form.Get("end") != want { + t.Fatalf("lookup %s end = %s, want %s", lookup.path, lookup.form.Get("end"), want) + } + if final.path != "/api/v1/query" || final.form.Get("time") != want { + t.Fatalf("query %s time = %q, want the lookup's instant %s", final.path, final.form.Get("time"), want) + } +} + // TestServerApplicationRefusesUnboundedSampleTime refuses a query whose // samples may lie at a time the lookup cannot be sized for, rather than // answer with series silently left out. From 19dbee15fc7cb2aef22fd36151061d18cb26a207 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Thu, 5 Nov 2026 18:15:09 +0700 Subject: [PATCH 118/174] docs(observability-proxy): explain why both application roots need the namespace root The reason given for the second access review held only for the metrics root. The log root needs it too, because an ownerless pod keeps the lineage labels its creator set and can claim any application. The tenant ClusterRole comment also named only the namespace roots, though the application roots now depend on the same rule. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/route.go | 12 ++++++++---- internal/observabilityproxy/server_app_test.go | 7 ++++--- .../templates/tenant-clusterroles.yaml | 6 +++++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/internal/observabilityproxy/route.go b/internal/observabilityproxy/route.go index 49a7c1770..bc12c6276 100644 --- a/internal/observabilityproxy/route.go +++ b/internal/observabilityproxy/route.go @@ -132,10 +132,14 @@ func ParseRoute(escapedPath string) (Route, *HTTPError) { } // Reviews lists every access a request on the route needs. An application -// root also needs read access to its namespace root: its pod and PVC -// branches match by name across the window, so a name another application -// held earlier reaches that application's series, and a grant narrowed by -// resourceNames would read more than it names. +// root also needs read access to its namespace root, because its scope +// narrows within the namespace but does not isolate: on both roots the +// identity comes from lineage labels that the lineage webhook keeps as set +// on an object it finds no owner for, so anyone who can create a pod there +// can label it as the application; on the metrics root the pod and PVC +// branches also match by name across the window, so a name another +// application held earlier reaches that application's series. A grant +// narrowed by resourceNames would read more than it names. func (r Route) Reviews() []authorizationv1.ResourceAttributes { switch r.Kind { case ApplicationLog: diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 7afaa3618..461cbc16d 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -282,9 +282,10 @@ func TestServerApplicationMetricsBranches(t *testing.T) { // A grant narrowed to application "a" by resourceNames must not reach "b" // through any endpoint of either root. Without read access to the -// namespace roots it does not reach "a" either: the pod and PVC branches -// match by name, and a name another application used earlier in the -// window would pull its series in. +// namespace roots it does not reach "a" either: an ownerless pod keeps the +// lineage labels its creator set, so it can claim to be "a" on both roots, +// and on the metrics root the pod and PVC branches match by name, so a name +// another application used earlier in the window would pull its series in. func TestServerApplicationNarrowedGrant(t *testing.T) { h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"a-0"}, nil, http.StatusOK), appPolicy, nil) paths := []string{ diff --git a/packages/system/observability-proxy/templates/tenant-clusterroles.yaml b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml index a2bf94f3c..ff5f3b987 100644 --- a/packages/system/observability-proxy/templates/tenant-clusterroles.yaml +++ b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml @@ -1,7 +1,11 @@ # Aggregated into cozy-tenant-view (and consequently use, admin, super-admin). # No API server serves cozystack.io logs or metrics: the rule exists only for # the SubjectAccessReview the observability proxy asks before it answers a -# read under /apis/cozystack.io/v1alpha1/namespaces//{logs,metrics}/. +# read under /apis/cozystack.io/v1alpha1/namespaces//{logs,metrics}/, +# and before a read under an application root, +# /apis/apps.cozystack.io/v1alpha1/namespaces////{log,metrics}/, +# which needs the namespace root too. Narrowing this role also cuts off the +# application roots. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: From 3a5eb4fbf200005c88da0e3c68da23d1de6679a0 Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 10 Nov 2026 03:40:26 +0700 Subject: [PATCH 119/174] test(observability-proxy): cover stalled reviews on the application roots Application roots take two access reviews before any body is read; a POST to either must answer 503 within the request timeout when kube-apiserver stalls on them. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/server_deadline_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/observabilityproxy/server_deadline_test.go b/internal/observabilityproxy/server_deadline_test.go index 6b367229c..5723365b3 100644 --- a/internal/observabilityproxy/server_deadline_test.go +++ b/internal/observabilityproxy/server_deadline_test.go @@ -428,6 +428,8 @@ func TestServerStalledReviewAnswersInTime(t *testing.T) { {"logs subject access review", logsRoot + "query", false}, {"metrics token review", metricsRoot + "api/v1/query", true}, {"metrics subject access review", metricsRoot + "api/v1/query", false}, + {"application logs subject access review", appRoot + "db/log/select/logsql/query", false}, + {"application metrics subject access review", appRoot + "db/metrics/api/v1/query", false}, } { t.Run(tc.name, func(t *testing.T) { h := newHarness(t, Config{RequestTimeout: 200 * time.Millisecond}, nil) From 6fe3e5f6ff831ab87b4af11f14db0e583716e5cb Mon Sep 17 00:00:00 2001 From: ZverGuy Date: Tue, 10 Nov 2026 03:46:57 +0700 Subject: [PATCH 120/174] fix(observability-proxy): refuse a partial application lookup A cluster vmselect answers label values with isPartial when a storage node does not respond, and the list it returns misses that node's names. Scoping the query to it would drop those pods' series without a word, which the lookup exists to prevent, so a partial list now fails the request as any other failed lookup does. Assisted-by: LLM Signed-off-by: ZverGuy --- internal/observabilityproxy/appscope.go | 10 ++++- .../observabilityproxy/server_app_test.go | 37 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go index e6df8ba02..659b55052 100644 --- a/internal/observabilityproxy/appscope.go +++ b/internal/observabilityproxy/appscope.go @@ -158,8 +158,9 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status) } var body struct { - Status string `json:"status"` - Data []string `json:"data"` + Status string `json:"status"` + IsPartial bool `json:"isPartial"` + Data []string `json:"data"` } if err := json.NewDecoder(io.LimitReader(resp.Body, maxLookupBytes)).Decode(&body); err != nil { return nil, err @@ -170,8 +171,13 @@ func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, if len(body.Data) > limit { // Over the cap before dropping empty values: the store cut the // list at limit+1, so a shorter filtered list could hide names. + // A partial list over the cap settles it too. return body.Data, nil } + // A storage node that did not answer leaves names out of the list. + if body.IsPartial { + return nil, errorf(http.StatusBadGateway, "metrics store answered a partial %s list", label) + } names := body.Data[:0] for _, n := range body.Data { if n != "" { diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go index 461cbc16d..056e1a555 100644 --- a/internal/observabilityproxy/server_app_test.go +++ b/internal/observabilityproxy/server_app_test.go @@ -516,6 +516,43 @@ func TestServerApplicationLookupStoreErrors(t *testing.T) { } } +// TestServerApplicationRefusesPartialLookup: vmselect answers a label +// values lookup with isPartial when a vmstorage node is down, and a short +// pod list would drop the missing pods' series without a word. +func TestServerApplicationRefusesPartialLookup(t *testing.T) { + for _, label := range []string{"pod", "persistentvolumeclaim"} { + h := newPolicyHarness(t, Config{}, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/label/"+label+"/values" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"status":"success","isPartial":true,"data":["db-0"]}`) + return + } + appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r) + }, appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusBadGateway { + t.Errorf("partial %s lookup answered %d, want 502", label, got) + } + for _, req := range h.requests() { + if req.path == "/api/v1/query" { + t.Errorf("partial %s lookup still sent the query", label) + } + } + } + // A partial list already over the cap settles it: the full one is longer. + cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}} + h := newPolicyHarness(t, cfg, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/label/pod/values" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"status":"success","isPartial":true,"data":["db-0","db-1"]}`) + return + } + appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r) + }, appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusUnprocessableEntity { + t.Errorf("partial pod list over the cap answered %d, want 422", got) + } +} + // TestServerApplicationRefusesEveryAt: no @ form can be bounded, and the // answer says so rather than naming forms that would pass. func TestServerApplicationRefusesEveryAt(t *testing.T) { From d48897cf8224661affbf6be7e8e5ce7bee63dba5 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 10 Nov 2026 10:28:28 +0100 Subject: [PATCH 121/174] feat(virtual-machine): let each disk set its own boot order The boot order of a VM's disks was the order of the disks list, so the only way to boot from installer or rescue media, such as an ISO attached as a CD-ROM, was to move that media to the front of the list. Callers that treat the first disk as the system disk then see the ISO in its place. An optional bootOrder per disk decides boot priority independently of the list. Disks that leave it unset keep their positional order, so existing VMs render unchanged. Duplicate effective boot orders are refused at render time with the names of both disks, since KubeVirt would otherwise deny the VirtualMachine with a less specific error. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- api/apps/v1alpha1/vminstance/types.go | 3 + .../vminstance/zz_generated.deepcopy.go | 9 +- packages/apps/vm-instance/README.md | 1 + packages/apps/vm-instance/templates/vm.yaml | 11 +- .../tests/disk_boot_order_test.yaml | 128 ++++++++++++++++++ packages/apps/vm-instance/values.schema.json | 5 + packages/apps/vm-instance/values.yaml | 8 ++ .../vm-instance-rd/cozyrds/vm-instance.yaml | 2 +- 8 files changed, 164 insertions(+), 3 deletions(-) create mode 100644 packages/apps/vm-instance/tests/disk_boot_order_test.yaml diff --git a/api/apps/v1alpha1/vminstance/types.go b/api/apps/v1alpha1/vminstance/types.go index 95c800aae..e457f4cac 100644 --- a/api/apps/v1alpha1/vminstance/types.go +++ b/api/apps/v1alpha1/vminstance/types.go @@ -74,6 +74,9 @@ type ConfigSpec struct { } type Disk struct { + // Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both. + // +kubebuilder:validation:Minimum=1 + BootOrder *int `json:"bootOrder,omitempty"` // Disk bus type (e.g. "sata"). Bus string `json:"bus,omitempty"` // Disk name. diff --git a/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go b/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go index 060281042..84f7315ac 100644 --- a/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go @@ -61,7 +61,9 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { if in.Disks != nil { in, out := &in.Disks, &out.Disks *out = make([]Disk, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.Networks != nil { in, out := &in.Networks, &out.Networks @@ -100,6 +102,11 @@ func (in *ConfigSpec) DeepCopy() *ConfigSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Disk) DeepCopyInto(out *Disk) { *out = *in + if in.BootOrder != nil { + in, out := &in.BootOrder, &out.BootOrder + *out = new(int) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Disk. diff --git a/packages/apps/vm-instance/README.md b/packages/apps/vm-instance/README.md index 2365ca9c1..f2a0d001d 100644 --- a/packages/apps/vm-instance/README.md +++ b/packages/apps/vm-instance/README.md @@ -48,6 +48,7 @@ virtctl ssh @ | `disks` | List of disks to attach. | `[]object` | `[]` | | `disks[i].name` | Disk name. | `string` | `""` | | `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` | +| `disks[i].bootOrder` | Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both. | `*int` | `null` | | `networks` | Networks to attach the VM to. | `[]object` | `[]` | | `networks[i].name` | Network attachment name. | `string` | `""` | | `subnets` | Deprecated: use networks instead. | `[]object` | `[]` | diff --git a/packages/apps/vm-instance/templates/vm.yaml b/packages/apps/vm-instance/templates/vm.yaml index 8153adf8f..0d7845621 100644 --- a/packages/apps/vm-instance/templates/vm.yaml +++ b/packages/apps/vm-instance/templates/vm.yaml @@ -92,7 +92,16 @@ spec: {{- end }} {{- end }} disks: + {{- /* KubeVirt rejects two devices sharing a bootOrder, so a clash is + reported here with the disk names rather than surfacing as a + webhook denial on the VirtualMachine. */}} + {{- $bootOrderOwners := dict }} {{- range $i, $disk := .Values.disks }} + {{- $bootOrder := int ($disk.bootOrder | default (add $i 1)) }} + {{- with get $bootOrderOwners (toString $bootOrder) }} + {{- fail (printf "disks %q and %q both have bootOrder %d; boot orders must be unique" . $disk.name $bootOrder) }} + {{- end }} + {{- $_ := set $bootOrderOwners (toString $bootOrder) $disk.name }} - name: disk-{{ $disk.name }} {{- $dv := lookup "cdi.kubevirt.io/v1beta1" "DataVolume" $.Release.Namespace (printf "vm-disk-%s" $disk.name) }} {{- if $dv }} @@ -104,7 +113,7 @@ spec: {{- else }} {{ $deviceKey }}: {} {{- end }} - bootOrder: {{ add $i 1 }} + bootOrder: {{ $bootOrder }} {{- else }} {{- fail (printf "Specified disk not exists in cluster: %s" .name) }} {{- end }} diff --git a/packages/apps/vm-instance/tests/disk_boot_order_test.yaml b/packages/apps/vm-instance/tests/disk_boot_order_test.yaml new file mode 100644 index 000000000..146cbaf53 --- /dev/null +++ b/packages/apps/vm-instance/tests/disk_boot_order_test.yaml @@ -0,0 +1,128 @@ +suite: vm-instance disk boot order + +# The first entry of `disks` is the one callers treat as the system disk, so +# booting from installer or rescue media must not require moving that media to +# the front of the list. These cases pin that an explicit bootOrder decides +# which device the VM boots from while list order stays as given, and that +# leaving it unset keeps the positional order existing VMs were rendered with. +release: + name: test-vm + namespace: tenant-test +templates: + - templates/vm.yaml +set: + fullnameOverride: test-vm + instanceType: "" + instanceProfile: "" + resources: + cpu: 1 + sockets: 1 + memory: 1Gi + _cluster: + scheduling: + dedicatedNodesForWindowsVMs: "false" +kubernetesProvider: + scheme: + "cdi.kubevirt.io/v1beta1/DataVolume": + gvr: + group: cdi.kubevirt.io + version: v1beta1 + resource: datavolumes + namespaced: true + objects: + - apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: vm-disk-system + namespace: tenant-test + annotations: {} + - apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: vm-disk-iso + namespace: tenant-test + annotations: + vm-disk.cozystack.io/optical: "true" + - apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: vm-disk-data + namespace: tenant-test + annotations: {} +tests: + - it: keeps the positional boot order when no disk sets bootOrder + set: + disks: + - name: system + - name: iso + - name: data + asserts: + - equal: + path: spec.template.spec.domain.devices.disks + value: + - name: disk-system + disk: {} + bootOrder: 1 + - name: disk-iso + cdrom: {} + bootOrder: 2 + - name: disk-data + disk: {} + bootOrder: 3 + + - it: boots from a CD-ROM in second position without reordering the disks + set: + disks: + - name: system + bootOrder: 2 + - name: iso + bus: sata + bootOrder: 1 + - name: data + asserts: + - equal: + path: spec.template.spec.domain.devices.disks + value: + - name: disk-system + disk: {} + bootOrder: 2 + - name: disk-iso + cdrom: + bus: sata + bootOrder: 1 + - name: disk-data + disk: {} + bootOrder: 3 + - equal: + path: spec.template.spec.volumes[0].dataVolume.name + value: vm-disk-system + + - it: refuses an explicit bootOrder that collides with another disk's positional one + set: + disks: + - name: system + - name: iso + bootOrder: 1 + asserts: + - failedTemplate: + errorMessage: 'disks "system" and "iso" both have bootOrder 1; boot orders must be unique' + + - it: refuses two disks given the same explicit bootOrder + set: + disks: + - name: system + bootOrder: 3 + - name: iso + bootOrder: 3 + asserts: + - failedTemplate: + errorMessage: 'disks "system" and "iso" both have bootOrder 3; boot orders must be unique' + + - it: rejects a bootOrder below 1 through the values schema + set: + disks: + - name: system + bootOrder: 0 + asserts: + - failedTemplate: + errorPattern: "minimum" diff --git a/packages/apps/vm-instance/values.schema.json b/packages/apps/vm-instance/values.schema.json index 7c24a9e52..45adaac72 100644 --- a/packages/apps/vm-instance/values.schema.json +++ b/packages/apps/vm-instance/values.schema.json @@ -69,6 +69,11 @@ "name" ], "properties": { + "bootOrder": { + "description": "Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both.", + "type": "integer", + "minimum": 1 + }, "bus": { "description": "Disk bus type (e.g. \"sata\").", "type": "string" diff --git a/packages/apps/vm-instance/values.yaml b/packages/apps/vm-instance/values.yaml index 88fc6f8dd..2730812a0 100644 --- a/packages/apps/vm-instance/values.yaml +++ b/packages/apps/vm-instance/values.yaml @@ -10,6 +10,8 @@ ## @field {string} name - Disk name. ## @x-cozystack-options {source: vmdisk} ## @field {string} [bus] - Disk bus type (e.g. "sata"). +## @field {*int} [bootOrder] - Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both. +## @minimum 1 ## @typedef {struct} Network - Network to attach the VM to. ## @field {string} [name] - Network attachment name. @@ -67,6 +69,12 @@ disks: [] ## - name: example-system ## - name: example-data ## bus: sata +## Boot from an attached ISO while keeping the system disk first in the list: +## disks: +## - name: example-system +## bootOrder: 2 +## - name: example-iso +## bootOrder: 1 ## @param {[]Network} networks - Networks to attach the VM to. networks: [] diff --git a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml index e91a334a1..dc533ee59 100644 --- a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml +++ b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml @@ -12,7 +12,7 @@ spec: singular: vminstance plural: vminstances openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"externalAllowICMP":{"description":"Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.","type":"boolean","default":true},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type. Ignored when `resources` sizes the VM, which needs whole-number cpu and sockets plus memory; the whole type is dropped then, so any dedicated CPU placement, hugepages, NUMA or realtime settings it carries go with it. While this field is set, a `resources` block that sizes only part of the VM is rejected outright instead of ignored, so complete it or clear it.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","x-cozystack-options":{"source":"instanceprofile"}},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string","x-cozystack-options":{"source":"vmdisk"}}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string","x-cozystack-options":{"source":"network"}}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string","x-cozystack-options":{"source":"network"}}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"firmware":{"description":"Firmware and boot configuration (UEFI/BIOS selection, Secure Boot, persistent EFI NVRAM).","type":"object","default":{},"properties":{"bootloader":{"description":"Bootloader to boot the VM with: \"uefi\" (OVMF) or \"bios\" (SeaBIOS). Empty inherits the instanceProfile default.","type":"string"},"efiPersistent":{"description":"Persist EFI NVRAM (e.g. enrolled Secure Boot keys, such as an updated Microsoft UEFI CA) across reboots. Only applies when bootloader is \"uefi\".","type":"boolean"},"secureBoot":{"description":"Enable UEFI Secure Boot. Only applies when bootloader is \"uefi\".","type":"boolean"}}},"resources":{"description":"Resource configuration for the virtual machine. Set whole-number cpu and sockets together with memory to size the VM directly, which overrides `instanceType`. Alongside an `instanceType`, a block that sizes only part of the VM is rejected; a whole-number cpu or sockets on its own sizes nothing and is ignored.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated, as a positive whole number such as `2`. Millicore or other unit-suffixed, fractional, zero, negative, above 4294967295 and quoted leading-zero values are rejected at render time, because the value becomes the VM's unsigned integer `domain.cpu.cores`.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated, as a quantity of at least `1M` such as `8Gi`. Less than 1M is rejected at render time, because KubeVirt refuses a non-zero amount below 1M and a zero or negative amount sizes nothing.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology), as a positive whole number such as `1`. Millicore or other unit-suffixed, fractional, zero, negative, above 4294967295 and quoted leading-zero values are rejected at render time, because the value becomes the VM's unsigned integer `domain.cpu.sockets`.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"netplanDhcp":{"description":"Supply cloud-init network-data that requests DHCP on every `en*` and `eth*` interface and keeps the DNS search domains DHCP offers. Needed for a netplan guest (Ubuntu, Debian) to address a secondary network such as a VPC subnet. Leave it off for a guest rendered by anything other than netplan, which would lose DHCP on every interface, and for a guest that configures an interface statically through netplan, which this configuration would shadow. Turning it on for a VM that has neither `cloudInit` nor `sshKeys` adds a cloud-init disk, and the guest's next boot is then treated as a first boot.","type":"boolean","default":false},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} + {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"externalAllowICMP":{"description":"Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.","type":"boolean","default":true},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type. Ignored when `resources` sizes the VM, which needs whole-number cpu and sockets plus memory; the whole type is dropped then, so any dedicated CPU placement, hugepages, NUMA or realtime settings it carries go with it. While this field is set, a `resources` block that sizes only part of the VM is rejected outright instead of ignored, so complete it or clear it.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","x-cozystack-options":{"source":"instanceprofile"}},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bootOrder":{"description":"Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both.","type":"integer","minimum":1},"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string","x-cozystack-options":{"source":"vmdisk"}}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string","x-cozystack-options":{"source":"network"}}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string","x-cozystack-options":{"source":"network"}}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"firmware":{"description":"Firmware and boot configuration (UEFI/BIOS selection, Secure Boot, persistent EFI NVRAM).","type":"object","default":{},"properties":{"bootloader":{"description":"Bootloader to boot the VM with: \"uefi\" (OVMF) or \"bios\" (SeaBIOS). Empty inherits the instanceProfile default.","type":"string"},"efiPersistent":{"description":"Persist EFI NVRAM (e.g. enrolled Secure Boot keys, such as an updated Microsoft UEFI CA) across reboots. Only applies when bootloader is \"uefi\".","type":"boolean"},"secureBoot":{"description":"Enable UEFI Secure Boot. Only applies when bootloader is \"uefi\".","type":"boolean"}}},"resources":{"description":"Resource configuration for the virtual machine. Set whole-number cpu and sockets together with memory to size the VM directly, which overrides `instanceType`. Alongside an `instanceType`, a block that sizes only part of the VM is rejected; a whole-number cpu or sockets on its own sizes nothing and is ignored.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated, as a positive whole number such as `2`. Millicore or other unit-suffixed, fractional, zero, negative, above 4294967295 and quoted leading-zero values are rejected at render time, because the value becomes the VM's unsigned integer `domain.cpu.cores`.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated, as a quantity of at least `1M` such as `8Gi`. Less than 1M is rejected at render time, because KubeVirt refuses a non-zero amount below 1M and a zero or negative amount sizes nothing.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology), as a positive whole number such as `1`. Millicore or other unit-suffixed, fractional, zero, negative, above 4294967295 and quoted leading-zero values are rejected at render time, because the value becomes the VM's unsigned integer `domain.cpu.sockets`.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"netplanDhcp":{"description":"Supply cloud-init network-data that requests DHCP on every `en*` and `eth*` interface and keeps the DNS search domains DHCP offers. Needed for a netplan guest (Ubuntu, Debian) to address a secondary network such as a VPC subnet. Leave it off for a guest rendered by anything other than netplan, which would lose DHCP on every interface, and for a guest that configures an interface statically through netplan, which this configuration would shadow. Turning it on for a VM that has neither `cloudInit` nor `sshKeys` adds a cloud-init disk, and the guest's next boot is then treated as a first boot.","type":"boolean","default":false},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} release: prefix: vm-instance- labels: From 47d9ca01f3f936f7ad02571b7a7dbf8fc9e3ec6e Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Tue, 10 Nov 2026 05:37:15 +0700 Subject: [PATCH 122/174] feat(observability-proxy): publish the proxy on its own host Clients outside the cluster, such as kubectl, scripts and third-party tools, need to call the proxy with their Kubernetes bearer token. The kube-apiserver service proxy cannot carry it: kube-apiserver drops the Authorization header once it has authenticated the request. The console route sits behind the dashboard gatekeeper, which with OIDC enabled answers an API client that has no session cookie with 401, and exists only where the dashboard itself is published. A route straight to the proxy Service is the path that keeps the token. It stays off until "observability-proxy" is listed in publishing.exposedServices, and follows the dashboard for TLS and for the Ingress versus Gateway API switch. The edge must not end a request the proxy still serves. The tenant ingress-nginx reads upstream with a 10s timeout, while a log tail may stay silent for the proxy's 5m idle limit, so the Ingress waits longer than that. Cilium leaves Envoy's 15s route timeout in place on an HTTPRoute that sets none, which would cut every tail, so the route disables it: the proxy bounds each request on its own. Assisted-by: LLM Signed-off-by: Maxim Belyy --- .../templates/httproute.yaml | 28 +++ .../templates/ingress.yaml | 52 +++++ .../tests/publishing_test.yaml | 196 ++++++++++++++++++ 3 files changed, 276 insertions(+) create mode 100644 packages/system/observability-proxy/templates/httproute.yaml create mode 100644 packages/system/observability-proxy/templates/ingress.yaml create mode 100644 packages/system/observability-proxy/tests/publishing_test.yaml diff --git a/packages/system/observability-proxy/templates/httproute.yaml b/packages/system/observability-proxy/templates/httproute.yaml new file mode 100644 index 000000000..5d4931031 --- /dev/null +++ b/packages/system/observability-proxy/templates/httproute.yaml @@ -0,0 +1,28 @@ +{{- $cluster := .Values._cluster | default dict }} +{{- $host := index $cluster "root-host" }} +{{- $exposeServices := splitList "," ((index $cluster "expose-services") | default "") }} +{{- $exposeIngress := (index $cluster "expose-ingress") | default "tenant-root" }} +{{- $gatewayEnabled := (index $cluster "gateway-enabled") | default "false" }} +{{- if and (has "observability-proxy" $exposeServices) (eq $gatewayEnabled "true") }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: observability-proxy +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: cozystack + namespace: {{ $exposeIngress }} + hostnames: + - {{ printf "observability-proxy.%s" $host | quote }} + rules: + - backendRefs: + - name: observability-proxy + port: 80 + {{- /* Without a request timeout Cilium leaves Envoy's 15s route default + in place, which would cut every tail. The proxy bounds each + request itself, so the route sets none. */}} + timeouts: + request: 0s +{{- end }} diff --git a/packages/system/observability-proxy/templates/ingress.yaml b/packages/system/observability-proxy/templates/ingress.yaml new file mode 100644 index 000000000..59e79a32b --- /dev/null +++ b/packages/system/observability-proxy/templates/ingress.yaml @@ -0,0 +1,52 @@ +{{- $cluster := .Values._cluster | default dict }} +{{- $solver := (index $cluster "solver") | default "http01" }} +{{- $clusterIssuer := (index $cluster "issuer-name") | default "letsencrypt-prod" }} +{{- $host := index $cluster "root-host" }} +{{- $exposeServices := splitList "," ((index $cluster "expose-services") | default "") }} +{{- $exposeIngress := (index $cluster "expose-ingress") | default "tenant-root" }} +{{- $gatewayEnabled := (index $cluster "gateway-enabled") | default "false" }} +{{- $wildcardSecret := (index $cluster "wildcard-secret-name") | default "" }} +{{- if and (has "observability-proxy" $exposeServices) (ne $gatewayEnabled "true") }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: observability-proxy + annotations: + {{- if not $wildcardSecret }} + cert-manager.io/cluster-issuer: {{ $clusterIssuer }} + {{- if eq $solver "http01" }} + acme.cert-manager.io/http01-ingress-ingressclassname: {{ $exposeIngress }} + {{- end }} + {{- end }} + {{- /* The proxy refuses a body above 1 MiB itself; nginx's 1m is the + same bound, so an oversized body stops at the edge. */}} + nginx.ingress.kubernetes.io/proxy-body-size: 1m + {{- /* A tail flushes every line; buffered, lines would reach the client + in batches. */}} + nginx.ingress.kubernetes.io/proxy-buffering: "off" + {{- /* The controller default is 10s. The longest silence the proxy + allows is a tail with no new line for --tail-idle-timeout (5m), + after which it ends the response itself. */}} + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" +spec: + ingressClassName: {{ $exposeIngress }} + rules: + - host: observability-proxy.{{ $host }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: observability-proxy + port: + name: http + tls: + - hosts: + - observability-proxy.{{ $host }} + {{- /* Wildcard mode: ingress-nginx serves its default certificate, and + the tls section stays to keep the HTTP-to-HTTPS redirect. */}} + {{- if not $wildcardSecret }} + secretName: observability-proxy-tls + {{- end }} +{{- end }} diff --git a/packages/system/observability-proxy/tests/publishing_test.yaml b/packages/system/observability-proxy/tests/publishing_test.yaml new file mode 100644 index 000000000..0efc3ca82 --- /dev/null +++ b/packages/system/observability-proxy/tests/publishing_test.yaml @@ -0,0 +1,196 @@ +suite: observability-proxy public endpoint +# Clients outside the cluster reach the proxy on its own host with a +# Kubernetes bearer token, so the route must pass Authorization and the +# /apis/... path through untouched, and must not end a request before the +# proxy does: a non-tail request may take up to --request-timeout (1m), and +# a tail may stay silent up to --tail-idle-timeout (5m). Whole objects are +# compared rather than single keys, so an annotation or a field that should +# not be there fails the test too. +templates: + - templates/ingress.yaml + - templates/httproute.yaml + +release: + name: observability-proxy + namespace: cozy-observability-proxy + +tests: + - it: publishes nothing without the cluster values + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: publishes nothing while observability-proxy is not in expose-services + set: + _cluster: + root-host: example.org + expose-services: "api,dashboard" + gateway-enabled: "false" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: publishes nothing in Gateway mode while observability-proxy is not in expose-services + set: + _cluster: + root-host: example.org + expose-services: "api,dashboard" + gateway-enabled: "true" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: renders an Ingress with a per-host certificate when gateway is off + set: + _cluster: + root-host: example.org + expose-services: "dashboard,observability-proxy" + expose-ingress: tenant-public + gateway-enabled: "false" + asserts: + - template: templates/httproute.yaml + hasDocuments: + count: 0 + - template: templates/ingress.yaml + isKind: + of: Ingress + - template: templates/ingress.yaml + equal: + path: metadata.name + value: observability-proxy + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-ingress-ingressclassname: tenant-public + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec + value: + ingressClassName: tenant-public + rules: + - host: observability-proxy.example.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: observability-proxy + port: + name: http + tls: + - hosts: + - observability-proxy.example.org + secretName: observability-proxy-tls + + - it: leaves the http01 ingress class out under the dns01 solver + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "false" + solver: dns01 + issuer-name: letsencrypt-stage + asserts: + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-stage + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec.ingressClassName + value: tenant-root + + - it: relies on the wildcard certificate and keeps the TLS section for the redirect + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "false" + wildcard-secret-name: wildcard-tls + asserts: + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec.tls + value: + - hosts: + - observability-proxy.example.org + + - it: renders an HTTPRoute without a route timeout when gateway is on + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + expose-ingress: tenant-public + gateway-enabled: "true" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + isKind: + of: HTTPRoute + - template: templates/httproute.yaml + isAPIVersion: + of: gateway.networking.k8s.io/v1 + - template: templates/httproute.yaml + equal: + path: metadata.name + value: observability-proxy + - template: templates/httproute.yaml + equal: + path: spec + value: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: cozystack + namespace: tenant-public + hostnames: + - observability-proxy.example.org + rules: + - backendRefs: + - name: observability-proxy + port: 80 + timeouts: + request: 0s + + - it: attaches to the tenant-root Gateway when expose-ingress is unset + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "true" + asserts: + - template: templates/httproute.yaml + equal: + path: spec.parentRefs[?(@.name == "cozystack")].namespace + value: tenant-root From 77acd23bdda7a3f514a2b090aa2a5d5096c38a86 Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Tue, 10 Nov 2026 05:38:18 +0700 Subject: [PATCH 123/174] feat(platform): let the observability proxy attach to the tenant Gateway With the Gateway API enabled, the publishing Gateway admits routes only from the publishing tenant and the namespaces in gateway.attachedNamespaces. The observability proxy publishes its HTTPRoute from its own namespace, so without an entry the route renders and is never served. Clusters that override gateway.attachedNamespaces replace the whole list and have to add the namespace themselves. Assisted-by: LLM Signed-off-by: Maxim Belyy --- .../tests/observability_proxy_wiring_test.yaml | 10 ++++++++++ packages/core/platform/values.yaml | 1 + 2 files changed, 11 insertions(+) diff --git a/packages/core/platform/tests/observability_proxy_wiring_test.yaml b/packages/core/platform/tests/observability_proxy_wiring_test.yaml index 9e6ec583d..84777ca7f 100644 --- a/packages/core/platform/tests/observability_proxy_wiring_test.yaml +++ b/packages/core/platform/tests/observability_proxy_wiring_test.yaml @@ -5,6 +5,7 @@ suite: observability-proxy package wiring templates: - templates/sources.yaml - templates/bundles/system.yaml + - templates/apps.yaml release: name: cozystack namespace: cozy-system @@ -67,3 +68,12 @@ tests: kind: Package name: cozystack.observability-proxy any: true + + # The proxy's HTTPRoute lives in its own namespace, and the tenant Gateway + # admits routes only from the namespaces listed here. + - it: lets the proxy namespace attach a route to the publishing Gateway + template: templates/apps.yaml + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: 'gateway-attached-namespaces: "([^"]*,)?cozy-observability-proxy(,[^"]*)?"' diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index f2ce252ed..5ca66ac17 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -668,6 +668,7 @@ gateway: - cozy-kubevirt-cdi - cozy-monitoring - cozy-linstor-gui + - cozy-observability-proxy - default # Platform monitoring configuration. monitoring: From 773a3a96d3d7a378914c72823857580f3920d8b7 Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Tue, 10 Nov 2026 05:42:46 +0700 Subject: [PATCH 124/174] feat(tenant): allow tenant egress to the observability proxy The observability proxy's Ingress is served by the ingress controller in the publishing tenant, and tenant egress is denied except for the allow-to-* policies. Without one for the proxy namespace, every request through the Ingress and every HTTP-01 challenge for its certificate times out at the controller. The policy matches the whole namespace because cert-manager runs the HTTP-01 solver pods there, next to the proxy. Assisted-by: LLM Signed-off-by: Maxim Belyy --- .../apps/tenant/templates/networkpolicy.yaml | 14 ++++ ...etworkpolicy_observability_proxy_test.yaml | 64 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml diff --git a/packages/apps/tenant/templates/networkpolicy.yaml b/packages/apps/tenant/templates/networkpolicy.yaml index 5285b495a..bc6904e0a 100644 --- a/packages/apps/tenant/templates/networkpolicy.yaml +++ b/packages/apps/tenant/templates/networkpolicy.yaml @@ -259,6 +259,20 @@ spec: - ports: - port: "8443" protocol: TCP +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + # The whole namespace, as for linstor-gui: the ingress controller also + # has to reach the ACME HTTP-01 solver pods cert-manager starts there. + name: allow-to-observability-proxy + namespace: {{ include "tenant.name" . }} +spec: + endpointSelector: {} + egress: + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": cozy-observability-proxy {{- if .Values.monitoring }} --- apiVersion: cilium.io/v2 diff --git a/packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml b/packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml new file mode 100644 index 000000000..3f3e027fd --- /dev/null +++ b/packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml @@ -0,0 +1,64 @@ +suite: tenant egress to the observability proxy +# The observability-proxy Ingress in cozy-observability-proxy is served by +# the ingress controller running in tenant-root. Tenant egress is +# default-deny plus explicit allow-to-* policies, so without this one the +# controller reaches neither the proxy nor the ACME HTTP-01 solver pod that +# cert-manager starts in the same namespace; hence the whole namespace, as +# allow-to-linstor-gui does. +templates: + - templates/networkpolicy.yaml +tests: + - it: lets every pod of the root tenant reach the proxy namespace + release: + name: tenant-root + namespace: tenant-root + asserts: + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + isKind: + of: CiliumNetworkPolicy + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + equal: + path: metadata.namespace + value: tenant-root + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + equal: + path: spec + value: + endpointSelector: {} + egress: + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": cozy-observability-proxy + + - it: renders in a nested tenant with every module off + release: + name: tenant-bar + namespace: tenant-foo + set: + monitoring: false + ingress: false + etcd: false + asserts: + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + equal: + path: metadata.namespace + value: tenant-foo-bar + + - it: renders nothing where the tenant Cilium policies are off + release: + name: tenant-root + namespace: tenant-root + set: + _cluster: + bundle-name: isp-hosted + asserts: + - hasDocuments: + count: 0 From d21d9de641db88f2e9fec6fbbe45d1d8500dc5a4 Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Tue, 10 Nov 2026 05:43:31 +0700 Subject: [PATCH 125/174] fix(observability-proxy): end silent tails before the Gateway does Behind a Cilium Gateway, Envoy resets a stream that carries no data for 300s, the default stream idle timeout, and the proxy's own tail idle limit was the same 5m. A tail that went quiet could then be cut by Envoy rather than ended by the proxy, which the client sees as a reset stream instead of a finished response. The HTTPRoute cannot raise Envoy's limit, so the proxy ends a silent tail a little earlier. Assisted-by: LLM Signed-off-by: Maxim Belyy --- .../system/observability-proxy/templates/deployment.yaml | 4 ++++ .../system/observability-proxy/templates/httproute.yaml | 4 +++- .../system/observability-proxy/templates/ingress.yaml | 5 +++-- .../observability-proxy/tests/deployment_test.yaml | 9 +++++++++ .../observability-proxy/tests/publishing_test.yaml | 2 +- 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/packages/system/observability-proxy/templates/deployment.yaml b/packages/system/observability-proxy/templates/deployment.yaml index 581c15fde..dd95dfbb6 100644 --- a/packages/system/observability-proxy/templates/deployment.yaml +++ b/packages/system/observability-proxy/templates/deployment.yaml @@ -38,6 +38,10 @@ spec: - --cluster-domain={{ $clusterDomain }} - --root-tenant={{ .Values.observabilityProxy.rootTenant }} - --auth-cache-ttl={{ .Values.observabilityProxy.authCacheTTL }} + {{- /* Below the 300s stream idle timeout of a Cilium Gateway, so a + published tail that stops sending lines is ended by the proxy, + not reset by Envoy. */}} + - --tail-idle-timeout=4m30s ports: - name: http containerPort: 8080 diff --git a/packages/system/observability-proxy/templates/httproute.yaml b/packages/system/observability-proxy/templates/httproute.yaml index 5d4931031..1ddd2f720 100644 --- a/packages/system/observability-proxy/templates/httproute.yaml +++ b/packages/system/observability-proxy/templates/httproute.yaml @@ -22,7 +22,9 @@ spec: port: 80 {{- /* Without a request timeout Cilium leaves Envoy's 15s route default in place, which would cut every tail. The proxy bounds each - request itself, so the route sets none. */}} + request itself, so the route disables it. Envoy's 300s stream + idle timeout still applies; the proxy's tail idle limit stays + below it (see deployment.yaml). */}} timeouts: request: 0s {{- end }} diff --git a/packages/system/observability-proxy/templates/ingress.yaml b/packages/system/observability-proxy/templates/ingress.yaml index 59e79a32b..7c6b1a029 100644 --- a/packages/system/observability-proxy/templates/ingress.yaml +++ b/packages/system/observability-proxy/templates/ingress.yaml @@ -25,8 +25,9 @@ metadata: in batches. */}} nginx.ingress.kubernetes.io/proxy-buffering: "off" {{- /* The controller default is 10s. The longest silence the proxy - allows is a tail with no new line for --tail-idle-timeout (5m), - after which it ends the response itself. */}} + allows is a tail with no new line for --tail-idle-timeout (4m30s), + after which it ends the response itself; before the first byte + the token and access reviews may add up to --request-timeout (1m). */}} nginx.ingress.kubernetes.io/proxy-read-timeout: "360" spec: ingressClassName: {{ $exposeIngress }} diff --git a/packages/system/observability-proxy/tests/deployment_test.yaml b/packages/system/observability-proxy/tests/deployment_test.yaml index ddf9d34b8..978f9e968 100644 --- a/packages/system/observability-proxy/tests/deployment_test.yaml +++ b/packages/system/observability-proxy/tests/deployment_test.yaml @@ -40,6 +40,15 @@ tests: path: spec.template.spec.containers[0].args content: --auth-cache-ttl=10s + # Envoy behind a Cilium Gateway resets a stream after 300s without data + # (envoy.streamIdleTimeoutDurationSeconds), so a silent tail has to be + # ended by the proxy first, or the client sees a reset instead of an end. + - it: ends a silent tail before the Gateway stream idle timeout + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --tail-idle-timeout=4m30s + - it: falls back to cozy.local when the platform passes no cluster domain asserts: - contains: diff --git a/packages/system/observability-proxy/tests/publishing_test.yaml b/packages/system/observability-proxy/tests/publishing_test.yaml index 0efc3ca82..676c38e08 100644 --- a/packages/system/observability-proxy/tests/publishing_test.yaml +++ b/packages/system/observability-proxy/tests/publishing_test.yaml @@ -3,7 +3,7 @@ suite: observability-proxy public endpoint # Kubernetes bearer token, so the route must pass Authorization and the # /apis/... path through untouched, and must not end a request before the # proxy does: a non-tail request may take up to --request-timeout (1m), and -# a tail may stay silent up to --tail-idle-timeout (5m). Whole objects are +# a tail may stay silent up to --tail-idle-timeout (4m30s). Whole objects are # compared rather than single keys, so an annotation or a field that should # not be there fails the test too. templates: From c6f1d6c55938725cf631c2ea92584602af036dfd Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 10 Nov 2026 14:37:16 +0100 Subject: [PATCH 126/174] fix(vm-instance): keep an external PortList VM valid with no ports The API server rejects a Service that declares no port, so a VM switched to external PortList with an empty port list could not be upgraded: the release failed and was retried forever, while the old Service kept the previously listed ports open. Render a single SCTP placeholder in that case. cozy-proxy only admits TCP and UDP ports into its PortList filter, so the VM keeps its external IP for egress and ICMP while all inbound TCP and UDP traffic is dropped, which is what an empty port list means. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../apps/vm-instance/templates/service.yaml | 7 ++ .../vm-instance/tests/service_ports_test.yaml | 70 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 packages/apps/vm-instance/tests/service_ports_test.yaml diff --git a/packages/apps/vm-instance/templates/service.yaml b/packages/apps/vm-instance/templates/service.yaml index cfeffa810..b757ab96d 100644 --- a/packages/apps/vm-instance/templates/service.yaml +++ b/packages/apps/vm-instance/templates/service.yaml @@ -30,6 +30,13 @@ spec: {{- if .Values.external }} {{- if and (eq .Values.externalMethod "WholeIP") (not .Values.externalPorts) }} - port: 65535 + {{- else if not .Values.externalPorts }} + {{- /* A Service needs at least one port. cozy-proxy only admits TCP and + UDP ports into its PortList filter, so an SCTP placeholder keeps the + external IP while every inbound TCP and UDP packet is dropped. */}} + - name: no-ports + port: 65535 + protocol: SCTP {{- else }} {{- range .Values.externalPorts }} - name: port-{{ . }} diff --git a/packages/apps/vm-instance/tests/service_ports_test.yaml b/packages/apps/vm-instance/tests/service_ports_test.yaml new file mode 100644 index 000000000..d7689d957 --- /dev/null +++ b/packages/apps/vm-instance/tests/service_ports_test.yaml @@ -0,0 +1,70 @@ +suite: vm-instance external Service ports + +# The API server rejects a Service without ports, so an external PortList VM +# whose port list was emptied could not be upgraded at all. The placeholder +# must be SCTP: cozy-proxy only admits TCP and UDP ports into its filter, so +# any TCP or UDP placeholder would open that port on the VM. +release: + name: test-vm + namespace: tenant-test +templates: + - templates/service.yaml +set: + fullnameOverride: test-vm + external: true + +tests: + - it: renders an SCTP placeholder for PortList with no ports + set: + externalMethod: PortList + externalPorts: [] + asserts: + - equal: + path: spec.type + value: LoadBalancer + - equal: + path: metadata.annotations["networking.cozystack.io/wholeIP"] + value: "false" + - lengthEqual: + path: spec.ports + count: 1 + - equal: + path: spec.ports[0] + value: + name: no-ports + port: 65535 + protocol: SCTP + + - it: renders the listed ports for PortList + set: + externalMethod: PortList + externalPorts: [22, 443] + asserts: + - lengthEqual: + path: spec.ports + count: 2 + - equal: + path: spec.ports[0] + value: + name: port-22 + port: 22 + targetPort: 22 + - notContains: + path: spec.ports + content: + name: no-ports + port: 65535 + protocol: SCTP + + - it: keeps the WholeIP placeholder when no ports are listed + set: + externalMethod: WholeIP + externalPorts: [] + asserts: + - equal: + path: metadata.annotations["networking.cozystack.io/wholeIP"] + value: "true" + - equal: + path: spec.ports + value: + - port: 65535 From f1dd2d17fc864ac5b8a7417fdb3d272b0fbc5c57 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Wed, 11 Nov 2026 09:20:19 +0100 Subject: [PATCH 127/174] fix(backups): give the barman-cloud sidecar an explicit CPU limit The sidecar was left without a CPU limit so that a throttled sidecar would slow WAL archiving down rather than fail it. In a tenant with resourceQuotas that does not hold: the tenant LimitRange gives every container a 250m CPU limit by default, so the sidecar gets it and compresses full backups and WAL segments on a quarter of a core. A 1.6 GB database took 30 minutes to back up. Set an explicit one-core limit in both places that build the sidecar configuration, the backup driver and cozy-lib, the same way the memory request and limit already override the LimitRange defaults. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- internal/backupcontroller/cnpgstrategy_controller.go | 10 +++++++--- .../backupcontroller/cnpgstrategy_controller_test.go | 7 +++++-- .../apps/postgres/tests/barman_resources_test.yaml | 8 ++------ packages/library/cozy-lib/templates/_barman.tpl | 11 +++++++---- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 19f255c6c..3e2f05c0d 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -2224,9 +2224,12 @@ func buildBarmanPlugin(objectStoreName, serverName string) cnpgtypes.PluginConfi // high-water mark on a 38 MB database is 254 MiB, so 128Mi cannot hold it. A // tenant that leaves resourceQuotas empty has no LimitRange, and there the // sidecar had no requests and no limit at all. 256Mi holds the measured working -// set and 1Gi is four times the measurement. No CPU limit is set, matching the -// entityOperator precedent in packages/apps/kafka: a throttled sidecar stalls -// WAL archiving instead of failing it. +// set and 1Gi is four times the measurement. The CPU limit is explicit for the +// same reason: left unset, that LimitRange gives the sidecar its 250m default, +// so gzip compression of full backups and WAL segments runs on a quarter of a +// core (a 1.6 GB database took 30 minutes to back up). One core keeps the +// sidecar from starving the instance while removing that ceiling; it counts +// against the tenant's limits.cpu budget, one core per instance pod. // // Checksum: since botocore ~1.36 the default (when_supported) attaches a // flexible checksum to every PutObject, which non-AWS S3-compatible backends @@ -2246,6 +2249,7 @@ func barmanSidecarConfiguration() *cnpgtypes.InstanceSidecarConfiguration { corev1.ResourceMemory: resource.MustParse("256Mi"), }, Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), }, }, diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 551209fac..bc2827371 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -2152,8 +2152,11 @@ func TestApplyClusterPluginBackup_PatchesExistingCluster(t *testing.T) { sc.Resources.Requests.Cpu().MilliValue() != 100 { t.Errorf("ObjectStore leaves Barman subject to undersized tenant resource defaults: %+v", sc.Resources) } - if _, ok := sc.Resources.Limits[corev1.ResourceCPU]; ok { - t.Errorf("ObjectStore caps sidecar CPU, which throttles WAL archiving instead of failing it: %+v", sc.Resources.Limits) + // Without an explicit CPU limit the tenant LimitRange caps the sidecar at + // its 250m default, and every backup and WAL upload compresses on a + // quarter of a core. + if sc.Resources.Limits.Cpu().MilliValue() != 1000 { + t.Errorf("ObjectStore leaves the sidecar CPU limit to the tenant LimitRange default: %+v", sc.Resources.Limits) } // The ObjectStore must be owner-referenced to the Cluster so Kubernetes GC // removes it when the Cluster is deleted (no orphan in the platform flow, diff --git a/packages/apps/postgres/tests/barman_resources_test.yaml b/packages/apps/postgres/tests/barman_resources_test.yaml index 905fde2bb..1903fe8b1 100644 --- a/packages/apps/postgres/tests/barman_resources_test.yaml +++ b/packages/apps/postgres/tests/barman_resources_test.yaml @@ -13,7 +13,7 @@ set: s3CredentialsSecret: name: postgres-backup tests: - - it: sets resources for native backup without inheriting the tenant 128Mi limit + - it: sets resources for native backup without inheriting the tenant 128Mi and 250m limits set: backup.enabled: true asserts: @@ -24,15 +24,11 @@ tests: cpu: 100m memory: 256Mi limits: + cpu: "1" memory: 1Gi documentSelector: path: kind value: ObjectStore - - notExists: - path: spec.instanceSidecarConfiguration.resources.limits.cpu - documentSelector: - path: kind - value: ObjectStore - equal: path: spec.configuration.s3Credentials.accessKeyId.name value: postgres-backup diff --git a/packages/library/cozy-lib/templates/_barman.tpl b/packages/library/cozy-lib/templates/_barman.tpl index c37b9673f..3b187dda6 100644 --- a/packages/library/cozy-lib/templates/_barman.tpl +++ b/packages/library/cozy-lib/templates/_barman.tpl @@ -13,10 +13,12 @@ LimitRange, which is cozy-keycloak and any tenant that leaves resourceQuotas empty, gave the sidecar no requests and no limit at all, so nothing reserved memory for it and nothing bounded it. The same request and limit settle both: 256Mi holds the measured working set and 1Gi is four times the measurement. -No CPU limit is set, matching the entityOperator precedent in -packages/apps/kafka: a throttled sidecar stalls WAL archiving instead of -failing it. Inside a tenant the limit counts against the ResourceQuota -limits.memory budget, 1Gi per instance pod. +The CPU limit is explicit for the same reason: left unset, that LimitRange +gives the sidecar its 250m default, so gzip compression of full backups and +WAL segments runs on a quarter of a core (a 1.6 GB database took 30 minutes +to back up). One core keeps the sidecar from starving the instance while +removing that ceiling. Inside a tenant both limits count against the +ResourceQuota budget, one core and 1Gi per instance pod. Checksum: since botocore ~1.36 (early 2025) the default RequestChecksumCalculation is when_supported, which attaches a flexible checksum @@ -39,6 +41,7 @@ instanceSidecarConfiguration: cpu: 100m memory: 256Mi limits: + cpu: "1" memory: 1Gi env: - name: AWS_REQUEST_CHECKSUM_CALCULATION From 320d0299ad8e4ebb4d34568fdc561473f2808b86 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:03:59 +0300 Subject: [PATCH 128/174] fix(backups): refuse a MariaDB restore from a Backup being deleted Deleting a MariaDB Backup now deletes its dump, and mariadb-operator restores the dump nearest the requested time when the exact one is gone. A restore that starts while the Backup's cleanup runs can lose its dump between the check and the operator's read, and then completes on another run's data without any error. The restore now reads the Backup from the API server before creating the operator Restore and fails when it is being deleted or is gone. The copy the reconcile was handed comes from the cache and can predate the delete; a transient read error keeps the RestoreJob waiting like any other failed dump check. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadb_restore_dump_test.go | 52 +++++++++++++++++++ .../mariadbstrategy_controller.go | 14 +++++ 2 files changed, 66 insertions(+) diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go index 6c8202adf..218c91de6 100644 --- a/internal/backupcontroller/mariadb_restore_dump_test.go +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -14,6 +14,7 @@ import ( "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" @@ -184,6 +185,57 @@ func recordedDump(name string) map[string]string { return map[string]string{mariadbDumpFileKey: name, mariadbDumpTimeKey: mdbRestoreTime} } +// A Backup being deleted is deleting its dump, so a restore that starts +// now can find it gone once the operator reads it, and the operator then +// restores the nearest other dump. The Backup is read from the API server, +// since the copy the reconcile was handed can predate the delete. +func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + present := func() *fakeS3Lister { + return &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}}} + } + cases := map[string]map[string]string{"recorded dump": recordedDump(mdbRestoreDump), "legacy Backup": nil} + for name, md := range cases { + t.Run(name+", being deleted", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, md, present(), creds) + ctx := context.Background() + live := &backupsv1alpha1.Backup{} + if err := f.c.Get(ctx, client.ObjectKeyFromObject(f.backup), live); err != nil { + t.Fatal(err) + } + live.Finalizers = []string{backupFinalizer} + if err := f.c.Update(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Delete(ctx, live); err != nil { + t.Fatal(err) + } + f.reconcile(t) + f.wantFailed(t, "src-bk", "being deleted") + }) + t.Run(name+", already gone", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, md, present(), creds) + if err := f.c.Delete(context.Background(), f.backup.DeepCopy()); err != nil { + t.Fatal(err) + } + f.reconcile(t) + f.wantFailed(t, "src-bk", "is gone") + }) + } + t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + f.wantWaiting(t, f.reconcile(t), "src-bk", "apiserver unavailable") + }) +} + func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) { creds := s3CredsSecret("tenant", "creds") diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 6d6312eb4..e3e00dbec 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -669,6 +669,20 @@ func (e mariadbRestoreRefused) Error() string { return string(e) } // checked to still exist. A Backup that records no dump is resolved by // legacyMariaDBRestoreTargetTime. func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) { + // Deleting a Backup deletes its dump, and the operator restores the + // nearest other dump once its own is gone, so a restore must not start + // from a Backup being deleted. The copy in hand can predate the delete; + // the cleanup holds the Backup only for RestoreJobs it can already see. + live := &backupsv1alpha1.Backup{} + if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(backup), live); err != nil { + if apierrors.IsNotFound(err) { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is gone, and its dump may be gone with it", backup.Namespace, backup.Name)) + } + return "", fmt.Errorf("read Backup %s/%s: %w", backup.Namespace, backup.Name, err) + } + if !live.DeletionTimestamp.IsZero() { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is being deleted, and its dump with it", backup.Namespace, backup.Name)) + } // A failed Job is Complete=True too, with reason JobFailed, and a // BackupJob that took Complete=True for success made a Backup of it. if cond := apimeta.FindStatusCondition(src.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed { From 654c543264bbcba9c939a7c8b436ffbc02340c4d Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:04:53 +0300 Subject: [PATCH 129/174] fix(backups): see a MariaDB restore the cache has not caught up with A MariaDB restore now checks on the API server that its Backup is not being deleted before it starts, so a restore that started first exists on the API server before the delete. The cleanup read RestoreJobs from the informer cache, which can lag behind, and could delete the dump under a restore it had not seen yet. Reading them uncached closes that window from both sides. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/backup_controller.go | 15 +++++++++++++++ .../backupcontroller/mariadbstrategy_cleanup.go | 2 +- .../mariadbstrategy_cleanup_test.go | 12 ++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index 9d755059e..d8c69bf1c 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -43,6 +43,10 @@ type BackupReconciler struct { // cleanups project them into the Backup's namespace, and the MariaDB // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig + // APIReader is the manager's uncached reader. The MariaDB cleanup lists + // the RestoreJobs of a Backup through it: a restore that started before + // the delete may not be in the cache yet. Wired in SetupWithManager. + APIReader client.Reader // newS3ObjectLister and newS3ObjectRemover override the S3 client the // MariaDB cleanup deletes a dump with; nil means a real minio client. @@ -294,7 +298,18 @@ func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backup // SetupWithManager registers the BackupReconciler with the Manager. func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.APIReader = mgr.GetAPIReader() return ctrl.NewControllerManagedBy(mgr). For(&backupsv1alpha1.Backup{}). Complete(r) } + +// apiReader returns the uncached reader. Reconcilers built by hand in unit +// tests have none and fall back to their fake Client, which has no cache to +// bypass. +func (r *BackupReconciler) apiReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 94a7d10f4..531b806c9 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -200,7 +200,7 @@ func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader cli // backup and has not finished, or "". func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { list := &backupsv1alpha1.RestoreJobList{} - if err := r.List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { + if err := r.apiReader().List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { return "", fmt.Errorf("list RestoreJobs: %w", err) } for i := range list.Items { diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index ddca34523..20f2757d4 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -763,6 +763,18 @@ func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { } }) } + // A restore checks that its Backup is not being deleted before it + // starts, so one that started first is on the API server before the + // delete, though not necessarily in this controller's cache yet. + t.Run("a restore the cache has not seen yet holds it", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, restoreJobOf("rj", "bk", backupsv1alpha1.RestoreJobPhaseRunning)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the restore runs", res, err, bucket.removed) + } + }) t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) { bucket := &fakeS3Bucket{} r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), From 37a74e59394a69700e99b04a697f50b9a78e2625 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:05:51 +0300 Subject: [PATCH 130/174] test(backups): reach the S3 calls the MariaDB cleanup timeout bounds Contesting a recorded dump lists the bucket before the delete, and the timeout test supplied no lister, so a real client failed on DNS before the blocking delete was ever called and the suite stayed green with the timeout removed. The test now lists the dump through the fake bucket, blocks in the listing and in the delete in turn, and requires that the blocking call was entered. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup_test.go | 75 +++++++++++++------ 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 20f2757d4..0ce6b2d01 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -961,36 +961,69 @@ func TestCleanupMariaDBBackup_PlatformEndpointFallsBackToTheConfiguredOne(t *tes } } -// blockingBucket answers only when its context ends, as an unreachable -// endpoint does. -type blockingBucket struct{ fakeS3Bucket } +// blockingBucket lists the dump it holds and answers the call named by +// blockOn only when its context ends, as an unreachable endpoint does. +type blockingBucket struct { + fakeS3Bucket + blockOn string + entered chan struct{} +} -func (b *blockingBucket) RemoveObject(ctx context.Context, _, _ string) error { +func (b *blockingBucket) block(ctx context.Context, call string) bool { + if b.blockOn != call { + return false + } + close(b.entered) <-ctx.Done() - return ctx.Err() + return true +} + +func (b *blockingBucket) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { + if b.block(ctx, "list") { + return nil, ctx.Err() + } + return b.fakeS3Bucket.ListObjects(ctx, bucket, prefix, recursive) +} + +func (b *blockingBucket) RemoveObject(ctx context.Context, bucket, key string) error { + if b.block(ctx, "remove") { + return ctx.Err() + } + return b.fakeS3Bucket.RemoveObject(ctx, bucket, key) } func TestCleanupMariaDBBackup_BoundsAnUnreachableEndpoint(t *testing.T) { prev := mariadbDumpCleanupTimeout mariadbDumpCleanupTimeout = 50 * time.Millisecond t.Cleanup(func() { mariadbDumpCleanupTimeout = prev }) - bucket := &blockingBucket{} - c := newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) - r := &BackupReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(20), - newS3ObjectRemover: func(s3Connection) (s3ObjectRemover, error) { return bucket, nil }} + for _, call := range []string{"list", "remove"} { + t.Run(call, func(t *testing.T) { + bucket := &blockingBucket{blockOn: call, entered: make(chan struct{})} + bucket.objects = []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}} + c := newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r := &BackupReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(20), + newS3ObjectLister: func(s3Connection) (s3ObjectLister, error) { return bucket, nil }, + newS3ObjectRemover: func(s3Connection) (s3ObjectRemover, error) { return bucket, nil }} - done := make(chan ctrl.Result, 1) - go func() { - res, _ := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) - done <- res - }() - select { - case res := <-done: - if !requeues(res) { - t.Errorf("res = %+v, want the Backup held and retried", res) - } - case <-time.After(5 * time.Second): - t.Fatal("cleanup did not give up on an unreachable endpoint") + done := make(chan ctrl.Result, 1) + go func() { + res, _ := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + done <- res + }() + select { + case res := <-done: + select { + case <-bucket.entered: + default: + t.Fatalf("cleanup returned without reaching the blocking %s", call) + } + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + case <-time.After(5 * time.Second): + t.Fatalf("cleanup did not give up on an unreachable endpoint blocking %s", call) + } + }) } } From bbeca3453534124975682bda008f0d025aaed65c Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:07:03 +0300 Subject: [PATCH 131/174] fix(backups): retry a failed read of the platform backup credentials The MariaDB cleanup reaches the platform bucket through the platform's current endpoint so that a Backup taken before the endpoint moved is still deleted. A failed read of the platform credentials made it fall back to the endpoint recorded on the operator Backup, which may be the old address: there the dump is not found, a missing dump counts as already deleted, and the operator Backup that pointed at the real dump was deleted with a success log. A read that fails with an API error now holds the Backup and retries. A missing or malformed platform Secret still falls back, since it is a standing state that a retry would not change. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup.go | 10 +++++-- .../mariadbstrategy_cleanup_test.go | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 531b806c9..b4617918e 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -220,12 +220,18 @@ func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv // Secret the platform credentials are projected to, is reached as the // platform reaches it now, so a Backup taken before the endpoint moved is // still deleted. Any other bucket, or the platform bucket while the -// platform credentials cannot be read, is reached as op recorded it; a -// delete that fails there is handled by retryMariaDBCleanup like any other. +// platform credentials are missing or malformed, is reached as op recorded +// it; a delete that fails there is handled by retryMariaDBCleanup like any +// other. A failed read of the platform credentials is retried instead: the +// recorded endpoint may be the one the platform moved off, where the dump +// would be found missing and reported deleted. func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) { if r.CredentialsConfig.IsEnabled() { conn, bucket, err := r.platformS3Connection(ctx) + var projErr *ProjectionError switch { + case errors.As(err, &projErr) && projErr.Reason == ReasonAPIError: + return s3Connection{}, err case err != nil: getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err) case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName: diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 0ce6b2d01..1eecfeaff 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -869,6 +869,36 @@ func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsFallBackToTheOperator } } +// A missing or malformed platform Secret is a standing state, and the +// operator Backup's own credentials are the best left. A failed read says +// nothing about the Secret: falling back then can list a moved endpoint's +// old address, find no dump there and report it deleted. +func TestCleanupMariaDBBackup_TransientPlatformCredentialsReadIsRetried(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.Secret); ok && key.Name == "cozy-backups-credentials" { + return errors.New("apiserver unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.conns) != 0 || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("conns=%v removed=%v operatorBackupExists=%v, want the bucket untouched", bucket.conns, bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteHold(t *testing.T) { bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) From 3334152da7d78e755ab7d3cc85ed8d459c0ba86e Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:08:43 +0300 Subject: [PATCH 132/174] fix(backups): report a recorded MariaDB dump that was not found A recorded dump missing from the listing was logged as deleted, with no event. That is right when an earlier attempt or maxRetention removed it, but the listing goes through the platform's current endpoint, and after an endpoint move the dump can still sit on the old storage: the cleanup then deleted the operator Backup that pointed at it and said it had deleted the dump. The cleanup now records a Normal ArtifactAlreadyAbsent event naming the key and the endpoint it listed, so whoever later looks for that dump can tell where it was not found. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup.go | 36 ++++++++++++------- .../mariadbstrategy_cleanup_test.go | 14 ++++++-- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index b4617918e..25775185b 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -118,8 +118,10 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac if err != nil { return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) } + absent := false if name != "" { - other, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name) + other, found, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name) + absent = !found if err != nil { return r.retryMariaDBCleanup(ctx, backup, location+name, err) } @@ -158,42 +160,52 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac if err := r.Delete(ctx, op, &client.DeleteOptions{PropagationPolicy: &policy}); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, fmt.Errorf("delete k8s.mariadb.com/Backup %s/%s: %w", op.Namespace, op.Name, err) } + if absent { + logger.Info("MariaDB dump already absent; deleted its operator Backup", "backup", backup.Name, "dump", location+name, "endpoint", conn.Endpoint, "operatorBackup", opName) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactAlreadyAbsent", + "%s was not found when listed through %s, so it was deleted earlier or never reached that storage; k8s.mariadb.com/Backup %s is deleted", + location+name, conn.Endpoint, opName) + } + return ctrl.Result{}, nil + } logger.Info("deleted MariaDB dump and its operator Backup", "backup", backup.Name, "dump", location+name, "operatorBackup", opName) return ctrl.Result{}, nil } // contestedMariaDBDump returns the name of another operator Backup on op's // bucket and prefix whose run, finished or not, could have written the dump -// name, or "" when none could. The size probe that recorded name let only -// runs finished by then contest it, and a run can upload its dump and prune -// op's through maxRetention before it completes. A dump no longer in the -// bucket is contested by nobody: deleting it again removes nothing. -func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, error) { +// name, or "" when none could, and whether the listing found the dump. The +// size probe that recorded name let only runs finished by then contest it, +// and a run can upload its dump and prune op's through maxRetention before +// it completes. A dump no longer in the bucket is contested by nobody: +// deleting it again removes nothing. +func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, bool, error) { ts, ok := parseMariaDBLogicalBackupFile(name) if !ok { - return "", fmt.Errorf("%q is not the name of a mariadb-operator dump", name) + return "", false, fmt.Errorf("%q is not the name of a mariadb-operator dump", name) } s3 := op.Spec.Storage.S3 key := mariadbOperatorS3Prefix(s3.Prefix) + name objects, err := lister.ListObjects(ctx, s3.Bucket, key, false) if err != nil { - return "", fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) + return "", false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) } i := slices.IndexFunc(objects, func(obj s3ObjectInfo) bool { return obj.Key == key }) if i < 0 { - return "", nil + return "", false, nil } others, err := neighbourMariaDBRuns(ctx, reader, op, true) if err != nil { - return "", fmt.Errorf("list operator Backups: %w", err) + return "", true, fmt.Errorf("list operator Backups: %w", err) } d := mariadbDump{Key: key, Name: name, Timestamp: ts, LastModified: objects[i].LastModified} for _, other := range others { if other.placesOnEitherClock(d) { - return other.name, nil + return other.name, true, nil } } - return "", nil + return "", true, nil } // activeRestoreOf returns the name of a RestoreJob that restores from diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 1eecfeaff..da9e0902e 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -165,7 +165,7 @@ func assertReleasedUntouched(t *testing.T, r *BackupReconciler, rec *record.Fake } func TestCleanupMariaDBBackup_DeletesRecordedDumpThenOperatorBackup(t *testing.T) { - bucket := &fakeS3Bucket{} + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}} r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) @@ -512,7 +512,7 @@ func TestContestedMariaDBDump_RejectsANameWithoutADumpTime(t *testing.T) { name := "backup.not-a-time.sql" bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + name, LastModified: cleanupCreated}}} c := newMariaDBStrategyTestClient(t, op) - if _, err := contestedMariaDBDump(context.Background(), bucket, c, op, name); err == nil { + if _, _, err := contestedMariaDBDump(context.Background(), bucket, c, op, name); err == nil { t.Fatal("err = nil, want the unreadable dump time reported") } } @@ -534,9 +534,17 @@ func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T if operatorBackupExists(t, r.Client) { t.Error("operator Backup kept, want it deleted") } - if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") { + events := drainEvents(rec) + if hasEvent(events, "Warning ArtifactNotDeleted") { t.Errorf("events = %v, want no ArtifactNotDeleted", events) } + // The listing may have reached other storage than the dump went to, as + // after an endpoint move, so the absence is reported with where it was + // looked for rather than logged as a delete. + if joined := strings.Join(events, "\n"); !hasEvent(events, "Normal ArtifactAlreadyAbsent") || + !strings.Contains(joined, "s3://bkt/"+cleanupPrefix+cleanupDumpName) || !strings.Contains(joined, "through s3.example:9000") { + t.Errorf("events = %v, want a Normal ArtifactAlreadyAbsent naming the key and the endpoint listed", events) + } } // A run started after the recorded dump was uploaded cannot have written it. From d57ebbe808e831637062a920ba9f12fd5580a541 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:09:15 +0300 Subject: [PATCH 133/174] docs(backups): state what the MariaDB dump delete does not cover The comment on who deletes a MariaDB Backup listed namespace teardown without saying that teardown can remove the operator Backup first, in which case the dump is left behind with a Warning, so it promised a delete the code does not make. The remover contract claimed a missing key is never an error, which holds only for a backend that answers the delete with 204, the one status minio-go accepts. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/artifact_size.go | 4 +++- internal/backupcontroller/backup_controller.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 40d827c62..97a67a729 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -39,7 +39,9 @@ type s3ObjectLister interface { } type s3ObjectRemover interface { - // RemoveObject deletes key; a key that does not exist is not an error. + // RemoveObject deletes key. S3 answers 204 for a key that does not + // exist and minio-go takes 204 as success; a backend answering a + // missing key with any other status makes it an error. RemoveObject(ctx context.Context, bucket, key string) error } diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index d8c69bf1c..a632f851d 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -142,7 +142,9 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // a dump on the platform bucket is deleted here or never. A dump in // a tenant's own bucket is deleted too: the Backup proves it owns // it, and tenants cannot delete Backups, so the delete comes from an - // administrator or from namespace teardown (see cleanupMariaDBBackup). + // administrator or from namespace teardown. Teardown can remove the + // operator Backup first, and the dump is then left behind with a + // Warning, since its storage is read from it (see cleanupMariaDBBackup). return r.cleanupMariaDBBackup(ctx, backup) case strategyv1alpha1.MongoDBStrategyKind: // Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the From 14a842ad168c0fbe3c650a6ff541c4af163ee114 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:15:18 +0300 Subject: [PATCH 134/174] fix(backups): find a MariaDB operator Restore the cache has not seen The MariaDB restore creates its operator Restore once and then follows it, finding it by label in the informer cache. Nothing watches operator Restores, so a reconcile that ran before the new Restore reached the cache saw none: it created a second one, or, now that a restore refuses a Backup being deleted, failed the RestoreJob while its Restore was still running, after which the cleanup deleted the dump that Restore was reading. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadb_restore_dump_test.go | 38 +++++++++++++++++++ .../mariadbstrategy_controller.go | 6 ++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go index 218c91de6..1919d41c9 100644 --- a/internal/backupcontroller/mariadb_restore_dump_test.go +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -222,6 +222,44 @@ func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) { f.wantFailed(t, "src-bk", "is gone") }) } + // The refusal applies only before the operator Restore exists. One + // created by an earlier reconcile but not yet in the cache must be + // found and followed, or the RestoreJob fails under a running restore + // and the cleanup deletes the dump that restore reads. + t.Run("an operator Restore the cache has not seen yet is followed", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + ctx := context.Background() + live := &backupsv1alpha1.Backup{} + if err := f.c.Get(ctx, client.ObjectKeyFromObject(f.backup), live); err != nil { + t.Fatal(err) + } + live.Finalizers = []string{backupFinalizer} + if err := f.c.Update(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Delete(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Get(ctx, client.ObjectKeyFromObject(live), live); err != nil || live.DeletionTimestamp.IsZero() { + t.Fatalf("Backup not left being deleted: %v", err) + } + live.ResourceVersion = "" + apiServer := newMariaDBStrategyTestClient(t, live, &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }}) + f.r.APIReader = apiServer + f.reconcile(t) + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed under a running restore: %s", f.rj.Status.Message) + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want the existing one followed", len(got)) + } + }) t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index e3e00dbec..b97f7548b 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -756,10 +756,12 @@ func (r *RestoreJobReconciler) awaitMariaDBDumpCheck(ctx context.Context, rj *ba } // findMariaDBRestoreForJob returns the k8s.mariadb.com/Restore labelled -// with the RestoreJob, if a previous reconcile already created one. +// with the RestoreJob, if a previous reconcile already created one. It is +// read uncached: one the cache has not caught up with would be created +// again, or, once its Backup is being deleted, refused while it runs. func (r *RestoreJobReconciler) findMariaDBRestoreForJob(ctx context.Context, rj *backupsv1alpha1.RestoreJob) (*mariadbtypes.Restore, error) { list := &mariadbtypes.RestoreList{} - if err := r.List(ctx, list, + if err := r.apiReader().List(ctx, list, client.InNamespace(rj.Namespace), client.MatchingLabels{ backupsv1alpha1.OwningJobNameLabel: rj.Name, From ee6404a0c88af0850910b9bbb25ad8cc292f30ed Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:28:45 +0300 Subject: [PATCH 135/174] fix(backups): retry a failed lookup of a MariaDB operator Restore The lookup now goes to the API server, which can time out or throttle, and a failed lookup failed the RestoreJob. It says nothing about whether the operator Restore exists, so the RestoreJob could be failed while its Restore was running, and the Backup's cleanup would then stop waiting for it and delete the dump the Restore reads. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadb_restore_dump_test.go | 22 +++++++++++++++++++ .../mariadbstrategy_controller.go | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go index 1919d41c9..5ec96e663 100644 --- a/internal/backupcontroller/mariadb_restore_dump_test.go +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -260,6 +260,28 @@ func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) { t.Errorf("created %d operator Restores; want the existing one followed", len(got)) } }) + // A failed lookup says nothing about whether the operator Restore + // exists, so the RestoreJob must not be failed under one that runs. + t.Run("an operator Restore lookup that fails is retried", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup); err == nil { + t.Error("err = nil, want the failed lookup returned for a retry") + } + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed on a lookup error: %s", f.rj.Status.Message) + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores without knowing whether one exists", len(got)) + } + }) t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index b97f7548b..28a24a14e 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -598,7 +598,9 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest mdbRestore, err := r.findMariaDBRestoreForJob(ctx, restoreJob) if err != nil { - return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) + // Retried: failing the RestoreJob here could fail it under an + // operator Restore that is running. + return ctrl.Result{}, fmt.Errorf("find k8s.mariadb.com/Restore of RestoreJob %s/%s: %w", restoreJob.Namespace, restoreJob.Name, err) } // The dump is chosen once, as the operator Restore is created; one that // already exists has made its choice. From 82468082890a9e6a587ae6c94f7ac02622f6610d Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:30:41 +0300 Subject: [PATCH 136/174] fix(backups): hold a MariaDB dump while an operator Restore reads it The cleanup waited for a restore by the phase of its RestoreJob, but the phase is not the restore: a RestoreJob fails on its deadline while the operator Restore it created keeps running, and that Restore still reads the dump. Deleting the dump then made mariadb-operator restore the nearest other one. The cleanup now also holds the Backup while an operator Restore of its operator Backup has not been marked Complete, which mariadb-operator does once the Restore's Job has completed or failed, so what keeps the dump is the restore that reads it rather than the status of the request. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup.go | 32 +++++-- .../mariadbstrategy_cleanup_test.go | 83 +++++++++++++++++++ 2 files changed, 108 insertions(+), 7 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 25775185b..17d612955 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -59,15 +59,15 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac // The operator Restore reads the dump through the operator Backup, so // neither may go while a restore from this Backup runs. - restore, err := r.activeRestoreOf(ctx, backup) + restore, err := r.activeRestoreOf(ctx, backup, opName) if err != nil { return ctrl.Result{}, err } if restore != "" { - logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restorejob", restore) + logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restore", restore) if r.Recorder != nil { r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactCleanupWaiting", - "RestoreJob %s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump", + "%s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump", restore, mariadbSkipArtifactCleanupAnnotation) } return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil @@ -208,9 +208,14 @@ func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader cli return "", true, nil } -// activeRestoreOf returns the name of a RestoreJob that restores from -// backup and has not finished, or "". -func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup) (string, error) { +// activeRestoreOf names a restore from backup that has not finished: a +// RestoreJob of it, or an operator Restore reading the dump of opName, the +// operator Backup it was made of; "" when there is none. The operator +// Restore is checked on its own because a RestoreJob can fail, on its +// deadline, while the Restore it created keeps reading the dump. +// mariadb-operator marks a Restore Complete=True once its Job has +// completed or failed. +func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup, opName string) (string, error) { list := &backupsv1alpha1.RestoreJobList{} if err := r.apiReader().List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { return "", fmt.Errorf("list RestoreJobs: %w", err) @@ -221,7 +226,20 @@ func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv continue } if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded && rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { - return rj.Name, nil + return "RestoreJob " + rj.Name, nil + } + } + restores := &mariadbtypes.RestoreList{} + if err := r.apiReader().List(ctx, restores, client.InNamespace(backup.Namespace)); err != nil { + return "", fmt.Errorf("list k8s.mariadb.com/Restores: %w", err) + } + for i := range restores.Items { + restore := &restores.Items[i] + if restore.Spec.BackupRef == nil || restore.Spec.BackupRef.Name != opName { + continue + } + if !apimeta.IsStatusConditionTrue(restore.Status.Conditions, mariadbtypes.ConditionTypeComplete) { + return "k8s.mariadb.com/Restore " + restore.Name, nil } } return "", nil diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index da9e0902e..ec2dea1a8 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -753,6 +753,89 @@ func restoreJobOf(name, backupName string, phase backupsv1alpha1.RestoreJobPhase } } +// operatorRestoreOf is a k8s.mariadb.com/Restore reading the dump of the +// operator Backup backupRef, with the given Complete condition. +func operatorRestoreOf(name, backupRef string, complete *metav1.Condition) *mariadbtypes.Restore { + restore := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: name}} + if backupRef != "" { + restore.Spec.BackupRef = &mariadbtypes.BackupReference{Name: backupRef} + } + if complete != nil { + restore.Status.Conditions = []metav1.Condition{*complete} + } + return restore +} + +func restoreComplete(status metav1.ConditionStatus, reason string) *metav1.Condition { + return &metav1.Condition{Type: mariadbtypes.ConditionTypeComplete, Status: status, Reason: reason, LastTransitionTime: metav1.Now()} +} + +// The RestoreJob's phase is not the restore: one failed on its deadline +// leaves its operator Restore running, and that Restore still reads the +// dump. Until the operator marks it Complete, as it does a finished or a +// failed Job, the dump stays. +func TestCleanupMariaDBBackup_WaitsForAnOperatorRestoreOfItsDump(t *testing.T) { + held := map[string]*metav1.Condition{ + "running": restoreComplete(metav1.ConditionFalse, mariadbtypes.ConditionReasonJobRunning), + "suspended": restoreComplete(metav1.ConditionFalse, mariadbtypes.ConditionReasonJobSuspended), + "not yet reconciled": nil, + } + for name, cond := range held { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("rj", "bk", backupsv1alpha1.RestoreJobPhaseFailed), operatorRestoreOf("rj-abcde", "op-bk", cond)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the operator Restore runs", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Normal ArtifactCleanupWaiting") || !strings.Contains(strings.Join(events, "\n"), "k8s.mariadb.com/Restore rj-abcde") { + t.Errorf("events = %v, want a Normal ArtifactCleanupWaiting naming the operator Restore", events) + } + }) + } + released := map[string]*mariadbtypes.Restore{ + "completed": operatorRestoreOf("rj-abcde", "op-bk", restoreComplete(metav1.ConditionTrue, mariadbtypes.ConditionReasonJobComplete)), + "failed": operatorRestoreOf("rj-abcde", "op-bk", restoreComplete(metav1.ConditionTrue, mariadbtypes.ConditionReasonJobFailed)), + "of another operator Backup": operatorRestoreOf("rj-abcde", "op-other", nil), + "from storage, not a Backup": operatorRestoreOf("rj-abcde", "", nil), + } + for name, restore := range released { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), restore) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + }) + } + t.Run("one the cache has not seen yet", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, operatorRestoreOf("rj-abcde", "op-bk", nil)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held", res, err, bucket.removed) + } + }) + t.Run("a failed listing deletes nothing", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil || len(bucket.removed) != 0 { + t.Fatalf("err=%v removed=%v, want the error returned and nothing deleted", err, bucket.removed) + } + }) +} + func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { for _, phase := range []backupsv1alpha1.RestoreJobPhase{backupsv1alpha1.RestoreJobPhaseEmpty, backupsv1alpha1.RestoreJobPhasePending, backupsv1alpha1.RestoreJobPhaseRunning} { t.Run(string(phase), func(t *testing.T) { From 0195a66faeea7a143ccf3aea2ba2f95df67d9178 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:35:33 +0300 Subject: [PATCH 137/174] fix(backups): release a MariaDB Backup when the operator CRDs are gone Listing the operator Restores of a Backup before reading its operator Backup made the cleanup fail on a cluster without the mariadb-operator CRDs, retried for good, so the Backup and its namespace never went away. The operator Backup read that follows already releases the Backup in that case; with no Restore kind no restore can run, so the listing now lets the cleanup reach it. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/mariadbstrategy_cleanup.go | 5 +++++ .../backupcontroller/mariadbstrategy_cleanup_test.go | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 17d612955..3ff8adfbe 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -231,6 +231,11 @@ func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv } restores := &mariadbtypes.RestoreList{} if err := r.apiReader().List(ctx, restores, client.InNamespace(backup.Namespace)); err != nil { + // Without the operator's CRDs no Restore can run; the operator + // Backup read that follows releases the Backup. + if apimeta.IsNoMatchError(err) { + return "", nil + } return "", fmt.Errorf("list k8s.mariadb.com/Restores: %w", err) } for i := range restores.Items { diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index ec2dea1a8..e9a90029b 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -1148,6 +1148,9 @@ func TestCleanupMariaDBBackup_BoundsAnUnreachableEndpoint(t *testing.T) { } } +// Without the mariadb-operator CRDs every k8s.mariadb.com read fails, the +// listing of operator Restores included: no Restore can run then, so the +// Backup is released rather than held for good. func TestCleanupMariaDBBackup_ReleasesWhenTheOperatorCRDIsGone(t *testing.T) { bucket := &fakeS3Bucket{} c := interceptor.NewClient(newMariaDBStrategyTestClient(t, tenantNamespace()).(client.WithWatch), interceptor.Funcs{ @@ -1157,6 +1160,12 @@ func TestCleanupMariaDBBackup_ReleasesWhenTheOperatorCRDIsGone(t *testing.T) { } return c.Get(ctx, key, obj, opts...) }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return &apimeta.NoKindMatchError{GroupKind: mariadbtypes.GroupVersion.WithKind("Restore").GroupKind()} + } + return c.List(ctx, list, opts...) + }, }) r, rec := cleanupReconcilerWithClient(c, bucket) res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) From f2507b4b84f905e13c0dea1565c64f9cf5699a7e Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:35:33 +0300 Subject: [PATCH 138/174] docs(backups): name what the MariaDB cleanup reads uncached The reader's comment named only RestoreJobs, and the restore path said the cleanup holds a Backup only for RestoreJobs it can see, while the cleanup now lists both RestoreJobs and operator Restores straight from the API server. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- internal/backupcontroller/backup_controller.go | 5 +++-- internal/backupcontroller/mariadbstrategy_controller.go | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index a632f851d..b67c640ab 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -44,8 +44,9 @@ type BackupReconciler struct { // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig // APIReader is the manager's uncached reader. The MariaDB cleanup lists - // the RestoreJobs of a Backup through it: a restore that started before - // the delete may not be in the cache yet. Wired in SetupWithManager. + // the RestoreJobs and operator Restores of a Backup through it: a + // restore that started before the delete may not be in the cache yet. + // Wired in SetupWithManager. APIReader client.Reader // newS3ObjectLister and newS3ObjectRemover override the S3 client the diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 28a24a14e..edff9d187 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -674,7 +674,8 @@ func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, rj // Deleting a Backup deletes its dump, and the operator restores the // nearest other dump once its own is gone, so a restore must not start // from a Backup being deleted. The copy in hand can predate the delete; - // the cleanup holds the Backup only for RestoreJobs it can already see. + // the cleanup holds the Backup only for restores that exist when it + // lists them. live := &backupsv1alpha1.Backup{} if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(backup), live); err != nil { if apierrors.IsNotFound(err) { From a6a704768ce0deb6013a6ab33772d46d0e16da6e Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Wed, 11 Nov 2026 15:41:03 +0300 Subject: [PATCH 139/174] test(backups): pin a failed RestoreJob listing in the MariaDB cleanup A RestoreJob listing that fails says nothing about whether a restore reads the dump. Swallowing that error left the whole suite green, while it would let the cleanup delete the dump under a running restore. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../mariadbstrategy_cleanup_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index e9a90029b..354bf83f3 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -866,6 +866,21 @@ func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the restore runs", res, err, bucket.removed) } }) + t.Run("a failed listing deletes nothing", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Fatalf("err=%v removed=%v, want the error returned and nothing deleted", err, bucket.removed) + } + }) t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) { bucket := &fakeS3Bucket{} r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), From b3ca798bc438f0854d624b19d11fb1bab67a73d7 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Thu, 12 Nov 2026 17:01:29 +0300 Subject: [PATCH 140/174] fix(backups): read the runs that may own a MariaDB dump uncached Before deleting a dump the cleanup checks every other operator Backup on the same prefix that could have written it, and it read them from the informer cache. A run whose operator Backup had not reached the cache yet but had already uploaded its dump was missed, and its dump deleted as this Backup's. Both the recorded and the legacy paths now list the runs from the API server, as the restore checks already do. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../backupcontroller/backup_controller.go | 7 ++--- .../mariadbstrategy_cleanup.go | 4 +-- .../mariadbstrategy_cleanup_test.go | 27 +++++++++++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index b67c640ab..5e6292635 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -44,9 +44,10 @@ type BackupReconciler struct { // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig // APIReader is the manager's uncached reader. The MariaDB cleanup lists - // the RestoreJobs and operator Restores of a Backup through it: a - // restore that started before the delete may not be in the cache yet. - // Wired in SetupWithManager. + // the RestoreJobs and operator Restores of a Backup, and the operator + // Backups that may own its dump, through it: a restore that started + // before the delete, or a run that has already uploaded its dump, may + // not be in the cache yet. Wired in SetupWithManager. APIReader client.Reader // newS3ObjectLister and newS3ObjectRemover override the S3 client the diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go index 3ff8adfbe..912bb0038 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -120,7 +120,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac } absent := false if name != "" { - other, found, err := contestedMariaDBDump(s3ctx, lister, r.Client, op, name) + other, found, err := contestedMariaDBDump(s3ctx, lister, r.apiReader(), op, name) absent = !found if err != nil { return r.retryMariaDBCleanup(ctx, backup, location+name, err) @@ -130,7 +130,7 @@ func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *bac fmt.Sprintf("the recorded dump also falls in the lifetime of k8s.mariadb.com/Backup %s, which may have written it; the operator Backup is kept", other)), nil } } else { - dump, err := findMariaDBDumpWith(s3ctx, lister, r.Client, op, backup.Spec.TakenAt.Time, true) + dump, err := findMariaDBDumpWith(s3ctx, lister, r.apiReader(), op, backup.Spec.TakenAt.Time, true) if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 354bf83f3..1d2f10cbe 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -547,6 +547,33 @@ func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T } } +// The runs that may own a dump are read from the API server: a run whose +// operator Backup has not reached the cache yet can already have uploaded +// its dump, and missing it would delete that dump as this Backup's. +func TestCleanupMariaDBBackup_ContestsWithRunsTheCacheHasNotSeen(t *testing.T) { + started := cleanupCreated.Add(time.Minute) + theirs := dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip") + unseen := func() *mariadbtypes.Backup { + b := cleanupOperatorBackup(t) + b.Name = "op-unseen" + b.CreationTimestamp = metav1.NewTime(started) + b.Status.Conditions = nil + return b + } + cases := map[string]string{"recorded dump": strings.TrimPrefix(theirs, cleanupPrefix), "legacy Backup": ""} + for name, recorded := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: started.Add(time.Minute)}}} + backup := cleanupBackup(recorded) + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), unseen()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + // A run started after the recorded dump was uploaded cannot have written it. func TestCleanupMariaDBBackup_DeletesItsRecordedDumpDespiteALaterUnfinishedRun(t *testing.T) { key := cleanupPrefix + cleanupDumpName From 6297ebd70865dfd8a5f084521715a7720c7b8971 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 02:55:31 +0300 Subject: [PATCH 141/174] fix(linstor): run LINSTOR init containers on the platform server image piraeus-operator fills in its own default LINSTOR server image for every container this package leaves unset, and two of those are init containers: run-migration in the controller pod and setup-lvm-configuration in the satellite pods. Only the main containers were pinned to the platform build. run-migration upgrades the LINSTOR database in place. When the operator default is newer than the platform build, it migrates the schema ahead of the controller, the controller then fails to read the migrated database and crash-loops, and the migration cannot be rolled back. This stayed hidden while the operator default was older than the version built here; an operator update that moves the default past it exposes it during the window between the operator and linstor releases reconciling. Pin both init containers to the same image as the main containers, so the database moves only together with the controller. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../system/linstor/templates/cluster.yaml | 6 +++ .../linstor/templates/satellites-cozy.yaml | 3 ++ .../linstor/tests/server-image_test.yaml | 40 +++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 packages/system/linstor/tests/server-image_test.yaml diff --git a/packages/system/linstor/templates/cluster.yaml b/packages/system/linstor/templates/cluster.yaml index ba5154375..c648c0a8c 100644 --- a/packages/system/linstor/templates/cluster.yaml +++ b/packages/system/linstor/templates/cluster.yaml @@ -36,6 +36,12 @@ spec: enabled: true podTemplate: spec: + {{- /* run-migration upgrades the database in place: left on the + operator's default image it migrates ahead of the controller, + irreversibly. */}} + initContainers: + - name: run-migration + image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} containers: - name: linstor-controller image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} diff --git a/packages/system/linstor/templates/satellites-cozy.yaml b/packages/system/linstor/templates/satellites-cozy.yaml index 7897189f4..bb6240128 100644 --- a/packages/system/linstor/templates/satellites-cozy.yaml +++ b/packages/system/linstor/templates/satellites-cozy.yaml @@ -11,6 +11,9 @@ spec: spec: # host-network is recommended by Piraeus while it is not default in the upstream hostNetwork: true + initContainers: + - name: setup-lvm-configuration + image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} containers: - name: linstor-satellite image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} diff --git a/packages/system/linstor/tests/server-image_test.yaml b/packages/system/linstor/tests/server-image_test.yaml new file mode 100644 index 000000000..d63d19ae4 --- /dev/null +++ b/packages/system/linstor/tests/server-image_test.yaml @@ -0,0 +1,40 @@ +suite: every LINSTOR server container runs the platform image + +# piraeus-operator ships its own default for the LINSTOR server image and +# applies it to every container this package leaves unset. The init +# containers matter most: run-migration upgrades the LINSTOR database in +# place, so an operator default newer than the platform build migrates the +# schema ahead of the controller, which then refuses to start, and the +# migration cannot be undone. Pinning them to the same image as the main +# containers makes the database move only together with the controller. + +templates: + - templates/cluster.yaml + - templates/satellites-cozy.yaml + +set: + piraeusServer: + image: + repository: registry.example/piraeus-server + tag: v0.0.0-test + +tests: + - it: the controller migration runs the platform image + template: templates/cluster.yaml + asserts: + - equal: + path: spec.controller.podTemplate.spec.initContainers[?(@.name=="run-migration")].image + value: registry.example/piraeus-server:v0.0.0-test + - equal: + path: spec.controller.podTemplate.spec.containers[?(@.name=="linstor-controller")].image + value: registry.example/piraeus-server:v0.0.0-test + + - it: the satellite LVM setup runs the platform image + template: templates/satellites-cozy.yaml + asserts: + - equal: + path: spec.podTemplate.spec.initContainers[?(@.name=="setup-lvm-configuration")].image + value: registry.example/piraeus-server:v0.0.0-test + - equal: + path: spec.podTemplate.spec.containers[?(@.name=="linstor-satellite")].image + value: registry.example/piraeus-server:v0.0.0-test From acfae30136fa9dfdd1afeab5dead9401f74d92fe Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 12 Nov 2026 23:20:34 +0300 Subject: [PATCH 142/174] fix(linstor): update piraeus-operator to v2.12.1 The nfs-server DaemonSet runs piraeus-csi-nfs-server v1.10.3, which boots systemd as PID 1 in a privileged container with the host /dev mounted. Its getty units hang up the host's getty@tty1 about a second after the pod starts; systemd restarts the host getty until it hits the start limit and gives up, so login on the local or out-of-band console stops working on nodes that run the NFS server. Seen on bare-metal Ubuntu 24.04 nodes with Cozystack 1.6.5 and 1.7.0. Upstream fixed the image by disabling the services the NFS server does not need (piraeusdatastore/linstor-csi#417, reported as piraeusdatastore/piraeus-operator#961); v1.10.6 includes the fix. v2.12.1 is the current stable release. Its image set is LINSTOR 1.35.3 and LINSTOR CSI v1.13.2, which this repository builds itself; both builds follow in the next commits. The CSI release line is what carries IO QoS limits (qos.linbit.com/* StorageClass parameters, added in v1.11.3), which operators have asked for. Behaviour that changes with this range: - The CSIDriver moves to fsGroupPolicy: File, so fsGroup is applied to ReadWriteMany filesystem volumes too. - The operator manages Secrets only in its own namespace, and the CSI controller loses its cluster-wide read on Secrets; nothing in this repository references snapshotter Secrets. - The csi-health-monitor sidecar is removed. - Volume group snapshots are enabled only when the groupsnapshot.storage.k8s.io/v1 API is served; the snapshot controller here still ships v1alpha1, so they stay off. The csi-provisioner arguments are unchanged in this range, so the restated strictTopology list stays complete. The chart is vendored the way `make update` does it, which also drops the stale CRD copy from the vendored chart; piraeus-operator-crds installs the CRDs. Declare the local chart dependency so Helm can validate the umbrella chart without changing its rendered resources. This commit is not deployable on its own. The operator now hands the CSI pods their client certificate through LS_USER_CERTIFICATE_FILE and LS_USER_KEY_FILE, which the linstor-wait-until in the current linstor-csi image does not read, so the CSI init containers retry forever until the linstor-csi update in the next commit. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../linstor/tests/csi-topology_test.yaml | 2 +- .../piraeus-operator-crds/templates/crds.yaml | 218 +- packages/system/piraeus-operator/Chart.yaml | 4 + .../charts/piraeus/Chart.yaml | 4 +- .../charts/piraeus/templates/config.yaml | 42 +- .../charts/piraeus/templates/crds.yaml | 2173 ----------------- .../charts/piraeus/templates/deployment.yaml | 18 +- .../piraeus/templates/extra-objects.yaml | 15 + .../piraeus/templates/metrics-service.yaml | 1 + .../charts/piraeus/templates/rbac.yaml | 47 +- .../piraeus/templates/rolebindings.yaml | 16 + .../piraeus/templates/serviceaccount.yaml | 1 + .../validating-webhook-configuration.yaml | 2 + .../piraeus/templates/webhook-service.yaml | 1 + .../charts/piraeus/values.yaml | 43 + 15 files changed, 372 insertions(+), 2215 deletions(-) delete mode 100644 packages/system/piraeus-operator/charts/piraeus/templates/crds.yaml create mode 100644 packages/system/piraeus-operator/charts/piraeus/templates/extra-objects.yaml diff --git a/packages/system/linstor/tests/csi-topology_test.yaml b/packages/system/linstor/tests/csi-topology_test.yaml index fbde97a2d..6dd9bb3ea 100644 --- a/packages/system/linstor/tests/csi-topology_test.yaml +++ b/packages/system/linstor/tests/csi-topology_test.yaml @@ -34,7 +34,7 @@ tests: # Until there is, the re-check on a bump is a human obligation, stated in # templates/cluster.yaml beside the list. # - # The list was last verified complete against piraeus-operator v2.10.2: + # The list was last verified complete against piraeus-operator v2.12.1: # eleven upstream flags plus --strict-topology. - it: the restated argument list is complete template: templates/cluster.yaml diff --git a/packages/system/piraeus-operator-crds/templates/crds.yaml b/packages/system/piraeus-operator-crds/templates/crds.yaml index 5bd9212c6..d190bc2b9 100644 --- a/packages/system/piraeus-operator-crds/templates/crds.yaml +++ b/packages/system/piraeus-operator-crds/templates/crds.yaml @@ -151,13 +151,17 @@ spec: *SecretName using the issuer configured here. properties: group: - description: Group of the resource being referred to. + description: |- + Group of the issuer being referred to. + Defaults to 'cert-manager.io'. type: string kind: - description: Kind of the resource being referred to. + description: |- + Kind of the issuer being referred to. + Defaults to 'Issuer'. type: string name: - description: Name of the resource being referred to. + description: Name of the issuer being referred to. type: string required: - name @@ -201,6 +205,11 @@ spec: type: object x-kubernetes-map-type: atomic x-kubernetes-preserve-unknown-fields: true + replicas: + description: Number of desired pods. Defaults to 1. + format: int32 + minimum: 0 + type: integer type: object csiController: description: CSIController controls the deployment of the CSI Controller @@ -320,13 +329,17 @@ spec: issuer configured here. properties: group: - description: Group of the resource being referred to. + description: |- + Group of the issuer being referred to. + Defaults to 'cert-manager.io'. type: string kind: - description: Kind of the resource being referred to. + description: |- + Kind of the issuer being referred to. + Defaults to 'Issuer'. type: string name: - description: Name of the resource being referred to. + description: Name of the issuer being referred to. type: string required: - name @@ -345,6 +358,18 @@ spec: * Store credentials for accessing remotes for backups. See https://linbit.com/drbd-user-guide/linstor-guide-1_0-en/#s-encrypt_commands for more information. type: string + maxConcurrentEvacuations: + description: |- + MaxConcurrentEvacuations limits how many Satellites are evacuated at the same time when using the + "Evacuate" deletion policy. + + This applies both to Satellites removed because their node no longer matches the cluster, and to nodes + removed through ClusterAPI. While the limit is reached, additional Satellites wait for a free slot before + their volumes are moved and, for ClusterAPI managed nodes, before the node is allowed to drain. + A value of 0 (the default) places no limit on the number of concurrent evacuations. + format: int32 + minimum: 0 + type: integer nfsServer: description: NFSServer controls the deployment of the LINSTOR CSI NFS Server DaemonSet. @@ -568,9 +593,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -1054,13 +1080,17 @@ spec: issuer configured here. properties: group: - description: Group of the resource being referred to. + description: |- + Group of the issuer being referred to. + Defaults to 'cert-manager.io'. type: string kind: - description: Kind of the resource being referred to. + description: |- + Kind of the issuer being referred to. + Defaults to 'Issuer'. type: string name: - description: Name of the resource being referred to. + description: Name of the issuer being referred to. type: string required: - name @@ -1321,6 +1351,15 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + resourceNameSuffixSeparator: + description: |- + ResourceNameSuffixSeparator is inserted between the base resource name and the Satellite name. + + Defaults to ".". + enum: + - . + - '-' + type: string storagePools: description: StoragePools is a list of storage pools to configure on the node. @@ -1347,18 +1386,66 @@ spec: lvmPool: description: Configures a LVM Volume Group as storage pool. properties: - volumeGroup: + externalLocking: + description: |- + ExternalLocking skips LINSTOR's internal locking for the shared VG, relying on an external + lock manager such as lvmlockd on the hosts instead. + type: boolean + physicalVolumeCreateArguments: + description: |- + PhysicalVolumeCreateArguments are arguments to pass to "pvcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array + sharedSpace: + description: |- + SharedSpace marks the VG as shared between multiple nodes, identified by the given name. + All storage pools backed by the same physical storage must use the same shared space name. + A shared VG must be set up manually on the hosts, it cannot be created from source devices. type: string + volumeGroup: + description: VolumeGroup is the name of the Volume Group + (VG) to use. + type: string + volumeGroupCreateArguments: + description: |- + VolumeGroupCreateArguments are arguments to pass to "vgcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array type: object lvmThinPool: description: Configures a LVM Thin Pool as storage pool. properties: + logicalVolumeCreateArguments: + description: |- + LogicalVolumeCreateArguments are arguments to pass to "lvcreate". + This has no effect on an existing thinpool, it only applies when the thinpool initially gets created. + items: + type: string + type: array + physicalVolumeCreateArguments: + description: |- + PhysicalVolumeCreateArguments are arguments to pass to "pvcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array thinPool: description: ThinPool is the name of the thinpool LV (without VG prefix). type: string volumeGroup: type: string + volumeGroupCreateArguments: + description: |- + VolumeGroupCreateArguments are arguments to pass to "vgcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array type: object name: description: Name of the storage pool in linstor. @@ -1447,6 +1534,16 @@ spec: zPool: description: ZPool is the name of the ZFS zpool. type: string + zpoolCreateArguments: + description: |- + ZpoolCreateArguments are arguments to pass to "zpool create". + This has no effect on an existing zpool, it only applies when the zpool initially gets created. + + If not set, "-o failmode=continue" is automatically added to ensure a failed zpool does not stop replication + to and from other nodes in the cluster. + items: + type: string + type: array type: object zfsThinPool: description: Configures a ZFS system based storage pool, allocating @@ -1455,6 +1552,16 @@ spec: zPool: description: ZPool is the name of the ZFS zpool. type: string + zpoolCreateArguments: + description: |- + ZpoolCreateArguments are arguments to pass to "zpool create". + This has no effect on an existing zpool, it only applies when the zpool initially gets created. + + If not set, "-o failmode=continue" is automatically added to ensure a failed zpool does not stop replication + to and from other nodes in the cluster. + items: + type: string + type: array type: object required: - name @@ -1762,13 +1869,17 @@ spec: issuer configured here. properties: group: - description: Group of the resource being referred to. + description: |- + Group of the issuer being referred to. + Defaults to 'cert-manager.io'. type: string kind: - description: Kind of the resource being referred to. + description: |- + Kind of the issuer being referred to. + Defaults to 'Issuer'. type: string name: - description: Name of the resource being referred to. + description: Name of the issuer being referred to. type: string required: - name @@ -1925,6 +2036,15 @@ spec: repository: description: Repository used to pull workload images. type: string + resourceNameSuffixSeparator: + description: |- + ResourceNameSuffixSeparator is inserted between the base resource name and the Satellite name. + + Defaults to ".". + enum: + - . + - '-' + type: string storagePools: description: StoragePools is a list of storage pools to configure on the node. @@ -1951,18 +2071,66 @@ spec: lvmPool: description: Configures a LVM Volume Group as storage pool. properties: - volumeGroup: + externalLocking: + description: |- + ExternalLocking skips LINSTOR's internal locking for the shared VG, relying on an external + lock manager such as lvmlockd on the hosts instead. + type: boolean + physicalVolumeCreateArguments: + description: |- + PhysicalVolumeCreateArguments are arguments to pass to "pvcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array + sharedSpace: + description: |- + SharedSpace marks the VG as shared between multiple nodes, identified by the given name. + All storage pools backed by the same physical storage must use the same shared space name. + A shared VG must be set up manually on the hosts, it cannot be created from source devices. type: string + volumeGroup: + description: VolumeGroup is the name of the Volume Group + (VG) to use. + type: string + volumeGroupCreateArguments: + description: |- + VolumeGroupCreateArguments are arguments to pass to "vgcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array type: object lvmThinPool: description: Configures a LVM Thin Pool as storage pool. properties: + logicalVolumeCreateArguments: + description: |- + LogicalVolumeCreateArguments are arguments to pass to "lvcreate". + This has no effect on an existing thinpool, it only applies when the thinpool initially gets created. + items: + type: string + type: array + physicalVolumeCreateArguments: + description: |- + PhysicalVolumeCreateArguments are arguments to pass to "pvcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array thinPool: description: ThinPool is the name of the thinpool LV (without VG prefix). type: string volumeGroup: type: string + volumeGroupCreateArguments: + description: |- + VolumeGroupCreateArguments are arguments to pass to "vgcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array type: object name: description: Name of the storage pool in linstor. @@ -2051,6 +2219,16 @@ spec: zPool: description: ZPool is the name of the ZFS zpool. type: string + zpoolCreateArguments: + description: |- + ZpoolCreateArguments are arguments to pass to "zpool create". + This has no effect on an existing zpool, it only applies when the zpool initially gets created. + + If not set, "-o failmode=continue" is automatically added to ensure a failed zpool does not stop replication + to and from other nodes in the cluster. + items: + type: string + type: array type: object zfsThinPool: description: Configures a ZFS system based storage pool, allocating @@ -2059,6 +2237,16 @@ spec: zPool: description: ZPool is the name of the ZFS zpool. type: string + zpoolCreateArguments: + description: |- + ZpoolCreateArguments are arguments to pass to "zpool create". + This has no effect on an existing zpool, it only applies when the zpool initially gets created. + + If not set, "-o failmode=continue" is automatically added to ensure a failed zpool does not stop replication + to and from other nodes in the cluster. + items: + type: string + type: array type: object required: - name diff --git a/packages/system/piraeus-operator/Chart.yaml b/packages/system/piraeus-operator/Chart.yaml index 40b3fce28..7a1925e25 100644 --- a/packages/system/piraeus-operator/Chart.yaml +++ b/packages/system/piraeus-operator/Chart.yaml @@ -1,3 +1,7 @@ apiVersion: v2 name: cozy-piraeus-operator version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process +dependencies: + - name: piraeus + version: "*" + repository: file://charts/piraeus diff --git a/packages/system/piraeus-operator/charts/piraeus/Chart.yaml b/packages/system/piraeus-operator/charts/piraeus/Chart.yaml index 777722f3f..1565e6a27 100644 --- a/packages/system/piraeus-operator/charts/piraeus/Chart.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/Chart.yaml @@ -3,8 +3,8 @@ name: piraeus description: | The Piraeus Operator manages software defined storage clusters using LINSTOR in Kubernetes. type: application -version: 2.10.2 -appVersion: "v2.10.2" +version: 2.12.1 +appVersion: "v2.12.1" maintainers: - name: Piraeus Datastore url: https://piraeus.io diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/config.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/config.yaml index 2e6963f0a..4788ffba0 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/config.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/config.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ include "piraeus-operator.fullname" . }}-image-config + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} data: @@ -17,38 +18,38 @@ data: # quay.io/piraeusdatastore/piraeus-server:v1.24.2 components: linstor-controller: - tag: v1.32.3 + tag: v1.35.3 image: piraeus-server linstor-satellite: - tag: v1.32.3 + tag: v1.35.3 image: piraeus-server linstor-csi: - tag: v1.10.3 + tag: v1.13.2 image: piraeus-csi nfs-server: - tag: v1.10.3 + tag: v1.13.2 image: piraeus-csi-nfs-server drbd-reactor: - tag: v1.10.0 + tag: v1.12.0 image: drbd-reactor ha-controller: - tag: v1.3.1 + tag: v1.3.4 image: piraeus-ha-controller drbd-shutdown-guard: - tag: v1.1.1 + tag: v1.1.2 image: drbd-shutdown-guard ktls-utils: - tag: v1.2.1 + tag: v1.5.0 image: ktls-utils linstor-affinity-controller: - tag: v1.3.0 + tag: v1.5.0 image: linstor-affinity-controller drbd-module-loader: - tag: v9.2.16 + tag: v9.3.4 # The special "match" attribute is used to select an image based on the node's reported OS. # The operator will first check the k8s node's ".status.nodeInfo.osImage" field, and compare it against the list # here. If one matches, that specific image name will be used instead of the fallback image. - image: drbd9-noble # Fallback image: chose a recent kernel, which can hopefully compile whatever config is actually in use + image: drbd9-resolute # Fallback image: chose a recent kernel, which can hopefully compile whatever config is actually in use match: - osImage: Red Hat Enterprise Linux 8\. image: drbd9-almalinux8 @@ -84,6 +85,10 @@ data: image: drbd9-jammy - osImage: Ubuntu 24\.04 image: drbd9-noble + - osImage: Ubuntu 26\.04 + image: drbd9-resolute + - osImage: Debian GNU/Linux 13 + image: drbd9-trixie - osImage: Debian GNU/Linux 12 image: drbd9-bookworm - osImage: Debian GNU/Linux 11 @@ -93,25 +98,22 @@ data: base: registry.k8s.io/sig-storage components: csi-attacher: - tag: v4.10.0 + tag: v4.13.0 image: csi-attacher csi-livenessprobe: - tag: v2.17.0 + tag: v2.20.0 image: livenessprobe csi-provisioner: - tag: v6.1.0 + tag: v6.4.0 image: csi-provisioner csi-snapshotter: - tag: v8.4.0 + tag: v8.7.0 image: csi-snapshotter csi-resizer: - tag: v2.0.0 + tag: v2.3.0 image: csi-resizer - csi-external-health-monitor-controller: - tag: v0.16.0 - image: csi-external-health-monitor-controller csi-node-driver-registrar: - tag: v2.15.0 + tag: v2.18.0 image: csi-node-driver-registrar {{- range $idx, $value := .Values.imageConfigOverride }} {{ add $idx 1 }}_helm_override.yaml: | diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/crds.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/crds.yaml deleted file mode 100644 index 5bd9212c6..000000000 --- a/packages/system/piraeus-operator/charts/piraeus/templates/crds.yaml +++ /dev/null @@ -1,2173 +0,0 @@ - {{ if .Values.installCRDs }} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.18.0 - name: linstorclusters.piraeus.io -spec: - group: piraeus.io - names: - kind: LinstorCluster - listKind: LinstorClusterList - plural: linstorclusters - singular: linstorcluster - scope: Cluster - versions: - - additionalPrinterColumns: - - description: If the LINSTOR Cluster is available - jsonPath: .status.conditions[?(@.type=='Available')].status - name: Available - type: string - - description: If the LINSTOR Cluster is fully configured - jsonPath: .status.conditions[?(@.type=='Configured')].status - name: Configured - type: string - - description: The version of the LINSTOR Cluster - jsonPath: .status.version - name: Version - priority: 10 - type: string - - description: The number of running/expected Satellites - jsonPath: .status.satellites - name: Satellites - type: string - - description: The used capacity in all storage pools - jsonPath: .status.capacity - name: Used Capacity - type: string - - description: The number of volumes in the cluster - jsonPath: .status.numberOfVolumes - name: Volumes - type: integer - - description: The number of snapshots in the cluster - jsonPath: .status.numberOfSnapshots - name: Snapshots - priority: 10 - type: integer - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: LinstorCluster is the Schema for the linstorclusters API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LinstorClusterSpec defines the desired state of LinstorCluster - properties: - affinityController: - description: AffinityController controls the deployment of the Affinity - Controller Deployment. - properties: - enabled: - default: true - description: Enable the component. - type: boolean - podTemplate: - description: |- - Template to apply to Pods of the component. - - The template is applied as a patch to the default deployment, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - replicas: - description: Number of desired pods. Defaults to 1. - format: int32 - minimum: 0 - type: integer - type: object - apiTLS: - description: |- - ApiTLS secures the LINSTOR API. - - This configures the TLS key and certificate used to secure the LINSTOR API. - nullable: true - properties: - affinityControllerSecretName: - description: |- - AffinityControllerSecretName references a secret holding the TLS key and certificate used by the Affinity - Controller to monitor volume state. Defaults to "linstor-affinity-controller-tls". - type: string - apiSecretName: - description: |- - ApiSecretName references a secret holding the TLS key and certificate used to protect the API. - Defaults to "linstor-api-tls". - type: string - caReference: - description: |- - CAReference configures the CA certificate to use when validating TLS certificates. - If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate. - properties: - key: - default: ca.crt - description: |- - Key to select in the resource. - Defaults to ca.crt if not specified. - type: string - kind: - default: Secret - description: Kind of the resource containing the CA Certificate, - either a ConfigMap or Secret. - enum: - - ConfigMap - - Secret - type: string - name: - description: Name of the resource containing the CA Certificate. - type: string - optional: - description: Optional specifies whether the resource and its - key must exist. - type: boolean - required: - - name - type: object - certManager: - description: |- - CertManager references a cert-manager Issuer or ClusterIssuer. - If set, cert-manager.io/Certificate resources will be created, provisioning the secrets referenced in - *SecretName using the issuer configured here. - properties: - group: - description: Group of the resource being referred to. - type: string - kind: - description: Kind of the resource being referred to. - type: string - name: - description: Name of the resource being referred to. - type: string - required: - - name - type: object - clientSecretName: - description: |- - ClientSecretName references a secret holding the TLS key and certificate used by the operator to configure - the cluster. Defaults to "linstor-client-tls". - type: string - csiControllerSecretName: - description: |- - CsiControllerSecretName references a secret holding the TLS key and certificate used by the CSI Controller - to provision volumes. Defaults to "linstor-csi-controller-tls". - type: string - csiNodeSecretName: - description: |- - CsiNodeSecretName references a secret holding the TLS key and certificate used by the CSI Nodes to query - the volume state. Defaults to "linstor-csi-node-tls". - type: string - nfsServerSecretName: - description: |- - NFSServerSecretName references a secret holding the TLS key and certificate used by the NFS Server to query - the cluster state. Defaults to "linstor-csi-nfs-server-tls". - type: string - type: object - controller: - description: Controller controls the deployment of the LINSTOR Controller - Deployment. - properties: - enabled: - default: true - description: Enable the component. - type: boolean - podTemplate: - description: |- - Template to apply to Pods of the component. - - The template is applied as a patch to the default deployment, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - type: object - csiController: - description: CSIController controls the deployment of the CSI Controller - Deployment. - properties: - enabled: - default: true - description: Enable the component. - type: boolean - podTemplate: - description: |- - Template to apply to Pods of the component. - - The template is applied as a patch to the default deployment, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - replicas: - description: Number of desired pods. Defaults to 1. - format: int32 - minimum: 0 - type: integer - type: object - csiNode: - description: CSINode controls the deployment of the CSI Node DaemonSet. - properties: - enabled: - default: true - description: Enable the component. - type: boolean - podTemplate: - description: |- - Template to apply to Pods of the component. - - The template is applied as a patch to the default deployment, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - type: object - externalController: - description: |- - ExternalController references an external controller. - When set, the Operator will skip deploying a LINSTOR Controller and instead use the external cluster - to register satellites. - properties: - url: - description: URL of the external controller. - minLength: 3 - type: string - required: - - url - type: object - highAvailabilityController: - description: HighAvailabilityController controls the deployment of - the High Availability Controller DaemonSet. - properties: - enabled: - default: true - description: Enable the component. - type: boolean - podTemplate: - description: |- - Template to apply to Pods of the component. - - The template is applied as a patch to the default deployment, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - type: object - internalTLS: - description: |- - InternalTLS secures the connection between LINSTOR Controller and Satellite. - - This configures the client certificate used when the Controller connects to a Satellite. This only has an effect - when the Satellite is configured to for secure connections using `LinstorSatellite.spec.internalTLS`. - nullable: true - properties: - caReference: - description: |- - CAReference configures the CA certificate to use when validating TLS certificates. - If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate. - properties: - key: - default: ca.crt - description: |- - Key to select in the resource. - Defaults to ca.crt if not specified. - type: string - kind: - default: Secret - description: Kind of the resource containing the CA Certificate, - either a ConfigMap or Secret. - enum: - - ConfigMap - - Secret - type: string - name: - description: Name of the resource containing the CA Certificate. - type: string - optional: - description: Optional specifies whether the resource and its - key must exist. - type: boolean - required: - - name - type: object - certManager: - description: |- - CertManager references a cert-manager Issuer or ClusterIssuer. - If set, a Certificate resource will be created, provisioning the secret references in SecretName using the - issuer configured here. - properties: - group: - description: Group of the resource being referred to. - type: string - kind: - description: Kind of the resource being referred to. - type: string - name: - description: Name of the resource being referred to. - type: string - required: - - name - type: object - secretName: - description: SecretName references a secret holding the TLS key - and certificates. - type: string - type: object - linstorPassphraseSecret: - description: |- - LinstorPassphraseSecret used to configure the LINSTOR master passphrase. - - The referenced secret must contain a single key "MASTER_PASSPHRASE". The master passphrase is used to - * Derive encryption keys for volumes using the LUKS layer. - * Store credentials for accessing remotes for backups. - See https://linbit.com/drbd-user-guide/linstor-guide-1_0-en/#s-encrypt_commands for more information. - type: string - nfsServer: - description: NFSServer controls the deployment of the LINSTOR CSI - NFS Server DaemonSet. - properties: - enabled: - default: true - description: Enable the component. - type: boolean - podTemplate: - description: |- - Template to apply to Pods of the component. - - The template is applied as a patch to the default deployment, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - type: object - nodeAffinity: - description: |- - NodeAffinity selects the nodes on which LINSTOR Satellites will be deployed. - See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. The terms - are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements by node's - labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies - to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements by node's - fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies - to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - nodeSelector: - additionalProperties: - type: string - description: |- - NodeSelector selects the nodes on which LINSTOR Satellites will be deployed. - See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - type: object - patches: - description: |- - Patches is a list of kustomize patches to apply. - - See https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/ for how to create patches. - items: - description: Patch represent either a Strategic Merge Patch or a - JSON patch and its targets. - properties: - options: - description: Options is a list of options for the patch - properties: - allowKindChange: - description: AllowKindChange allows kind changes to the - resource. - type: boolean - allowNameChange: - description: AllowNameChange allows name changes to the - resource. - type: boolean - type: object - patch: - description: Patch is the content of a patch. - minLength: 1 - type: string - target: - description: Target points to the resources that the patch is - applied to - properties: - annotationSelector: - description: |- - AnnotationSelector is a string that follows the label selection expression - https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api - It matches against the resource annotations. - type: string - group: - type: string - kind: - type: string - labelSelector: - description: |- - LabelSelector is a string that follows the label selection expression - https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api - It matches against the resource labels. - type: string - name: - description: Name of the resource. - type: string - namespace: - description: Namespace the resource belongs to, if it can - belong to a namespace. - type: string - version: - type: string - type: object - required: - - patch - type: object - type: array - properties: - description: |- - Properties to apply on the cluster level. - - Use to create default settings for DRBD that should apply to all resources or to configure some other cluster - wide default. - items: - properties: - name: - description: Name of the property to set. - minLength: 1 - type: string - value: - description: Value to set the property to. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - repository: - description: Repository used to pull workload images. - type: string - tolerations: - description: |- - Tolerations selects the nodes on which LINSTOR Satellites will be deployed. - - The default tolerations for DaemonSets are automatically added. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - type: object - status: - description: LinstorClusterStatus defines the observed state of LinstorCluster - properties: - availableCapacityBytes: - description: The number of bytes in total in all storage pools in - the LINSTOR Cluster. - format: int64 - type: integer - capacity: - description: Capacity mirrors the information from TotalCapacityBytes - and FreeCapacityBytes in a human-readable string - type: string - conditions: - description: Current LINSTOR Cluster state - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - freeCapacityBytes: - description: The number of bytes free in all storage pools in the - LINSTOR Cluster. - format: int64 - type: integer - numberOfSnapshots: - description: The number of snapshots in the LINSTOR Cluster. - format: int32 - type: integer - numberOfVolumes: - description: The number of volumes in the LINSTOR Cluster. - format: int32 - type: integer - runningSatellites: - description: The number of LINSTOR Satellites currently running. - format: int32 - type: integer - satellites: - description: Satellites mirrors the information from ScheduledSatellites - and RunningSatellites in a human-readable string - type: string - scheduledSatellites: - description: The number of LINSTOR Satellites that are expected to - run. - format: int32 - type: integer - version: - description: The Version of the LINSTOR Cluster. - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.18.0 - name: linstornodeconnections.piraeus.io -spec: - group: piraeus.io - names: - kind: LinstorNodeConnection - listKind: LinstorNodeConnectionList - plural: linstornodeconnections - singular: linstornodeconnection - scope: Cluster - versions: - - additionalPrinterColumns: - - description: If the LINSTOR Node Connection is fully configured - jsonPath: .status.conditions[?(@.type=='Configured')].status - name: Configured - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: LinstorNodeConnection is the Schema for the linstornodeconnections - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LinstorNodeConnectionSpec defines the desired state of LinstorNodeConnection - properties: - paths: - description: Paths configure the network path used when connecting - two nodes. - items: - properties: - interface: - description: Interface to use on both nodes. - type: string - name: - description: Name of the path. - type: string - required: - - interface - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - properties: - description: |- - Properties to apply for the node connection. - - Use to create default settings for DRBD that should apply to all resources connections between a set of - cluster nodes. - items: - properties: - name: - description: Name of the property to set. - minLength: 1 - type: string - value: - description: Value to set the property to. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - selector: - description: |- - Selector selects which pair of Satellites the connection should apply to. - If not given, the connection will be applied to all connections. - items: - description: SelectorTerm matches pairs of nodes by checking that - the nodes match all specified requirements. - properties: - matchLabels: - description: MatchLabels is a list of match expressions that - the node pairs must meet. - items: - properties: - key: - description: Key is the name of a node label. - minLength: 1 - type: string - op: - default: Exists - description: |- - Op to apply to the label. - Exists (default) checks for the presence of the label on both nodes in the pair. - DoesNotExist checks that the label is not present on either node in the pair. - In checks for the presence of the label value given by Values on both nodes in the pair. - NotIn checks that both nodes in the pair do not have any of the label values given by Values. - Same checks that the label value is equal in the node pair. - NotSame checks that the label value is not equal in the node pair. - enum: - - Exists - - DoesNotExist - - In - - NotIn - - Same - - NotSame - type: string - values: - description: Values to match on, using the provided Op. - items: - type: string - type: array - required: - - key - type: object - type: array - required: - - matchLabels - type: object - type: array - type: object - status: - description: LinstorNodeConnectionStatus defines the observed state of - LinstorNodeConnection - properties: - conditions: - description: Current LINSTOR Node Connection state - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.18.0 - name: linstorsatelliteconfigurations.piraeus.io -spec: - group: piraeus.io - names: - kind: LinstorSatelliteConfiguration - listKind: LinstorSatelliteConfigurationList - plural: linstorsatelliteconfigurations - singular: linstorsatelliteconfiguration - scope: Cluster - versions: - - additionalPrinterColumns: - - description: The node selector used - jsonPath: .spec.nodeSelector - name: Selector - type: string - - description: If the Configuration was applied - jsonPath: .status.conditions[?(@.type=='Applied')].status - name: Applied - type: string - - description: Number of Satellites this Configuration has been applied to - jsonPath: .status.matched - name: Matched - type: integer - - description: Satellites this Configuration has been applied to - jsonPath: .status.appliedTo - name: Satellites - priority: 10 - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: LinstorSatelliteConfiguration is the Schema for the linstorsatelliteconfigurations - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - LinstorSatelliteConfigurationSpec defines a partial, desired state of a LinstorSatelliteSpec. - - All the LinstorSatelliteConfiguration resources with matching NodeSelector will - be merged into a single LinstorSatelliteSpec. - properties: - deletionPolicy: - description: |- - DeletionPolicy configures the way LinstorSatellite resources are deleted. - - A LinstorSatellite may be deleted because: - * It no longer matches the affinity and node selector of the LinstorCluster resource. - * The node it references has been removed from Kubernetes. - * It was manually deleted outside the Operator. - - A LinstorSatellite may store the last copy of a volume, in which case it is not desirable to unconditionally remove - the satellite from the cluster. For this reason, the following deletion policies exist: - - * DeletionPolicyEvacuate will start evacuation of the LINSTOR Satellite and wait until it completes before removing the LinstorSatellite object, comparable to the "linstor node evacuate" command. - * DeletionPolicyRetain will retain the LINSTOR Satellite, keeping it registered in LINSTOR, but removing associated Kubernetes resources. - * DeletionPolicyDelete will remove the LINSTOR Satellite from the LINSTOR Cluster without prior eviction, comparable to the "linstor node lost" command. - enum: - - Evacuate - - Retain - - Delete - type: string - evacuationStrategy: - description: EvacuationStrategy configures the evacuation of volumes - from a Satellite when DeletionPolicy "Evacuate" is used. - nullable: true - properties: - attachedVolumeReattachTimeout: - default: 5m - description: |- - AttachedVolumeReattachTimeout configures how long evacuation waits for attached volumes to reattach on - different nodes. Setting this to 0 disable this evacuation step. - type: string - unattachedVolumeAttachTimeout: - default: 5m - description: |- - UnattachedVolumeAttachTimeout configures how long evacuation waits for unattached volumes to attach on - different nodes. Setting this to 0 disable this evacuation step. - type: string - type: object - internalTLS: - description: |- - InternalTLS configures secure communication for the LINSTOR Satellite. - - If set, the control traffic between LINSTOR Controller and Satellite will be encrypted using mTLS. - nullable: true - properties: - caReference: - description: |- - CAReference configures the CA certificate to use when validating TLS certificates. - If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate. - properties: - key: - default: ca.crt - description: |- - Key to select in the resource. - Defaults to ca.crt if not specified. - type: string - kind: - default: Secret - description: Kind of the resource containing the CA Certificate, - either a ConfigMap or Secret. - enum: - - ConfigMap - - Secret - type: string - name: - description: Name of the resource containing the CA Certificate. - type: string - optional: - description: Optional specifies whether the resource and its - key must exist. - type: boolean - required: - - name - type: object - certManager: - description: |- - CertManager references a cert-manager Issuer or ClusterIssuer. - If set, a Certificate resource will be created, provisioning the secret references in SecretName using the - issuer configured here. - properties: - group: - description: Group of the resource being referred to. - type: string - kind: - description: Kind of the resource being referred to. - type: string - name: - description: Name of the resource being referred to. - type: string - required: - - name - type: object - secretName: - description: SecretName references a secret holding the TLS key - and certificates. - type: string - tlsHandshakeDaemon: - description: |- - TLSHandshakeDaemon enables tlshd for establishing TLS sessions for use by DRBD. - - If enabled, adds a new sidecar to the LINSTOR Satellite that runs the tlshd handshake daemon. - The daemon uses the TLS certificate and key to establish secure connections on behalf of DRBD. - type: boolean - type: object - ipFamilies: - description: |- - IPFamilies configures the IP Family (IPv4 or IPv6) to use to connect to the LINSTOR Satellite. - - If set, the control traffic between LINSTOR Controller and Satellite will use only the given IP Family. - If not set, the Operator will configure all families found in the Satellites Pods' Status. - items: - description: IPFamily represents the IP Family (IPv4 or IPv6). - enum: - - IPv4 - - IPv6 - type: string - type: array - nodeAffinity: - description: |- - NodeAffinity selects which LinstorSatellite resources this spec should be applied to. - See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. The terms - are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements by node's - labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies - to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements by node's - fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies - to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - nodeSelector: - additionalProperties: - type: string - description: |- - NodeSelector selects which LinstorSatellite resources this spec should be applied to. - See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - type: object - patches: - description: |- - Patches is a list of kustomize patches to apply. - - See https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/ for how to create patches. - items: - description: Patch represent either a Strategic Merge Patch or a - JSON patch and its targets. - properties: - options: - description: Options is a list of options for the patch - properties: - allowKindChange: - description: AllowKindChange allows kind changes to the - resource. - type: boolean - allowNameChange: - description: AllowNameChange allows name changes to the - resource. - type: boolean - type: object - patch: - description: Patch is the content of a patch. - minLength: 1 - type: string - target: - description: Target points to the resources that the patch is - applied to - properties: - annotationSelector: - description: |- - AnnotationSelector is a string that follows the label selection expression - https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api - It matches against the resource annotations. - type: string - group: - type: string - kind: - type: string - labelSelector: - description: |- - LabelSelector is a string that follows the label selection expression - https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api - It matches against the resource labels. - type: string - name: - description: Name of the resource. - type: string - namespace: - description: Namespace the resource belongs to, if it can - belong to a namespace. - type: string - version: - type: string - type: object - required: - - patch - type: object - type: array - podTemplate: - description: |- - Template to apply to Satellite Pods. - - The template is applied as a patch to the default resource, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - properties: - description: Properties is a list of properties to set on the node. - items: - properties: - expandFrom: - description: |- - ExpandFrom can reference multiple resource fields at once. - It either sets the property to an aggregate value based on matched resource fields, or expands to multiple - properties. - properties: - delimiter: - description: Delimiter used to join multiple key and value - pairs together. - type: string - nameTemplate: - description: |- - NameTemplate defines how the property key is expanded. - If set, the template is appended to the defined property name, creating multiple properties instead of one - aggregate. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - nodeFieldRef: - description: Select a field of the node. Supports `metadata.name`, - `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - valueTemplate: - description: |- - ValueTemplate defines how the property value is expanded. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - required: - - nodeFieldRef - type: object - name: - description: Name of the property to set. - minLength: 1 - type: string - optional: - description: Optional values are only set if they have a non-empty - value - type: boolean - value: - description: Value to set the property to. - type: string - valueFrom: - description: ValueFrom sets the value from an existing resource. - properties: - nodeFieldRef: - description: Select a field of the node. Supports `metadata.name`, - `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - required: - - nodeFieldRef - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - storagePools: - description: StoragePools is a list of storage pools to configure - on the node. - items: - properties: - filePool: - description: Configures a file system based storage pool, allocating - a regular file per volume. - properties: - directory: - description: Directory is the path to the host directory - used to store volume data. - type: string - type: object - fileThinPool: - description: Configures a file system based storage pool, allocating - a sparse file per volume. - properties: - directory: - description: Directory is the path to the host directory - used to store volume data. - type: string - type: object - lvmPool: - description: Configures a LVM Volume Group as storage pool. - properties: - volumeGroup: - type: string - type: object - lvmThinPool: - description: Configures a LVM Thin Pool as storage pool. - properties: - thinPool: - description: ThinPool is the name of the thinpool LV (without - VG prefix). - type: string - volumeGroup: - type: string - type: object - name: - description: Name of the storage pool in linstor. - minLength: 3 - type: string - properties: - description: Properties to set on the storage pool. - items: - properties: - expandFrom: - description: |- - ExpandFrom can reference multiple resource fields at once. - It either sets the property to an aggregate value based on matched resource fields, or expands to multiple - properties. - properties: - delimiter: - description: Delimiter used to join multiple key and - value pairs together. - type: string - nameTemplate: - description: |- - NameTemplate defines how the property key is expanded. - If set, the template is appended to the defined property name, creating multiple properties instead of one - aggregate. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - nodeFieldRef: - description: Select a field of the node. Supports - `metadata.name`, `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - valueTemplate: - description: |- - ValueTemplate defines how the property value is expanded. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - required: - - nodeFieldRef - type: object - name: - description: Name of the property to set. - minLength: 1 - type: string - optional: - description: Optional values are only set if they have - a non-empty value - type: boolean - value: - description: Value to set the property to. - type: string - valueFrom: - description: ValueFrom sets the value from an existing - resource. - properties: - nodeFieldRef: - description: Select a field of the node. Supports - `metadata.name`, `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - required: - - nodeFieldRef - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - source: - properties: - hostDevices: - description: HostDevices is a list of device paths used - to configure the given pool. - items: - type: string - minItems: 1 - type: array - type: object - zfsPool: - description: Configures a ZFS system based storage pool, allocating - zvols from the given zpool. - properties: - zPool: - description: ZPool is the name of the ZFS zpool. - type: string - type: object - zfsThinPool: - description: Configures a ZFS system based storage pool, allocating - sparse zvols from the given zpool. - properties: - zPool: - description: ZPool is the name of the ZFS zpool. - type: string - type: object - required: - - name - type: object - type: array - type: object - status: - description: LinstorSatelliteConfigurationStatus defines the observed - state of LinstorSatelliteConfiguration - properties: - appliedTo: - description: AppliedTo lists the LinstorSatellite resource this configuration - was applied to - items: - type: string - type: array - conditions: - description: Current LINSTOR Satellite Config state - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - matched: - description: Number of configured LinstorSatellite resource. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.18.0 - name: linstorsatellites.piraeus.io -spec: - group: piraeus.io - names: - kind: LinstorSatellite - listKind: LinstorSatelliteList - plural: linstorsatellites - singular: linstorsatellite - scope: Cluster - versions: - - additionalPrinterColumns: - - description: If the LINSTOR Satellite is connected - jsonPath: .status.conditions[?(@.type=='Available')].status - name: Connected - type: string - - description: If the LINSTOR Satellite is fully configured - jsonPath: .status.conditions[?(@.type=='Configured')].status - name: Configured - type: string - - description: The Satellite Configurations applied to this Satellite - jsonPath: .metadata.annotations.piraeus\.io/applied-configurations - name: Applied Configurations - priority: 10 - type: string - - description: The deletion policy of the Satellite - jsonPath: .spec.deletionPolicy - name: Deletion Policy - type: string - - description: The used capacity on the node - jsonPath: .status.capacity - name: Used Capacity - type: string - - description: The number of volumes on the node - jsonPath: .status.numberOfVolumes - name: Volumes - type: integer - - description: The number of snapshots on the node - jsonPath: .status.numberOfSnapshots - name: Snapshots - priority: 10 - type: integer - - description: The storage providers supported by the node - jsonPath: .status.storageProviders - name: Storage Providers - priority: 10 - type: string - - description: The device layers supported by the node - jsonPath: .status.deviceLayers - name: Device Layers - priority: 10 - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: LinstorSatellite is the Schema for the linstorsatellites API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LinstorSatelliteSpec defines the desired state of LinstorSatellite - properties: - clusterRef: - description: ClusterRef references the LinstorCluster used to create - this LinstorSatellite. - properties: - caReference: - description: |- - CAReference configures the CA certificate to use when validating TLS certificates. - If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate. - properties: - key: - default: ca.crt - description: |- - Key to select in the resource. - Defaults to ca.crt if not specified. - type: string - kind: - default: Secret - description: Kind of the resource containing the CA Certificate, - either a ConfigMap or Secret. - enum: - - ConfigMap - - Secret - type: string - name: - description: Name of the resource containing the CA Certificate. - type: string - optional: - description: Optional specifies whether the resource and its - key must exist. - type: boolean - required: - - name - type: object - clientSecretName: - description: ClientSecretName references the secret used by the - operator to validate the https endpoint. - type: string - externalController: - description: |- - ExternalController references an external controller. - When set, the Operator uses the external cluster to register satellites. - properties: - url: - description: URL of the external controller. - minLength: 3 - type: string - required: - - url - type: object - name: - description: Name of the LinstorCluster resource controlling this - satellite. - type: string - type: object - deletionPolicy: - default: Retain - description: |- - DeletionPolicy configures the way LinstorSatellite resources are deleted. - - A LinstorSatellite may be deleted because: - * It no longer matches the affinity and node selector of the LinstorCluster resource. - * The node it references has been removed from Kubernetes. - * It was manually deleted outside the Operator. - - A LinstorSatellite may store the last copy of a volume, in which case it is not desirable to unconditionally remove - the satellite from the cluster. For this reason, the following deletion policies exist: - - * DeletionPolicyEvacuate will start evacuation of the LINSTOR Satellite and wait until it completes before removing the LinstorSatellite object, comparable to the "linstor node evacuate" command. - * DeletionPolicyRetain will retain the LINSTOR Satellite, keeping it registered in LINSTOR, but removing associated Kubernetes resources. - * DeletionPolicyDelete will remove the LINSTOR Satellite from the LINSTOR Cluster without prior eviction, comparable to the "linstor node lost" command. - enum: - - Evacuate - - Retain - - Delete - type: string - evacuationStrategy: - description: EvacuationStrategy configures the evacuation of volumes - from a Satellite when DeletionPolicy "Evacuate" is used. - properties: - attachedVolumeReattachTimeout: - default: 5m - description: |- - AttachedVolumeReattachTimeout configures how long evacuation waits for attached volumes to reattach on - different nodes. Setting this to 0 disable this evacuation step. - type: string - unattachedVolumeAttachTimeout: - default: 5m - description: |- - UnattachedVolumeAttachTimeout configures how long evacuation waits for unattached volumes to attach on - different nodes. Setting this to 0 disable this evacuation step. - type: string - type: object - internalTLS: - description: |- - InternalTLS configures secure communication for the LINSTOR Satellite. - - If set, the control traffic between LINSTOR Controller and Satellite will be encrypted using mTLS. - The Controller will use the client key from `LinstorCluster.spec.internalTLS` when connecting. - nullable: true - properties: - caReference: - description: |- - CAReference configures the CA certificate to use when validating TLS certificates. - If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate. - properties: - key: - default: ca.crt - description: |- - Key to select in the resource. - Defaults to ca.crt if not specified. - type: string - kind: - default: Secret - description: Kind of the resource containing the CA Certificate, - either a ConfigMap or Secret. - enum: - - ConfigMap - - Secret - type: string - name: - description: Name of the resource containing the CA Certificate. - type: string - optional: - description: Optional specifies whether the resource and its - key must exist. - type: boolean - required: - - name - type: object - certManager: - description: |- - CertManager references a cert-manager Issuer or ClusterIssuer. - If set, a Certificate resource will be created, provisioning the secret references in SecretName using the - issuer configured here. - properties: - group: - description: Group of the resource being referred to. - type: string - kind: - description: Kind of the resource being referred to. - type: string - name: - description: Name of the resource being referred to. - type: string - required: - - name - type: object - secretName: - description: SecretName references a secret holding the TLS key - and certificates. - type: string - tlsHandshakeDaemon: - description: |- - TLSHandshakeDaemon enables tlshd for establishing TLS sessions for use by DRBD. - - If enabled, adds a new sidecar to the LINSTOR Satellite that runs the tlshd handshake daemon. - The daemon uses the TLS certificate and key to establish secure connections on behalf of DRBD. - type: boolean - type: object - ipFamilies: - description: |- - IPFamilies configures the IP Family (IPv4 or IPv6) to use to connect to the LINSTOR Satellite. - - If set, the control traffic between LINSTOR Controller and Satellite will use only the given IP Family. - If not set, the Operator will configure all families found in the Satellites Pods' Status. - items: - description: IPFamily represents the IP Family (IPv4 or IPv6). - enum: - - IPv4 - - IPv6 - type: string - type: array - patches: - description: |- - Patches is a list of kustomize patches to apply. - - See https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/ for how to create patches. - items: - description: Patch represent either a Strategic Merge Patch or a - JSON patch and its targets. - properties: - options: - description: Options is a list of options for the patch - properties: - allowKindChange: - description: AllowKindChange allows kind changes to the - resource. - type: boolean - allowNameChange: - description: AllowNameChange allows name changes to the - resource. - type: boolean - type: object - patch: - description: Patch is the content of a patch. - minLength: 1 - type: string - target: - description: Target points to the resources that the patch is - applied to - properties: - annotationSelector: - description: |- - AnnotationSelector is a string that follows the label selection expression - https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api - It matches against the resource annotations. - type: string - group: - type: string - kind: - type: string - labelSelector: - description: |- - LabelSelector is a string that follows the label selection expression - https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api - It matches against the resource labels. - type: string - name: - description: Name of the resource. - type: string - namespace: - description: Namespace the resource belongs to, if it can - belong to a namespace. - type: string - version: - type: string - type: object - required: - - patch - type: object - type: array - properties: - description: Properties is a list of properties to set on the node. - items: - properties: - expandFrom: - description: |- - ExpandFrom can reference multiple resource fields at once. - It either sets the property to an aggregate value based on matched resource fields, or expands to multiple - properties. - properties: - delimiter: - description: Delimiter used to join multiple key and value - pairs together. - type: string - nameTemplate: - description: |- - NameTemplate defines how the property key is expanded. - If set, the template is appended to the defined property name, creating multiple properties instead of one - aggregate. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - nodeFieldRef: - description: Select a field of the node. Supports `metadata.name`, - `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - valueTemplate: - description: |- - ValueTemplate defines how the property value is expanded. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - required: - - nodeFieldRef - type: object - name: - description: Name of the property to set. - minLength: 1 - type: string - optional: - description: Optional values are only set if they have a non-empty - value - type: boolean - value: - description: Value to set the property to. - type: string - valueFrom: - description: ValueFrom sets the value from an existing resource. - properties: - nodeFieldRef: - description: Select a field of the node. Supports `metadata.name`, - `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - required: - - nodeFieldRef - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - repository: - description: Repository used to pull workload images. - type: string - storagePools: - description: StoragePools is a list of storage pools to configure - on the node. - items: - properties: - filePool: - description: Configures a file system based storage pool, allocating - a regular file per volume. - properties: - directory: - description: Directory is the path to the host directory - used to store volume data. - type: string - type: object - fileThinPool: - description: Configures a file system based storage pool, allocating - a sparse file per volume. - properties: - directory: - description: Directory is the path to the host directory - used to store volume data. - type: string - type: object - lvmPool: - description: Configures a LVM Volume Group as storage pool. - properties: - volumeGroup: - type: string - type: object - lvmThinPool: - description: Configures a LVM Thin Pool as storage pool. - properties: - thinPool: - description: ThinPool is the name of the thinpool LV (without - VG prefix). - type: string - volumeGroup: - type: string - type: object - name: - description: Name of the storage pool in linstor. - minLength: 3 - type: string - properties: - description: Properties to set on the storage pool. - items: - properties: - expandFrom: - description: |- - ExpandFrom can reference multiple resource fields at once. - It either sets the property to an aggregate value based on matched resource fields, or expands to multiple - properties. - properties: - delimiter: - description: Delimiter used to join multiple key and - value pairs together. - type: string - nameTemplate: - description: |- - NameTemplate defines how the property key is expanded. - If set, the template is appended to the defined property name, creating multiple properties instead of one - aggregate. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - nodeFieldRef: - description: Select a field of the node. Supports - `metadata.name`, `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - valueTemplate: - description: |- - ValueTemplate defines how the property value is expanded. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - required: - - nodeFieldRef - type: object - name: - description: Name of the property to set. - minLength: 1 - type: string - optional: - description: Optional values are only set if they have - a non-empty value - type: boolean - value: - description: Value to set the property to. - type: string - valueFrom: - description: ValueFrom sets the value from an existing - resource. - properties: - nodeFieldRef: - description: Select a field of the node. Supports - `metadata.name`, `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - required: - - nodeFieldRef - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - source: - properties: - hostDevices: - description: HostDevices is a list of device paths used - to configure the given pool. - items: - type: string - minItems: 1 - type: array - type: object - zfsPool: - description: Configures a ZFS system based storage pool, allocating - zvols from the given zpool. - properties: - zPool: - description: ZPool is the name of the ZFS zpool. - type: string - type: object - zfsThinPool: - description: Configures a ZFS system based storage pool, allocating - sparse zvols from the given zpool. - properties: - zPool: - description: ZPool is the name of the ZFS zpool. - type: string - type: object - required: - - name - type: object - type: array - required: - - clusterRef - type: object - status: - description: LinstorSatelliteStatus defines the observed state of LinstorSatellite - properties: - availableCapacityBytes: - description: The number of bytes in total in all storage pools on - this Satellite. - format: int64 - type: integer - capacity: - description: Capacity mirrors the information from TotalCapacityBytes - and FreeCapacityBytes in a human-readable string. - type: string - conditions: - description: Current LINSTOR Satellite state - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - deviceLayers: - description: DeviceLayers lists the device layers (LUKS, CACHE, etc...) - this Satellite supports. - items: - type: string - type: array - freeCapacityBytes: - description: The number of bytes free in all storage pools on this - Satellite. - format: int64 - type: integer - numberOfSnapshots: - description: The number of snapshots on this Satellite. - format: int32 - type: integer - numberOfVolumes: - description: The number of volumes on this Satellite. - format: int32 - type: integer - storageProviders: - description: StorageProviders lists the storage providers (LVM, ZFS, - etc...) this Satellite supports. - items: - type: string - type: array - type: object - type: object - served: true - storage: true - subresources: - status: {} -{{ end }} diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/deployment.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/deployment.yaml index c67501972..0c3209e19 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/deployment.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/deployment.yaml @@ -3,6 +3,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "piraeus-operator.fullname" . }}-controller-manager + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} spec: @@ -70,6 +71,9 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true + {{- with .Values.operator.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- if .Values.kubeRbacProxy.enabled }} - args: - --secure-listen-address=0.0.0.0:8443 @@ -91,8 +95,11 @@ spec: securityContext: {{- toYaml .Values.kubeRbacProxy.securityContext | nindent 12}} volumeMounts: - - mountPath: /etc/tls - name: cert + - mountPath: /etc/tls + name: cert + {{- with .Values.kubeRbacProxy.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} securityContext: runAsNonRoot: true @@ -100,6 +107,10 @@ spec: {{ toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ include "piraeus-operator.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} terminationGracePeriodSeconds: 10 priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} nodeSelector: @@ -113,3 +124,6 @@ spec: secret: defaultMode: 420 secretName: {{ include "piraeus-operator.certifcateName" . }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 6 }} + {{- end }} diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/extra-objects.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/extra-objects.yaml new file mode 100644 index 000000000..b0ec6fad2 --- /dev/null +++ b/packages/system/piraeus-operator/charts/piraeus/templates/extra-objects.yaml @@ -0,0 +1,15 @@ +{{- /* Normalize extraObjects to a list, easier to loop over */ -}} +{{- $extraObjects := .Values.extraManifests | default (list) -}} + +{{- if kindIs "map" $extraObjects -}} + {{- $extraObjects = values $extraObjects -}} +{{- end -}} + +{{- range $extraObjects }} +--- + {{- if kindIs "map" . }} + {{- tpl (toYaml .) $ | nindent 0 }} + {{- else if kindIs "string" . }} + {{- tpl . $ | nindent 0 }} + {{- end }} +{{- end }} diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/metrics-service.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/metrics-service.yaml index d1a24e3ca..49dee196a 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/metrics-service.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/metrics-service.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Service metadata: name: {{ include "piraeus-operator.fullname" . }}-controller-manager-metrics-service + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} spec: diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/rbac.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/rbac.yaml index c532e02d4..56b7d019b 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/rbac.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/rbac.yaml @@ -15,7 +15,6 @@ rules: - events - persistentvolumes - pods - - secrets - serviceaccounts - services verbs: @@ -59,6 +58,13 @@ rules: - pods/eviction verbs: - create +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create + - get - apiGroups: - apiextensions.k8s.io resources: @@ -71,6 +77,12 @@ rules: - patch - update - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions/status + verbs: + - patch - apiGroups: - apps resources: @@ -235,7 +247,6 @@ rules: - snapshot.storage.k8s.io resources: - volumesnapshotclasses - - volumesnapshots verbs: - get - list @@ -257,6 +268,15 @@ rules: verbs: - patch - update +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - update + - watch - apiGroups: - storage.k8s.io resources: @@ -283,6 +303,7 @@ rules: - storage.k8s.io resources: - storageclasses + - volumeattributesclasses verbs: - get - list @@ -306,8 +327,30 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role +metadata: + name: {{ include "piraeus-operator.fullname" . }}-controller-manager + namespace: {{ .Release.Namespace }} + labels: + {{- include "piraeus-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role metadata: name: {{ include "piraeus-operator.fullname" . }}-leader-election + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} rules: diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/rolebindings.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/rolebindings.yaml index 97e53b952..fc5212cc4 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/rolebindings.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/rolebindings.yaml @@ -17,6 +17,22 @@ subjects: namespace: {{ .Release.Namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "piraeus-operator.fullname" . }}-controller-manager + namespace: {{ .Release.Namespace }} + labels: + {{- include "piraeus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "piraeus-operator.fullname" . }}-controller-manager +subjects: +- kind: ServiceAccount + name: {{ include "piraeus-operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: {{ include "piraeus-operator.fullname" . }}-controller-manager diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/serviceaccount.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/serviceaccount.yaml index 89315de95..c4997dd24 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/serviceaccount.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/serviceaccount.yaml @@ -4,6 +4,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "piraeus-operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} {{ end }} diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/validating-webhook-configuration.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/validating-webhook-configuration.yaml index 5a6c5c9e6..04d05bfaf 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/validating-webhook-configuration.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/validating-webhook-configuration.yaml @@ -9,6 +9,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: {{ include "piraeus-operator.fullname" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} spec: @@ -36,6 +37,7 @@ apiVersion: v1 kind: Secret metadata: name: {{ include "piraeus-operator.certifcateName" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} type: kubernetes.io/tls diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/webhook-service.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/webhook-service.yaml index d360269bc..57f8966c9 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/webhook-service.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/webhook-service.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Service metadata: name: {{ include "piraeus-operator.fullname" . }}-webhook-service + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} spec: diff --git a/packages/system/piraeus-operator/charts/piraeus/values.yaml b/packages/system/piraeus-operator/charts/piraeus/values.yaml index bcc4dc9af..ff0d92b5a 100644 --- a/packages/system/piraeus-operator/charts/piraeus/values.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/values.yaml @@ -29,6 +29,12 @@ operator: # cpu: 100m # memory: 128Mi + ## Additional volume mounts + extraVolumeMounts: [ ] + # - name: clusterapi-kubeconfig + # mountPath: /etc/operator + # readOnly: true + kubeRbacProxy: enabled: false image: @@ -54,6 +60,16 @@ kubeRbacProxy: # cpu: 100m # memory: 128Mi + ## Additional volume mounts + extraVolumeMounts: [ ] + + +## Additional volumes +extraVolumes: [] +# - name: clusterapi-kubeconfig +# secret: +# secretName: super-secret + webhook: timeoutSeconds: 2 failurePolicy: Fail @@ -111,3 +127,30 @@ imageConfigOverride: [] # tag: my-custom-tag # Results in the image example.com/piraeus/linstor-csi:my-custom-tag being used. # See templates/config.yaml for available components. + +## Extra manifests to deploy. Can be of type dict or list. +## If dict, keys are ignored and only values are used. +## Items contained within extraObjects can be defined as dict or string and are passed through tpl. +extraManifests: null + # - apiVersion: v1 + # kind: ConfigMap + # metadata: + # labels: + # name: piraeus-extra + # data: + # extra-data: "value" + # + # can also be defined as a string, useful for templating field names + # - | + # apiVersion: v1 + # kind: Secret + # type: Opaque + # metadata: + # name: super-secret + # labels: + # {{- range $key, $value := .Values.commonLabels }} + # {{ $key }}: {{ $value }} + # {{- end }} + # data: + # plaintext: Zm9vYmFy +# templated: '{{ print "foobar" | upper | b64enc }}' From e1ed8c771938e31c9130b06931d649ef85761e49 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 02:23:32 +0300 Subject: [PATCH 143/174] fix(linstor): update linstor-csi to v1.13.2 piraeus-operator v2.12.1 is released against LINSTOR CSI v1.13.2, and the release line carries IO QoS limits through the qos.linbit.com/* StorageClass parameters. The operator gives the CSI pods their client certificate as mounted files through LS_USER_CERTIFICATE_FILE and LS_USER_KEY_FILE, instead of inline values in LS_USER_CERTIFICATE and LS_USER_KEY. The linstor-wait-until v0.3.1 in this image is built on a golinstor release that reads only the inline variables, so its init containers present no client certificate, the LINSTOR API closes the TLS connection, and they retry on EOF forever. Move to linstor-wait-until v1.0.1, the version upstream v1.13.2 pins. The Protocol C override for dual attach is dropped. Since v1.12 the driver no longer sets allow-two-primaries itself: it asks LINSTOR to manage dual-primary through make-available, and LINSTOR 1.35 sets protocol C on the connections that need it. Upstream closed the equivalent change as superseded for that reason (piraeusdatastore/linstor-csi#435). The relocate-after-clone patch is carried forward. Volume IDs became a struct for consistency groups; Clone and VolFromSnap handle standalone resources, so relocation keys on the resource name. The enumer output is regenerated rather than hand-patched. Upstream now pins grpc v1.84.0, which builds on Go 1.27 and includes the fix for GO-2026-6443, so the grpc override is no longer needed. The golden-image clone guard in vm-disk was re-checked: parameter parsing for allowRemoteVolumeAccess and layerList, and the default derived from the layer list, are unchanged in this range. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- packages/system/linstor/Makefile | 2 +- .../linstor/images/linstor-csi/Dockerfile | 11 +- .../001-relocate-after-clone-restore.diff | 103 +++++++------- ...2-protocol-c-override-for-dual-attach.diff | 132 ------------------ 4 files changed, 57 insertions(+), 191 deletions(-) delete mode 100644 packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff diff --git a/packages/system/linstor/Makefile b/packages/system/linstor/Makefile index 74e3f5aef..5e44ebade 100644 --- a/packages/system/linstor/Makefile +++ b/packages/system/linstor/Makefile @@ -8,7 +8,7 @@ LINSTOR_VERSION ?= 1.33.3 # The golden-image clone guard in packages/apps/vm-disk/templates/dv.yaml # reads allowRemoteVolumeAccess and layerList the way this release does and # has to be re-checked against the new one. -LINSTOR_CSI_VERSION ?= v1.11.2 +LINSTOR_CSI_VERSION ?= v1.13.2 test: helm unittest . diff --git a/packages/system/linstor/images/linstor-csi/Dockerfile b/packages/system/linstor/images/linstor-csi/Dockerfile index 50e755090..3ff1d154d 100644 --- a/packages/system/linstor/images/linstor-csi/Dockerfile +++ b/packages/system/linstor/images/linstor-csi/Dockerfile @@ -1,8 +1,8 @@ FROM --platform=$BUILDPLATFORM golang:1.27@sha256:3680233e3204827fbdc66088528ae6d4b3d034f51d03a99d454f6de034888244 AS builder ARG GOPROXY -ARG VERSION=v1.11.2 -ARG LINSTOR_WAIT_UNTIL_VERSION=v0.3.1 +ARG VERSION=v1.13.2 +ARG LINSTOR_WAIT_UNTIL_VERSION=v1.0.1 ARG TARGETARCH ARG TARGETOS @@ -15,13 +15,6 @@ RUN git apply /patches/*.diff RUN for attempt in 1 2 3; do go mod download && break; [ "$attempt" = 3 ] && exit 1; sleep $((attempt * 10)); done -# Upstream pins grpc v1.81.0, which uses http2.TrailerPrefix; on Go 1.27 -# golang.org/x/net builds http2 on top of net/http and no longer defines it. -# v1.83.2 also carries the fix for GO-2026-6443, a server panic reachable -# through the CSI gRPC endpoint. -RUN go get google.golang.org/grpc@v1.83.2 && \ - go mod tidy - # go build never compiles a _test.go, so nothing else runs the tests of the # packages the patches touch. Each package is tested from its own directory: # that finds the module it belongs to, and an empty list runs nothing. diff --git a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff index 0e2726708..29c209444 100644 --- a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff +++ b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff @@ -1,38 +1,37 @@ diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go -index ac4651d..46d565d 100644 --- a/pkg/client/linstor.go +++ b/pkg/client/linstor.go -@@ -462,6 +462,14 @@ func (s *Linstor) Clone(ctx context.Context, vol, src *volume.Info, params *volu +@@ -498,6 +498,14 @@ return err } + if params.RelocateAfterClone { + logger.Debug("relocate resources to optimal nodes") + -+ if err := s.relocateResources(ctx, vol.ID, rGroup.Name); err != nil { ++ if err := s.relocateResources(ctx, vol.ResourceName, rGroup.Name); err != nil { + logger.WithError(err).Warn("resource relocation failed, volume is still usable") + } + } + logger.Debug("reconcile extra properties") - err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties}) -@@ -1297,6 +1305,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v + err = s.client.ResourceDefinitions.Modify(ctx, vol.ResourceName, lapi.GenericPropsModify{OverrideProps: vol.Properties}) +@@ -1614,6 +1622,14 @@ return err } + if snapParams != nil && snapParams.RelocateAfterRestore { + logger.Debug("relocate resources to optimal nodes") + -+ if err := s.relocateResources(ctx, vol.ID, rGroup.Name); err != nil { ++ if err := s.relocateResources(ctx, vol.ResourceName, rGroup.Name); err != nil { + logger.WithError(err).Warn("resource relocation failed, volume is still usable") + } + } + logger.Debug("reconcile extra properties") - err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties}) -@@ -1487,9 +1503,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu + err = s.client.ResourceDefinitions.Modify(ctx, vol.ResourceName, lapi.GenericPropsModify{OverrideProps: vol.Properties}) +@@ -1818,9 +1834,8 @@ return fmt.Errorf("snapshot '%s' not deployed on any node", snap.Name) } @@ -44,7 +43,7 @@ index ac4651d..46d565d 100644 for _, snapNode := range snap.Nodes { if err := s.NodeAvailable(ctx, snapNode); err != nil { logger.WithField("selected node candidate", snapNode).WithError(err).Debug("node is not available") -@@ -1497,13 +1512,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu +@@ -1828,15 +1843,25 @@ } if slices.Contains(preferredNodes, snapNode) { @@ -58,8 +57,8 @@ index ac4651d..46d565d 100644 } + + available = append(available, snapNode) -+ } -+ + } + + // Pick a random node from preferred candidates, falling back to any available node. + // Randomization distributes restore load across snapshot nodes, preventing all + // clones of the same source from concentrating on a single node. @@ -71,13 +70,20 @@ index ac4651d..46d565d 100644 + var selectedNode string + if len(candidates) > 0 { + selectedNode = candidates[rand.Intn(len(candidates))] - } - ++ } ++ if selectedNode == "" { -@@ -1696,6 +1721,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In - return nil - } - + return fmt.Errorf("no node [%s] available for snapshot '%s'", strings.Join(snap.Nodes, ","), snap.Name) + } +@@ -2030,6 +2055,114 @@ + err = volumeScheduler.Create(ctx, vol.ResourceName, params, topologies) + if err != nil { + return err ++ } ++ ++ return nil ++} ++ +const propertyRelocationTriggered = "Aux/csi-relocation-triggered" + +// relocateResources migrates replicas from their current nodes to the nodes that @@ -181,19 +187,13 @@ index ac4651d..46d565d 100644 + logger.WithError(err).Warn("migrate-disk failed, skipping this pair") + continue + } -+ } -+ -+ return nil -+} -+ - // FindSnapsByID searches the snapshot in the backend - func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) { - snapshotId, err := volume.ParseSnapshotId(id) + } + + return nil diff --git a/pkg/volume/parameter.go b/pkg/volume/parameter.go -index 39acd95..54d1dfb 100644 --- a/pkg/volume/parameter.go +++ b/pkg/volume/parameter.go -@@ -50,6 +50,7 @@ const ( +@@ -51,6 +51,7 @@ nfsservicename nfssquash nfsrecoveryvolumesize @@ -201,24 +201,24 @@ index 39acd95..54d1dfb 100644 ) // Parameters configuration for linstor volumes. -@@ -118,6 +119,8 @@ type Parameters struct { - // NfsRecoveryVolumeSize sets the volume size (in bytes) of the recovery volume used by the NFS server. - // Defaults to 300MiB. - NfsRecoveryVolumeBytes int64 +@@ -122,6 +123,8 @@ + // IOQoSLimits holds the IO QoS limits parsed from the qos.linbit.com/* parameters. They are stored on + // the LINSTOR resource definition and passed to the nri-volume-qos plugin via VolumeAttachment metadata. + IOQoSLimits meta.Limits + // RelocateAfterClone triggers asynchronous relocation of replicas to optimal nodes after cloning. + RelocateAfterClone bool } const DefaultDisklessStoragePoolName = "DfltDisklessStorPool" -@@ -140,6 +143,7 @@ func NewParameters(params map[string]string, topologyPrefix string) (Parameters, +@@ -144,6 +147,7 @@ NfsServiceName: "linstor-csi-nfs", NfsSquash: "no_root_squash", NfsRecoveryVolumeBytes: 300 * 1024 * 1024, + RelocateAfterClone: true, } - for k, v := range params { -@@ -287,6 +291,13 @@ func NewParameters(params map[string]string, topologyPrefix string) (Parameters, + limits, err := meta.ParseLimits(params) +@@ -298,6 +302,13 @@ } p.NfsRecoveryVolumeBytes = s.Value() @@ -233,10 +233,9 @@ index 39acd95..54d1dfb 100644 } diff --git a/pkg/volume/paramkey_enumer.go b/pkg/volume/paramkey_enumer.go -index 5474963..2eb8a7c 100644 --- a/pkg/volume/paramkey_enumer.go +++ b/pkg/volume/paramkey_enumer.go -@@ -7,11 +7,11 @@ import ( +@@ -7,11 +7,11 @@ "strings" ) @@ -251,7 +250,7 @@ index 5474963..2eb8a7c 100644 func (i paramKey) String() string { if i < 0 || i >= paramKey(len(_paramKeyIndex)-1) { -@@ -50,9 +50,10 @@ func _paramKeyNoOp() { +@@ -50,9 +50,10 @@ _ = x[nfsservicename-(23)] _ = x[nfssquash-(24)] _ = x[nfsrecoveryvolumesize-(25)] @@ -262,17 +261,24 @@ index 5474963..2eb8a7c 100644 +var _paramKeyValues = []paramKey{allowremotevolumeaccess, autoplace, clientlist, disklessonremaining, disklessstoragepool, donotplacewithregex, encryption, fsopts, layerlist, mountopts, nodelist, placementcount, placementpolicy, replicasondifferent, replicasonsame, sizekib, storagepool, postmountxfsopts, resourcegroup, usepvcname, overprovision, xreplicasondifferent, nfsconfigtemplate, nfsservicename, nfssquash, nfsrecoveryvolumesize, relocateafterclone} var _paramKeyNameToValueMap = map[string]paramKey{ - _paramKeyName[0:23]: allowremotevolumeaccess, -@@ -107,6 +108,8 @@ var _paramKeyNameToValueMap = map[string]paramKey{ + _paramKeyName[0:23]: allowremotevolumeaccess, +@@ -81,6 +82,7 @@ + _paramKeyName[310:324]: nfsservicename, + _paramKeyName[324:333]: nfssquash, + _paramKeyName[333:354]: nfsrecoveryvolumesize, ++ _paramKeyName[354:372]: relocateafterclone, + } + + var _paramKeyLowerNameToValueMap = map[string]paramKey{ +@@ -110,6 +112,7 @@ + _paramKeyLowerName[310:324]: nfsservicename, _paramKeyLowerName[324:333]: nfssquash, - _paramKeyName[333:354]: nfsrecoveryvolumesize, _paramKeyLowerName[333:354]: nfsrecoveryvolumesize, -+ _paramKeyName[354:372]: relocateafterclone, + _paramKeyLowerName[354:372]: relocateafterclone, } var _paramKeyNames = []string{ -@@ -136,6 +139,7 @@ var _paramKeyNames = []string{ +@@ -139,6 +142,7 @@ _paramKeyName[310:324], _paramKeyName[324:333], _paramKeyName[333:354], @@ -281,18 +287,17 @@ index 5474963..2eb8a7c 100644 // paramKeyString retrieves an enum value from the enum constants string name. diff --git a/pkg/volume/snapshot_params.go b/pkg/volume/snapshot_params.go -index d167cb8..50b70fb 100644 --- a/pkg/volume/snapshot_params.go +++ b/pkg/volume/snapshot_params.go -@@ -35,6 +35,7 @@ type SnapshotParameters struct { - LinstorTargetUrl string `json:"linstor-target-url,omitempty"` - LinstorTargetClusterID string `json:"linstor-target-cluster-id,omitempty"` - LinstorTargetStoragePool string `json:"linstor-target-storage-pool,omitempty"` -+ RelocateAfterRestore bool `json:"relocate-after-restore,omitempty"` +@@ -38,6 +38,7 @@ + LinstorTargetUrl string `json:"linstor-target-url,omitempty"` + LinstorTargetClusterID string `json:"linstor-target-cluster-id,omitempty"` + LinstorTargetStoragePool string `json:"linstor-target-storage-pool,omitempty"` ++ RelocateAfterRestore bool `json:"relocate-after-restore,omitempty"` } func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParameters, error) { -@@ -91,6 +92,13 @@ func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParamete +@@ -116,6 +117,13 @@ p.LinstorTargetClusterID = v case "/linstor-target-storage-pool": p.LinstorTargetStoragePool = v diff --git a/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff b/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff deleted file mode 100644 index 279c636b1..000000000 --- a/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff +++ /dev/null @@ -1,132 +0,0 @@ -diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go -index ac4651d..9d75c79 100644 ---- a/pkg/client/linstor.go -+++ b/pkg/client/linstor.go -@@ -653,11 +653,35 @@ func (s *Linstor) Attach(ctx context.Context, volId, node string, rwxBlock bool) - } - - if otherResInUse > 0 && rwxBlock { -- rdPropsModify := lapi.GenericPropsModify{OverrideProps: map[string]string{ -+ rdProps := map[string]string{ - linstor.PropertyAllowTwoPrimaries: "yes", -- }} -+ } - -- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify) -+ // DRBD requires Protocol C whenever allow-two-primaries is enabled. -+ // If the resource is configured with Protocol A or B (typically -+ // through its resource-group), drbdadm adjust on the satellites -+ // fails with "Protocol C required" (errno 139) and live migration -+ // cannot proceed. Override Protocol to C on the resource-definition -+ // itself so it applies to every connection (including diskless -+ // peers, e.g. a TieBreaker). The marker lets Detach revert exactly -+ // the override we installed without disturbing operator-set props. -+ proto, protoErr := s.getEffectiveDrbdProtocol(ctx, volId) -+ if protoErr != nil { -+ s.log.WithError(protoErr).WithField("volume", volId). -+ Warn("failed to determine effective DRBD protocol; skipping Protocol=C override for dual-attach") -+ } else if proto == "A" || proto == "B" { -+ s.log.WithFields(logrus.Fields{ -+ "volume": volId, -+ "protocol": proto, -+ }).Info("installing Protocol=C override on resource-definition for dual-attach") -+ -+ rdProps[linstor.PropertyDrbdNetProtocol] = "C" -+ rdProps[linstor.PropertyCsiProtocolOverride] = "yes" -+ } -+ -+ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{ -+ OverrideProps: rdProps, -+ }) - if err != nil { - return "", err - } -@@ -774,11 +798,26 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error { - } - - if resInUse == 1 { -- rdPropsModify := lapi.GenericPropsModify{DeleteProps: []string{ -- linstor.PropertyAllowTwoPrimaries, -- }} -+ deleteProps := []string{linstor.PropertyAllowTwoPrimaries} -+ -+ // Drop the Protocol=C override only if Attach installed it (gated by -+ // our marker), so an operator-set Protocol property on the -+ // resource-definition is preserved. -+ rd, getErr := s.client.ResourceDefinitions.Get(ctx, volId) -+ if getErr != nil { -+ log.WithError(getErr).Warn("failed to fetch resource-definition props; skipping Protocol=C override removal") -+ } else if rd.Props[linstor.PropertyCsiProtocolOverride] != "" { -+ log.Info("removing Protocol=C override from resource-definition") - -- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify) -+ deleteProps = append(deleteProps, -+ linstor.PropertyDrbdNetProtocol, -+ linstor.PropertyCsiProtocolOverride, -+ ) -+ } -+ -+ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{ -+ DeleteProps: deleteProps, -+ }) - if err != nil { - return nil404(err) - } -@@ -805,6 +844,33 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error { - return nil404(s.client.Resources.Delete(ctx, volId, node)) - } - -+// getEffectiveDrbdProtocol returns the DRBD network protocol ("A", "B" or "C") -+// that LINSTOR will hand to drbdadm for the given resource. Resource-definition -+// properties take precedence; otherwise the value is inherited from the -+// resource-group. An empty string is returned when neither level sets the -+// property (LINSTOR's compiled-in default is "C"). -+func (s *Linstor) getEffectiveDrbdProtocol(ctx context.Context, volId string) (string, error) { -+ rd, err := s.client.ResourceDefinitions.Get(ctx, volId) -+ if err != nil { -+ return "", err -+ } -+ -+ if v, ok := rd.Props[linstor.PropertyDrbdNetProtocol]; ok { -+ return v, nil -+ } -+ -+ if rd.ResourceGroupName == "" { -+ return "", nil -+ } -+ -+ rg, err := s.client.ResourceGroups.Get(ctx, rd.ResourceGroupName) -+ if err != nil { -+ return "", err -+ } -+ -+ return rg.Props[linstor.PropertyDrbdNetProtocol], nil -+} -+ - // CapacityBytes returns the amount of free space in the storage pool specified by the params and topology. - func (s *Linstor) CapacityBytes(ctx context.Context, storagePools []string, overProvision *float64, segments map[string]string) (int64, error) { - log := s.log.WithField("storage-pools", storagePools).WithField("segments", segments) -diff --git a/pkg/linstor/const.go b/pkg/linstor/const.go -index 9a5f79c..c8bc9c3 100644 ---- a/pkg/linstor/const.go -+++ b/pkg/linstor/const.go -@@ -44,6 +44,19 @@ const ( - // PropertyAllowTwoPrimaries is DRBD option to allow second primary. Mainly used for live-migration. - PropertyAllowTwoPrimaries = lc.NamespcDrbdNetOptions + "/allow-two-primaries" - -+ // PropertyDrbdNetProtocol is the DRBD network replication protocol option -+ // (A = async, B = semi-sync, C = sync). DRBD requires Protocol C whenever -+ // allow-two-primaries is enabled, otherwise drbdadm adjust fails with -+ // "Protocol C required" (errno 139). -+ PropertyDrbdNetProtocol = lc.NamespcDrbdNetOptions + "/protocol" -+ -+ // PropertyCsiProtocolOverride marks a resource-connection where this driver -+ // has installed a temporary Protocol=C override to make a Protocol-A/B -+ // resource compatible with allow-two-primaries during live migration. The -+ // marker lets us safely remove only the overrides we set, without touching -+ // connection-level overrides installed by the operator. -+ PropertyCsiProtocolOverride = lc.NamespcAuxiliary + "/csi-protocol-override" -+ - // CreatedForTemporaryDisklessAttach marks a resource as temporary, i.e. it should be removed after it is no longer - // needed. - CreatedForTemporaryDisklessAttach = "temporary-diskless-attach" From c1652a730d77bd3217597a152cdcaafcf1e68970 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 02:41:08 +0300 Subject: [PATCH 144/174] fix(linstor): update LINSTOR server to v1.35.3 piraeus-operator v2.12.1 ships LINSTOR 1.35.3 as its controller and satellite version, and the in-repo piraeus-server build has to match it. LINSTOR 1.34 raised the minimum Java to 21, so the build and runtime stages move from bookworm to trixie, and the system Gradle used by debian/rules follows the upstream wrapper to 8.14.2. Six carried patches are part of upstream by v1.35.3 and are dropped; the stale-bitmap adjust retry is rebased onto the new DrbdAdm. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- packages/system/linstor/Makefile | 2 +- .../linstor/images/piraeus-server/Dockerfile | 9 +- .../images/piraeus-server/patches/README.md | 34 +- .../patches/allow-toggle-disk-retry.diff | 261 ------- .../patches/fix-duplicate-tcp-ports.diff | 137 ---- .../patches/fix-luks-header-size.diff | 570 --------------- .../patches/fix-min-io-probe-device-wait.diff | 660 ------------------ .../piraeus-server/patches/native-protoc.diff | 10 +- .../retry-adjust-after-stale-bitmap.diff | 25 +- .../patches/retry-secondary-after-mkfs.diff | 104 --- .../snapshot-restore-drbd-initialized.diff | 67 -- 11 files changed, 33 insertions(+), 1846 deletions(-) delete mode 100644 packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff delete mode 100644 packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff delete mode 100644 packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff delete mode 100644 packages/system/linstor/images/piraeus-server/patches/fix-min-io-probe-device-wait.diff delete mode 100644 packages/system/linstor/images/piraeus-server/patches/retry-secondary-after-mkfs.diff delete mode 100644 packages/system/linstor/images/piraeus-server/patches/snapshot-restore-drbd-initialized.diff diff --git a/packages/system/linstor/Makefile b/packages/system/linstor/Makefile index 5e44ebade..e58fc8578 100644 --- a/packages/system/linstor/Makefile +++ b/packages/system/linstor/Makefile @@ -4,7 +4,7 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/common-envs.mk include ../../../hack/package.mk -LINSTOR_VERSION ?= 1.33.3 +LINSTOR_VERSION ?= 1.35.3 # The golden-image clone guard in packages/apps/vm-disk/templates/dv.yaml # reads allowRemoteVolumeAccess and layerList the way this release does and # has to be re-checked against the new one. diff --git a/packages/system/linstor/images/piraeus-server/Dockerfile b/packages/system/linstor/images/piraeus-server/Dockerfile index 2c8a5500a..631303b72 100644 --- a/packages/system/linstor/images/piraeus-server/Dockerfile +++ b/packages/system/linstor/images/piraeus-server/Dockerfile @@ -1,9 +1,10 @@ -ARG DISTRO=bookworm@sha256:f37a335e82bca302e955fa39f9dfe28f1be618f016f8a2b56318e5a5111afc26 +# LINSTOR 1.35 builds and runs on Java 21, which bookworm does not ship. +ARG DISTRO=trixie@sha256:9c6e381421d0f48d1b8133e573f33c185ee16cbf505eda9c4a8a4791b25ea8a6 ARG LINSTOR_VERSION # ------------------------------------------------------------------------------ # Build linstor-server from source -FROM --platform=$BUILDPLATFORM debian:bookworm@sha256:f37a335e82bca302e955fa39f9dfe28f1be618f016f8a2b56318e5a5111afc26 AS builder +FROM --platform=$BUILDPLATFORM debian:trixie@sha256:9c6e381421d0f48d1b8133e573f33c185ee16cbf505eda9c4a8a4791b25ea8a6 AS builder ARG LINSTOR_VERSION ARG VERSION=${LINSTOR_VERSION} @@ -13,10 +14,10 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get -y upgrade \ && apt-get -y install build-essential git default-jdk-headless python3-all debhelper wget unzip && \ - wget https://services.gradle.org/distributions/gradle-8.9-bin.zip -O /tmp/gradle.zip && \ + wget https://services.gradle.org/distributions/gradle-8.14.2-bin.zip -O /tmp/gradle.zip && \ unzip -d /opt /tmp/gradle.zip && \ rm /tmp/gradle.zip && \ - ln -s /opt/gradle-8.9/bin/gradle /usr/local/bin/gradle + ln -s /opt/gradle-8.14.2/bin/gradle /usr/local/bin/gradle RUN git clone https://github.com/LINBIT/linstor-server.git /linstor-server WORKDIR /linstor-server diff --git a/packages/system/linstor/images/piraeus-server/patches/README.md b/packages/system/linstor/images/piraeus-server/patches/README.md index d2ded4285..f0b80dc92 100644 --- a/packages/system/linstor/images/piraeus-server/patches/README.md +++ b/packages/system/linstor/images/piraeus-server/patches/README.md @@ -1,36 +1,22 @@ # LINSTOR Server Patches -Custom patches for piraeus-server (linstor-server) v1.33.3. +Custom patches for piraeus-server (linstor-server) v1.35.3. -- **allow-toggle-disk-retry.diff** — Backport maintainer implementation of toggle-disk retry/abort - - Source PR/comment: [#475](https://github.com/LINBIT/linstor-server/pull/475), [maintainer note](https://github.com/LINBIT/linstor-server/pull/475#issuecomment-3949630419) - - Backported from upstream commit: [`3d97f71c9`](https://github.com/LINBIT/linstor-server/commit/3d97f71c95a493588d3d521c63eac4d846935fb3) -- **fix-duplicate-tcp-ports.diff** — Preserve DRBD TCP ports during toggle-disk and avoid redundant `ensureStackDataExists()` - - Source PR/review: [#476](https://github.com/LINBIT/linstor-server/pull/476), [review suggestion](https://github.com/LINBIT/linstor-server/pull/476#discussion_r3007725079) - - Backported from commits: [`79d6375c5`](https://github.com/kvaps/linstor-server/commit/79d6375c55d6181b35a7b7f0fe8dbdfb86e126cd), [`bcc89902f`](https://github.com/kvaps/linstor-server/commit/bcc89902f4f61ac1589dd07ebb7f5aae1935370d) -- **fix-luks-header-size.diff** — Account for LUKS2 `--offset`, metadata/keyslots sizing, and device `optimal_io_size` - - Source PR/comment: [#472](https://github.com/LINBIT/linstor-server/pull/472), [maintainer note](https://github.com/LINBIT/linstor-server/pull/472#issuecomment-3949687603) - - Backported from commits: [`ccc85fbd2`](https://github.com/LINBIT/linstor-server/commit/ccc85fbd2c65f0b97c52403fa80f1efdb886ec4e), [`71b601554`](https://github.com/LINBIT/linstor-server/commit/71b601554f41bcb50cd5bd06989c5b0d3a814acd) - - Note: upstream commit [`3d0402a0c`](https://github.com/LINBIT/linstor-server/commit/3d0402a0c25f0a4b57b380321f10e89982f26e7a) is already included in `v1.33.1` - **fix-k8s-create-already-exists.diff** — Adopt the existing object when a create in the Kubernetes CRD backend returns 409 AlreadyExists. The HTTP client retries a POST after a 5xx or a dropped connection, so a create the apiserver already persisted comes back as 409; a single-object transaction has no rollback entry, the key is dropped from memory while the CR stays, and every later write of that key fails until the controller restarts. Seen as a PV stuck in `Released` because linstor-csi could not set `BalanceResourcesEnabled` on the resource definition. - Source PR: [#531](https://github.com/LINBIT/linstor-server/pull/531) - Backported from commit: [`8badf8f05`](https://github.com/lexfrei/linstor-server/commit/8badf8f0526ff3b1b391857e4a046ed46b95a79c) - **retry-adjust-after-stale-bitmap.diff** — Retry `drbdadm adjust` after detaching a stale local bitmap state - Source PR: [#491](https://github.com/LINBIT/linstor-server/pull/491) - Backported from commit: [`51ae50a84`](https://github.com/kvaps/linstor-server/commit/51ae50a84dcb98093f543b819652c750a94d96c9) -- **retry-secondary-after-mkfs.diff** — Retry `drbdadm secondary` after mkfs when DRBD reports the device as held open by an external probe (Talos block-controller, udev, multipathd, etc.). Without this retry, a transient `Device is held open by someone` aborts resource initialization, leaves the satellite in an intermediate state, and prevents the controller from receiving the final `UpToDate` event — orphan PVs in `Released` then cannot be cleaned up. - - Related upstream issue: [#268](https://github.com/LINBIT/linstor-server/issues/268) - - Related upstream issue: [drbd #74](https://github.com/LINBIT/drbd/issues/74) (same EBUSY pattern with multipathd) - -- **fix-min-io-probe-device-wait.diff** — Wait for temporary probe devices before reading pool I/O properties; retry failures and fall back to a probe when ZFS volumes have no usable device path. - - Source PR: [#528](https://github.com/LINBIT/linstor-server/pull/528), fixes [#527](https://github.com/LINBIT/linstor-server/issues/527) - - Adapted for v1.33.3; inactive LVM volumes retain the existing no-probe behavior. - + - Rebased onto v1.35.3, where `DrbdAdm` gained the `DFLT_WINDOWS_TIMEOUT_MS` constant; `DrbdadmAdjust` still receives the failed `OutputData` through `ExtCmdFailedException`, so its own retry on "Resize not allowed during resync" keeps working on top of this one. - **native-protoc.diff** — Download the protoc build for the build host's architecture instead of always `linux-x86_64`. The builder stage runs on `$BUILDPLATFORM`, so on a native arm64 builder the x86_64 protoc cannot execute and `./gradlew generateJava` fails with `Syntax error: ")" unexpected`. - Not proposed upstream yet. -- **snapshot-restore-drbd-initialized.diff** — Mark the volume definitions of a restored snapshot as DRBD-initialized, unless the restored resource definition carries `DrbdOptions/ForceInitialSyncPermanent`, which keeps the existing path that recreates metadata, as for a backup. Without the flag the initial UpToDate race is still open on a restored volume, and `Linstor/Drbd/InitialUptoDateOn`, copied from the source volume through the snapshot, names whichever node won it for the source. When linstor-csi autoplaces a replica on that node, the empty new volume wins the race, gets mkfs and goes StandAlone against the restored replica, so a pod attached there sees an empty filesystem. With the flag set, nothing wins the race, and an autoplaced replica resyncs from the restored one like any replica added to an existing volume. - - Backported from upstream commit: [`9d8cdf511`](https://github.com/LINBIT/linstor-server/commit/9d8cdf5118338fc72a34bb99799e452e031ea8f6), first released in v1.34.0; the `CHANGELOG.md` hunk is left out - - Drop when LINSTOR is upgraded to v1.34.0 or later - - On ZFS and thin pools the autoplaced replicas still skip the initial sync and rely on the source volume's day0 UUID being in the restored replica's history, upstream included; if it is not, DRBD reports unrelated data and goes StandAlone rather than syncing - - Fixes [cozystack/cozystack#4862](https://github.com/cozystack/cozystack/issues/4862) +Dropped on the move from v1.33.3, because v1.35.3 already carries them: + +- `allow-toggle-disk-retry.diff` — upstream [`3d97f71c9`](https://github.com/LINBIT/linstor-server/commit/3d97f71c95a493588d3d521c63eac4d846935fb3), v1.34.0 +- `fix-luks-header-size.diff` — upstream [`ccc85fbd2`](https://github.com/LINBIT/linstor-server/commit/ccc85fbd2c65f0b97c52403fa80f1efdb886ec4e) and [`71b601554`](https://github.com/LINBIT/linstor-server/commit/71b601554f41bcb50cd5bd06989c5b0d3a814acd), v1.34.0 +- `snapshot-restore-drbd-initialized.diff` — upstream [`9d8cdf511`](https://github.com/LINBIT/linstor-server/commit/9d8cdf5118338fc72a34bb99799e452e031ea8f6), v1.34.0 +- `fix-duplicate-tcp-ports.diff` — upstream [`521c6d211`](https://github.com/LINBIT/linstor-server/commit/521c6d21121dbbcc26cf6eda723c24a5b34889be) (from [#476](https://github.com/LINBIT/linstor-server/pull/476)), v1.34.0 +- `retry-secondary-after-mkfs.diff` — upstream [`677789c02`](https://github.com/LINBIT/linstor-server/commit/677789c028ac), v1.34.0 +- `fix-min-io-probe-device-wait.diff` — upstream [`6e5557839`](https://github.com/LINBIT/linstor-server/commit/6e5557839c911c7a1db240dcb6fc348383a74266) (from [#528](https://github.com/LINBIT/linstor-server/pull/528)), v1.35.1 diff --git a/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff b/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff deleted file mode 100644 index 952ba16c5..000000000 --- a/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff +++ /dev/null @@ -1,261 +0,0 @@ -diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -index d93a18014..a944cb809 100644 ---- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -+++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -@@ -111,6 +111,14 @@ import reactor.util.function.Tuple2; - @Singleton - public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionListener - { -+ private enum ToggleDiskAction -+ { -+ NOOP, -+ NORMAL, -+ ABORT, -+ RETRY -+ } -+ - private final AccessContext apiCtx; - private final ScopeRunner scopeRunner; - private final BackgroundRunner backgroundRunner; -@@ -386,69 +394,33 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - "Toggle Disk on %s/%s %s", nodeNameStr, rscNameStr, removeDisk ? "removing disk" : "adding disk"); - - Resource rsc = ctrlApiDataLoader.loadRsc(nodeName, rscName, true); -+ ResourceDefinition rscDfn = rsc.getResourceDefinition(); - -- if (hasDiskAddRequested(rsc)) -- { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.FAIL_RSC_BUSY, -- "Addition of disk to resource already requested", -- true -- )); -- } -- if (hasDiskRemoveRequested(rsc)) -- { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.FAIL_RSC_BUSY, -- "Removal of disk from resource already requested", -- true -- )); -- } -+ ToggleDiskAction action = determineToggleDiskAction(rsc, removeDisk); -+ errorReporter.logDebug("Toggle Disk action: %s", action); - -- if (!removeDisk && !ctrlVlmCrtApiHelper.isDiskless(rsc)) -+ switch (action) - { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.WARN_RSC_ALREADY_HAS_DISK, -- "Resource already has disk", -- true -- )); -- } -- if (removeDisk && ctrlVlmCrtApiHelper.isDiskless(rsc)) -- { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.WARN_RSC_ALREADY_DISKLESS, -- "Resource already diskless", -- true -- )); -+ case NOOP: -+ return handleNoopAction(rsc, removeDisk); -+ case RETRY: -+ return handleRetryAction(rsc, removeDisk, toggleIntoTiebreakerRef, context); -+ case ABORT: -+ clearToggleDiskFlags(rsc); -+ errorReporter.logInfo( -+ "Aborting previous toggle disk transition, starting new transition to %s", -+ removeDisk ? "diskless" : "diskful" -+ ); -+ break; -+ case NORMAL: -+ break; -+ default: -+ throw new ImplementationError("Unhandled case: " + action); - } - -- ResourceDefinition rscDfn = rsc.getResourceDefinition(); -- AccessContext peerCtx = peerAccCtx.get(); -- if (removeDisk) -- { -- // Prevent removal of the last disk -- int haveDiskCount = countDisksAndIsOnline(rscDfn); -- if (haveDiskCount <= 1) -- { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.FAIL_INSUFFICIENT_REPLICA_COUNT, -- "Cannot remove the disk from the only online resource with a disk", -- true -- )); -- } -+ validateToggleDiskPreconditions(rsc, removeDisk); - -- if (!LayerUtils.hasLayer(getLayerData(peerCtx, rsc), DeviceLayerKind.DRBD)) -- { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.FAIL_INVLD_LAYER_STACK, -- "Toggle disk is only supported in combination with DRBD", -- true -- )); -- } -- } -- else -- { -- ensureAllPeersHavePeerSlotLeft(rscDfn); -- } -+ AccessContext peerCtx = peerAccCtx.get(); - - // Save the requested storage pool in the resource properties. - // This does not cause the storage pool to be used automatically. -@@ -628,10 +600,10 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - - ctrlTransactionHelper.commit(); - -- String action = removeDisk ? "Removal of disk from" : "Addition of disk to"; -+ String actionStr = removeDisk ? "Removal of disk from" : "Addition of disk to"; - responses.addEntry(ApiCallRcImpl.simpleEntry( - ApiConsts.MODIFIED, -- action + " resource '" + rscDfn.getName().displayValue + "' " + -+ actionStr + " resource '" + rscDfn.getName().displayValue + "' " + - "on node '" + rsc.getNode().getName().displayValue + "' registered" - )); - -@@ -651,6 +623,40 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - ); - } - -+ private Flux handleNoopAction(Resource rscRef, boolean removeDiskRef) -+ { -+ String state = removeDiskRef ? "diskless" : "diskful"; -+ ApiCallRcImpl responses = new ApiCallRcImpl(); -+ responses.addEntry(ApiCallRcImpl.simpleEntry( -+ ApiConsts.INFO_NOOP, -+ "Resource '" + rscRef.getResourceDefinition().getName().displayValue + "' on node '" + -+ rscRef.getNode().getName().displayValue + "' is already " + state -+ )); -+ return Flux.just(responses); -+ } -+ -+ private Flux handleRetryAction( -+ Resource rscRef, -+ boolean removeDisk, -+ boolean toggleIntoTiebreakerRef, -+ ResponseContext context -+ ) -+ { -+ String direction = removeDisk ? "diskless" : "diskful"; -+ ApiCallRcImpl responses = new ApiCallRcImpl(); -+ NodeName nodeName = rscRef.getNode().getName(); -+ ResourceName rscName = rscRef.getResourceDefinition().getName(); -+ responses.addEntry(ApiCallRcImpl.simpleEntry( -+ ApiConsts.INFO_NOOP, -+ "Retrying toggle disk to " + direction + " for resource '" + rscName.displayValue + -+ "' on node '" + nodeName.displayValue + "'" -+ )); -+ -+ return Flux -+ .just(responses) -+ .concatWith(updateAndAdjustDisk(nodeName, rscName, removeDisk, toggleIntoTiebreakerRef, context)); -+ } -+ - private long getVlmDfnSizePrivileged(VolumeDefinition vlmDfnRef) - { - try -@@ -781,6 +787,96 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - } - } - -+ private ToggleDiskAction determineToggleDiskAction(Resource rsc, boolean removeDisk) -+ { -+ boolean isDiskless = ctrlVlmCrtApiHelper.isDiskless(rsc); -+ boolean diskAddRequested = hasDiskAddRequested(rsc); -+ boolean diskRemoveRequested = hasDiskRemoveRequested(rsc); -+ -+ ToggleDiskAction action; -+ if (isDiskless) -+ { -+ if (diskAddRequested) -+ { -+ action = removeDisk ? ToggleDiskAction.ABORT : ToggleDiskAction.RETRY; -+ } -+ else if (diskRemoveRequested) -+ { -+ action = removeDisk ? ToggleDiskAction.RETRY : ToggleDiskAction.ABORT; -+ } -+ else -+ { -+ action = removeDisk ? ToggleDiskAction.NOOP : ToggleDiskAction.NORMAL; -+ } -+ } -+ else -+ { -+ if (diskRemoveRequested) -+ { -+ action = removeDisk ? ToggleDiskAction.RETRY : ToggleDiskAction.ABORT; -+ } -+ else -+ { -+ action = removeDisk ? ToggleDiskAction.NORMAL : ToggleDiskAction.NOOP; -+ } -+ } -+ -+ return action; -+ } -+ -+ private void validateToggleDiskPreconditions(Resource rsc, boolean removeDisk) -+ { -+ ResourceDefinition rscDfn = rsc.getResourceDefinition(); -+ if (removeDisk) -+ { -+ validateDiskRemovalAllowed(rsc, rscDfn); -+ } -+ else -+ { -+ ensureAllPeersHavePeerSlotLeft(rscDfn); -+ } -+ } -+ -+ private void clearToggleDiskFlags(Resource rsc) -+ { -+ try -+ { -+ rsc.getStateFlags().disableFlags( -+ apiCtx, -+ Resource.Flags.DISK_ADD_REQUESTED, -+ Resource.Flags.DISK_ADDING, -+ Resource.Flags.DISK_REMOVE_REQUESTED, -+ Resource.Flags.DISK_REMOVING -+ ); -+ } -+ catch (AccessDeniedException | DatabaseException exc) -+ { -+ throw new ImplementationError(exc); -+ } -+ } -+ -+ private void validateDiskRemovalAllowed(Resource rsc, ResourceDefinition rscDfn) -+ { -+ int haveDiskCount = countDisksAndIsOnline(rscDfn); -+ if (haveDiskCount <= 1) -+ { -+ throw new ApiRcException(ApiCallRcImpl.simpleEntry( -+ ApiConsts.FAIL_INSUFFICIENT_REPLICA_COUNT, -+ "Cannot remove the disk from the only online resource with a disk", -+ true -+ )); -+ } -+ -+ if (!LayerUtils.hasLayer(getLayerData(peerAccCtx.get(), rsc), DeviceLayerKind.DRBD)) -+ { -+ throw new ApiRcException(ApiCallRcImpl.simpleEntry( -+ ApiConsts.FAIL_INVLD_LAYER_STACK, -+ "Toggle disk is only supported in combination with DRBD", -+ true -+ )); -+ } -+ } -+ - /** - * Although we need to rebuild the layerData as the layerList might have changed, if we do not - * deactivate (i.e. down) the current resource, we need to make sure that deleting DrbdRscData diff --git a/packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff b/packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff deleted file mode 100644 index ed106f98c..000000000 --- a/packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff +++ /dev/null @@ -1,137 +0,0 @@ -diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -index d93a18014..01cfbbacf 100644 ---- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -+++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -@@ -37,6 +37,7 @@ import com.linbit.linstor.core.objects.StorPool; - import com.linbit.linstor.core.objects.Volume; - import com.linbit.linstor.core.objects.VolumeDefinition; - import com.linbit.linstor.core.objects.utils.MixedStorPoolHelper; -+import com.linbit.linstor.core.types.TcpPortNumber; - import com.linbit.linstor.dbdrivers.DatabaseException; - import com.linbit.linstor.event.EventWaiter; - import com.linbit.linstor.event.ObjectIdentifier; -@@ -85,6 +86,7 @@ import java.util.List; - import java.util.Map; - import java.util.Map.Entry; - import java.util.Set; -+import java.util.TreeSet; - - import org.reactivestreams.Publisher; - import reactor.core.publisher.Flux; -@@ -575,12 +577,13 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - - /* - * We also have to remove the currently diskless DrbdRscData and free up the node-id as now we must -- * use the shared resource's node-id -+ * use the shared resource's node-id. We still need to preserve TCP ports though. - */ -+ copyDrbdTcpPortsIfExists(rsc, payload); - } - else - { -- copyDrbdNodeIdIfExists(rsc, payload); -+ copyDrbdSettings(rsc, payload); - } - /* - * rebuilds the layerdata in case we just removed it.. -@@ -782,10 +785,20 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - } - - /** -- * Although we need to rebuild the layerData as the layerList might have changed, if we do not -- * deactivate (i.e. down) the current resource, we need to make sure that deleting DrbdRscData -- * and recreating a new DrbdRscData ends up with the same node-id as before. -+ * Copies DRBD settings (node-id and TCP ports) from the existing DrbdRscData into the payload -+ * before removeLayerData() deletes them. This ensures that recreated DrbdRscData ends up with -+ * the same node-id and TCP ports as before. -+ * -+ * TCP ports must be preserved because if the satellite misses the update (e.g. due to controller -+ * restart or connectivity issues), it will keep the old ports while peers receive the new ones, -+ * causing DRBD connections to fail with StandAlone state. - */ -+ private void copyDrbdSettings(Resource rsc, LayerPayload payload) throws ImplementationError -+ { -+ copyDrbdNodeIdIfExists(rsc, payload); -+ copyDrbdTcpPortsIfExists(rsc, payload); -+ } -+ - private void copyDrbdNodeIdIfExists(Resource rsc, LayerPayload payload) throws ImplementationError - { - Set> drbdRscDataSet = LayerRscUtils.getRscDataByLayer( -@@ -804,6 +817,28 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - } - } - -+ private void copyDrbdTcpPortsIfExists(Resource rsc, LayerPayload payload) throws ImplementationError -+ { -+ Set> drbdRscDataSet = LayerRscUtils.getRscDataByLayer( -+ getLayerData(apiCtx, rsc), -+ DeviceLayerKind.DRBD -+ ); -+ if (!drbdRscDataSet.isEmpty()) -+ { -+ DrbdRscData drbdRscData = (DrbdRscData) drbdRscDataSet.iterator().next(); -+ Collection tcpPorts = drbdRscData.getTcpPortList(); -+ if (tcpPorts != null && !tcpPorts.isEmpty()) -+ { -+ Set portInts = new TreeSet<>(); -+ for (TcpPortNumber port : tcpPorts) -+ { -+ portInts.add(port.value); -+ } -+ payload.drbdRsc.tcpPorts = portInts; -+ } -+ } -+ } -+ - private List removeLayerData(Resource rscRef) - { - List layerList; -@@ -1058,15 +1093,15 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - /* - * We also have to remove the possible meta-children of previous StorageRscData. - * LayerData will be recreated with ensureStackDataExists. -- * However, we still need to remember our node-id if we had / have DRBD in the list -+ * However, we still need to remember our DRBD settings if we had / have DRBD in the list - */ -- copyDrbdNodeIdIfExists(rsc, payload); -+ copyDrbdSettings(rsc, payload); - layerList = removeLayerData(rsc); - } - else - { - markDiskAdded(rsc); -- ctrlLayerStackHelper.resetStoragePools(rsc); -+ ctrlLayerStackHelper.resetStoragePools(rsc, false); - } - ctrlLayerStackHelper.ensureStackDataExists(rsc, layerList, payload); - -diff --git a/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java b/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java -index 3538b380c..f9733b6f1 100644 ---- a/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java -+++ b/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java -@@ -263,6 +263,11 @@ public class CtrlRscLayerDataFactory - } - - public void resetStoragePools(Resource rscRef) -+ { -+ resetStoragePools(rscRef, true); -+ } -+ -+ public void resetStoragePools(Resource rscRef, boolean callEnsureStackDataExistsRef) - { - try - { -@@ -276,8 +281,10 @@ public class CtrlRscLayerDataFactory - - rscDataToProcess.addAll(rscData.getChildren()); - } -- -- ensureStackDataExists(rscRef, null, new LayerPayload()); -+ if (callEnsureStackDataExistsRef) -+ { -+ ensureStackDataExists(rscRef, null, new LayerPayload()); -+ } - } - catch (AccessDeniedException exc) - { diff --git a/packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff b/packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff deleted file mode 100644 index 8611825bc..000000000 --- a/packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff +++ /dev/null @@ -1,570 +0,0 @@ -diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java ---- a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java -+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java -@@ -2081,7 +2081,6 @@ public abstract class AbsStorageProvider< - ) - throws IOException, AccessDeniedException - { -- final StorPoolName storPoolObjName = storPoolObj.getName(); - if (storDevicePath != null) - { - errorReporter.logDebug("updateMinIoSize: Have storDevicePath \"%s\"", storDevicePath); -@@ -2094,35 +2093,18 @@ public abstract class AbsStorageProvider< - "updateMinIoSize: Block device path is \"%s\"", - blockDevicePath.toString() - ); -- final long minIoSize = BlockSizeInfo.getBlockSize(blockDevicePath); -- -- boolean updateValue = true; -- -- final String propKey = StorageConstants.NAMESPACE_INTERNAL + '/' + -- StorageConstants.BLK_DEV_MIN_IO_SIZE; -- final Props storPoolProps = storPoolObj.getProps(storDriverAccCtx); -- final @Nullable String currentPropValue = storPoolProps.getProp(propKey); -- if (currentPropValue != null) -- { -- try -- { -- final long currentPropMinIoSize = Long.parseLong(currentPropValue); -- updateValue = currentPropMinIoSize != minIoSize; -- } -- catch (NumberFormatException ignored) -- { -- } -- } -- -- if (updateValue) -- { -- final String propValue = Long.toString(minIoSize); -- errorReporter.logDebug( -- "Storage pool \"%s\": Set property \"%s\" = \"%s\"", -- storPoolObjName.displayValue, propKey, propValue -- ); -- propsChange.changeStorPoolProp(storPoolObj, propKey, propValue); -- } -+ updatePropIfNeeded( -+ propsChange, -+ storPoolObj, -+ StorageConstants.NAMESPACE_INTERNAL + '/' + StorageConstants.BLK_DEV_MIN_IO_SIZE, -+ Long.toString(BlockSizeInfo.getBlockSize(blockDevicePath)) -+ ); -+ updatePropIfNeeded( -+ propsChange, -+ storPoolObj, -+ StorageConstants.NAMESPACE_INTERNAL + '/' + StorageConstants.BLK_DEV_OPT_IO_SIZE, -+ Long.toString(BlockSizeInfo.getOptimalIoSize(blockDevicePath)) -+ ); - } - else - { -@@ -2130,6 +2112,28 @@ public abstract class AbsStorageProvider< - } - } - -+ private void updatePropIfNeeded( -+ LocalPropsChangePojo propsChangeRef, -+ StorPool storPoolObjRef, -+ String propKeyRef, -+ String propValueRef -+ ) -+ throws AccessDeniedException -+ { -+ final Props storPoolProps = storPoolObjRef.getProps(storDriverAccCtx); -+ final @Nullable String currentPropValue = storPoolProps.getProp(propKeyRef); -+ if (currentPropValue == null || !currentPropValue.equals(propValueRef)) -+ { -+ errorReporter.logDebug( -+ "Storage pool \"%s\": Set property \"%s\" = \"%s\"", -+ storPoolObjRef.getName().displayValue, -+ propKeyRef, -+ propValueRef -+ ); -+ propsChangeRef.changeStorPoolProp(storPoolObjRef, propKeyRef, propValueRef); -+ } -+ } -+ - @SuppressWarnings("unchecked") - @Override - public void updateAllocatedSize(VlmProviderObject vlmDataRef) -diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java ---- a/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java -+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java -@@ -3,8 +3,11 @@ package com.linbit.linstor.layer.storage.utils; - import com.linbit.utils.MathUtils; - import com.linbit.utils.SymbolicLinkResolver; - -+import static com.linbit.linstor.layer.storage.BlockSizeConsts.DFLT_OPT_IO_SIZE; - import static com.linbit.linstor.layer.storage.BlockSizeConsts.DFLT_IO_SIZE; -+import static com.linbit.linstor.layer.storage.BlockSizeConsts.MAX_OPT_IO_SIZE; - import static com.linbit.linstor.layer.storage.BlockSizeConsts.MAX_IO_SIZE; -+import static com.linbit.linstor.layer.storage.BlockSizeConsts.MIN_OPT_IO_SIZE; - import static com.linbit.linstor.layer.storage.BlockSizeConsts.MIN_IO_SIZE; - - import java.io.FileInputStream; -@@ -13,6 +16,8 @@ import java.nio.file.Path; - - public class BlockSizeInfo - { -+ private static final int NUMBER_BUFFER_SIZE = 32; -+ - /** - * Determines the blocksize, aka minimum I/O size, for the specified backing storage path. - * -@@ -51,7 +56,7 @@ public class BlockSizeInfo - final Path infoSourceName = blockDevice.getFileName(); - final Path infoSource = Path.of("/sys/block", infoSourceName.toString(), "queue/physical_block_size"); - -- final byte[] data = new byte[32]; -+ final byte[] data = new byte[NUMBER_BUFFER_SIZE]; - try (final FileInputStream fileIn = new FileInputStream(infoSource.toString())) - { - final int readCount = fileIn.read(data); -@@ -75,4 +80,38 @@ public class BlockSizeInfo - } - return blockSize; - } -+ -+ public static long getOptimalIoSize(final Path storageObj) -+ { -+ long optIoSize = DFLT_OPT_IO_SIZE; -+ try -+ { -+ final Path blockDevice = SymbolicLinkResolver.resolveSymLink(storageObj); -+ final Path infoSourceName = blockDevice.getFileName(); -+ final Path infoSource = Path.of("/sys/block", infoSourceName.toString(), "queue/optimal_io_size"); -+ -+ final byte[] data = new byte[NUMBER_BUFFER_SIZE]; -+ try (final FileInputStream fileIn = new FileInputStream(infoSource.toString())) -+ { -+ final int readCount = fileIn.read(data); -+ if (readCount > 0) -+ { -+ String numberStr = new String(data, 0, readCount); -+ numberStr = numberStr.trim(); -+ try -+ { -+ final long unboundedOptIoSize = Long.parseLong(numberStr); -+ optIoSize = MathUtils.bounds(MIN_OPT_IO_SIZE, unboundedOptIoSize, MAX_OPT_IO_SIZE); -+ } -+ catch (NumberFormatException ignored) -+ { -+ } -+ } -+ } -+ } -+ catch (IOException ignored) -+ { -+ } -+ return optIoSize; -+ } - } -diff --git a/server/src/main/java/com/linbit/SizeConv.java b/server/src/main/java/com/linbit/SizeConv.java ---- a/server/src/main/java/com/linbit/SizeConv.java -+++ b/server/src/main/java/com/linbit/SizeConv.java -@@ -16,6 +16,7 @@ public class SizeConv - public enum SizeUnit - { - UNIT_B, -+ UNIT_SECTORS, - UNIT_KiB, - UNIT_MiB, - UNIT_GiB, -@@ -41,6 +42,9 @@ public class SizeConv - case UNIT_B: - factor = FACTOR_B; - break; -+ case UNIT_SECTORS: -+ factor = FACTOR_SECTORS; -+ break; - case UNIT_KiB: - factor = FACTOR_KiB; - break; -@@ -111,6 +115,9 @@ public class SizeConv - case "b": - unit = SizeUnit.UNIT_B; - break; -+ case "s": -+ unit = SizeUnit.UNIT_SECTORS; -+ break; - case "k": - // fall-through - case "kb": -@@ -217,6 +224,9 @@ public class SizeConv - // Factor 1 - public static final BigInteger FACTOR_B = BigInteger.valueOf(1L); - -+ // Factor 512 -+ public static final BigInteger FACTOR_SECTORS = BigInteger.valueOf(512L); -+ - // Factor 1,024 - // Naming convention exception: SI unit capitalization rules - @SuppressWarnings("checkstyle:constantname") -diff --git a/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java b/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java ---- a/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java -+++ b/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java -@@ -1,27 +1,62 @@ - package com.linbit.linstor.layer.luks; - -+import com.linbit.SizeConv; -+import com.linbit.SizeConv.SizeUnit; - import com.linbit.exceptions.InvalidSizeException; -+import com.linbit.linstor.PriorityProps; -+import com.linbit.linstor.annotation.Nullable; -+import com.linbit.linstor.api.ApiConsts; -+import com.linbit.linstor.core.objects.AbsResource; -+import com.linbit.linstor.core.objects.AbsVolume; -+import com.linbit.linstor.core.objects.Resource; -+import com.linbit.linstor.core.objects.ResourceDefinition; -+import com.linbit.linstor.core.objects.ResourceGroup; -+import com.linbit.linstor.core.objects.Snapshot; -+import com.linbit.linstor.core.objects.SnapshotVolume; -+import com.linbit.linstor.core.objects.StorPool; -+import com.linbit.linstor.core.objects.Volume; -+import com.linbit.linstor.core.objects.VolumeDefinition; - import com.linbit.linstor.dbdrivers.DatabaseException; - import com.linbit.linstor.layer.AbsLayerSizeCalculator; -+import com.linbit.linstor.netcom.Peer; -+import com.linbit.linstor.propscon.InvalidKeyException; -+import com.linbit.linstor.propscon.ReadOnlyProps; - import com.linbit.linstor.security.AccessDeniedException; -+import com.linbit.linstor.storage.StorageConstants; - import com.linbit.linstor.storage.data.adapter.luks.LuksVlmData; - import com.linbit.linstor.storage.interfaces.categories.resource.VlmProviderObject; - import com.linbit.linstor.storage.kinds.DeviceLayerKind; - import com.linbit.linstor.storage.kinds.ExtTools; - import com.linbit.linstor.storage.kinds.ExtToolsInfo; - import com.linbit.linstor.storage.kinds.ExtToolsInfo.Version; -+import com.linbit.linstor.utils.layer.LayerVlmUtils; -+import com.linbit.utils.ShellUtils; -+import com.linbit.utils.SignedAlign; - - import javax.inject.Inject; - import javax.inject.Singleton; - -+import java.util.Iterator; -+import java.util.LinkedList; -+import java.util.List; -+import java.util.Set; -+import java.util.regex.Matcher; -+import java.util.regex.Pattern; -+ - @Singleton - public class LuksLayerSizeCalculator extends AbsLayerSizeCalculator> - { -+ public static final String LUKS2_OPT_METADATA_SIZE = "--luks2-metadata-size"; -+ public static final String LUKS2_OPT_KEYSLOTS_SIZE = "--luks2-keyslots-size"; -+ public static final String LUKS2_OPT_OFFSET = "--offset"; -+ private static final String LUKS2_OPT_ALIGN_PAYLOAD = "--align-payload"; -+ private static final long DFLT_LUKS2_METADATA_SIZE_IN_BYTES = 16L << 10; -+ private static final long DFLT_ALIGNMENT_1MIB_IN_BYTES = 1L << 20; -+ private static final long LUKS1_HEADER_SIZE_IN_KIB = 2L << 10; -+ private static final long LUKS2_HEADER_SIZE_IN_KIB = 16L << 10; -+ private static final long LUKS2_HEADER_SIZE_IN_BYTES = LUKS2_HEADER_SIZE_IN_KIB << 10; - -- // linstor calculates in KiB -- private static final int MIB = 1024; -- private static final int LUKS1_HEADER_SIZE = 2 * MIB; -- private static final int LUKS2_HEADER_SIZE = 16 * MIB; -+ private static final Pattern PATTERN_SIZE = Pattern.compile("(\\d+)([kKmMgGtT]i?[bB]?|[sS]|)"); - - @Inject - public LuksLayerSizeCalculator(AbsLayerSizeCalculatorInit initRef) -@@ -74,30 +109,266 @@ public class LuksLayerSizeCalculator extends AbsLayerSizeCalculator vlmDataRef) -- throws AccessDeniedException -+ throws AccessDeniedException, InvalidSizeException - { -- ExtToolsInfo cryptSetupInfo = vlmDataRef.getRscLayerObject() -+ @Nullable Peer peer = vlmDataRef.getRscLayerObject() - .getAbsResource() - .getNode() -- .getPeer(sysCtx) -- .getExtToolsManager() -+ .getPeer(sysCtx); -+ if (peer == null) -+ { -+ throw new InvalidSizeException( -+ "Could not calculate size of LUKS volume, since cryptsetup's version could not be determined", -+ null -+ ); -+ } -+ ExtToolsInfo cryptSetupInfo = peer.getExtToolsManager() - .getExtToolInfo(ExtTools.CRYPT_SETUP); -- long luksHeaderSize; -+ -+ final long luksHeaderSize; - if (cryptSetupInfo != null && cryptSetupInfo.isSupported()) - { - if (cryptSetupInfo.hasVersionOrHigher(new Version(2, 1))) - { -- luksHeaderSize = LUKS2_HEADER_SIZE; -+ luksHeaderSize = calcLuks2HeaderSize(vlmDataRef); - } - else - { -- luksHeaderSize = LUKS1_HEADER_SIZE; -+ luksHeaderSize = LUKS1_HEADER_SIZE_IN_KIB; - } - } - else - { -- luksHeaderSize = -1; -+ throw new InvalidSizeException( -+ "Could not calculate size of LUKS volume, since cryptsetup's version could not be determined", -+ null -+ ); - } - return luksHeaderSize; - } -+ -+ private long calcLuks2HeaderSize(VlmProviderObject vlmDataRef) throws AccessDeniedException -+ { -+ PriorityProps prioProps = getPrioProps(vlmDataRef); -+ -+ final @Nullable String userOptProp = prioProps.getProp( -+ ApiConsts.KEY_STOR_DRIVER_LUKS_FORMAT_OPTIONS, -+ ApiConsts.NAMESPC_STORAGE_DRIVER -+ ); -+ List userOptions = userOptProp != null ? -+ ShellUtils.shellSplit(userOptProp) : -+ new LinkedList<>(); -+ -+ final long alignedHeaderSizeInBytes; -+ final long unalignedHeaderSizeInBytes; -+ @Nullable Long cryptsetupOffsetInBytes = getLongOptionValue( -+ userOptions, -+ LUKS2_OPT_OFFSET, -+ SizeUnit.UNIT_SECTORS -+ ); -+ if (cryptsetupOffsetInBytes != null) -+ { -+ alignedHeaderSizeInBytes = cryptsetupOffsetInBytes; -+ } -+ else -+ { -+ @Nullable Long cryptsetupLuks2KeyslotsSize = getLongOptionValue( -+ userOptions, -+ LUKS2_OPT_KEYSLOTS_SIZE -+ ); -+ if (cryptsetupLuks2KeyslotsSize == null) -+ { -+ unalignedHeaderSizeInBytes = LUKS2_HEADER_SIZE_IN_BYTES; -+ } -+ else -+ { -+ @Nullable Long cryptsetupLuks2MetadataSize = getLongOptionValue( -+ userOptions, -+ LUKS2_OPT_METADATA_SIZE -+ ); -+ cryptsetupLuks2MetadataSize = cryptsetupLuks2MetadataSize == null ? -+ DFLT_LUKS2_METADATA_SIZE_IN_BYTES : -+ cryptsetupLuks2MetadataSize; -+ -+ unalignedHeaderSizeInBytes = 2 * cryptsetupLuks2MetadataSize + cryptsetupLuks2KeyslotsSize; -+ } -+ -+ long alignment = getAlignment(vlmDataRef, userOptions); -+ alignedHeaderSizeInBytes = new SignedAlign(alignment).ceiling(unalignedHeaderSizeInBytes); -+ } -+ return SizeConv.convert( -+ alignedHeaderSizeInBytes, -+ SizeUnit.UNIT_B, -+ SizeUnit.UNIT_KiB -+ ); -+ } -+ -+ private long getAlignment(VlmProviderObject vlmDataRef, List userOptions) throws AccessDeniedException -+ { -+ @Nullable Long cryptsetupAlignPayloadInBytes = getLongOptionValue( -+ userOptions, -+ LUKS2_OPT_ALIGN_PAYLOAD, -+ SizeUnit.UNIT_SECTORS -+ ); -+ -+ long alignment = DFLT_ALIGNMENT_1MIB_IN_BYTES; -+ if (cryptsetupAlignPayloadInBytes != null) -+ { -+ alignment = cryptsetupAlignPayloadInBytes; -+ } -+ else -+ { -+ final long maxOptIoSize = getMaxOptIoSize(vlmDataRef); -+ alignment = Math.max(alignment, maxOptIoSize); -+ } -+ return alignment; -+ } -+ -+ private long getMaxOptIoSize(VlmProviderObject vlmDataRef) throws InvalidKeyException, AccessDeniedException -+ { -+ long ret = 0; -+ Set storPoolSet = LayerVlmUtils.getStorPoolSet(vlmDataRef, sysCtx); -+ for (StorPool sp : storPoolSet) -+ { -+ @Nullable String strValue = sp.getProps(sysCtx) -+ .getProp( -+ StorageConstants.BLK_DEV_OPT_IO_SIZE, -+ StorageConstants.NAMESPACE_INTERNAL -+ ); -+ if (strValue != null) -+ { -+ try -+ { -+ long parsed = Long.parseLong(strValue); -+ ret = Math.max(parsed, ret); -+ } -+ catch (NumberFormatException ignored) -+ { -+ errorReporter.logWarning( -+ "LuksHeaderSize: Failed to parse '%s' from prop %s. Defaulting to 0 opt_io_size " + -+ "(no recommendation/hint)", -+ strValue, -+ StorageConstants.NAMESPACE_INTERNAL + "/" + StorageConstants.BLK_DEV_OPT_IO_SIZE -+ ); -+ } -+ } -+ } -+ return ret; -+ } -+ -+ private @Nullable Long getLongOptionValue(List userOptionsRef, String optRef) -+ { -+ return getLongOptionValue(userOptionsRef, optRef, SizeUnit.UNIT_B); -+ } -+ -+ @SuppressWarnings("checkstyle:magicnumber") -+ private @Nullable Long getLongOptionValue(List userOptionsRef, String optRef, SizeUnit dfltSizeUnit) -+ { -+ @Nullable Long ret = null; -+ @Nullable String val = findLastValue(userOptionsRef, optRef); -+ if (val != null && !val.isBlank()) -+ { -+ Matcher matcher = PATTERN_SIZE.matcher(val); -+ if (matcher.matches()) -+ { -+ try -+ { -+ ret = Long.parseLong(matcher.group(1)); -+ String unit = matcher.group(2); -+ SizeUnit sizeUnit; -+ if (unit.isBlank()) -+ { -+ sizeUnit = dfltSizeUnit; -+ } -+ else -+ { -+ boolean forcePowerOfTwo = unit.length() == 1 || unit.length() == 3; -+ sizeUnit = SizeUnit.parse(unit, forcePowerOfTwo); -+ } -+ -+ ret = SizeConv.convert(ret, sizeUnit, SizeUnit.UNIT_B); -+ } -+ catch (NumberFormatException ignored) -+ { -+ errorReporter.logWarning( -+ "LuksHeaderSize: Failed to parse '%s' from option '%s %s'.", -+ matcher.group(1), -+ optRef, -+ val -+ ); -+ } -+ } -+ } -+ return ret; -+ } -+ -+ private @Nullable String findLastValue(List userOptionsRef, String optRef) -+ { -+ Iterator it = userOptionsRef.iterator(); -+ @Nullable String val = null; -+ while (it.hasNext()) -+ { -+ String opt = it.next(); -+ if (opt.equals(optRef)) -+ { -+ if (it.hasNext()) -+ { -+ val = it.next(); -+ } -+ else -+ { -+ val = null; -+ } -+ } -+ else if (opt.startsWith(optRef + "=")) -+ { -+ val = opt.substring(optRef.length() + 1); -+ if (val.isBlank()) -+ { -+ val = null; -+ } -+ } -+ } -+ return val; -+ } -+ -+ private PriorityProps getPrioProps(VlmProviderObject vlmDataRef) throws AccessDeniedException -+ { -+ final AbsVolume vlm = vlmDataRef.getVolume(); -+ final AbsResource rsc = vlm.getAbsResource(); -+ final ResourceDefinition rscDfn = vlm.getResourceDefinition(); -+ final VolumeDefinition vlmDfn = vlm.getVolumeDefinition(); -+ final ResourceGroup rscGrp = rscDfn.getResourceGroup(); -+ -+ final ReadOnlyProps vlmProps; -+ final ReadOnlyProps rscProps; -+ if (vlm instanceof Volume) -+ { -+ vlmProps = ((Volume) vlm).getProps(sysCtx); -+ rscProps = ((Resource) rsc).getProps(sysCtx); -+ } -+ else -+ { -+ vlmProps = ((SnapshotVolume) vlm).getVlmProps(sysCtx); -+ rscProps = ((Snapshot) rsc).getRscProps(sysCtx); -+ } -+ -+ final PriorityProps prioProps = new PriorityProps( -+ vlmProps, -+ rscProps -+ ); -+ for (StorPool storPool : LayerVlmUtils.getStorPoolSet(vlmDataRef, sysCtx)) -+ { -+ prioProps.addProps(storPool.getProps(sysCtx)); -+ } -+ prioProps.addProps( -+ rsc.getNode().getProps(sysCtx), -+ vlmDfn.getProps(sysCtx), -+ rscDfn.getProps(sysCtx), -+ rscGrp.getVolumeGroupProps(sysCtx, vlmDfn.getVolumeNumber()), -+ rscGrp.getProps(sysCtx), -+ stltProps -+ ); -+ return prioProps; -+ } - } -diff --git a/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java b/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java ---- a/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java -+++ b/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java -@@ -13,4 +13,9 @@ public class BlockSizeConsts - - // Default value for the minimum_io_size value of non-storage layers - public static final long DFLT_SPECIAL_IO_SIZE = (1L << 12); -+ -+ // optimal_io_size may be 0 to indicate no recommendation. -+ public static final long MIN_OPT_IO_SIZE = 0; -+ public static final long DFLT_OPT_IO_SIZE = 0; -+ public static final long MAX_OPT_IO_SIZE = Long.MAX_VALUE; - } -diff --git a/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java b/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java ---- a/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java -+++ b/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java -@@ -11,6 +11,7 @@ public class StorageConstants - public static final String NAMESPACE_NVME = ApiConsts.NAMESPC_STORAGE_DRIVER + "/NVME"; - public static final String NAMESPACE_INTERNAL = NAMESPACE_STOR_DRIVER + "/internal/"; - -+ public static final String BLK_DEV_OPT_IO_SIZE = "optIoSize"; - public static final String BLK_DEV_MIN_IO_SIZE = "minIoSize"; - public static final String BLK_DEV_MIN_IO_SIZE_AUTO = "minIoSizeAuto"; - public static final String BLK_DEV_MAX_BIO_SIZE = "maxBioSize"; diff --git a/packages/system/linstor/images/piraeus-server/patches/fix-min-io-probe-device-wait.diff b/packages/system/linstor/images/piraeus-server/patches/fix-min-io-probe-device-wait.diff deleted file mode 100644 index bb6185b88..000000000 --- a/packages/system/linstor/images/piraeus-server/patches/fix-min-io-probe-device-wait.diff +++ /dev/null @@ -1,660 +0,0 @@ -diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java ---- a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java -+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java -@@ -163,6 +163,8 @@ - } - - private static final long DFLT_WAIT_UNTIL_DEVICE_CREATED_TIMEOUT_IN_MS = 5000; -+ private static final int PROBE_VLM_ATTEMPTS = 3; -+ private static final long PROBE_VLM_RETRY_DELAY_IN_MS = 1000; - protected static final int DFLT_STRIPES = 1; - public static final long SIZE_OF_NOT_FOUND_STOR_POOL = ApiConsts.VAL_STOR_POOL_SPACE_NOT_FOUND; - -@@ -184,6 +186,7 @@ - protected final Set changedStoragePoolStrings = new HashSet<>(); - private final String typeDescr; - private final FileSystemWatch fsWatch; -+ private final Object probeVlmLock = new Object(); - protected final DeviceProviderKind kind; - private final DrbdInvalidateUtils drbdInvalidateUtils; - -@@ -2022,61 +2025,126 @@ - final StorPoolName storPoolObjName = storPoolObj.getName(); - errorReporter.logDebug("ENTER updateMinIoSize method: Storage pool \"%s\"", storPoolObjName.displayValue); - Collection> vlmProviderList = storPoolObj.getVolumes(storDriverAccCtx); -+ boolean haveInfo = false; - if (vlmProviderList != null && vlmProviderList.size() >= 1) - { - errorReporter.logDebug("updateMinIoSize: Have vlmProviderList with %d items", vlmProviderList.size()); - Iterator> vlmProviderIter = vlmProviderList.iterator(); -- boolean haveInfo = false; - while (vlmProviderIter.hasNext() && !haveInfo) - { - VlmProviderObject vlmProvider = vlmProviderIter.next(); - errorReporter.logDebug("updateMinIoSize: Have vlmProvider"); - final String storDevicePath = vlmProvider.getDevicePath(); -+ if (storDevicePath != null) -+ { -+ try -+ { -+ updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange); -+ haveInfo = true; -+ } -+ catch (IOException ignored) -+ { -+ } -+ } -+ } -+ } -+ // An inactive shared LVM volume can be owned by another node; do not probe it by creating a volume. -+ if (!haveInfo && this instanceof ProbeVlmStorageProvider && -+ (vlmProviderList == null || vlmProviderList.isEmpty() || -+ kind == DeviceProviderKind.ZFS || kind == DeviceProviderKind.ZFS_THIN)) -+ { -+ updateMinIoSizeByProbeVlm((ProbeVlmStorageProvider) this, storPoolObj, propsChange); -+ } -+ errorReporter.logDebug("EXIT updateMinIoSize method"); -+ } -+ -+ private void updateMinIoSizeByProbeVlm( -+ ProbeVlmStorageProvider storPrv, -+ StorPool storPoolObj, -+ LocalPropsChangePojo propsChange -+ ) -+ throws AccessDeniedException -+ { -+ // Providers use a fixed probe name. Keep creation, probing and cleanup in one critical section. -+ synchronized (probeVlmLock) -+ { -+ boolean haveInfo = false; -+ boolean interrupted = false; -+ String lastFailure = ""; -+ int attempts = 0; -+ while (attempts < PROBE_VLM_ATTEMPTS && !haveInfo && !interrupted) -+ { -+ if (attempts > 0) -+ { -+ try -+ { -+ Thread.sleep(PROBE_VLM_RETRY_DELAY_IN_MS); -+ } -+ catch (InterruptedException exc) -+ { -+ Thread.currentThread().interrupt(); -+ lastFailure = "interrupted before the next probe attempt"; -+ break; -+ } -+ } -+ attempts++; -+ boolean cleaned = false; - try - { -- updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange); -- haveInfo = true; -- } -- catch (IOException ignored) -- { -- } -- } -- } -- else -- { -- errorReporter.logDebug("updateMinIoSize: Don't have vlmProviderList, using temporary volumes"); -- if (this instanceof ProbeVlmStorageProvider) -- { -- final ProbeVlmStorageProvider storPrv = (ProbeVlmStorageProvider) this; -- try -- { -- final String storDevicePath = storPrv.createTmpProbeVlm(storPoolObj); -+ @Nullable String storDevicePath = storPrv.createTmpProbeVlm(storPoolObj); -+ if (storDevicePath == null) -+ { -+ lastFailure = "temporary volume has no device path"; -+ } -+ else -+ { -+ waitUntilDeviceCreated(storDevicePath, getWaitTimeoutAfterCreate(storPoolObj)); -+ updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange); -+ haveInfo = true; -+ } -+ } -+ catch (StorageException | IOException exc) -+ { -+ lastFailure = exc.getMessage(); -+ interrupted = exc.getCause() instanceof InterruptedException; -+ errorReporter.logDebug("Temporary volume probe attempt %d failed: %s", attempts, lastFailure); -+ } -+ finally -+ { - try - { -- updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange); -- } -- catch (IOException ignored) -- { -- } -- } -- catch (StorageException ignored) -- { -- errorReporter.logDebug("updateMinIoSize: Temporary volume creation failed"); -- } -- finally -- { -- try -- { - storPrv.deleteTmpProbeVlm(storPoolObj); -- } -- catch (StorageException ignored) -- { -- errorReporter.logDebug("updateMinIoSize: Temporary volume deletion failed"); -- } -- } -- } -- } -- errorReporter.logDebug("EXIT updateMinIoSize method"); -+ cleaned = true; -+ } -+ catch (StorageException exc) -+ { -+ interrupted |= exc.getCause() instanceof InterruptedException; -+ errorReporter.logWarning( -+ "Failed to delete temporary probe volume for storage pool '%s': %s", -+ storPoolObj.getName().displayValue, -+ exc.getMessage() -+ ); -+ } -+ } -+ if (interrupted) -+ { -+ Thread.currentThread().interrupt(); -+ } -+ if (!cleaned) -+ { -+ break; -+ } -+ } -+ if (!haveInfo) -+ { -+ errorReporter.logWarning( -+ "Could not determine block device information for storage pool '%s' after %d probe attempts: %s", -+ storPoolObj.getName().displayValue, -+ attempts, -+ lastFailure -+ ); -+ } -+ } - } - - private void updateMinIoSizeByDevice( -diff --git a/src/test/java/com/linbit/fsevent/ProbeVolumeDeviceWaitTest.java b/src/test/java/com/linbit/fsevent/ProbeVolumeDeviceWaitTest.java -new file mode 100644 ---- /dev/null -+++ b/src/test/java/com/linbit/fsevent/ProbeVolumeDeviceWaitTest.java -@@ -0,0 +1,467 @@ -+package com.linbit.fsevent; -+ -+import com.linbit.Platform; -+import com.linbit.linstor.core.identifier.StorPoolName; -+import com.linbit.linstor.core.objects.Resource; -+import com.linbit.linstor.core.objects.StorPool; -+import com.linbit.linstor.core.pojos.LocalPropsChangePojo; -+import com.linbit.linstor.layer.storage.AbsStorageProvider.AbsStorageProviderInit; -+import com.linbit.linstor.layer.storage.BlockSizeConsts; -+import com.linbit.linstor.layer.storage.lvm.LvmProvider; -+import com.linbit.linstor.layer.storage.zfs.ZfsThinProvider; -+import com.linbit.linstor.logging.ErrorReporter; -+import com.linbit.linstor.propscon.Props; -+import com.linbit.linstor.storage.StorageConstants; -+import com.linbit.linstor.storage.StorageException; -+import com.linbit.linstor.storage.interfaces.categories.resource.VlmProviderObject; -+ -+import java.io.IOException; -+import java.nio.file.Files; -+import java.nio.file.Path; -+import java.util.Collections; -+import java.util.List; -+import java.util.Map; -+import java.util.concurrent.Callable; -+import java.util.concurrent.CountDownLatch; -+import java.util.concurrent.ExecutorService; -+import java.util.concurrent.Executors; -+import java.util.concurrent.Future; -+import java.util.concurrent.ScheduledExecutorService; -+import java.util.concurrent.ScheduledFuture; -+import java.util.concurrent.TimeUnit; -+import java.util.concurrent.atomic.AtomicInteger; -+ -+import org.junit.After; -+import org.junit.Before; -+import org.junit.Rule; -+import org.junit.Test; -+import org.junit.rules.TemporaryFolder; -+import org.junit.rules.Timeout; -+ -+import static org.junit.Assert.assertEquals; -+import static org.junit.Assert.assertFalse; -+import static org.junit.Assert.assertTrue; -+import static org.junit.Assume.assumeTrue; -+import static org.mockito.Mockito.doReturn; -+import static org.mockito.Mockito.mock; -+import static org.mockito.Mockito.mockingDetails; -+import static org.mockito.Mockito.when; -+ -+public class ProbeVolumeDeviceWaitTest -+{ -+ @Rule -+ public final TemporaryFolder directory = new TemporaryFolder(); -+ -+ @Rule -+ public final Timeout timeout = Timeout.seconds(15); -+ -+ private FileSystemWatch watch; -+ private Path device; -+ private Path link; -+ private StorPool pool; -+ private ProbeProvider provider; -+ private ErrorReporter reporter; -+ private ScheduledExecutorService executor; -+ -+ @Before -+ public void setUp() throws Exception -+ { -+ assumeTrue(Platform.isLinux()); -+ device = directory.newFile("zd-test").toPath(); -+ link = directory.getRoot().toPath().resolve("probe"); -+ executor = Executors.newSingleThreadScheduledExecutor(); -+ reporter = mock(ErrorReporter.class); -+ watch = new FileSystemWatch(reporter); -+ watch.start(); -+ pool = mock(StorPool.class); -+ when(pool.getName()).thenReturn(new StorPoolName("data")); -+ when(pool.getVolumes(null)).thenReturn(Collections.emptyList()); -+ when(pool.getProps(null)).thenReturn(mock(Props.class)); -+ provider = new ProbeProvider(providerInit()); -+ } -+ -+ @After -+ public void tearDown() throws Exception -+ { -+ if (executor != null) -+ { -+ executor.shutdownNow(); -+ assertTrue(executor.awaitTermination(2, TimeUnit.SECONDS)); -+ } -+ if (watch != null) -+ { -+ watch.shutdown(false); -+ watch.awaitShutdown(2000); -+ watch.cancelAllWatchKeys(); -+ } -+ } -+ -+ @Test -+ public void delayedDevicePublishesPropertiesBeforeCleanup() throws Exception -+ { -+ provider.delayMillis = 50; -+ assertPublished(probe()); -+ assertCleaned(1); -+ assertEquals(0, warningCount()); -+ } -+ -+ @Test -+ public void existingProbeDevicePublishesProperties() throws Exception -+ { -+ assertPublished(probe()); -+ assertCleaned(1); -+ } -+ -+ @Test -+ public void missingDeviceTimesOutAndCleansEveryAttempt() throws Exception -+ { -+ provider.delayMillis = -1; -+ provider.timeoutMillis = 20; -+ assertTrue(probe().isEmpty()); -+ assertCleaned(3); -+ assertEquals(1, warningCount()); -+ } -+ -+ @Test -+ public void failedCreationStillCleansEveryAttempt() throws Exception -+ { -+ provider.failCreateTimes = Integer.MAX_VALUE; -+ assertTrue(probe().isEmpty()); -+ assertCleaned(3); -+ assertEquals(1, warningCount()); -+ } -+ -+ @Test -+ public void transientCreationFailureIsRetried() throws Exception -+ { -+ provider.failCreateTimes = 1; -+ provider.delayMillis = 50; -+ assertPublished(probe()); -+ assertCleaned(2); -+ assertEquals(0, warningCount()); -+ } -+ -+ @Test -+ public void nullProbePathDoesNotPublishProperties() throws Exception -+ { -+ provider.nullPath = true; -+ assertTrue(probe().isEmpty()); -+ assertCleaned(3); -+ assertEquals(1, warningCount()); -+ } -+ -+ @Test -+ public void nullFirstPathDoesNotSkipUsableVolume() throws Exception -+ { -+ Files.createSymbolicLink(link, device); -+ doReturn(List.of(volume(null), volume(link))).when(pool).getVolumes(null); -+ assertPublished(probe()); -+ assertEquals(0, provider.createCalls); -+ assertEquals(0, provider.deleteCalls); -+ } -+ -+ @Test -+ public void missingFirstPathDoesNotSkipUsableVolume() throws Exception -+ { -+ Files.createSymbolicLink(link, device); -+ doReturn(List.of(volume(link.resolveSibling("missing")), volume(link))).when(pool).getVolumes(null); -+ assertPublished(probe()); -+ assertEquals(0, provider.createCalls); -+ assertEquals(0, provider.deleteCalls); -+ } -+ -+ @Test -+ public void allNullPathsFallBackToProbe() throws Exception -+ { -+ doReturn(List.of(volume(null))).when(pool).getVolumes(null); -+ provider.delayMillis = 50; -+ assertPublished(probe()); -+ assertCleaned(1); -+ } -+ -+ @Test -+ public void allMissingPathsFallBackToProbe() throws Exception -+ { -+ doReturn(List.of( -+ volume(link.resolveSibling("missing-a")), volume(link.resolveSibling("missing-b")) -+ )).when(pool).getVolumes(null); -+ provider.delayMillis = 50; -+ assertPublished(probe()); -+ assertCleaned(1); -+ } -+ -+ @Test -+ public void cleanupFailureAfterSuccessIsReported() throws Exception -+ { -+ provider.failDelete = true; -+ assertPublished(probe()); -+ assertEquals(1, provider.createCalls); -+ assertEquals(1, provider.deleteCalls); -+ assertEquals(1, warningCount()); -+ } -+ -+ @Test -+ public void cleanupFailurePreventsAnotherCreationAttempt() throws Exception -+ { -+ provider.delayMillis = -1; -+ provider.timeoutMillis = 20; -+ provider.failDelete = true; -+ assertTrue(probe().isEmpty()); -+ assertEquals(1, provider.createCalls); -+ assertEquals(1, provider.deleteCalls); -+ assertEquals(2, warningCount()); -+ } -+ -+ @Test -+ public void interruptedDeviceWaitCleansUpAndPreservesInterrupt() throws Exception -+ { -+ provider.delayMillis = -1; -+ Thread callingThread = Thread.currentThread(); -+ executor.schedule(callingThread::interrupt, 50, TimeUnit.MILLISECONDS); -+ try -+ { -+ assertTrue(probe().isEmpty()); -+ assertTrue(callingThread.isInterrupted()); -+ assertCleaned(1); -+ assertEquals(1, warningCount()); -+ } -+ finally -+ { -+ Thread.interrupted(); -+ } -+ } -+ -+ @Test -+ public void interruptedRetryDelayDoesNotCreateAnotherProbe() throws Exception -+ { -+ provider.failCreateTimes = Integer.MAX_VALUE; -+ Thread callingThread = Thread.currentThread(); -+ executor.schedule(callingThread::interrupt, 100, TimeUnit.MILLISECONDS); -+ try -+ { -+ assertTrue(probe().isEmpty()); -+ assertTrue(callingThread.isInterrupted()); -+ assertCleaned(1); -+ assertEquals(1, warningCount()); -+ } -+ finally -+ { -+ Thread.interrupted(); -+ } -+ } -+ -+ @Test -+ public void interruptedCleanupPreservesInterruptAndStopsRetries() throws Exception -+ { -+ provider.failCreateTimes = Integer.MAX_VALUE; -+ provider.interruptDelete = true; -+ try -+ { -+ assertTrue(probe().isEmpty()); -+ assertTrue(Thread.currentThread().isInterrupted()); -+ assertEquals(1, provider.createCalls); -+ assertEquals(1, provider.deleteCalls); -+ } -+ finally -+ { -+ Thread.interrupted(); -+ } -+ } -+ -+ @Test -+ public void concurrentProbesDoNotShareTheTemporaryVolume() throws Exception -+ { -+ provider.creationDelayMillis = 200; -+ ExecutorService callers = Executors.newFixedThreadPool(2); -+ CountDownLatch ready = new CountDownLatch(2); -+ CountDownLatch start = new CountDownLatch(1); -+ Callable call = () -> -+ { -+ ready.countDown(); -+ start.await(); -+ return probe(); -+ }; -+ try -+ { -+ Future first = callers.submit(call); -+ Future second = callers.submit(call); -+ assertTrue(ready.await(2, TimeUnit.SECONDS)); -+ start.countDown(); -+ assertFalse(first.get(10, TimeUnit.SECONDS).isEmpty()); -+ assertFalse(second.get(10, TimeUnit.SECONDS).isEmpty()); -+ assertEquals(1, provider.peakActiveProbes.get()); -+ assertCleaned(2); -+ } -+ finally -+ { -+ callers.shutdownNow(); -+ assertTrue(callers.awaitTermination(2, TimeUnit.SECONDS)); -+ } -+ } -+ -+ @Test -+ public void inactiveLvmVolumesDoNotCreateProbe() throws Exception -+ { -+ doReturn(List.of(volume(null))).when(pool).getVolumes(null); -+ LvmProvider lvm = new LvmProvider(providerInit()) -+ { -+ @Override -+ public String createTmpProbeVlm(StorPool ignored) -+ { -+ throw new AssertionError("Inactive shared LVM volumes must not trigger a probe volume"); -+ } -+ -+ @Override -+ public void deleteTmpProbeVlm(StorPool ignored) -+ { -+ throw new AssertionError("No probe volume was created"); -+ } -+ }; -+ LocalPropsChangePojo changes = new LocalPropsChangePojo(); -+ lvm.updateMinIoSize(pool, changes); -+ assertTrue(changes.isEmpty()); -+ } -+ -+ private AbsStorageProviderInit providerInit() -+ { -+ return new AbsStorageProviderInit( -+ reporter, null, null, null, null, null, null, null, null, null, watch, null, null, null -+ ); -+ } -+ -+ private LocalPropsChangePojo probe() throws Exception -+ { -+ LocalPropsChangePojo changes = new LocalPropsChangePojo(); -+ provider.updateMinIoSize(pool, changes); -+ return changes; -+ } -+ -+ private void assertPublished(LocalPropsChangePojo changesRef) -+ { -+ String namespace = StorageConstants.NAMESPACE_INTERNAL + '/'; -+ assertEquals(Map.of( -+ namespace + StorageConstants.BLK_DEV_MIN_IO_SIZE, Long.toString(BlockSizeConsts.DFLT_IO_SIZE), -+ namespace + StorageConstants.BLK_DEV_OPT_IO_SIZE, Long.toString(BlockSizeConsts.DFLT_OPT_IO_SIZE) -+ ), changesRef.changedStorPoolProps.get(pool.getName())); -+ } -+ -+ private void assertCleaned(int attemptsRef) -+ { -+ assertEquals(attemptsRef, provider.createCalls); -+ assertEquals(attemptsRef, provider.deleteCalls); -+ assertFalse(Files.exists(link)); -+ } -+ -+ @SuppressWarnings("unchecked") -+ private VlmProviderObject volume(Path pathRef) -+ { -+ VlmProviderObject volume = mock(VlmProviderObject.class); -+ when(volume.getDevicePath()).thenReturn(pathRef == null ? null : pathRef.toString()); -+ return volume; -+ } -+ -+ private long warningCount() -+ { -+ return mockingDetails(reporter).getInvocations().stream() -+ .filter(invocation -> invocation.getMethod().getName().equals("logWarning")) -+ .count(); -+ } -+ -+ private class ProbeProvider extends ZfsThinProvider -+ { -+ private long delayMillis; -+ private long creationDelayMillis; -+ private long timeoutMillis = 4000; -+ private int failCreateTimes; -+ private boolean nullPath; -+ private boolean failDelete; -+ private boolean interruptDelete; -+ private int createCalls; -+ private int deleteCalls; -+ private ScheduledFuture create; -+ private final AtomicInteger activeProbes = new AtomicInteger(); -+ private final AtomicInteger peakActiveProbes = new AtomicInteger(); -+ -+ ProbeProvider(AbsStorageProviderInit initRef) -+ { -+ super(initRef); -+ } -+ -+ @Override -+ protected long getWaitTimeoutAfterCreate(StorPool ignored) -+ { -+ return timeoutMillis; -+ } -+ -+ @Override -+ public String createTmpProbeVlm(StorPool ignored) throws StorageException -+ { -+ peakActiveProbes.accumulateAndGet(activeProbes.incrementAndGet(), Math::max); -+ createCalls++; -+ if (creationDelayMillis > 0) -+ { -+ try -+ { -+ Thread.sleep(creationDelayMillis); -+ } -+ catch (InterruptedException exc) -+ { -+ throw new StorageException("Test probe creation interrupted", exc); -+ } -+ } -+ if (failCreateTimes > 0) -+ { -+ failCreateTimes--; -+ throw new StorageException("Probe creation failed"); -+ } -+ if (nullPath) -+ { -+ return null; -+ } -+ if (delayMillis == 0) -+ { -+ try -+ { -+ Files.createSymbolicLink(link, device); -+ } -+ catch (IOException exc) -+ { -+ throw new StorageException("Test device creation failed", exc); -+ } -+ } -+ else if (delayMillis > 0) -+ { -+ create = executor.schedule(() -> Files.createSymbolicLink(link, device), -+ delayMillis, TimeUnit.MILLISECONDS); -+ } -+ return link.toString(); -+ } -+ -+ @Override -+ public void deleteTmpProbeVlm(StorPool ignored) throws StorageException -+ { -+ deleteCalls++; -+ if (interruptDelete) -+ { -+ throw new StorageException("Probe cleanup interrupted", new InterruptedException()); -+ } -+ if (failDelete) -+ { -+ throw new StorageException("Probe cleanup failed"); -+ } -+ try -+ { -+ if (create != null && !create.cancel(false)) -+ { -+ create.get(); -+ } -+ Files.deleteIfExists(link); -+ activeProbes.decrementAndGet(); -+ } -+ catch (Exception exc) -+ { -+ throw new StorageException("Test device cleanup failed", exc); -+ } -+ } -+ } -+} diff --git a/packages/system/linstor/images/piraeus-server/patches/native-protoc.diff b/packages/system/linstor/images/piraeus-server/patches/native-protoc.diff index f3e601698..fa8596177 100644 --- a/packages/system/linstor/images/piraeus-server/patches/native-protoc.diff +++ b/packages/system/linstor/images/piraeus-server/patches/native-protoc.diff @@ -1,8 +1,8 @@ diff --git a/build.gradle b/build.gradle -index 75b7f64..d2dfec9 100644 +index 9df6c840cdc0..9894a436712c 100644 --- a/build.gradle +++ b/build.gradle -@@ -37,6 +37,9 @@ configurations { +@@ -34,6 +34,9 @@ configurations { } def protobufVersion = '31.1' @@ -12,7 +12,7 @@ index 75b7f64..d2dfec9 100644 def protoc = file("${projectDir}/tools/protoc-" + protobufVersion + '/bin/protoc') def logbackConfig = "logback.xml" def jvmOpts = ["-Xms32M"] -@@ -577,7 +580,7 @@ tasks.register('buildTestCommands', Exec) { +@@ -650,7 +653,7 @@ tasks.register('buildTestCommands', Exec) { tasks.register('unzipProtoc', Copy) { dependsOn(downloadProtoc) @@ -21,7 +21,7 @@ index 75b7f64..d2dfec9 100644 def outputDir = file("tools/protoc-" + protobufVersion) from zipTree(zipFile) -@@ -591,7 +594,7 @@ tasks.register('unzipProtoc', Copy) { +@@ -664,7 +667,7 @@ tasks.register('unzipProtoc', Copy) { tasks.register('downloadProtoc') { outputs.dir "tools" @@ -30,7 +30,7 @@ index 75b7f64..d2dfec9 100644 outputs.file protozip doLast { -@@ -599,7 +602,7 @@ tasks.register('downloadProtoc') { +@@ -672,7 +675,7 @@ tasks.register('downloadProtoc') { mkdir "tools" println "downloading protoc..." new URL('https://github.com/google/protobuf/releases/download/v' diff --git a/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff b/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff index 009591674..e5e0ec013 100644 --- a/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff +++ b/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff @@ -1,28 +1,27 @@ diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java -index 5627d1be8..ece191292 100644 +index 0a19f875b702..d44e19b70d88 100644 --- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java +++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java -@@ -42,6 +42,9 @@ import java.util.Arrays; - import java.util.List; +@@ -45,6 +45,8 @@ import java.util.List; + import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.TimeUnit; -+import java.nio.charset.StandardCharsets; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; @Singleton -@@ -56,6 +58,9 @@ public class DrbdAdm - - public static final int WAIT_CONNECT_RES_TIME = 10; - private static final long DOWN_WAIT_TIMEOUT_SEC = 5; +@@ -68,6 +70,9 @@ public class DrbdAdm + private static final long SECONDARY_RETRY_INITIAL_BACKOFF_MS = 250L; + private static final long SECONDARY_RETRY_MAX_BACKOFF_MS = 2_000L; + private static final String DEVICE_HELD_OPEN_ERR_MSG = "Device is held open by someone"; + private static final long FORCE_DETACH_RETRY_WAIT_MS = 250; + private static final String BITMAP_LEAK_ERR_MSG = "already has a bitmap, this should not happen"; + private static final Pattern BITMAP_LEAK_MINOR_PATTERN = Pattern.compile("\\bminor\\s+(\\d+)\\b"); private final ExtCmdFactory extCmdFactory; - private final AccessContext sysCtx; -@@ -131,8 +136,38 @@ public class DrbdAdm + private final StltConfigAccessor stltCfgAccessor; +@@ -143,8 +148,38 @@ public class DrbdAdm // command.add(resName); command.add(drbdRscData.getSuffixedResourceName()); // execute(Arrays.asList("drbdsetup", "show", drbdRscData.getSuffixedResourceName())); @@ -35,7 +34,7 @@ index 5627d1be8..ece191292 100644 + ExtCmd extCmd = extCmdFactory.create(); + if (Platform.isWindows()) + { -+ extCmd.setTimeout(TimeoutType.WAIT, 5 * 60 * 1000); ++ extCmd.setTimeout(TimeoutType.WAIT, DFLT_WINDOWS_TIMEOUT_MS); + } + + OutputData outputData = extCmd.pipeExec(ProcessBuilder.Redirect.from(nullDevice), commandArr); @@ -63,8 +62,8 @@ index 5627d1be8..ece191292 100644 drbdRscData.setAdjustRequired(false); } -@@ -805,6 +840,75 @@ public class DrbdAdm - } +@@ -955,6 +990,75 @@ public class DrbdAdm + return ret; } + static boolean isBitmapLeakOnAttach(OutputData outputData) diff --git a/packages/system/linstor/images/piraeus-server/patches/retry-secondary-after-mkfs.diff b/packages/system/linstor/images/piraeus-server/patches/retry-secondary-after-mkfs.diff deleted file mode 100644 index f213e9037..000000000 --- a/packages/system/linstor/images/piraeus-server/patches/retry-secondary-after-mkfs.diff +++ /dev/null @@ -1,104 +0,0 @@ -diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java -index 71c1df6..f91903b 100644 ---- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java -+++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java -@@ -62,6 +62,10 @@ public class DrbdAdm - private static final long FORCE_DETACH_RETRY_WAIT_MS = 250; - private static final String BITMAP_LEAK_ERR_MSG = "already has a bitmap, this should not happen"; - private static final Pattern BITMAP_LEAK_MINOR_PATTERN = Pattern.compile("\\bminor\\s+(\\d+)\\b"); -+ private static final int SECONDARY_RETRY_MAX_ATTEMPTS = 7; -+ private static final long SECONDARY_RETRY_INITIAL_BACKOFF_MS = 250L; -+ private static final long SECONDARY_RETRY_MAX_BACKOFF_MS = 2000L; -+ private static final String DEVICE_HELD_OPEN_ERR_MSG = "Device is held open by someone"; - - private final ExtCmdFactory extCmdFactory; - private final AccessContext sysCtx; -@@ -945,6 +949,79 @@ public class DrbdAdm - return true; - } - -+ /** -+ * Switches a resource to secondary mode after mkfs, retrying when DRBD reports the device as held -+ * open by another process. On Talos and other systems with active block-device probing -+ * (machined/udev), the kernel uevent for a freshly-promoted device may cause an external probe -+ * to open the device just before LINSTOR demotes it back to Secondary, producing -+ * "Device is held open by someone" (errno -12) and aborting resource initialization. -+ * Probes typically release the fd within milliseconds, so a short retry with exponential backoff -+ * is sufficient to win the race without changing the contract of {@link #secondary}. -+ */ -+ public void secondaryAfterMkfs(DrbdRscData drbdRscData) throws ExtCmdFailedException -+ { -+ String[] commandArr = new String[] { -+ DRBDADM_UTIL, "-vvv", "secondary", drbdRscData.getSuffixedResourceName() -+ }; -+ -+ File nullDevice = new File(Platform.nullDevice()); -+ -+ OutputData lastOutput = null; -+ for (int attempt = 1; attempt <= SECONDARY_RETRY_MAX_ATTEMPTS; attempt++) -+ { -+ try -+ { -+ ExtCmd extCmd = extCmdFactory.create(); -+ if (Platform.isWindows()) -+ { -+ extCmd.setTimeout(TimeoutType.WAIT, 5 * 60 * 1000); -+ } -+ -+ lastOutput = extCmd.pipeExec(ProcessBuilder.Redirect.from(nullDevice), commandArr); -+ if (lastOutput.exitCode == 0) -+ { -+ return; -+ } -+ if (!isDeviceHeldOpenError(lastOutput) || attempt == SECONDARY_RETRY_MAX_ATTEMPTS) -+ { -+ throw new ExtCmdFailedException(commandArr, lastOutput); -+ } -+ } -+ catch (ChildProcessTimeoutException timeoutExc) -+ { -+ throw new ExtCmdFailedException(commandArr, timeoutExc); -+ } -+ catch (IOException ioExc) -+ { -+ throw new ExtCmdFailedException(commandArr, ioExc); -+ } -+ -+ long sleepMs = Math.min( -+ SECONDARY_RETRY_INITIAL_BACKOFF_MS * (1L << (attempt - 1)), -+ SECONDARY_RETRY_MAX_BACKOFF_MS -+ ); -+ try -+ { -+ Thread.sleep(sleepMs); -+ } -+ catch (InterruptedException ie) -+ { -+ Thread.currentThread().interrupt(); -+ throw new ExtCmdFailedException(commandArr, lastOutput); -+ } -+ } -+ } -+ -+ static boolean isDeviceHeldOpenError(OutputData outputData) -+ { -+ if (outputData == null || outputData.stderrData == null) -+ { -+ return false; -+ } -+ String stderr = new String(outputData.stderrData, StandardCharsets.UTF_8); -+ return stderr.contains(DEVICE_HELD_OPEN_ERR_MSG); -+ } -+ - public static class DrbdPrimary implements AutoCloseable - { - private final DrbdAdm drbdAdm; -@@ -963,7 +1040,7 @@ public class DrbdAdm - @Override - public void close() throws ExtCmdFailedException - { -- drbdAdm.secondary(drbdRscData); -+ drbdAdm.secondaryAfterMkfs(drbdRscData); - } - } - } diff --git a/packages/system/linstor/images/piraeus-server/patches/snapshot-restore-drbd-initialized.diff b/packages/system/linstor/images/piraeus-server/patches/snapshot-restore-drbd-initialized.diff deleted file mode 100644 index 7c72aab69..000000000 --- a/packages/system/linstor/images/piraeus-server/patches/snapshot-restore-drbd-initialized.diff +++ /dev/null @@ -1,67 +0,0 @@ -diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java -index a639c27cc..4dabc1a83 100644 ---- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java -+++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java -@@ -434,26 +434,39 @@ public class CtrlSnapshotRestoreApiCallHandler - } - } - -- /* -- * When a snapshot was received as a backup, it is marked to re-create its local metadata. Before the -- * snapshot was sent the new DRBD_INITIALIZED flag and the VlmDfn property InitialUpToDateOn were -+ /** -+ *

Sets the {@value InternalApiConsts#KEY_LINSTOR_DRBD_INITIAL_UPTODATE_ON} property to the nodeName -+ * of the first Snapshot (which has to have a disk) of the given SnapshotDefinition.

-+ * -+ *

If this SnapshotDefinition is a backup, we also unset the {@link VolumeDefinition.Flags#DRBD_INITIALIZED} -+ * flag. Otherwise we ensure the same flag is set.

-+ * -+ *

Backup path: When a snapshot was received as a backup, it is marked to re-create its local metadata. -+ * Before the snapshot was sent the new DRBD_INITIALIZED flag and the VlmDfn property InitialUpToDateOn were - * cleared. If DRBD_INITIALZED would be set, no peer would even read the property (on purpose). - * Without the property, no peer would think of itself as a winner, thus the DRBD resource would never - * become UpToDate. - * To avoid this, this method sets the InitialUpToDateOn property to let the first resource win the -- * race, so it can become UpToDate and therefore also pull the other peers eventually into UpToDate state. -+ * race, so it can become UpToDate and therefore also pull the other peers eventually into UpToDate state.

- * -- * If the given SnapDfn had also non-backup snapshots, those should be able to keep their metadata. -+ *

If the given SnapDfn had also non-backup snapshots, those should be able to keep their metadata. - * In that case we do not have to do anything. If however all snapshots are descendants from backups - * we must pick one and decide the UpToDate race upfront so that the winner can set its local DRBD -- * as UpToDate, which also triggers the controller to set the DRBD_INITIALZED flag eventually. -+ * as UpToDate, which also triggers the controller to set the DRBD_INITIALZED flag eventually.

-+ * -+ *

Non-Backup path: When a regular snapshot (i.e. not from a backup) is restored, we can or even should -+ * assume that the backing snapshot LVs have correct metadata which do not need be be initialized. When -+ * a SnapshotVolumeDefinition is created it unfortunately does not properly store the VolumeDefinition's flags, -+ * therefore the restored VolumeDefinition will also not have the DRBD_INITIALIZED flag set, although it -+ * refers to a device with healthy and UpToDate metadata. To fix this issue we simply set the flag in this case.

- */ - private void checkDrbdInitializedState(ResourceDefinition rscDfnRef) throws AccessDeniedException - { - AccessContext accCtx = peerAccCtx.get(); -- if (DrbdLayerUtils.isForceInitialSyncSet(accCtx, rscDfnRef) && rscDfnRef.getDiskfulCount(accCtx) > 0) -+ if (rscDfnRef.getDiskfulCount(accCtx) > 0) - { - String winnerNodeName = rscDfnRef.getDiskfulResources(accCtx).get(0).getNode().getName().value; -+ boolean uninitializeDrbd = DrbdLayerUtils.isForceInitialSyncSet(accCtx, rscDfnRef); - Iterator vlmDfnIt = rscDfnRef.iterateVolumeDfn(accCtx); - while (vlmDfnIt.hasNext()) - { -@@ -462,7 +475,14 @@ public class CtrlSnapshotRestoreApiCallHandler - { - vlmDfn.getProps(accCtx) - .setProp(InternalApiConsts.KEY_LINSTOR_DRBD_INITIAL_UPTODATE_ON, winnerNodeName); -- vlmDfn.getFlags().disableFlags(accCtx, VolumeDefinition.Flags.DRBD_INITIALIZED); -+ if (uninitializeDrbd) -+ { -+ vlmDfn.getFlags().disableFlags(accCtx, VolumeDefinition.Flags.DRBD_INITIALIZED); -+ } -+ else -+ { -+ vlmDfn.getFlags().enableFlags(accCtx, VolumeDefinition.Flags.DRBD_INITIALIZED); -+ } - } - catch (InvalidKeyException | InvalidValueException exc) - { From 9e861c117840cbc4f3886bc7f53dbdd0fb692427 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 13 Nov 2026 09:19:25 +0100 Subject: [PATCH 145/174] fix(seaweedfs): stop the master crashing on the crowded volume map SeaweedFS 4.45 updates the crowded-volume map of a volume layout under the read lock, while Assign reads that map under the same read lock to count writable volumes. Under a steady write load the Go runtime aborts the master with "concurrent map read and map write", a fatal error that recover cannot catch, and the S3 gateway answers 500 until a master is elected again. Taking the write lock for the update, as every other mutation of the map already does, removes the race. Upstream fixed it in seaweedfs/seaweedfs#11216, released in 4.46. The patch carries that change and its regression test unchanged, so the build runs the test and the patch stops applying, and is dropped, once the vendored release carries the fix. Signed-off-by: Mattia Eleuteri Assisted-by: LLM --- .../seaweedfs/images/seaweedfs/Dockerfile | 9 ++ .../patches/volume-layout-crowded-lock.diff | 87 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 packages/system/seaweedfs/images/seaweedfs/patches/volume-layout-crowded-lock.diff diff --git a/packages/system/seaweedfs/images/seaweedfs/Dockerfile b/packages/system/seaweedfs/images/seaweedfs/Dockerfile index 8f42f3ea2..d52ee7329 100644 --- a/packages/system/seaweedfs/images/seaweedfs/Dockerfile +++ b/packages/system/seaweedfs/images/seaweedfs/Dockerfile @@ -16,6 +16,15 @@ # seaweedfs/seaweedfs#11674 after 4.48; drop the patch on the first vendored # release that carries it. # +# patches/volume-layout-crowded-lock.diff makes SetVolumeCrowded take the write +# lock of the volume layout. Upstream 4.45 mutates the crowded map under the +# read lock, while Assign reads it under the same read lock through +# GetWritableVolumeCount, so under a steady stream of writes the master dies on +# a fatal "concurrent map read and map write", which recover cannot catch, and +# S3 answers 500 until it is back. Backport of seaweedfs/seaweedfs#11216, +# released in 4.46; drop the patch on the first vendored release that carries +# it. +# # `make update` re-pins VERSION from the vendored chart, SOURCE_SHA256 from the # source archive of that tag, UPSTREAM_DIGEST from the upstream image of that # tag and the builder from upstream's docker/Dockerfile.go_build, and checks diff --git a/packages/system/seaweedfs/images/seaweedfs/patches/volume-layout-crowded-lock.diff b/packages/system/seaweedfs/images/seaweedfs/patches/volume-layout-crowded-lock.diff new file mode 100644 index 000000000..12e09ca67 --- /dev/null +++ b/packages/system/seaweedfs/images/seaweedfs/patches/volume-layout-crowded-lock.diff @@ -0,0 +1,87 @@ +diff --git a/weed/topology/crowded_writable_test.go b/weed/topology/crowded_writable_test.go +--- a/weed/topology/crowded_writable_test.go ++++ b/weed/topology/crowded_writable_test.go +@@ -1,6 +1,7 @@ + package topology + + import ( ++ "sync" + "testing" + "time" + +@@ -73,3 +74,52 @@ func TestCrowdedCountStillMatchesWritables(t *testing.T) { + t.Errorf("writable=%d crowded=%d, want 1 and 1", writable, crowded) + } + } ++ ++// SetVolumeCrowded mutates the crowded map while GetWritableVolumeCount reads ++// it on the Assign hot path. Under RLock both could run at once and the Go ++// runtime aborts with "concurrent map read and map write". Run with -race to ++// reproduce the original bug; the write lock makes this pass. ++func TestSetVolumeCrowdedNoRaceWithGetWritableVolumeCount(t *testing.T) { ++ rp, _ := super_block.NewReplicaPlacementFromString("000") ++ vl := NewVolumeLayout(rp, needle.EMPTY_TTL, types.HardDriveType, 10000, false) ++ // GetWritableVolumeCount iterates vl.writables, so give it something to ++ // read while the crowded map is being mutated. ++ vl.writables = []needle.VolumeId{1, 2, 3} ++ ++ var wg sync.WaitGroup ++ stop := make(chan struct{}) ++ for range 8 { ++ wg.Add(1) ++ go func() { ++ defer wg.Done() ++ for { ++ select { ++ case <-stop: ++ return ++ default: ++ vl.SetVolumeCrowded(needle.VolumeId(2)) ++ } ++ } ++ }() ++ } ++ for range 8 { ++ wg.Add(1) ++ go func() { ++ defer wg.Done() ++ for { ++ select { ++ case <-stop: ++ return ++ default: ++ vl.GetWritableVolumeCount() ++ } ++ } ++ }() ++ } ++ ++ // The race detector needs only a brief window to flag a concurrent ++ // map read/write; let the goroutines hammer the lock briefly. ++ time.Sleep(50 * time.Millisecond) ++ close(stop) ++ wg.Wait() ++} +diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go +--- a/weed/topology/volume_layout.go ++++ b/weed/topology/volume_layout.go +@@ -1019,11 +1019,14 @@ func (vl *VolumeLayout) setVolumeCrowded(vid needle.VolumeId) { + } + + func (vl *VolumeLayout) SetVolumeCrowded(vid needle.VolumeId) { +- // since delete is guarded by accessLock.Lock(), +- // and is always called in sequential order, +- // RLock() should be safe enough +- vl.accessLock.RLock() +- defer vl.accessLock.RUnlock() ++ // setVolumeCrowded mutates the crowded map, and GetWritableVolumeCount ++ // reads it under RLock on the Assign hot path. Two concurrent RLock ++ // holders with one writing the map triggers a fatal ++ // "concurrent map read and map write". Take the write lock: this path ++ // is a low-frequency single consumer driven by the crowded-volume ++ // event loop, and every other mutation of crowded already holds Lock(). ++ vl.accessLock.Lock() ++ defer vl.accessLock.Unlock() + + vl.setVolumeCrowded(vid) + } From 0f16bf9c7336c03f3236ad68528106964d02c2c0 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 13 Nov 2026 09:30:50 +0100 Subject: [PATCH 146/174] feat(redis): add maxmemory, eviction policy and snapshot toggle A Redis used purely as a cache had no durable way to bound its memory or stop writing RDB snapshots: the chart only disabled snapshots together with the persistent volume, and a CONFIG SET is lost when the pod restarts. Snapshots and replica full syncs then filled the volume and left the instance crashlooping. maxmemory, maxmemory-policy and save are always rendered with an explicit value, because the operator applies customConfig with CONFIG SET on every reconcile and never resets a directive whose line disappears; leaving the defaults implicit would make reverting a setting a no-op until the next pod restart. maxmemory is refused above 90% of the memory request, the threshold above which the operator silently skips the line and leaves Redis unbounded. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- api/apps/v1alpha1/redis/types.go | 11 ++ .../v1alpha1/redis/zz_generated.deepcopy.go | 1 + packages/apps/redis/README.md | 13 +- .../apps/redis/templates/redisfailover.yaml | 25 ++++ .../apps/redis/tests/cache_config_test.yaml | 118 ++++++++++++++++++ .../apps/redis/tests/min_replicas_test.yaml | 6 +- packages/apps/redis/values.schema.json | 33 +++++ packages/apps/redis/values.yaml | 18 +++ packages/system/redis-rd/cozyrds/redis.yaml | 4 +- 9 files changed, 219 insertions(+), 10 deletions(-) create mode 100644 packages/apps/redis/tests/cache_config_test.yaml diff --git a/api/apps/v1alpha1/redis/types.go b/api/apps/v1alpha1/redis/types.go index c2363749c..03ff0175d 100644 --- a/api/apps/v1alpha1/redis/types.go +++ b/api/apps/v1alpha1/redis/types.go @@ -51,6 +51,14 @@ type ConfigSpec struct { // Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it. // +kubebuilder:validation:Minimum=0 MinReplicasMaxLag *int `json:"minReplicasMaxLag,omitempty"` + // Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers. + Maxmemory resource.Quantity `json:"maxmemory,omitempty"` + // Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset. + // +kubebuilder:default:="noeviction" + MaxmemoryPolicy MaxmemoryPolicy `json:"maxmemoryPolicy"` + // Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty. + // +kubebuilder:default:=true + Snapshots bool `json:"snapshots"` // Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does. // +kubebuilder:default:={} Sentinel Sentinel `json:"sentinel,omitempty"` @@ -80,6 +88,9 @@ type TLS struct { // +kubebuilder:validation:Enum="no";"optional";"yes" type AuthClients string +// +kubebuilder:validation:Enum="noeviction";"allkeys-lru";"allkeys-lfu";"allkeys-random";"volatile-lru";"volatile-lfu";"volatile-random";"volatile-ttl" +type MaxmemoryPolicy string + // +kubebuilder:validation:Enum="t1.nano";"t1.micro";"t1.small";"t1.medium";"t1.large";"t1.xlarge";"t1.2xlarge";"t1.4xlarge";"c1.nano";"c1.micro";"c1.small";"c1.medium";"c1.large";"c1.xlarge";"c1.2xlarge";"c1.4xlarge";"s1.nano";"s1.micro";"s1.small";"s1.medium";"s1.large";"s1.xlarge";"s1.2xlarge";"s1.4xlarge";"u1.nano";"u1.micro";"u1.small";"u1.medium";"u1.large";"u1.xlarge";"u1.2xlarge";"u1.4xlarge";"m1.nano";"m1.micro";"m1.small";"m1.medium";"m1.large";"m1.xlarge";"m1.2xlarge";"m1.4xlarge";"nano";"micro";"small";"medium";"large";"xlarge";"2xlarge" type ResourcesPreset string diff --git a/api/apps/v1alpha1/redis/zz_generated.deepcopy.go b/api/apps/v1alpha1/redis/zz_generated.deepcopy.go index 489c28e85..86437ea58 100644 --- a/api/apps/v1alpha1/redis/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/redis/zz_generated.deepcopy.go @@ -66,6 +66,7 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { *out = new(int) **out = **in } + out.Maxmemory = in.Maxmemory.DeepCopy() in.Sentinel.DeepCopyInto(&out.Sentinel) } diff --git a/packages/apps/redis/README.md b/packages/apps/redis/README.md index 84cb04930..4044ce951 100644 --- a/packages/apps/redis/README.md +++ b/packages/apps/redis/README.md @@ -39,11 +39,14 @@ Service utilizes the freshworks-oss Redis Operator (a maintained fork of the arc ### Application-specific parameters -| Name | Description | Type | Value | -| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | ------ | -| `authEnabled` | Enable password generation. | `bool` | `true` | -| `minReplicasToWrite` | Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value. | `*int` | `null` | -| `minReplicasMaxLag` | Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it. | `*int` | `null` | +| Name | Description | Type | Value | +| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ------------ | +| `authEnabled` | Enable password generation. | `bool` | `true` | +| `minReplicasToWrite` | Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value. | `*int` | `null` | +| `minReplicasMaxLag` | Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it. | `*int` | `null` | +| `maxmemory` | Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers. | `quantity` | `""` | +| `maxmemoryPolicy` | Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset. | `string` | `noeviction` | +| `snapshots` | Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty. | `bool` | `true` | ### Sentinel parameters diff --git a/packages/apps/redis/templates/redisfailover.yaml b/packages/apps/redis/templates/redisfailover.yaml index 05a96e3cc..ba36e3f6c 100644 --- a/packages/apps/redis/templates/redisfailover.yaml +++ b/packages/apps/redis/templates/redisfailover.yaml @@ -116,8 +116,33 @@ spec: - loglevel notice {{- if not .Values.size }} - appendonly no + {{- end }} + {{- /* The operator applies these lines with CONFIG SET on every reconcile and never + resets a line that disappears, so each one is always rendered with an explicit + value: switching back to the default must reach the running pods too. The + schedule is the Redis 7 and 8 default. */}} + {{- if and .Values.size .Values.snapshots }} + - save 3600 1 300 100 60 10000 + {{- else }} - save "" {{- end }} + {{- $maxmemory := 0 }} + {{- if not (kindIs "invalid" .Values.maxmemory) }} + {{- $maxmemory = include "cozy-lib.resources.toFloat" (.Values.maxmemory | toString) | float64 | int64 }} + {{- $redisResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.resourcesPreset .Values.resources $) | fromYaml }} + {{- $memoryRequest := include "cozy-lib.resources.toFloat" (dig "requests" "memory" "0" $redisResources | toString) | float64 }} + {{- /* The operator compares maxmemory with 90% of the pod's memory request + (reservedPodMemoryPercent) and silently skips the line when it is larger. */}} + {{- $allowed := mulf $memoryRequest 0.9 | int64 }} + {{- if lt $maxmemory 0 }} + {{- fail (printf "maxmemory (%v) must not be negative" .Values.maxmemory) }} + {{- end }} + {{- if gt $maxmemory $allowed }} + {{- fail (printf "maxmemory (%v, %d bytes) must be at most 90%% of the Redis memory request (%d bytes): the operator ignores a larger value" .Values.maxmemory $maxmemory $allowed) }} + {{- end }} + {{- end }} + - maxmemory {{ $maxmemory }} + - maxmemory-policy {{ .Values.maxmemoryPolicy | default "noeviction" }} {{- /* A master with `replicas` pods has replicas-1 replicas, so a requirement of `replicas` or more can never be met and the master would refuse every write. */}} {{- if and (not (kindIs "invalid" .Values.minReplicasToWrite)) (ge (.Values.minReplicasToWrite | int) (.Values.replicas | int)) }} diff --git a/packages/apps/redis/tests/cache_config_test.yaml b/packages/apps/redis/tests/cache_config_test.yaml new file mode 100644 index 000000000..557b5d3d4 --- /dev/null +++ b/packages/apps/redis/tests/cache_config_test.yaml @@ -0,0 +1,118 @@ +suite: Redis cache tuning (maxmemory, eviction, snapshots) + +# The operator applies customConfig with CONFIG SET and never resets a line +# that disappears, so every directive here is rendered with an explicit value +# even at its default: reverting a setting has to reach the running pods. + +release: + name: my-redis + namespace: tenant-test + +set: + _cluster: + cluster-domain: cluster.local + _namespace: + host: example.com + +templates: + - templates/redisfailover.yaml + +tests: + - it: "renders the Redis defaults explicitly" + documentSelector: + path: kind + value: RedisFailover + asserts: + - contains: + path: spec.redis.customConfig + content: save 3600 1 300 100 60 10000 + - contains: + path: spec.redis.customConfig + content: maxmemory 0 + - contains: + path: spec.redis.customConfig + content: maxmemory-policy noeviction + - notContains: + path: spec.redis.customConfig + content: appendonly no + + - it: "disables snapshots while keeping the volume" + set: + snapshots: false + documentSelector: + path: kind + value: RedisFailover + asserts: + - contains: + path: spec.redis.customConfig + content: save "" + - notContains: + path: spec.redis.customConfig + content: save 3600 1 300 100 60 10000 + - equal: + path: spec.redis.storage.persistentVolumeClaim.spec.resources.requests.storage + value: 1Gi + + - it: "keeps snapshots off without a volume even when enabled" + set: + size: null + snapshots: true + documentSelector: + path: kind + value: RedisFailover + asserts: + - contains: + path: spec.redis.customConfig + content: save "" + - contains: + path: spec.redis.customConfig + content: appendonly no + - notExists: + path: spec.redis.storage + + - it: "renders maxmemory in bytes with the chosen policy" + set: + resourcesPreset: t1.large + maxmemory: 1Gi + maxmemoryPolicy: allkeys-lru + documentSelector: + path: kind + value: RedisFailover + asserts: + - contains: + path: spec.redis.customConfig + content: maxmemory 1073741824 + - contains: + path: spec.redis.customConfig + content: maxmemory-policy allkeys-lru + + - it: "sizes the maxmemory ceiling from explicit resources" + set: + resources: + memory: 1Gi + maxmemory: 900Mi + documentSelector: + path: kind + value: RedisFailover + asserts: + - contains: + path: spec.redis.customConfig + content: maxmemory 943718400 + + # The operator compares maxmemory with 90% of the pod's memory request and + # silently skips a larger line, which would leave Redis unbounded. + - it: "refuses maxmemory above 90% of the memory request" + set: + resources: + memory: 1Gi + maxmemory: 1000Mi + asserts: + - failedTemplate: + errorPattern: must be at most 90% of the Redis memory request + + - it: "refuses a negative maxmemory" + set: + maxmemory: -1Mi + asserts: + - failedTemplate: + errorPattern: must not be negative diff --git a/packages/apps/redis/tests/min_replicas_test.yaml b/packages/apps/redis/tests/min_replicas_test.yaml index f0de83e86..13217884d 100644 --- a/packages/apps/redis/tests/min_replicas_test.yaml +++ b/packages/apps/redis/tests/min_replicas_test.yaml @@ -1,7 +1,7 @@ suite: Redis min-replicas write safety -# Both settings are opt-in: unset keeps the Redis defaults, so existing -# instances render the same customConfig as before. +# Both settings are opt-in: unset keeps the Redis defaults, so neither +# directive is rendered. release: name: my-redis @@ -30,7 +30,7 @@ tests: pattern: ^tcp-keepalive 0$ - lengthEqual: path: spec.redis.customConfig - count: 2 + count: 5 - it: "renders min-replicas-to-write when set" set: diff --git a/packages/apps/redis/values.schema.json b/packages/apps/redis/values.schema.json index 1a301615f..b3b143320 100644 --- a/packages/apps/redis/values.schema.json +++ b/packages/apps/redis/values.schema.json @@ -171,6 +171,39 @@ "type": "integer", "minimum": 0 }, + "maxmemory": { + "description": "Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers.", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "x-kubernetes-int-or-string": true + }, + "maxmemoryPolicy": { + "description": "Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.", + "type": "string", + "default": "noeviction", + "enum": [ + "noeviction", + "allkeys-lru", + "allkeys-lfu", + "allkeys-random", + "volatile-lru", + "volatile-lfu", + "volatile-random", + "volatile-ttl" + ] + }, + "snapshots": { + "description": "Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty.", + "type": "boolean", + "default": true + }, "sentinel": { "description": "Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.", "type": "object", diff --git a/packages/apps/redis/values.yaml b/packages/apps/redis/values.yaml index 7917ca34d..0f8380ae7 100644 --- a/packages/apps/redis/values.yaml +++ b/packages/apps/redis/values.yaml @@ -119,6 +119,24 @@ authEnabled: true ## @param {*int} [minReplicasMaxLag] - Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it. ## @minimum 0 +## @param {quantity} [maxmemory] - Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers. + +## @enum {string} MaxmemoryPolicy - Values accepted by the Redis `maxmemory-policy` directive. +## @value noeviction +## @value allkeys-lru +## @value allkeys-lfu +## @value allkeys-random +## @value volatile-lru +## @value volatile-lfu +## @value volatile-random +## @value volatile-ttl + +## @param {MaxmemoryPolicy} maxmemoryPolicy="noeviction" - Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset. +maxmemoryPolicy: "noeviction" + +## @param {bool} snapshots - Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty. +snapshots: true + ## ## @section Sentinel parameters ## diff --git a/packages/system/redis-rd/cozyrds/redis.yaml b/packages/system/redis-rd/cozyrds/redis.yaml index cfcaeb01d..02046b97e 100644 --- a/packages/system/redis-rd/cozyrds/redis.yaml +++ b/packages/system/redis-rd/cozyrds/redis.yaml @@ -8,7 +8,7 @@ spec: plural: redises singular: redis openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"maxmemory":{"description":"Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"maxmemoryPolicy":{"description":"Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.","type":"string","default":"noeviction","enum":["noeviction","allkeys-lru","allkeys-lfu","allkeys-random","volatile-lru","volatile-lfu","volatile-random","volatile-ttl"]},"snapshots":{"description":"Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty.","type":"boolean","default":true},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}} release: prefix: redis- labels: @@ -25,7 +25,7 @@ spec: tags: - cache icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODNfMzIxMykiLz4KPHBhdGggZD0iTTEyMC4xNDkgOTUuNTQ5MUMxMTQuNTg2IDk4LjQ0ODUgODUuNzcwOSAxMTAuMjk2IDc5LjYzNjMgMTEzLjQ5NUM3My41MDE2IDExNi42OTMgNzAuMDkzNyAxMTYuNjYyIDY1LjI0NzIgMTE0LjM0NkM2MC40MDEyIDExMi4wMjkgMjkuNzM2NCA5OS42NDIzIDI0LjIxMjUgOTcuMDAxOUMyMS40NTE5IDk1LjY4MjcgMjAgOTQuNTY4NyAyMCA5My41MTY2VjgyLjk4MDlDMjAgODIuOTgwOSA1OS45MjIgNzQuMjkwMSA2Ni4zNjY5IDcxLjk3NzhDNzIuODExNSA2OS42NjU2IDc1LjA0NzYgNjkuNTgyMSA4MC41MzIgNzEuNTkxQzg2LjAxNzMgNzMuNjAwOCAxMTguODEyIDc5LjUxNzYgMTI0LjIzMyA4MS41MDI5TDEyNC4yMyA5MS44ODk2QzEyNC4yMzEgOTIuOTMxMSAxMjIuOTggOTQuMDczNiAxMjAuMTQ5IDk1LjU0OTFaIiBmaWxsPSIjOTEyNjI2Ii8+CjxwYXRoIGQ9Ik0xMjAuMTQ3IDg1LjA3NTJDMTE0LjU4NSA4Ny45NzM0IDg1Ljc3IDk5LjgyMTggNzkuNjM1NCAxMDMuMDJDNzMuNTAxMSAxMDYuMjE5IDcwLjA5MzIgMTA2LjE4NyA2NS4yNDcyIDEwMy44NzFDNjAuNDAwNyAxMDEuNTU1IDI5LjczNzEgODkuMTY2OCAyNC4yMTM2IDg2LjUyOEMxOC42OTAxIDgzLjg4NzYgMTguNTc0NCA4Mi4wNzA0IDI0LjAwMDMgNzkuOTQ1OEMyOS40MjYxIDc3LjgyMDUgNTkuOTIxNSA2NS44NTYxIDY2LjM2NzIgNjMuNTQzOEM3Mi44MTE4IDYxLjIzMjQgNzUuMDQ3NSA2MS4xNDgxIDgwLjUzMTkgNjMuMTU3OEM4Ni4wMTY4IDY1LjE2NjggMTE0LjY2IDc2LjU2NzYgMTIwLjA3OSA3OC41NTI1QzEyNS41MDEgODAuNTM5OSAxMjUuNzA5IDgyLjE3NjMgMTIwLjE0NyA4NS4wNzUyWiIgZmlsbD0iI0M2MzAyQiIvPgo8cGF0aCBkPSJNMTIwLjE0OSA3OC41MDJDMTE0LjU4NiA4MS40MDE4IDg1Ljc3MDkgOTMuMjQ5MyA3OS42MzYzIDk2LjQ0ODhDNzMuNTAxNiA5OS42NDYyIDcwLjA5MzcgOTkuNjE1MiA2NS4yNDcyIDk3LjI5ODVDNjAuNDAwOCA5NC45ODMgMjkuNzM2NCA4Mi41OTUyIDI0LjIxMjUgNzkuOTU0N0MyMS40NTE5IDc4LjYzNTUgMjAgNzcuNTIzMiAyMCA3Ni40NzA3VjY1LjkzMzhDMjAgNjUuOTMzOCA1OS45MjIgNTcuMjQzNCA2Ni4zNjY5IDU0LjkzMTFDNzIuODExNSA1Mi42MTg5IDc1LjA0NzYgNTIuNTM1IDgwLjUzMiA1NC41NDQzQzg2LjAxNzcgNTYuNTUzNiAxMTguODEzIDYyLjQ2OTMgMTI0LjIzMyA2NC40NTVMMTI0LjIzIDc0Ljg0MjhDMTI0LjIzMSA3NS44ODQgMTIyLjk4IDc3LjAyNjQgMTIwLjE0OSA3OC41MDJaIiBmaWxsPSIjOTEyNjI2Ii8+CjxwYXRoIGQ9Ik0xMjAuMTQ3IDY4LjAyODJDMTE0LjU4NSA3MC45MjcxIDg1Ljc3IDgyLjc3NDcgNzkuNjM1NCA4NS45NzM3QzczLjUwMTEgODkuMTcxNiA3MC4wOTMyIDg5LjE0MDIgNjUuMjQ3MiA4Ni44MjM1QzYwLjQwMDcgODQuNTA4NCAyOS43MzcxIDcyLjEyMDEgMjQuMjEzNiA2OS40ODA5QzE4LjY5MDEgNjYuODQxMyAxOC41NzQ0IDY1LjAyMzcgMjQuMDAwMyA2Mi44OTg0QzI5LjQyNjEgNjAuNzc0MiA1OS45MjE5IDQ4LjgwOSA2Ni4zNjcyIDQ2LjQ5NzJDNzIuODExOCA0NC4xODUzIDc1LjA0NzUgNDQuMTAxNCA4MC41MzE5IDQ2LjExMDhDODYuMDE2OCA0OC4xMTk3IDExNC42NiA1OS41MTk3IDEyMC4wNzkgNjEuNTA1NUMxMjUuNTAxIDYzLjQ5MjQgMTI1LjcwOSA2NS4xMjkyIDEyMC4xNDcgNjguMDI4MloiIGZpbGw9IiNDNjMwMkIiLz4KPHBhdGggZD0iTTEyMC4xNDkgNjAuODIyNEMxMTQuNTg2IDYzLjcyMTQgODUuNzcwOSA3NS41Njk4IDc5LjYzNjMgNzguNzY5MkM3My41MDE2IDgxLjk2NzEgNzAuMDkzNyA4MS45MzU3IDY1LjI0NzIgNzkuNjE5QzYwLjQwMDggNzcuMzAzNSAyOS43MzY0IDY0LjkxNTIgMjQuMjEyNSA2Mi4yNzZDMjEuNDUxOSA2MC45NTU2IDIwIDU5Ljg0MjggMjAgNTguNzkxNVY0OC4yNTQyQzIwIDQ4LjI1NDIgNTkuOTIyIDM5LjU2NDIgNjYuMzY2OSAzNy4yNTI0QzcyLjgxMTUgMzQuOTM5NyA3NS4wNDc2IDM0Ljg1NjcgODAuNTMyIDM2Ljg2NTZDODYuMDE3NyAzOC44NzQ5IDExOC44MTMgNDQuNzkwNSAxMjQuMjMzIDQ2Ljc3NjNMMTI0LjIzIDU3LjE2MzdDMTI0LjIzMSA1OC4yMDQgMTIyLjk4IDU5LjM0NjUgMTIwLjE0OSA2MC44MjI0WiIgZmlsbD0iIzkxMjYyNiIvPgo8cGF0aCBkPSJNMTIwLjE0NyA1MC4zNDlDMTE0LjU4NSA1My4yNDc5IDg1Ljc2OTggNjUuMDk2MyA3OS42MzUyIDY4LjI5NDFDNzMuNTAwOSA3MS40OTIgNzAuMDkzIDcxLjQ2MDYgNjUuMjQ2OSA2OS4xNDUxQzYwLjQwMDkgNjYuODI4MyAyOS43MzY5IDU0LjQ0MDkgMjQuMjEzOCA1MS44MDEzQzE4LjY4OTkgNDkuMTYyMSAxOC41NzQ2IDQ3LjM0NDEgMjQgNDUuMjE5MkMyOS40MjU5IDQzLjA5NDYgNTkuOTIxNyAzMS4xMzEgNjYuMzY3IDI4LjgxODRDNzIuODExNiAyNi41MDYxIDc1LjA0NzMgMjYuNDIzIDgwLjUzMTcgMjguNDMyNEM4Ni4wMTY2IDMwLjQ0MTcgMTE0LjY1OSA0MS44NDE4IDEyMC4wNzkgNDMuODI3NUMxMjUuNTAxIDQ1LjgxMjggMTI1LjcwOSA0Ny40NSAxMjAuMTQ3IDUwLjM0OVoiIGZpbGw9IiNDNjMwMkIiLz4KPHBhdGggZD0iTTg0Ljg1NDEgNDAuMDk5NEw3NS44OTI2IDQxLjAyOThMNzMuODg2NSA0NS44NTdMNzAuNjQ2MyA0MC40NzAzTDYwLjI5ODMgMzkuNTQwNEw2OC4wMTk3IDM2Ljc1NThMNjUuNzAzIDMyLjQ4MTRMNzIuOTMyMSAzNS4zMDg4TDc5Ljc0NzEgMzMuMDc3NUw3Ny45MDUyIDM3LjQ5NzJMODQuODU0MSA0MC4wOTk0Wk03My4zNTE1IDYzLjUxODRMNTYuNjI2NiA1Ni41ODE2TDgwLjU5MiA1Mi45MDI5TDczLjM1MTUgNjMuNTE4NFpNNTAuMTYzNyA0Mi43ODI2QzU3LjIzODEgNDIuNzgyNiA2Mi45NzMgNDUuMDA1NyA2Mi45NzMgNDcuNzQ3NUM2Mi45NzMgNTAuNDkwMSA1Ny4yMzgxIDUyLjcxMjggNTAuMTYzNyA1Mi43MTI4QzQzLjA4OTMgNTIuNzEyOCAzNy4zNTQ1IDUwLjQ4OTcgMzcuMzU0NSA0Ny43NDc1QzM3LjM1NDUgNDUuMDA1NyA0My4wODkzIDQyLjc4MjYgNTAuMTYzNyA0Mi43ODI2WiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTk1LjQ0MzQgNDEuNDE3NEwxMDkuNjI3IDQ3LjAyMjRMOTUuNDU1NiA1Mi42MjJMOTUuNDQzNCA0MS40MTc0WiIgZmlsbD0iIzYyMUIxQyIvPgo8cGF0aCBkPSJNNzkuNzUyOSA0Ny42MjYxTDk1LjQ0NDkgNDEuNDE4OUw5NS40NTcxIDUyLjYyMzZMOTMuOTE4NCA1My4yMjU0TDc5Ljc1MjkgNDcuNjI2MVoiIGZpbGw9IiM5QTI5MjgiLz4KPGRlZnM+CjxsaW5lYXJHcmFkaWVudCBpZD0icGFpbnQwX2xpbmVhcl82ODNfMzIxMyIgeDE9IjE4OSIgeTE9IjIxMC41IiB4Mj0iMCIgeTI9IjAiIGdyYWRpZW50VW5pdHM9InVzZXJTcGFjZU9uVXNlIj4KPHN0b3Agc3RvcC1jb2xvcj0iI0E4MDAwMCIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiNGRkNGQ0YiLz4KPC9saW5lYXJHcmFkaWVudD4KPC9kZWZzPgo8L3N2Zz4K - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "tls"], ["spec", "authEnabled"], ["spec", "sentinel"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "tls"], ["spec", "authEnabled"], ["spec", "maxmemoryPolicy"], ["spec", "snapshots"], ["spec", "sentinel"]] secrets: # Backstop for the label-only include below, which matches on the label # alone with no name constraint. These two Secrets are the ones whose From 4c3a4b42ccc70343895237cce4de69fa6b7ea16b Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 12 Nov 2026 23:27:04 +0300 Subject: [PATCH 147/174] style: clear unused code and style findings reported by staticcheck Remove helpers and constants nothing references, escape format characters in test literals, and take the simplifications staticcheck suggests where they read the same. Error strings no longer start with a capital: they keep their meaning, with the kind or component name moved out of first position. Callers match errors by type, not text; the one test that matches text, for Kafka, keeps the phrase it looks for. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- cmd/backport-batch/injection_test.go | 6 ++--- cmd/backport-batch/onto_test.go | 4 +-- cmd/backport-batch/policy_test.go | 4 +-- cmd/cozypkg/cmd/add.go | 2 +- cmd/cozystack-operator/main.go | 2 +- .../altinitystrategy_controller.go | 2 +- .../etcdstrategy_controller.go | 4 +-- .../kafkastrategy_controller.go | 6 ++--- .../rabbitmqstrategy_controller.go | 2 +- internal/controller/kubeovnplunger/metrics.go | 2 +- internal/controller/workload_controller.go | 5 ---- internal/fluxinstall/install.go | 2 +- internal/lineagecontrollerwebhook/webhook.go | 2 +- internal/migrationcontroller/inventory.go | 4 +-- internal/operator/package_reconciler_test.go | 9 ------- internal/telemetry/collector.go | 4 +-- internal/telemetry/operator_collector.go | 22 +++++++-------- pkg/apiserver/apiserver.go | 6 ++--- pkg/cmd/server/openapi.go | 4 +-- pkg/ovnstatus/ovnstatus.go | 27 ------------------- pkg/ovnstatus/ovnstatus_test.go | 4 +-- pkg/reservation/evaluator.go | 6 ++--- 22 files changed, 43 insertions(+), 86 deletions(-) diff --git a/cmd/backport-batch/injection_test.go b/cmd/backport-batch/injection_test.go index f1d12b004..a1f5abbe6 100644 --- a/cmd/backport-batch/injection_test.go +++ b/cmd/backport-batch/injection_test.go @@ -142,11 +142,11 @@ func TestOptionInjection(t *testing.T) { // no bidi override reorders it and no zero-width character hides in it. func TestDeferredBlockCleansFileNames(t *testing.T) { l, _ := parseLine("release-1.6") - files := []string{"app/‮txt.exe", "app/zero​width.yaml"} + files := []string{"app/\u202etxt.exe", "app/zero\u200bwidth.yaml"} conflict := pick{v: &verdict{o: &original{number: 5}}, files: files, failed: strings.Repeat("a", 40)} waiting := pick{v: &verdict{o: &original{number: 6}}, files: files} block := deferredBlock(l, []pick{conflict}, nil, []pick{waiting}) - if strings.ContainsAny(block, "‮​") { + if strings.ContainsAny(block, "\u202e\u200b") { t.Errorf("the block carries format characters: %q", block) } want := []string{"app/txt.exe", "app/zerowidth.yaml"} @@ -154,7 +154,7 @@ func TestDeferredBlockCleansFileNames(t *testing.T) { if err != nil || !slices.Equal(got[5].Files, want) || !slices.Equal(got[6].Files, want) { t.Errorf("parseDeferred = %+v, %v; want both originals with %q", got, err, want) } - if files[0] != "app/‮txt.exe" { + if files[0] != "app/\u202etxt.exe" { t.Errorf("deferredBlock rewrote the pick's own file names: %q", files) } } diff --git a/cmd/backport-batch/onto_test.go b/cmd/backport-batch/onto_test.go index ee4d52d33..994059235 100644 --- a/cmd/backport-batch/onto_test.go +++ b/cmd/backport-batch/onto_test.go @@ -41,12 +41,12 @@ func TestOntoContinues(t *testing.T) { f.lines() head := pushed(t, f) f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) - code, stdout, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head, "--date", "2026-10-29") + code, _, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head, "--date", "2026-10-29") if code != 2 || !strings.Contains(stderr, "not the head of backport-batch/release-1.1-20261029") { t.Errorf("--onto with another date = %d (%s), want a refusal", code, stderr) } f.now = f.now.AddDate(0, 0, 3) - code, stdout, stderr = f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head) + code, stdout, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head) if code != 0 { t.Fatalf("run --onto = %d\n%s", code, stderr) } diff --git a/cmd/backport-batch/policy_test.go b/cmd/backport-batch/policy_test.go index 920c18a0b..2cd7d20f3 100644 --- a/cmd/backport-batch/policy_test.go +++ b/cmd/backport-batch/policy_test.go @@ -176,7 +176,7 @@ func TestPolicyPageMatchesCode(t *testing.T) { t.Errorf("the page's block gives %d released lines, want 2 written out", pol.released) } - rule2 := regexp.MustCompile("(?m)^2\\. Its title type is (.*)$").FindStringSubmatch(page) + rule2 := regexp.MustCompile(`(?m)^2\. Its title type is (.*)$`).FindStringSubmatch(page) if rule2 == nil { t.Fatal("the page has no rule 2") } @@ -187,7 +187,7 @@ func TestPolicyPageMatchesCode(t *testing.T) { if !slices.Equal(types, takenTypes) { t.Errorf("rule 2 on the page takes %v, the code %v", types, takenTypes) } - never := regexp.MustCompile("(?m)^### What waits for the next minor\\n\\n(.*)$").FindStringSubmatch(page) + never := regexp.MustCompile(`(?m)^### What waits for the next minor\n\n(.*)$`).FindStringSubmatch(page) if never == nil { t.Fatal("the page has no list of what never goes into a patch release") } diff --git a/cmd/cozypkg/cmd/add.go b/cmd/cozypkg/cmd/add.go index d882ce287..8c5e22026 100644 --- a/cmd/cozypkg/cmd/add.go +++ b/cmd/cozypkg/cmd/add.go @@ -390,7 +390,7 @@ func createPackageFromFile(ctx context.Context, k8sClient client.Client, filePat } } - return fmt.Errorf("Package %s not found in file", packageName) + return fmt.Errorf("no Package %s in file", packageName) } func installPackage(ctx context.Context, k8sClient client.Client, packageSourceName string) error { diff --git a/cmd/cozystack-operator/main.go b/cmd/cozystack-operator/main.go index e11290aed..e1a2aa1de 100644 --- a/cmd/cozystack-operator/main.go +++ b/cmd/cozystack-operator/main.go @@ -663,7 +663,7 @@ func validateGitRef(refMap map[string]string) error { return fmt.Errorf("commit SHA should be at least 7 characters, got: %s", commit) } for _, c := range commit { - if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { return fmt.Errorf("commit SHA should be a hexadecimal string, got: %s", commit) } } diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go index f0c597583..4314c3dbf 100644 --- a/internal/backupcontroller/altinitystrategy_controller.go +++ b/internal/backupcontroller/altinitystrategy_controller.go @@ -631,7 +631,7 @@ func (r *BackupJobReconciler) createAltinityBackupArtifact( // used name meets the earlier run's Backup here. Only one recording // this run's archive was created by an earlier reconcile of this run. if got := existing.Spec.DriverMetadata[altinityArchiveMetaKey]; got != archive { - return nil, fmt.Errorf("Backup %s/%s already exists and records archive %q, not %q that this run wrote", + return nil, fmt.Errorf("existing Backup %s/%s records archive %q, not %q that this run wrote", backup.Namespace, backup.Name, got, archive) } return existing, nil diff --git a/internal/backupcontroller/etcdstrategy_controller.go b/internal/backupcontroller/etcdstrategy_controller.go index 2f2f2732c..4f530ca2f 100644 --- a/internal/backupcontroller/etcdstrategy_controller.go +++ b/internal/backupcontroller/etcdstrategy_controller.go @@ -134,14 +134,14 @@ var etcdClusterGVR = schema.GroupVersionResource{ // the wrong CRD via the typed client. func validateEtcdApplicationRef(ref corev1.TypedLocalObjectReference) error { if ref.Kind != etcdAppKind { - return fmt.Errorf("Etcd strategy supports applicationRef.kind=%q, got %q", etcdAppKind, ref.Kind) + return fmt.Errorf("the Etcd strategy supports applicationRef.kind=%q, got %q", etcdAppKind, ref.Kind) } apiGroup := "" if ref.APIGroup != nil { apiGroup = *ref.APIGroup } if apiGroup != "" && apiGroup != etcdapp.GroupName { - return fmt.Errorf("Etcd strategy supports applicationRef.apiGroup=%q, got %q", etcdapp.GroupName, apiGroup) + return fmt.Errorf("the Etcd strategy supports applicationRef.apiGroup=%q, got %q", etcdapp.GroupName, apiGroup) } return nil } diff --git a/internal/backupcontroller/kafkastrategy_controller.go b/internal/backupcontroller/kafkastrategy_controller.go index 70f594d65..f7c34ce53 100644 --- a/internal/backupcontroller/kafkastrategy_controller.go +++ b/internal/backupcontroller/kafkastrategy_controller.go @@ -104,10 +104,10 @@ func kafkaClusterName(appName string) string { // apps.cozystack.io/Kafka application. func validateKafkaApplicationRef(ref corev1.TypedLocalObjectReference) error { if ref.Kind != kafkaApplicationKind { - return fmt.Errorf("Kafka strategy only supports applicationRef.kind=%q, got %q", kafkaApplicationKind, ref.Kind) + return fmt.Errorf("the Kafka strategy only supports applicationRef.kind=%q, got %q", kafkaApplicationKind, ref.Kind) } if ref.APIGroup != nil && *ref.APIGroup != "" && *ref.APIGroup != backupsv1alpha1.DefaultApplicationAPIGroup { - return fmt.Errorf("Kafka strategy only supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup) + return fmt.Errorf("the Kafka strategy only supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup) } return nil } @@ -528,7 +528,7 @@ func (r *BackupJobReconciler) createKafkaBackupArtifact( // RabbitMQ driver's guard). if existing.Spec.ApplicationRef.Kind != backup.Spec.ApplicationRef.Kind || existing.Spec.ApplicationRef.Name != backup.Spec.ApplicationRef.Name { - return nil, fmt.Errorf("Backup %s/%s already exists for a different application (%s/%s, not %s/%s)", + return nil, fmt.Errorf("existing Backup %s/%s belongs to a different application (%s/%s, not %s/%s)", backup.Namespace, backup.Name, existing.Spec.ApplicationRef.Kind, existing.Spec.ApplicationRef.Name, backup.Spec.ApplicationRef.Kind, backup.Spec.ApplicationRef.Name) diff --git a/internal/backupcontroller/rabbitmqstrategy_controller.go b/internal/backupcontroller/rabbitmqstrategy_controller.go index adfe11d83..4c3d9a55c 100644 --- a/internal/backupcontroller/rabbitmqstrategy_controller.go +++ b/internal/backupcontroller/rabbitmqstrategy_controller.go @@ -513,7 +513,7 @@ func (r *BackupJobReconciler) createRabbitmqBackupArtifact( // this run's success while describing something else. if existing.Spec.ApplicationRef.Kind != backup.Spec.ApplicationRef.Kind || existing.Spec.ApplicationRef.Name != backup.Spec.ApplicationRef.Name { - return nil, fmt.Errorf("Backup %s/%s already exists for a different application (%s/%s, not %s/%s)", + return nil, fmt.Errorf("existing Backup %s/%s belongs to a different application (%s/%s, not %s/%s)", backup.Namespace, backup.Name, existing.Spec.ApplicationRef.Kind, existing.Spec.ApplicationRef.Name, backup.Spec.ApplicationRef.Kind, backup.Spec.ApplicationRef.Name) diff --git a/internal/controller/kubeovnplunger/metrics.go b/internal/controller/kubeovnplunger/metrics.go index 1330db814..523467fcd 100644 --- a/internal/controller/kubeovnplunger/metrics.go +++ b/internal/controller/kubeovnplunger/metrics.go @@ -219,7 +219,7 @@ func (r *KubeOVNPlunger) WriteClusterMetrics(db string, snaps []ovnstatus.Health r.metrics.allAgree.WithLabelValues(db, cid).Set(b2f(ecv.AllAgree)) r.metrics.membersExpected.WithLabelValues(db, cid).Set(float64(expectedReplicas)) r.metrics.membersObserved.WithLabelValues(db, cid).Set(float64(ecv.MembersCount)) - r.metrics.ipsExpected.WithLabelValues(db, cid).Set(float64(len(ecv.ConsensusResult.TruthView.Members))) // optional; or len(hints.ExpectedIPs) + r.metrics.ipsExpected.WithLabelValues(db, cid).Set(float64(len(ecv.TruthView.Members))) // optional; or len(hints.ExpectedIPs) r.metrics.ipsObserved.WithLabelValues(db, cid).Set(float64(ecv.DistinctIPCount)) r.metrics.excessMembers.WithLabelValues(db, cid).Set(float64(ecv.ExpectedExcess)) r.metrics.missingMembers.WithLabelValues(db, cid).Set(float64(ecv.ExpectedShortfall)) diff --git a/internal/controller/workload_controller.go b/internal/controller/workload_controller.go index d0d9e7430..052358a05 100644 --- a/internal/controller/workload_controller.go +++ b/internal/controller/workload_controller.go @@ -3,7 +3,6 @@ package controller import ( "context" "strings" - "time" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -15,10 +14,6 @@ import ( cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" ) -const ( - deletionRequeueDelay = 30 * time.Second -) - // WorkloadMonitorReconciler reconciles a WorkloadMonitor object type WorkloadReconciler struct { client.Client diff --git a/internal/fluxinstall/install.go b/internal/fluxinstall/install.go index d16d7d854..042bdfd5e 100644 --- a/internal/fluxinstall/install.go +++ b/internal/fluxinstall/install.go @@ -171,7 +171,7 @@ func extractNamespace(objects []*unstructured.Unstructured) (string, error) { if obj.GetKind() == "Namespace" { namespace := obj.GetName() if namespace == "" { - return "", fmt.Errorf("Namespace object has no name") + return "", fmt.Errorf("the Namespace object has no name") } return namespace, nil } diff --git a/internal/lineagecontrollerwebhook/webhook.go b/internal/lineagecontrollerwebhook/webhook.go index ea8b0d01b..7b29e1f76 100644 --- a/internal/lineagecontrollerwebhook/webhook.go +++ b/internal/lineagecontrollerwebhook/webhook.go @@ -176,7 +176,7 @@ func (h *LineageControllerWebhook) computeLabels(ctx context.Context, obj *unstr return s } s = s[:63] - for b := s[62]; !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9')); s = s[:len(s)-1] { + for b := s[62]; (b < 'a' || b > 'z') && (b < 'A' || b > 'Z') && (b < '0' || b > '9'); s = s[:len(s)-1] { b = s[len(s)-1] } return s diff --git a/internal/migrationcontroller/inventory.go b/internal/migrationcontroller/inventory.go index d56bec2da..52a5e4cd8 100644 --- a/internal/migrationcontroller/inventory.go +++ b/internal/migrationcontroller/inventory.go @@ -146,7 +146,7 @@ func (c *inventoryClient) VMs(ctx context.Context, providerUID string) ([]invent defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Forklift inventory returned %s listing VMs", resp.Status) + return nil, fmt.Errorf("listing VMs: the Forklift inventory returned %s", resp.Status) } var vms []inventoryVMRef @@ -195,7 +195,7 @@ func (c *inventoryClient) VM(ctx context.Context, providerUID, vmID string) (*in case http.StatusNotFound: return nil, ¬FoundError{vmID: vmID} default: - return nil, fmt.Errorf("Forklift inventory returned %s for VM %q", resp.Status, vmID) + return nil, fmt.Errorf("VM %q: the Forklift inventory returned %s", vmID, resp.Status) } vm := &inventoryVM{} diff --git a/internal/operator/package_reconciler_test.go b/internal/operator/package_reconciler_test.go index c9923e7dc..fd8a15fe5 100644 --- a/internal/operator/package_reconciler_test.go +++ b/internal/operator/package_reconciler_test.go @@ -790,15 +790,6 @@ func scanRecordingClient(scheme *runtime.Scheme, listed *[]string, writes *int, }).Build() } -func listedKind(listed []string, kind string) bool { - for _, l := range listed { - if strings.Contains(l, kind) { - return true - } - } - return false -} - // The comparison guarding that write has to be semantic rather than structural, or a // LimitRange written in a different but equivalent notation would re-apply on every // reconcile — invisible in a test that only looked at the resulting object, since every one diff --git a/internal/telemetry/collector.go b/internal/telemetry/collector.go index f4bb8dd88..a948c7495 100644 --- a/internal/telemetry/collector.go +++ b/internal/telemetry/collector.go @@ -118,11 +118,11 @@ func (c *Collector) collect(ctx context.Context) { // Write application count metrics for kind, count := range appKindCounts { - metrics.WriteString(fmt.Sprintf( + fmt.Fprintf(&metrics, "cozy_application_count{kind=\"%s\"} %d\n", kind, count, - )) + ) } // Send metrics only if there's something to send diff --git a/internal/telemetry/operator_collector.go b/internal/telemetry/operator_collector.go index 8b4ec3028..2c0e1b9bb 100644 --- a/internal/telemetry/operator_collector.go +++ b/internal/telemetry/operator_collector.go @@ -133,11 +133,11 @@ func (c *OperatorCollector) collect(ctx context.Context) { var metrics strings.Builder // Add cluster info metric - metrics.WriteString(fmt.Sprintf( + fmt.Fprintf(&metrics, "cozy_cluster_info{cozystack_version=\"%s\",kubernetes_version=\"%s\"} 1\n", version.Version, k8sVersion.GitVersion, - )) + ) // Collect node metrics grouped by OS and kernel nodeOSCount := make(map[string]map[string]int) // os -> kernel -> count @@ -153,12 +153,12 @@ func (c *OperatorCollector) collect(ctx context.Context) { for osKey, kernels := range nodeOSCount { for kernel, count := range kernels { - metrics.WriteString(fmt.Sprintf( + fmt.Fprintf(&metrics, "cozy_nodes_count{os=\"%s\",kernel=\"%s\"} %d\n", osKey, kernel, count, - )) + ) } } @@ -174,11 +174,11 @@ func (c *OperatorCollector) collect(ctx context.Context) { } for resourceName, total := range capacityTotals { - metrics.WriteString(fmt.Sprintf( + fmt.Fprintf(&metrics, "cozy_cluster_capacity{resource=\"%s\"} %d\n", resourceName, total, - )) + ) } // Collect LoadBalancer services metrics @@ -192,7 +192,7 @@ func (c *OperatorCollector) collect(ctx context.Context) { lbCount++ } } - metrics.WriteString(fmt.Sprintf("cozy_loadbalancers_count %d\n", lbCount)) + fmt.Fprintf(&metrics, "cozy_loadbalancers_count %d\n", lbCount) } // Collect PV metrics grouped by driver and size @@ -224,12 +224,12 @@ func (c *OperatorCollector) collect(ctx context.Context) { for size, drivers := range pvMetrics { for driver, count := range drivers { - metrics.WriteString(fmt.Sprintf( + fmt.Fprintf(&metrics, "cozy_pvs_count{driver=\"%s\",size=\"%s\"} %d\n", driver, size, count, - )) + ) } } } @@ -244,11 +244,11 @@ func (c *OperatorCollector) collect(ctx context.Context) { if variant == "" { variant = "default" } - metrics.WriteString(fmt.Sprintf( + fmt.Fprintf(&metrics, "cozy_package_info{name=\"%s\",variant=\"%s\"} 1\n", pkg.Name, variant, - )) + ) } } diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index c3555e46c..06f024bc6 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -78,7 +78,7 @@ func init() { // Register HelmRelease types. if err := helmv2.AddToScheme(mgrScheme); err != nil { - panic(fmt.Errorf("Failed to add HelmRelease types to scheme: %w", err)) + panic(fmt.Errorf("failed to add HelmRelease types to scheme: %w", err)) } // Register the in-tree CiliumNetworkPolicy mirror backing SecurityGroup. @@ -87,10 +87,10 @@ func init() { } if err := corev1.AddToScheme(mgrScheme); err != nil { - panic(fmt.Errorf("Failed to add core types to scheme: %w", err)) + panic(fmt.Errorf("failed to add core types to scheme: %w", err)) } if err := rbacv1.AddToScheme(mgrScheme); err != nil { - panic(fmt.Errorf("Failed to add RBAC types to scheme: %w", err)) + panic(fmt.Errorf("failed to add RBAC types to scheme: %w", err)) } // Register Cozystack types for WorkloadMonitor queries. diff --git a/pkg/cmd/server/openapi.go b/pkg/cmd/server/openapi.go index 90af89cd0..d0cbe3289 100644 --- a/pkg/cmd/server/openapi.go +++ b/pkg/cmd/server/openapi.go @@ -319,7 +319,7 @@ func BuildPostProcessV3(kindSchemas map[string]string) func(*spec3.OpenAPI) (*sp base, ok1 := doc.Components.Schemas[baseRef] list, ok2 := doc.Components.Schemas[baseListRef] stat, ok3 := doc.Components.Schemas[baseStatusRef] - if !(ok1 && ok2 && ok3) { + if !ok1 || !ok2 || !ok3 { return doc, nil // not the apps GV — nothing to patch } @@ -463,7 +463,7 @@ func BuildPostProcessV2(kindSchemas map[string]string) func(*spec.Swagger) (*spe base, ok1 := defs[baseRef] list, ok2 := defs[baseListRef] stat, ok3 := defs[baseStatusRef] - if !(ok1 && ok2 && ok3) { + if !ok1 || !ok2 || !ok3 { return sw, nil // not the apps GV — nothing to patch } diff --git a/pkg/ovnstatus/ovnstatus.go b/pkg/ovnstatus/ovnstatus.go index e6d682e51..b50c3f7ed 100644 --- a/pkg/ovnstatus/ovnstatus.go +++ b/pkg/ovnstatus/ovnstatus.go @@ -277,33 +277,6 @@ func defaultRunner(ctx context.Context, bin string, args ...string) (string, err return out, nil } -/************** Helpers **************/ - -func parseClusterStatusJSON(out string) (ClusterStatus, bool) { - var cs ClusterStatus - if json.Unmarshal([]byte(out), &cs) == nil && len(cs.Servers) > 0 { - return cs, true - } - var wrap struct { - Data ClusterStatus `json:"data"` - } - if json.Unmarshal([]byte(out), &wrap) == nil && len(wrap.Data.Servers) > 0 { - return wrap.Data, true - } - return ClusterStatus{}, false -} - -func portOf(db DB) string { - switch db { - case DBNorthbound: - return "6641" - case DBSouthbound: - return "6642" - default: - return "0" - } -} - /************** Minimal text fallback for "Servers" **************/ // Accepts variants like: diff --git a/pkg/ovnstatus/ovnstatus_test.go b/pkg/ovnstatus/ovnstatus_test.go index 8dde66b3a..8c44f5413 100644 --- a/pkg/ovnstatus/ovnstatus_test.go +++ b/pkg/ovnstatus/ovnstatus_test.go @@ -28,9 +28,7 @@ var expectedServersBlock = `` + func TestExtractServersBlock(t *testing.T) { if actual := extractServersBlock(testStdout); actual != expectedServersBlock { - fmt.Println([]byte(actual)) - fmt.Println([]byte(expectedServersBlock)) - t.Errorf("error extracting servers block from following string:\n%s\nexpected:\n%s\ngot:\n%s\n", testStdout, expectedServersBlock, actual) + t.Errorf("error extracting servers block from following string:\n%s\nexpected:\n%q\ngot:\n%q\n", testStdout, expectedServersBlock, actual) } } diff --git a/pkg/reservation/evaluator.go b/pkg/reservation/evaluator.go index 6927b0bf2..40d5c2ced 100644 --- a/pkg/reservation/evaluator.go +++ b/pkg/reservation/evaluator.go @@ -284,9 +284,9 @@ func mulQuantity(q, n ref.Val) ref.Val { if !ok { return types.MaybeNoSuchOverloadErr(n) } - out := qq.Quantity.DeepCopy() + out := qq.DeepCopy() if !out.Mul(int64(nn)) { - return types.NewErr("quantity %s * %d overflows", qq.Quantity.String(), int64(nn)) + return types.NewErr("quantity %s * %d overflows", qq.String(), int64(nn)) } return apiservercel.Quantity{Quantity: &out} } @@ -294,7 +294,7 @@ func mulQuantity(q, n ref.Val) ref.Val { func toQuantity(v ref.Val) (resource.Quantity, error) { switch t := v.(type) { case apiservercel.Quantity: - return t.Quantity.DeepCopy(), nil + return t.DeepCopy(), nil case types.Int: return *resource.NewQuantity(int64(t), resource.DecimalSI), nil case types.Uint: From 238e85577e63211c8124603ae87d4cab01931098 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 12 Nov 2026 23:28:51 +0300 Subject: [PATCH 148/174] refactor(lineage): resolve a HelmRelease's application in a helper The walk used a loop that always ended in its first pass to get "break to the fallback" control flow. A function returning whether it resolved the application says the same without the loop. A test pins that each failure to resolve still falls back to the HelmRelease's own label and that a resolved one ends the walk. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- pkg/lineage/helmrelease_test.go | 70 +++++++++++++++++++++++++++++++++ pkg/lineage/lineage.go | 59 ++++++++++++++------------- 2 files changed, 99 insertions(+), 30 deletions(-) create mode 100644 pkg/lineage/helmrelease_test.go diff --git a/pkg/lineage/helmrelease_test.go b/pkg/lineage/helmrelease_test.go new file mode 100644 index 000000000..c8ef2676e --- /dev/null +++ b/pkg/lineage/helmrelease_test.go @@ -0,0 +1,70 @@ +package lineage + +import ( + "context" + "fmt" + "strings" + "testing" + + helmv2 "github.com/fluxcd/helm-controller/api/v2" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" +) + +// namedAppMapper maps a HelmRelease to the Kubernetes application it names; +// a HelmRelease missing from the map fails to map. +type namedAppMapper map[string]string + +func (m namedAppMapper) Map(hr *helmv2.HelmRelease) (string, string, string, error) { + app, ok := m[hr.Name] + if !ok { + return "", "", "", fmt.Errorf("no application for HelmRelease %s", hr.Name) + } + return "apps.cozystack.io/v1alpha1", "Kubernetes", strings.TrimSuffix(hr.Name, app), nil +} + +// A HelmRelease that cannot be resolved to its application hands the walk to +// the HelmRelease its label names; one that resolves ends the walk there. +func TestWalkHelmReleaseFallsBackToItsLabel(t *testing.T) { + parent := map[string]string{HRLabel: "kubernetes-b"} + unconvertible := capkObject(HRAPIVersion, HRKind, "kubernetes-a", parent, nil) + unconvertible.Object["spec"] = "not an object" + for _, tc := range []struct { + name string + hr *unstructured.Unstructured + mapped namedAppMapper + want []string + }{ + {name: "mapped", hr: capkObject(HRAPIVersion, HRKind, "kubernetes-a", parent, nil), mapped: namedAppMapper{"kubernetes-a": "a", "kubernetes-b": "b"}, want: []string{"a"}}, + {name: "unmappable", hr: capkObject(HRAPIVersion, HRKind, "kubernetes-a", parent, nil), mapped: namedAppMapper{"kubernetes-b": "b"}, want: []string{"b"}}, + {name: "application missing", hr: capkObject(HRAPIVersion, HRKind, "kubernetes-a", parent, nil), mapped: namedAppMapper{"kubernetes-a": "absent", "kubernetes-b": "b"}, want: []string{"b"}}, + {name: "unconvertible", hr: unconvertible, mapped: namedAppMapper{"kubernetes-a": "a", "kubernetes-b": "b"}, want: []string{"b"}}, + {name: "unmappable without a label", hr: capkObject(HRAPIVersion, HRKind, "kubernetes-a", nil, nil), mapped: namedAppMapper{"kubernetes-b": "b"}, want: nil}, + } { + t.Run(tc.name, func(t *testing.T) { + objs := []runtime.Object{ + capkObject("apps.cozystack.io/v1alpha1", "Kubernetes", "a", nil, nil), + capkObject("apps.cozystack.io/v1alpha1", "Kubernetes", "b", nil, nil), + capkObject(HRAPIVersion, HRKind, "kubernetes-b", nil, nil), + } + mapper := meta.NewDefaultRESTMapper(nil) + for _, o := range objs { + mapper.Add(o.GetObjectKind().GroupVersionKind(), meta.RESTScopeNamespace) + } + dyn := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), objs...) + var got []string + for _, id := range WalkOwnershipGraph(context.Background(), dyn, mapper, tc.mapped, tc.hr) { + got = append(got, id.Kind+"/"+id.Name) + } + var want []string + for _, name := range tc.want { + want = append(want, "Kubernetes/"+name) + } + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("WalkOwnershipGraph(%s) = %v, want %v", tc.name, got, want) + } + }) + } +} diff --git a/pkg/lineage/lineage.go b/pkg/lineage/lineage.go index 1546df647..8d8614b68 100644 --- a/pkg/lineage/lineage.go +++ b/pkg/lineage/lineage.go @@ -113,36 +113,8 @@ func WalkOwnershipGraph( return } - // I want "if err1 != nil go to next block, if err2 != nil, go to next block, etc semantics", - // like an early return from a function, but if all checks succeed, I don't want to do the rest - // of the function, so it's a `for { if err { break } if othererr { break } if allgood { return } - for { - if obj.GetAPIVersion() != HRAPIVersion || obj.GetKind() != HRKind { - break - } - hr := helmReleaseFromUnstructured(obj) - if hr == nil { - break - } - a, k, p, err := appMapper.Map(hr) - if err != nil { - l.Error(err, "failed to map HelmRelease to app") - break - } - ownerObj, err := getUnstructuredObject(ctx, client, mapper, state.cache, a, k, obj.GetNamespace(), strings.TrimPrefix(obj.GetName(), p)) - if err != nil { - l.Error(err, "couldn't get unstructured object", "APIVersion", a, "Kind", k, "Name", strings.TrimPrefix(obj.GetName(), p)) - break - } - // successfully mapped a HelmRelease to a custom app, no need to continue - out = append(out, - ObjectID{ - APIVersion: ownerObj.GetAPIVersion(), - Kind: ownerObj.GetKind(), - Namespace: ownerObj.GetNamespace(), - Name: ownerObj.GetName(), - }, - ) + if app, ok := helmReleaseApp(ctx, client, mapper, appMapper, state.cache, obj); ok { + out = append(out, app) return } @@ -160,6 +132,33 @@ func WalkOwnershipGraph( return } +// helmReleaseApp resolves a HelmRelease to the application it deploys. When +// it cannot, the walk goes on through the HelmRelease's own label instead. +func helmReleaseApp(ctx context.Context, client dynamic.Interface, mapper meta.RESTMapper, appMapper AppMapper, cache *ObjectCache, obj *unstructured.Unstructured) (ObjectID, bool) { + hr := helmReleaseFromUnstructured(obj) + if hr == nil { + return ObjectID{}, false + } + l := log.FromContext(ctx) + a, k, p, err := appMapper.Map(hr) + if err != nil { + l.Error(err, "failed to map HelmRelease to app") + return ObjectID{}, false + } + name := strings.TrimPrefix(obj.GetName(), p) + app, err := getUnstructuredObject(ctx, client, mapper, cache, a, k, obj.GetNamespace(), name) + if err != nil { + l.Error(err, "couldn't get unstructured object", "APIVersion", a, "Kind", k, "Name", name) + return ObjectID{}, false + } + return ObjectID{ + APIVersion: app.GetAPIVersion(), + Kind: app.GetKind(), + Namespace: app.GetNamespace(), + Name: app.GetName(), + }, true +} + // kubevirtMachineName returns the KubevirtMachine a CAPK-created // VirtualMachine belongs to. It only follows a KubevirtMachine in the VM's own // namespace: CAPK can also run against an external infrastructure cluster, From 04fa87ccd1edf6dfec048717a3bd1ee7d3ab1a1d Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 12 Nov 2026 23:30:02 +0300 Subject: [PATCH 149/174] style(backups): clear staticcheck findings in overlapping controllers Drop a field-manager constant nothing uses in the MariaDB driver, switch on the MongoDB operator's state as a tagged switch, and reword the error strings of the MongoDB driver and the workload monitor so they do not start with a capital. The errors keep their meaning and nothing matches their text. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../backupcontroller/mariadbstrategy_controller.go | 2 -- .../backupcontroller/psmdbstrategy_controller.go | 14 +++++++------- internal/controller/workloadmonitor_controller.go | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 9fd6780db..452fff071 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -30,8 +30,6 @@ import ( // --------------------------------------------------------------------------- const ( - mariadbFieldManager = "cozystack-mariadb-backup-driver" - // mariadb-operator's MariaDB ApplicationDefinition (release.prefix). The // cozystack mariadb-rd renders the HelmRelease with releaseName = // "mariadb-" + appName (see packages/system/mariadb-rd/cozyrds/mariadb.yaml, diff --git a/internal/backupcontroller/psmdbstrategy_controller.go b/internal/backupcontroller/psmdbstrategy_controller.go index 9caa218a7..ccf3e7735 100644 --- a/internal/backupcontroller/psmdbstrategy_controller.go +++ b/internal/backupcontroller/psmdbstrategy_controller.go @@ -528,8 +528,8 @@ func (r *BackupJobReconciler) reconcileMongoDB(ctx context.Context, j *backupsv1 } } - switch state := mdbBackup.Status.State; { - case state == psmdbtypes.StateReady: + switch state := mdbBackup.Status.State; state { + case psmdbtypes.StateReady: // Backup completed successfully. Materialise the Cozystack artifact and // finalise the BackupJob. Idempotent: reuse the existing artifact if a // previous reconcile created it and then raced on the status update. @@ -570,7 +570,7 @@ func (r *BackupJobReconciler) reconcileMongoDB(ctx context.Context, j *backupsv1 } return ctrl.Result{}, nil - case state == psmdbtypes.StateError || state == psmdbtypes.StateRejected: + case psmdbtypes.StateError, psmdbtypes.StateRejected: message := mdbBackup.Status.Error // The residual half of the injection race: on the useSystemBucket flow the // operator can latch state=error because it resolved the injected storage @@ -1740,8 +1740,8 @@ func (r *RestoreJobReconciler) reconcileMongoDBRestore(ctx context.Context, rest return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure psmdb.percona.com/PerconaServerMongoDBRestore: %v", err)) } - switch state := mdbRestore.Status.State; { - case state == psmdbtypes.StateReady: + switch state := mdbRestore.Status.State; state { + case psmdbtypes.StateReady: now := metav1.Now() restoreJob.Status.CompletedAt = &now restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded @@ -1756,7 +1756,7 @@ func (r *RestoreJobReconciler) reconcileMongoDBRestore(ctx context.Context, rest } return ctrl.Result{}, nil - case state == psmdbtypes.StateError || state == psmdbtypes.StateRejected: + case psmdbtypes.StateError, psmdbtypes.StateRejected: message := mdbRestore.Status.Error if message == "" { message = fmt.Sprintf("psmdb.percona.com/PerconaServerMongoDBRestore %s/%s reported state=%s", mdbRestore.Namespace, mdbRestore.Name, state) @@ -1855,7 +1855,7 @@ func (r *RestoreJobReconciler) resolveMongoDBBackupSource(ctx context.Context, b } if destination == "" { return nil, fmt.Errorf( - "Backup has no restorable destination (the operator PerconaServerMongoDBBackup was reaped and no snapshot destination is persisted); re-run the BackupJob") + "no restorable destination for the Backup (the operator PerconaServerMongoDBBackup was reaped and no snapshot destination is persisted); re-run the BackupJob") } src := &psmdbtypes.BackupSource{ Type: psmdbtypes.BackupTypeLogical, diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 984ab7b21..217fd0bbf 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -528,7 +528,7 @@ func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, p defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Prometheus returned status %d", resp.StatusCode) + return nil, fmt.Errorf("unexpected status %d from Prometheus", resp.StatusCode) } body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) @@ -549,7 +549,7 @@ func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, p return nil, fmt.Errorf("parsing Prometheus response: %w", err) } if promResp.Status != "success" { - return nil, fmt.Errorf("Prometheus query finished with status %q", promResp.Status) + return nil, fmt.Errorf("the Prometheus query finished with status %q", promResp.Status) } for _, r := range promResp.Data.Result { From 2cda7d692e1bb2343ca6b71fc2a1335eefc2aade Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 12 Nov 2026 23:30:02 +0300 Subject: [PATCH 150/174] test(backport): drop an unused fixture helper Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- cmd/backport-batch/fixture_test.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cmd/backport-batch/fixture_test.go b/cmd/backport-batch/fixture_test.go index ac1307c46..08aa5f2ac 100644 --- a/cmd/backport-batch/fixture_test.go +++ b/cmd/backport-batch/fixture_test.go @@ -522,15 +522,6 @@ func (f *fixture) batchPR(n int, base, branch, body string) pullRequest { return pr } -// botBatchPR is batchPR opened by the bot the scheduled batch runs as. -func (f *fixture) botBatchPR(gh *fakeGH, n int, base, branch, body string) pullRequest { - f.t.Helper() - gh.bot = defaultBatchBot - pr := f.batchPR(n, base, branch, body) - pr.User.Login, pr.User.Type, pr.AuthorAssociation = defaultBatchBot, "Bot", "CONTRIBUTOR" - return pr -} - // mergeBatch merges the open batch into release-1.1 with a merge commit, as // GitHub does, and deletes its branch, as the repository's // delete_branch_on_merge does. The pull request stays, merged, with its body From d308e9e5e59aea5be1a41952fef9f8c3035845c5 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 13 Nov 2026 09:43:05 +0100 Subject: [PATCH 151/174] fix(redis): refuse sub-byte maxmemory and apply the policy first A quantity such as 512m is 0.512 bytes; the integer cast truncated it, or -512m, to 0, which Redis reads as no limit, so the bound a tenant set could vanish without an error. The operator also sends the directives in list order, and lowering maxmemory below the dataset evicts at once, so the policy has to be set before the bound. Turning snapshots off does not remove an RDB already on the volume, which Redis still loads on restart; the parameter no longer says the data is lost. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- api/apps/v1alpha1/redis/types.go | 2 +- packages/apps/redis/README.md | 2 +- .../apps/redis/templates/redisfailover.yaml | 18 ++++++++--- .../apps/redis/tests/cache_config_test.yaml | 32 +++++++++++++++++++ packages/apps/redis/values.schema.json | 2 +- packages/apps/redis/values.yaml | 2 +- packages/system/redis-rd/cozyrds/redis.yaml | 2 +- 7 files changed, 50 insertions(+), 10 deletions(-) diff --git a/api/apps/v1alpha1/redis/types.go b/api/apps/v1alpha1/redis/types.go index 03ff0175d..41cf03c14 100644 --- a/api/apps/v1alpha1/redis/types.go +++ b/api/apps/v1alpha1/redis/types.go @@ -56,7 +56,7 @@ type ConfigSpec struct { // Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset. // +kubebuilder:default:="noeviction" MaxmemoryPolicy MaxmemoryPolicy `json:"maxmemoryPolicy"` - // Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty. + // Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty. // +kubebuilder:default:=true Snapshots bool `json:"snapshots"` // Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does. diff --git a/packages/apps/redis/README.md b/packages/apps/redis/README.md index 4044ce951..be03bbea3 100644 --- a/packages/apps/redis/README.md +++ b/packages/apps/redis/README.md @@ -46,7 +46,7 @@ Service utilizes the freshworks-oss Redis Operator (a maintained fork of the arc | `minReplicasMaxLag` | Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it. | `*int` | `null` | | `maxmemory` | Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers. | `quantity` | `""` | | `maxmemoryPolicy` | Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset. | `string` | `noeviction` | -| `snapshots` | Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty. | `bool` | `true` | +| `snapshots` | Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty. | `bool` | `true` | ### Sentinel parameters diff --git a/packages/apps/redis/templates/redisfailover.yaml b/packages/apps/redis/templates/redisfailover.yaml index ba36e3f6c..330a2d517 100644 --- a/packages/apps/redis/templates/redisfailover.yaml +++ b/packages/apps/redis/templates/redisfailover.yaml @@ -128,21 +128,29 @@ spec: {{- end }} {{- $maxmemory := 0 }} {{- if not (kindIs "invalid" .Values.maxmemory) }} - {{- $maxmemory = include "cozy-lib.resources.toFloat" (.Values.maxmemory | toString) | float64 | int64 }} + {{- $maxmemoryBytes := include "cozy-lib.resources.toFloat" (.Values.maxmemory | toString) | float64 }} + {{- /* Checked before the int64 cast, which would truncate a sub-byte quantity such + as 512m (0.512 bytes) or -512m to 0, and Redis reads 0 as no limit. */}} + {{- if lt $maxmemoryBytes 0.0 }} + {{- fail (printf "maxmemory (%v) must not be negative" .Values.maxmemory) }} + {{- end }} + {{- if ne (floor $maxmemoryBytes) $maxmemoryBytes }} + {{- fail (printf "maxmemory (%v) must be a whole number of bytes, e.g. 512Mi rather than 512m (millibytes)" .Values.maxmemory) }} + {{- end }} + {{- $maxmemory = $maxmemoryBytes | int64 }} {{- $redisResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.resourcesPreset .Values.resources $) | fromYaml }} {{- $memoryRequest := include "cozy-lib.resources.toFloat" (dig "requests" "memory" "0" $redisResources | toString) | float64 }} {{- /* The operator compares maxmemory with 90% of the pod's memory request (reservedPodMemoryPercent) and silently skips the line when it is larger. */}} {{- $allowed := mulf $memoryRequest 0.9 | int64 }} - {{- if lt $maxmemory 0 }} - {{- fail (printf "maxmemory (%v) must not be negative" .Values.maxmemory) }} - {{- end }} {{- if gt $maxmemory $allowed }} {{- fail (printf "maxmemory (%v, %d bytes) must be at most 90%% of the Redis memory request (%d bytes): the operator ignores a larger value" .Values.maxmemory $maxmemory $allowed) }} {{- end }} {{- end }} - - maxmemory {{ $maxmemory }} + {{- /* The operator sends these in list order, and lowering maxmemory below the + dataset evicts at once, so the policy has to be in place first. */}} - maxmemory-policy {{ .Values.maxmemoryPolicy | default "noeviction" }} + - maxmemory {{ $maxmemory }} {{- /* A master with `replicas` pods has replicas-1 replicas, so a requirement of `replicas` or more can never be met and the master would refuse every write. */}} {{- if and (not (kindIs "invalid" .Values.minReplicasToWrite)) (ge (.Values.minReplicasToWrite | int) (.Values.replicas | int)) }} diff --git a/packages/apps/redis/tests/cache_config_test.yaml b/packages/apps/redis/tests/cache_config_test.yaml index 557b5d3d4..1b618bcac 100644 --- a/packages/apps/redis/tests/cache_config_test.yaml +++ b/packages/apps/redis/tests/cache_config_test.yaml @@ -116,3 +116,35 @@ tests: asserts: - failedTemplate: errorPattern: must not be negative + + # A sub-byte quantity would truncate to maxmemory 0, which Redis reads as no + # limit, so the bound the tenant asked for would silently disappear. + - it: "refuses a maxmemory that is not a whole number of bytes" + set: + maxmemory: 512m + asserts: + - failedTemplate: + errorPattern: must be a whole number of bytes + + - it: "refuses a negative sub-byte maxmemory" + set: + maxmemory: -512m + asserts: + - failedTemplate: + errorPattern: must not be negative + + - it: "applies the eviction policy before the memory bound" + set: + resourcesPreset: t1.large + maxmemory: 1Gi + maxmemoryPolicy: allkeys-lru + documentSelector: + path: kind + value: RedisFailover + asserts: + - equal: + path: spec.redis.customConfig[3] + value: maxmemory-policy allkeys-lru + - equal: + path: spec.redis.customConfig[4] + value: maxmemory 1073741824 diff --git a/packages/apps/redis/values.schema.json b/packages/apps/redis/values.schema.json index b3b143320..183fb6a53 100644 --- a/packages/apps/redis/values.schema.json +++ b/packages/apps/redis/values.schema.json @@ -200,7 +200,7 @@ ] }, "snapshots": { - "description": "Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty.", + "description": "Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty.", "type": "boolean", "default": true }, diff --git a/packages/apps/redis/values.yaml b/packages/apps/redis/values.yaml index 0f8380ae7..ed14c4f46 100644 --- a/packages/apps/redis/values.yaml +++ b/packages/apps/redis/values.yaml @@ -134,7 +134,7 @@ authEnabled: true ## @param {MaxmemoryPolicy} maxmemoryPolicy="noeviction" - Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset. maxmemoryPolicy: "noeviction" -## @param {bool} snapshots - Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty. +## @param {bool} snapshots - Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty. snapshots: true ## diff --git a/packages/system/redis-rd/cozyrds/redis.yaml b/packages/system/redis-rd/cozyrds/redis.yaml index 02046b97e..8c3045b0f 100644 --- a/packages/system/redis-rd/cozyrds/redis.yaml +++ b/packages/system/redis-rd/cozyrds/redis.yaml @@ -8,7 +8,7 @@ spec: plural: redises singular: redis openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"maxmemory":{"description":"Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"maxmemoryPolicy":{"description":"Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.","type":"string","default":"noeviction","enum":["noeviction","allkeys-lru","allkeys-lfu","allkeys-random","volatile-lru","volatile-lfu","volatile-random","volatile-ttl"]},"snapshots":{"description":"Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so the data is lost on restart but replicas still have room for the full-sync RDB they write to disk. Always off when `size` is empty.","type":"boolean","default":true},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"maxmemory":{"description":"Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"maxmemoryPolicy":{"description":"Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.","type":"string","default":"noeviction","enum":["noeviction","allkeys-lru","allkeys-lfu","allkeys-random","volatile-lru","volatile-lfu","volatile-random","volatile-ttl"]},"snapshots":{"description":"Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty.","type":"boolean","default":true},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}} release: prefix: redis- labels: From 05818aa3b221a670d94c332f461ae3bb046a5de3 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 12 Nov 2026 23:34:44 +0300 Subject: [PATCH 152/174] fix(cli): fail cozypkg and check-readiness on unwritable output For the commands this change touches (cozypkg list, the validation report of validate, push and tap, tap's summary, and check-readiness), a write error the process observes is a failure: the command exits non-zero and says whether the action it reports on already happened. Progress lines printed before the decisive step stay best-effort. A closed pipe ends the process with SIGPIPE, as for other Unix tools. `cozypkg list` and the validation report dropped write errors, and the list deferred its tabwriter flush, so a closed or full stdout still reported success with nothing printed. The list now writes to the command's writer and returns the flush error, and the report returns its write error. When validation refuses a push or a tap, or fails a validate run, the refusal stays the reported reason and the report's write error is joined to it. A push whose report cannot be written stops before flux runs and says "nothing was pushed"; a tap whose summary cannot be written fails with "tap applied, but writing its summary failed". check-readiness fails --help, --version and its readiness result the same way, in one-shot, --wait and --watch mode, with exit 1 as for a result that is not ready, which its help now lists among the exit statuses along with SIGPIPE for a closed pipe; the watch loop stops at the first frame it cannot write, and a timed-out wait still ends with its verdict. push writes its "Pushing" line before it runs flux and stops with "nothing was pushed" when that line cannot be written, with validation skipped or after the report. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- cmd/check-readiness/main.go | 55 ++++-- cmd/check-readiness/main_test.go | 68 +++++++ cmd/cozypkg/cmd/list.go | 48 +++-- cmd/cozypkg/cmd/output_test.go | 185 ++++++++++++++++++ cmd/cozypkg/cmd/push.go | 12 +- cmd/cozypkg/cmd/tap.go | 34 ++-- cmd/cozypkg/cmd/validate.go | 17 +- test/check-readiness/main_test.go | 79 ++++++++ .../testdata/golden/help-long.stdout | 4 + .../testdata/golden/help-short.stdout | 4 + .../testdata/golden/unknown-arg.stderr | 4 + .../golden/watch-nonnumeric-interval.stderr | 4 + 12 files changed, 458 insertions(+), 56 deletions(-) create mode 100644 cmd/check-readiness/main_test.go create mode 100644 cmd/cozypkg/cmd/output_test.go diff --git a/cmd/check-readiness/main.go b/cmd/check-readiness/main.go index 50fb8aaf8..eb13f62e1 100644 --- a/cmd/check-readiness/main.go +++ b/cmd/check-readiness/main.go @@ -22,6 +22,7 @@ package main import ( "fmt" + "io" "os" "os/exec" "strconv" @@ -58,6 +59,10 @@ Options: --no-color Disable color output (auto-disabled on non-TTY) -h, --help Show this help +Exit status: 0 when everything is ready, 1 when something is not ready or a +write to stdout fails, 2 on invalid arguments. A closed pipe ends the process +with SIGPIPE, like other Unix tools. + ` // entry is a resource catalog item. @@ -181,10 +186,16 @@ func run(args []string) int { cfg.coreOnly = true i++ case "-h", "--help": - fmt.Fprint(os.Stdout, usageText) + if _, err := fmt.Fprint(os.Stdout, usageText); err != nil { + fmt.Fprintf(os.Stderr, "writing usage: %v\n", err) + return 1 + } return 0 case "--version": - fmt.Fprintln(os.Stdout, Version) + if _, err := fmt.Fprintln(os.Stdout, Version); err != nil { + fmt.Fprintf(os.Stderr, "writing version: %v\n", err) + return 1 + } return 0 default: fmt.Fprintf(os.Stderr, "Unknown argument: %s\n", a) @@ -213,14 +224,14 @@ func run(args []string) int { cfg.timeoutS = sec if cfg.waitMode { - return cfg.runWait() + return cfg.runWait(os.Stdout) } if cfg.watch { - cfg.runWatch() - return 0 + return cfg.runWatch(os.Stdout) + } + if writeOutput(os.Stdout, cfg.runOnce()) != nil { + return 1 } - out := cfg.runOnce() - fmt.Print(out) if cfg.foundIssues { return 1 } @@ -281,18 +292,31 @@ func parseTimeout(raw string) (int, bool) { return 0, false } -func (cfg *config) runWait() int { +// writeOutput prints a result. A result its reader never got fails the run, +// whatever the resources' state, since the exit code alone would vouch for it. +func writeOutput(w io.Writer, parts ...string) error { + for _, part := range parts { + if _, err := io.WriteString(w, part); err != nil { + fmt.Fprintf(os.Stderr, "writing output: %v\n", err) + return err + } + } + return nil +} + +func (cfg *config) runWait(w io.Writer) int { start := time.Now() for { output := cfg.runOnce() elapsed := int(time.Since(start).Seconds()) if strings.Contains(output, "All resources are ready.") { - fmt.Print(output) - fmt.Printf("%sReady after %ds.%s\n", cfg.green, elapsed, cfg.reset) + if writeOutput(w, output, fmt.Sprintf("%sReady after %ds.%s\n", cfg.green, elapsed, cfg.reset)) != nil { + return 1 + } return 0 } if elapsed >= cfg.timeoutS { - fmt.Print(output) + _ = writeOutput(w, output) fmt.Fprintf(os.Stderr, "%s%sTimeout after %ds — resources still not ready.%s\n", cfg.red, cfg.bold, cfg.timeoutS, cfg.reset) return 1 } @@ -300,13 +324,14 @@ func (cfg *config) runWait() int { } } -func (cfg *config) runWatch() { +func (cfg *config) runWatch(w io.Writer) int { for { output := cfg.runOnce() // Smooth clear is cosmetic and untested; just reprint. - fmt.Printf("%sLast updated: %s (refreshing every %ds, Ctrl+C to stop)%s\n", cfg.bold, time.Now().Format(time.UnixDate), cfg.interval, cfg.reset) - fmt.Println() - fmt.Print(output) + header := fmt.Sprintf("%sLast updated: %s (refreshing every %ds, Ctrl+C to stop)%s\n\n", cfg.bold, time.Now().Format(time.UnixDate), cfg.interval, cfg.reset) + if writeOutput(w, header, output) != nil { + return 1 + } time.Sleep(time.Duration(cfg.interval) * time.Second) } } diff --git a/cmd/check-readiness/main_test.go b/cmd/check-readiness/main_test.go new file mode 100644 index 000000000..9afddeedf --- /dev/null +++ b/cmd/check-readiness/main_test.go @@ -0,0 +1,68 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "io" + "os" + "strings" + "testing" +) + +func TestHelpAndVersionSucceedOnWritableStdout(t *testing.T) { + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = r.Close() }) + stdout := os.Stdout + os.Stdout = w + t.Cleanup(func() { os.Stdout = stdout }) + read := make(chan string, 1) + go func() { + b, _ := io.ReadAll(r) + read <- string(b) + }() + + for _, arg := range []string{"--help", "--version"} { + if code := run([]string{arg}); code != 0 { + t.Errorf("run(%s) = %d, want 0", arg, code) + } + } + _ = w.Close() + if out := <-read; !strings.Contains(out, "Usage:") || !strings.Contains(out, Version) { + t.Errorf("stdout = %q, want the usage text and the version", out) + } +} + +func TestHelpAndVersionFailOnUnwritableStdout(t *testing.T) { + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + _ = r.Close() + _ = w.Close() + stdout := os.Stdout + os.Stdout = w + t.Cleanup(func() { os.Stdout = stdout }) + + for _, arg := range []string{"--help", "--version"} { + if code := run([]string{arg}); code == 0 { + t.Errorf("run(%s) with an unwritable stdout = 0, want a failure", arg) + } + } +} diff --git a/cmd/cozypkg/cmd/list.go b/cmd/cozypkg/cmd/list.go index 6f7375026..4b08a602b 100644 --- a/cmd/cozypkg/cmd/list.go +++ b/cmd/cozypkg/cmd/list.go @@ -19,7 +19,7 @@ package cmd import ( "context" "fmt" - "os" + "io" "strings" "text/tabwriter" @@ -78,24 +78,25 @@ Use --components flag to show components on separate lines.`, } if listCmdFlags.installed { - return listPackages(ctx, k8sClient, listCmdFlags.components) + return listPackages(ctx, k8sClient, cmd.OutOrStdout(), listCmdFlags.components) } - return listPackageSources(ctx, k8sClient, listCmdFlags.components) + return listPackageSources(ctx, k8sClient, cmd.OutOrStdout(), listCmdFlags.components) }, } -func listPackageSources(ctx context.Context, k8sClient client.Client, showComponents bool) error { +func listPackageSources(ctx context.Context, k8sClient client.Client, out io.Writer, showComponents bool) error { var psList cozyv1alpha1.PackageSourceList if err := k8sClient.List(ctx, &psList); err != nil { return fmt.Errorf("failed to list PackageSources: %w", err) } // Use tabwriter for better column alignment - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) - defer w.Flush() + w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0) // Print header - fmt.Fprintln(w, "NAME\tVARIANTS\tREADY\tSTATUS") + if _, err := fmt.Fprintln(w, "NAME\tVARIANTS\tREADY\tSTATUS"); err != nil { + return err + } // Print rows for _, ps := range psList.Items { @@ -123,24 +124,26 @@ func listPackageSources(ctx context.Context, k8sClient client.Client, showCompon } } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", ps.Name, variantsStr, ready, status) + if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", ps.Name, variantsStr, ready, status); err != nil { + return err + } // Show components if requested if showComponents { for _, variant := range ps.Spec.Variants { for _, component := range variant.Components { - fmt.Fprintf(w, " %s\t%s\t\t\n", - fmt.Sprintf("%s.%s", ps.Name, component.Name), - variant.Name) + if _, err := fmt.Fprintf(w, " %s.%s\t%s\t\t\n", ps.Name, component.Name, variant.Name); err != nil { + return err + } } } } } - return nil + return w.Flush() } -func listPackages(ctx context.Context, k8sClient client.Client, showComponents bool) error { +func listPackages(ctx context.Context, k8sClient client.Client, out io.Writer, showComponents bool) error { var pkgList cozyv1alpha1.PackageList if err := k8sClient.List(ctx, &pkgList); err != nil { return fmt.Errorf("failed to list Packages: %w", err) @@ -160,11 +163,12 @@ func listPackages(ctx context.Context, k8sClient client.Client, showComponents b } // Use tabwriter for better column alignment - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) - defer w.Flush() + w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0) // Print header - fmt.Fprintln(w, "NAME\tVARIANT\tREADY\tSTATUS") + if _, err := fmt.Fprintln(w, "NAME\tVARIANT\tREADY\tSTATUS"); err != nil { + return err + } // Print rows for _, pkg := range pkgList.Items { @@ -187,7 +191,9 @@ func listPackages(ctx context.Context, k8sClient client.Client, showComponents b } } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", pkg.Name, variant, ready, status) + if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", pkg.Name, variant, ready, status); err != nil { + return err + } // Show components if requested if showComponents { @@ -197,9 +203,9 @@ func listPackages(ctx context.Context, k8sClient client.Client, showComponents b for _, v := range ps.Spec.Variants { if v.Name == variant { for _, component := range v.Components { - fmt.Fprintf(w, " %s\t%s\t\t\n", - fmt.Sprintf("%s.%s", pkg.Name, component.Name), - variant) + if _, err := fmt.Fprintf(w, " %s.%s\t%s\t\t\n", pkg.Name, component.Name, variant); err != nil { + return err + } } break } @@ -208,7 +214,7 @@ func listPackages(ctx context.Context, k8sClient client.Client, showComponents b } } - return nil + return w.Flush() } func init() { diff --git a/cmd/cozypkg/cmd/output_test.go b/cmd/cozypkg/cmd/output_test.go new file mode 100644 index 000000000..9eed21639 --- /dev/null +++ b/cmd/cozypkg/cmd/output_test.go @@ -0,0 +1,185 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("broken pipe") } + +func TestPrintReportReturnsWriteError(t *testing.T) { + r := &Report{Findings: []Finding{{Severity: SeverityError, Code: "X", Message: "m"}}} + if err := printReport(r, failingWriter{}); err == nil { + t.Fatal("printReport to an unwritable writer returned nil") + } +} + +// The table is buffered until the tabwriter flushes, so a failing reader shows +// up only as a Flush error, which list has to return. +func TestListReturnsWriteError(t *testing.T) { + s := runtime.NewScheme() + utilruntime.Must(cozyv1alpha1.AddToScheme(s)) + ps := &cozyv1alpha1.PackageSource{ObjectMeta: metav1.ObjectMeta{Name: "acme.app"}} + pkg := &cozyv1alpha1.Package{ObjectMeta: metav1.ObjectMeta{Name: "acme.app"}} + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(ps, pkg).Build() + + for name, list := range map[string]func(io.Writer) error{ + "sources": func(w io.Writer) error { return listPackageSources(context.Background(), cl, w, true) }, + "packages": func(w io.Writer) error { return listPackages(context.Background(), cl, w, true) }, + } { + t.Run(name, func(t *testing.T) { + if err := list(failingWriter{}); err == nil { + t.Fatal("list to an unwritable writer returned nil") + } + var out strings.Builder + if err := list(&out); err != nil || !strings.Contains(out.String(), "acme.app") { + t.Fatalf("list = %v, output %q; want the row", err, out.String()) + } + }) + } +} + +// A command that cannot print its validation report still fails, and a +// refusal keeps validation as its reason with the write error joined to it. +func TestValidationCommandsReportUnwritableReport(t *testing.T) { + fakeFlux := t.TempDir() + if err := os.WriteFile(filepath.Join(fakeFlux, "flux"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", fakeFlux+string(os.PathListSeparator)+os.Getenv("PATH")) + + emptyRepo := func(t *testing.T) string { + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "packages"), 0o755); err != nil { + t.Fatal(err) + } + return root + } + run := func(c *cobra.Command, args ...string) error { + c.SetOut(failingWriter{}) + t.Cleanup(func() { c.SetOut(nil) }) + return c.RunE(c, args) + } + push := func(root string) error { + pushCmdFlags.path = root + t.Cleanup(func() { pushCmdFlags.path = "." }) + return run(pushCmd, "oci://registry.example/acme/apps:v1") + } + + for _, tc := range []struct { + name string + run func(t *testing.T) error + refusal string + }{ + {name: "validate, valid", run: func(t *testing.T) error { return run(validateCmd, validRepo(t)) }}, + {name: "validate, invalid", run: func(t *testing.T) error { return run(validateCmd, emptyRepo(t)) }, refusal: "validation failed"}, + {name: "push, valid", run: func(t *testing.T) error { return push(validRepo(t)) }, refusal: "nothing was pushed"}, + {name: "push, invalid", run: func(t *testing.T) error { return push(emptyRepo(t)) }, refusal: "refusing to push"}, + {name: "tap, invalid", run: func(t *testing.T) error { return run(tapCmd, "oci://registry.example/acme/apps:v1") }, refusal: "refusing to tap"}, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.run(t) + if err == nil || !strings.Contains(err.Error(), "broken pipe") { + t.Fatalf("err = %v, want the write error", err) + } + if tc.refusal != "" && !strings.Contains(err.Error(), tc.refusal) { + t.Errorf("err = %v, want the refusal %q kept", err, tc.refusal) + } + }) + } +} + +// A tap that applied is reported as applied even when its summary is lost, +// and the lost summary still fails the command. +func TestTapSummaryReportsUnwritableOutput(t *testing.T) { + err := writeTapSummary(failingWriter{}, "oci://registry.example/acme/apps:v1", "tap-acme", []string{"acme.app"}) + if err == nil || !strings.Contains(err.Error(), "tap applied") || !strings.Contains(err.Error(), "broken pipe") { + t.Fatalf("err = %v, want the write error with the tap reported as applied", err) + } + var out strings.Builder + if err := writeTapSummary(&out, "oci://registry.example/acme/apps:v1", "tap-acme", []string{"acme.app"}); err != nil || !strings.Contains(out.String(), "cozypkg add acme.app") { + t.Fatalf("writeTapSummary = %v, output %q", err, out.String()) + } +} + +// A clean report has no finding lines, so its summary is its only write and +// alone decides the result. +func TestPrintReportReturnsSummaryWriteError(t *testing.T) { + if err := printReport(&Report{}, failingWriter{}); err == nil { + t.Fatal("printReport of a clean report to an unwritable writer returned nil") + } +} + +// push runs flux only after its status line was written, with validation +// skipped and after a report that was written. +func TestPushStopsBeforeFluxWhenItsStatusIsLost(t *testing.T) { + fakeFlux := t.TempDir() + ran := filepath.Join(t.TempDir(), "flux-ran") + if err := os.WriteFile(filepath.Join(fakeFlux, "flux"), []byte("#!/bin/sh\ntouch "+ran+"\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", fakeFlux+string(os.PathListSeparator)+os.Getenv("PATH")) + for _, skip := range []bool{true, false} { + t.Run(fmt.Sprintf("skip-validate=%v", skip), func(t *testing.T) { + pushCmdFlags.path, pushCmdFlags.skipValidate = validRepo(t), skip + pushCmdFlags.source, pushCmdFlags.revision = "https://example.com/acme.git", "v1" + t.Cleanup(func() { + pushCmdFlags.path, pushCmdFlags.skipValidate = ".", false + pushCmdFlags.source, pushCmdFlags.revision = "", "" + }) + pushCmd.SetOut(&failOn{want: "Pushing"}) + t.Cleanup(func() { pushCmd.SetOut(nil) }) + err := pushCmd.RunE(pushCmd, []string{"oci://registry.example/acme/apps:v1"}) + if err == nil || !strings.Contains(err.Error(), "nothing was pushed") { + t.Fatalf("push losing its status = %v, want nothing pushed", err) + } + if _, statErr := os.Stat(ran); statErr == nil { + t.Fatal("flux ran after the status line was lost") + } + }) + } +} + +// failOn fails every write that contains want, and takes every other one. +type failOn struct { + want string + buf strings.Builder +} + +func (w *failOn) Write(p []byte) (int, error) { + if strings.Contains(string(p), w.want) { + return 0, errors.New("broken pipe") + } + return w.buf.Write(p) +} diff --git a/cmd/cozypkg/cmd/push.go b/cmd/cozypkg/cmd/push.go index 3f8e65cac..f60004034 100644 --- a/cmd/cozypkg/cmd/push.go +++ b/cmd/cozypkg/cmd/push.go @@ -17,6 +17,7 @@ limitations under the License. package cmd import ( + "errors" "fmt" "os" "os/exec" @@ -106,10 +107,13 @@ published. Source URL and revision are derived from git when not given.`, if err != nil { return err } - printReport(rep, cmd.OutOrStdout()) + printErr := printReport(rep, cmd.OutOrStdout()) if rep.HasErrors() { errs, _, _ := rep.Counts() - return fmt.Errorf("refusing to push: validation failed with %d error(s) (use --skip-validate to override)", errs) + return errors.Join(fmt.Errorf("refusing to push: validation failed with %d error(s) (use --skip-validate to override)", errs), printErr) + } + if printErr != nil { + return fmt.Errorf("nothing was pushed: writing the validation report failed: %w", printErr) } } @@ -129,7 +133,9 @@ published. Source URL and revision are derived from git when not given.`, } fluxArgs := buildFluxPushArgs(ref, packagesDir, source, revision, pushCmdFlags.reproducible) - fmt.Fprintf(cmd.OutOrStdout(), "Pushing %s (source=%s revision=%s)\n", ref, source, revision) + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "Pushing %s (source=%s revision=%s)\n", ref, source, revision); err != nil { + return fmt.Errorf("nothing was pushed: writing the push status failed: %w", err) + } flux := exec.Command("flux", fluxArgs...) flux.Stdout = cmd.OutOrStdout() flux.Stderr = cmd.ErrOrStderr() diff --git a/cmd/cozypkg/cmd/tap.go b/cmd/cozypkg/cmd/tap.go index e04edebd7..2d5f7bb9b 100644 --- a/cmd/cozypkg/cmd/tap.go +++ b/cmd/cozypkg/cmd/tap.go @@ -18,6 +18,7 @@ package cmd import ( "context" + "errors" "fmt" "io" "regexp" @@ -402,9 +403,8 @@ runs its charts in your management cluster: tap only sources you trust.`, return err } if rep.HasErrors() { - printReport(rep, cmd.OutOrStdout()) errs, _, _ := rep.Counts() - return fmt.Errorf("refusing to tap: artifact failed validation with %d error(s) (use --skip-validate to override)", errs) + return errors.Join(fmt.Errorf("refusing to tap: artifact failed validation with %d error(s) (use --skip-validate to override)", errs), printReport(rep, cmd.OutOrStdout())) } } @@ -487,19 +487,29 @@ runs its charts in your management cluster: tap only sources you trust.`, _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "warning: could not reset materialization state on OCIRepository/%s: %v\n", srcName, err) } - fmt.Fprintf(cmd.OutOrStdout(), "Tapped %s\n OCIRepository/%s\n", fullRef, srcName) - for _, n := range names { - fmt.Fprintf(cmd.OutOrStdout(), " PackageSource/%s\n", n) - } - _, _ = fmt.Fprintf(cmd.OutOrStdout(), "Tapping installs nothing. Install an app with:\n") - for _, n := range names { - fmt.Fprintf(cmd.OutOrStdout(), " cozypkg add %s\n", n) - } - _, _ = fmt.Fprintf(cmd.OutOrStdout(), "A privileged install component is not installed until you confirm it with 'cozypkg add --allow-privileged'.\n") - return nil + return writeTapSummary(cmd.OutOrStdout(), fullRef, srcName, names) }, } +// writeTapSummary reports what a tap created. The tap is applied by then, so a +// summary that cannot be written fails the command while saying so. +func writeTapSummary(w io.Writer, fullRef, srcName string, names []string) error { + var b strings.Builder + fmt.Fprintf(&b, "Tapped %s\n OCIRepository/%s\n", fullRef, srcName) + for _, n := range names { + fmt.Fprintf(&b, " PackageSource/%s\n", n) + } + b.WriteString("Tapping installs nothing. Install an app with:\n") + for _, n := range names { + fmt.Fprintf(&b, " cozypkg add %s\n", n) + } + b.WriteString("A privileged install component is not installed until you confirm it with 'cozypkg add --allow-privileged'.\n") + if _, err := io.WriteString(w, b.String()); err != nil { + return fmt.Errorf("tap applied, but writing its summary failed: %w", err) + } + return nil +} + var untapCmd = &cobra.Command{ Use: "untap ", Short: "Remove a tapped repository", diff --git a/cmd/cozypkg/cmd/validate.go b/cmd/cozypkg/cmd/validate.go index 809f0f246..f781cfd8b 100644 --- a/cmd/cozypkg/cmd/validate.go +++ b/cmd/cozypkg/cmd/validate.go @@ -21,6 +21,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "io/fs" @@ -535,16 +536,19 @@ func pullOCIArtifact(ref string) (string, func(), error) { return dir, cleanup, nil } -func printReport(r *Report, w io.Writer) { +func printReport(r *Report, w io.Writer) error { for _, f := range r.Findings { loc := f.Location if loc == "" { loc = "-" } - fmt.Fprintf(w, "[%s] %s (%s): %s\n", f.Severity, loc, f.Code, f.Message) + if _, err := fmt.Fprintf(w, "[%s] %s (%s): %s\n", f.Severity, loc, f.Code, f.Message); err != nil { + return err + } } errs, warns, infos := r.Counts() - fmt.Fprintf(w, "\nSummary: %d error(s), %d warning(s), %d info\n", errs, warns, infos) + _, err := fmt.Fprintf(w, "\nSummary: %d error(s), %d warning(s), %d info\n", errs, warns, infos) + return err } var validateCmdFlags struct { @@ -663,10 +667,13 @@ runs "helm lint" on every component chart.`, if err != nil { return err } - printReport(rep, cmd.OutOrStdout()) + printErr := printReport(rep, cmd.OutOrStdout()) if rep.HasErrors() { errs, _, _ := rep.Counts() - return fmt.Errorf("validation failed: %d error(s)", errs) + return errors.Join(fmt.Errorf("validation failed: %d error(s)", errs), printErr) + } + if printErr != nil { + return fmt.Errorf("writing the validation report failed: %w", printErr) } return nil }, diff --git a/test/check-readiness/main_test.go b/test/check-readiness/main_test.go index 622340c7f..01e6c1505 100644 --- a/test/check-readiness/main_test.go +++ b/test/check-readiness/main_test.go @@ -13,6 +13,7 @@ package checkreadiness_test import ( "bytes" + "context" "errors" "flag" "fmt" @@ -23,6 +24,7 @@ import ( "runtime" "strings" "testing" + "time" ) var update = flag.Bool("update", false, "regenerate golden files") @@ -335,3 +337,80 @@ func repoRoot(t *testing.T) string { } return root } + +// A result that never reaches stdout fails the run, whatever the cluster's +// state, and the watch loop stops instead of refreshing into the void. stdout +// is a read-only descriptor, so every write fails without a SIGPIPE. +func TestUnwritableStdoutFails(t *testing.T) { + root := repoRoot(t) + fakeBin := filepath.Join(root, "test", "check-readiness", "fakekubectl") + base := filepath.Join(root, "test", "check-readiness", "testdata", "fixtures", "_base") + for _, tc := range []struct { + name string + args []string + }{ + {name: "once", args: []string{"--no-color"}}, + {name: "wait", args: []string{"--wait", "--no-color"}}, + {name: "watch", args: []string{"-w", "1", "--no-color"}}, + } { + t.Run(tc.name, func(t *testing.T) { + stdout, err := os.Open(os.DevNull) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdout.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(cancel) + argv := subjectArgv(tc.args) + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Env = testEnv(fakeBin, base, base, filepath.Join(t.TempDir(), "kubectl.log")) + var stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = stdout, &stderr + runErr := cmd.Run() + if ctx.Err() != nil { + t.Fatalf("still running after its output failed\nstderr:\n%s", stderr.String()) + } + if code := exitCode(t, runErr); code != 1 { + t.Errorf("exit code = %d, want 1\nstderr:\n%s", code, stderr.String()) + } + if !strings.Contains(stderr.String(), "writing output") { + t.Errorf("stderr does not say the output was lost:\n%s", stderr.String()) + } + }) + } +} + +// A wait that times out ends with its verdict: the table comes first and the +// timeout line last, also when the table could not be written. +func TestWaitTimeoutVerdictComesLast(t *testing.T) { + root := repoRoot(t) + fakeBin := filepath.Join(root, "test", "check-readiness", "fakekubectl") + fixtures := filepath.Join(root, "test", "check-readiness", "testdata", "fixtures") + args := []string{"-w", "1", "--wait", "--timeout", "1s", "--no-color"} + for _, unwritable := range []bool{false, true} { + t.Run(fmt.Sprintf("unwritable=%v", unwritable), func(t *testing.T) { + cmd := scriptCommand(args) + cmd.Env = testEnv(fakeBin, filepath.Join(fixtures, "helmrelease-not-ready"), filepath.Join(fixtures, "_base"), filepath.Join(t.TempDir(), "kubectl.log")) + var combined bytes.Buffer + cmd.Stdout, cmd.Stderr = &combined, &combined + if unwritable { + stdout, err := os.Open(os.DevNull) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdout.Close() }) + cmd.Stdout = stdout + } + if code := exitCode(t, cmd.Run()); code != 1 { + t.Errorf("exit code = %d, want 1\n%s", code, combined.String()) + } + lines := strings.Split(strings.TrimRight(combined.String(), "\n"), "\n") + if last := lines[len(lines)-1]; !strings.HasPrefix(last, "Timeout after") { + t.Errorf("last line = %q, want the timeout verdict\n%s", last, combined.String()) + } + if unwritable && !strings.Contains(combined.String(), "writing output") { + t.Errorf("stderr does not say the table was lost:\n%s", combined.String()) + } + }) + } +} diff --git a/test/check-readiness/testdata/golden/help-long.stdout b/test/check-readiness/testdata/golden/help-long.stdout index f1f289e08..428f135aa 100644 --- a/test/check-readiness/testdata/golden/help-long.stdout +++ b/test/check-readiness/testdata/golden/help-long.stdout @@ -21,3 +21,7 @@ Options: --no-color Disable color output (auto-disabled on non-TTY) -h, --help Show this help +Exit status: 0 when everything is ready, 1 when something is not ready or a +write to stdout fails, 2 on invalid arguments. A closed pipe ends the process +with SIGPIPE, like other Unix tools. + diff --git a/test/check-readiness/testdata/golden/help-short.stdout b/test/check-readiness/testdata/golden/help-short.stdout index f1f289e08..428f135aa 100644 --- a/test/check-readiness/testdata/golden/help-short.stdout +++ b/test/check-readiness/testdata/golden/help-short.stdout @@ -21,3 +21,7 @@ Options: --no-color Disable color output (auto-disabled on non-TTY) -h, --help Show this help +Exit status: 0 when everything is ready, 1 when something is not ready or a +write to stdout fails, 2 on invalid arguments. A closed pipe ends the process +with SIGPIPE, like other Unix tools. + diff --git a/test/check-readiness/testdata/golden/unknown-arg.stderr b/test/check-readiness/testdata/golden/unknown-arg.stderr index d6ec7c0db..ca652d9ea 100644 --- a/test/check-readiness/testdata/golden/unknown-arg.stderr +++ b/test/check-readiness/testdata/golden/unknown-arg.stderr @@ -22,3 +22,7 @@ Options: --no-color Disable color output (auto-disabled on non-TTY) -h, --help Show this help +Exit status: 0 when everything is ready, 1 when something is not ready or a +write to stdout fails, 2 on invalid arguments. A closed pipe ends the process +with SIGPIPE, like other Unix tools. + diff --git a/test/check-readiness/testdata/golden/watch-nonnumeric-interval.stderr b/test/check-readiness/testdata/golden/watch-nonnumeric-interval.stderr index 3cb2c43bb..cc23fc457 100644 --- a/test/check-readiness/testdata/golden/watch-nonnumeric-interval.stderr +++ b/test/check-readiness/testdata/golden/watch-nonnumeric-interval.stderr @@ -22,3 +22,7 @@ Options: --no-color Disable color output (auto-disabled on non-TTY) -h, --help Show this help +Exit status: 0 when everything is ready, 1 when something is not ready or a +write to stdout fails, 2 on invalid arguments. A closed pipe ends the process +with SIGPIPE, like other Unix tools. + From f99e511eea92e311eaf68234f1d8e16cdcfb07d4 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 12 Nov 2026 23:42:56 +0300 Subject: [PATCH 153/174] fix(backport): fail a run whose output could not be written Every report, check and verify result goes to stdout, and every write error there was dropped, so a run whose output never reached its reader still exited 0. The terminal writer now keeps the first stdout error, and the run fails with it unless it already failed. Only stdout is tracked; a write error on stderr is still discarded. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- cmd/backport-batch/README.md | 2 ++ cmd/backport-batch/assemble.go | 24 ++++++------- cmd/backport-batch/check.go | 16 ++++----- cmd/backport-batch/main.go | 36 +++++++++++++++----- cmd/backport-batch/notify.go | 14 ++++---- cmd/backport-batch/output_test.go | 56 +++++++++++++++++++++++++++++++ cmd/backport-batch/report.go | 30 ++++++++--------- cmd/backport-batch/verify.go | 10 +++--- 8 files changed, 133 insertions(+), 55 deletions(-) create mode 100644 cmd/backport-batch/output_test.go diff --git a/cmd/backport-batch/README.md b/cmd/backport-batch/README.md index 81afd0ef6..4919035c0 100644 --- a/cmd/backport-batch/README.md +++ b/cmd/backport-batch/README.md @@ -27,6 +27,8 @@ Requires `git` and an authenticated `gh` on PATH, and a remote called `origin`. Build the binary wherever an exit code matters: `go run` turns any non-zero exit into its own `1`, so under it a check that could not be completed looks the same as one that found something missing. +A command whose output on stdout could not be written exits 2 where it would have exited 0, and says on stderr that the output was lost; anything the run did before that, a push or a published pull request included, has still happened. + ### run Switches the current working tree to `backport-batch/-`, created at `origin/`, and picks onto it one original at a time, in the order they merged into `main`, as one unit each on the branch's first-parent line. A pull request merged into `main` with a merge commit `M` goes on as its own commits, `git rev-list --reverse --topo-order --no-merges M^1..M^2`, each picked with `git cherry-pick -x --signoff`, so each keeps its author, message and sign-offs and gains the `(cherry picked from commit …)` line; the merges that brought `main` into its branch are left out, since `main` already has what they brought. A merge commit then joins the picks to the batch: its first parent is the batch before the original, its second the last pick, and its message is `M`'s with a paragraph of its own, `Backported-from: M`. A pull request squash-merged into `main` goes on as one pick. When any commit of an original conflicts, the pick is aborted, the branch goes back to where it was before that original, and the original is recorded with the commit that failed and its files; the run goes on with the next one. A commit the branch already carries is skipped as empty, and an original none of whose commits changes anything is reported as carried. The tree has to be clean. A local branch of that name is reset to where the run starts; when that drops commits only it has, the run first prints its tip and the `git branch` command that restores them, and so does `--defer`. Signing follows your git configuration; hooks are off for the picks and the merge, so each commit carries its message as git writes it. diff --git a/cmd/backport-batch/assemble.go b/cmd/backport-batch/assemble.go index 3c025517e..a0aab98db 100644 --- a/cmd/backport-batch/assemble.go +++ b/cmd/backport-batch/assemble.go @@ -153,7 +153,7 @@ func (a *app) run(args []string) int { } // The scheduled job has half an hour for every line; this says where it // goes. - fmt.Fprintf(a.stderr, "prepared %s in %s, %s of it judging partial units and what originals build on\n", l.branch(), time.Since(started).Round(time.Millisecond), b.detecting.Round(time.Millisecond)) + _, _ = fmt.Fprintf(a.stderr, "prepared %s in %s, %s of it judging partial units and what originals build on\n", l.branch(), time.Since(started).Round(time.Millisecond), b.detecting.Round(time.Millisecond)) if drops != nil { return a.redrop(b, drops, reason, *undropArg != "", *push) } @@ -449,20 +449,20 @@ func (a *app) finish(b *batch, r *report, branch string, date time.Time, push, r } body, left := r.body(template) p.body = body - fmt.Fprint(a.stdout, body) + _, _ = fmt.Fprint(a.stdout, body) skipped := 0 for _, items := range r.skipped { skipped += len(items) } for _, rule := range left { for _, v := range r.skipped[rule] { - fmt.Fprintf(a.stderr, "skipped by %s: %s %s (%s)\n", ruleName(rule), v.o.url(), terminalSafe(v.o.title), terminalSafe(v.reason)) + _, _ = fmt.Fprintf(a.stderr, "skipped by %s: %s %s (%s)\n", ruleName(rule), v.o.url(), terminalSafe(v.o.title), terminalSafe(v.reason)) } } for _, p := range r.waiting { - fmt.Fprintf(a.stderr, "waiting for a maintainer: %s %s changes %s\n", p.v.o.url(), terminalSafe(p.v.o.title), terminalSafe(strings.Join(p.files, ", "))) + _, _ = fmt.Fprintf(a.stderr, "waiting for a maintainer: %s %s changes %s\n", p.v.o.url(), terminalSafe(p.v.o.title), terminalSafe(strings.Join(p.files, ", "))) } - fmt.Fprintf(a.stderr, "%s at %.12s: %d on the branch, %d deferred, %d waiting for a maintainer, %d skipped\ntitle: %s\nreport: %s\n", + _, _ = fmt.Fprintf(a.stderr, "%s at %.12s: %d on the branch, %d deferred, %d waiting for a maintainer, %d skipped\ntitle: %s\nreport: %s\n", branch, head, len(r.onBranch), len(r.deferred)+len(r.held), len(r.waiting), skipped, p.title, doc.ID) if !push { return 0 @@ -471,8 +471,8 @@ func (a *app) finish(b *batch, r *report, branch string, date time.Time, push, r // Picking nothing is not a failure of the run; check still reports // whatever the line lacks. Merged and closed batches are never // written, so there is nowhere to keep what the run found. - fmt.Fprintf(a.stderr, "%s carries nothing beyond %s; nothing to push\n", branch, b.line.branch()) - fmt.Fprintf(a.stderr, "no batch pull request is open on %s; this run's classification is not retained\n", b.line.branch()) + _, _ = fmt.Fprintf(a.stderr, "%s carries nothing beyond %s; nothing to push\n", branch, b.line.branch()) + _, _ = fmt.Fprintf(a.stderr, "no batch pull request is open on %s; this run's classification is not retained\n", b.line.branch()) return 0 } if err := a.publish(b, p, rebuilt); err != nil { @@ -537,7 +537,7 @@ func (a *app) warnReset(branch, start string) error { if err != nil || kept { return err } - fmt.Fprintf(a.stderr, "resetting the local %s from %s, dropping commits %.12s does not carry; git branch %s-%.12s %s restores them\n", branch, old, start, branch, old, old) + _, _ = fmt.Fprintf(a.stderr, "resetting the local %s from %s, dropping commits %.12s does not carry; git branch %s-%.12s %s restores them\n", branch, old, start, branch, old, old) return nil } @@ -733,7 +733,7 @@ func (a *app) publish(b *batch, p publication, rebuilt bool) error { if err != nil { return err } - fmt.Fprintf(a.stderr, "opened draft %s\n", url) + _, _ = fmt.Fprintf(a.stderr, "opened draft %s\n", url) if err := a.postReport(n, p.report); err != nil { return fmt.Errorf("posting report %.12s on #%d: %w; the body keeps it, and the next run --push posts it", p.report.doc.ID, n, err) } @@ -745,7 +745,7 @@ func (a *app) publish(b *batch, p publication, rebuilt bool) error { if err := a.gh.editPull(b.pr.Number, p.title, name); err != nil { return err } - fmt.Fprintf(a.stderr, "updated %s\n", pullURL(b.pr)) + _, _ = fmt.Fprintf(a.stderr, "updated %s\n", pullURL(b.pr)) return nil } @@ -861,7 +861,7 @@ func (a *app) finishClose(l line, url string, c closing) error { if err := a.deleteBranch(l, c); err != nil { return err } - fmt.Fprintf(a.stderr, "closed %s: without what is deferred it carries nothing beyond %s; the next batch tries them again\n", url, l.branch()) + _, _ = fmt.Fprintf(a.stderr, "closed %s: without what is deferred it carries nothing beyond %s; the next batch tries them again\n", url, l.branch()) return nil } @@ -933,7 +933,7 @@ func (a *app) resumeClose(b *batch, push, changing bool) (bool, int) { if err := a.deleteBranch(b.line, c); err != nil { return false, a.fail(err) } - fmt.Fprintf(a.stderr, "deleted %s, which the close of #%d left\n", c.Branch, c.PR) + _, _ = fmt.Fprintf(a.stderr, "deleted %s, which the close of #%d left\n", c.Branch, c.PR) return false, a.finished(c, changing) } diff --git a/cmd/backport-batch/check.go b/cmd/backport-batch/check.go index d8acf4e37..2101d251f 100644 --- a/cmd/backport-batch/check.go +++ b/cmd/backport-batch/check.go @@ -73,14 +73,14 @@ func (a *app) check(args []string) int { case d.kind == outcomeWarning: a.warn("%s", d.text) case d.kind == outcomeNote: - fmt.Fprintf(a.stderr, "note: %s\n", d.text) + _, _ = fmt.Fprintf(a.stderr, "note: %s\n", d.text) } } if len(missing)+len(failing) == 0 { if *cut { - fmt.Fprintf(a.stdout, "%s: no change the policy takes is missing before the cut\n", l.branch()) + _, _ = fmt.Fprintf(a.stdout, "%s: no change the policy takes is missing before the cut\n", l.branch()) } else { - fmt.Fprintf(a.stdout, "%s: every change the policy takes is on the line or on the batch\n", l.branch()) + _, _ = fmt.Fprintf(a.stdout, "%s: every change the policy takes is on the line or on the batch\n", l.branch()) } return 0 } @@ -89,13 +89,13 @@ func (a *app) check(args []string) int { where = "nor on batch pull request #" + strconv.Itoa(b.pr.Number) } if len(missing) > 0 { - fmt.Fprintf(a.stdout, "%s: %d changes the policy takes are not on the line, %s:\n", l.branch(), len(missing), where) + _, _ = fmt.Fprintf(a.stdout, "%s: %d changes the policy takes are not on the line, %s:\n", l.branch(), len(missing), where) for _, text := range missing { - fmt.Fprintf(a.stdout, " %s\n", text) + _, _ = fmt.Fprintf(a.stdout, " %s\n", text) } } for _, text := range failing { - fmt.Fprintf(a.stdout, "%s: %s\n", l.branch(), text) + _, _ = fmt.Fprintf(a.stdout, "%s: %s\n", l.branch(), text) } return 1 } @@ -326,10 +326,10 @@ func (a *app) fold(args []string) int { return a.fail(err) } if why != "" { - fmt.Fprintf(a.stdout, "refused: %s\n", why) + _, _ = fmt.Fprintf(a.stdout, "refused: %s\n", why) return 1 } - fmt.Fprintln(a.stdout, "ok") + _, _ = fmt.Fprintln(a.stdout, "ok") return 0 } diff --git a/cmd/backport-batch/main.go b/cmd/backport-batch/main.go index 23b357dff..19102e57f 100644 --- a/cmd/backport-batch/main.go +++ b/cmd/backport-batch/main.go @@ -111,6 +111,9 @@ type app struct { policyRef string // tool is the commit backport-batch was built from, for the record. tool string + // unwritten is the first error writing to stdout, which fails the run: + // what it printed never reached its reader. + unwritten error } func main() { @@ -119,10 +122,21 @@ func main() { } func (a *app) main(args []string) int { - a.stdout, a.stderr = textWriter{a.stdout}, textWriter{a.stderr} + a.stdout, a.stderr = textWriter{w: a.stdout, failed: &a.unwritten}, textWriter{w: a.stderr} + code := a.command(args) + if a.unwritten != nil { + _, _ = fmt.Fprintf(a.stderr, "backport-batch: output was lost: %v; anything the run did, such as a push or a publish, still happened\n", a.unwritten) + if code == 0 { + code = 2 + } + } + return code +} + +func (a *app) command(args []string) int { a.gh = &permCache{github: a.gh, perms: map[string]string{}} if len(args) == 0 { - fmt.Fprint(a.stderr, usageText) + _, _ = fmt.Fprint(a.stderr, usageText) return 2 } if bot := a.gh.batchBot(); bot != "" && !botLoginRE.MatchString(bot) { @@ -144,20 +158,26 @@ func (a *app) main(args []string) int { case "lines": return a.lines(args[1:]) case "-h", "--help", "help": - fmt.Fprint(a.stdout, usageText) + _, _ = fmt.Fprint(a.stdout, usageText) return 0 } - fmt.Fprintf(a.stderr, "unknown command %q\n\n%s", args[0], usageText) + _, _ = fmt.Fprintf(a.stderr, "unknown command %q\n\n%s", args[0], usageText) return 2 } // textWriter keeps output fit for a terminal. Titles, file names and the // messages git relays come from other people, and a control sequence among // them would move the cursor or rewrite what was printed before. -type textWriter struct{ w io.Writer } +type textWriter struct { + w io.Writer + failed *error +} func (t textWriter) Write(p []byte) (int, error) { if _, err := io.WriteString(t.w, clean(string(p), true)); err != nil { + if t.failed != nil && *t.failed == nil { + *t.failed = err + } return 0, err } return len(p), nil @@ -166,7 +186,7 @@ func (t textWriter) Write(p []byte) (int, error) { func (a *app) flags(name string) *flag.FlagSet { fs := flag.NewFlagSet(name, flag.ContinueOnError) fs.SetOutput(a.stderr) - fs.Usage = func() { fmt.Fprint(a.stderr, usageText) } + fs.Usage = func() { _, _ = fmt.Fprint(a.stderr, usageText) } return fs } @@ -189,12 +209,12 @@ func parse(fs *flag.FlagSet, args []string) error { } func (a *app) fail(err error) int { - fmt.Fprintf(a.stderr, "backport-batch: %v\n", err) + _, _ = fmt.Fprintf(a.stderr, "backport-batch: %v\n", err) return 2 } func (a *app) warn(format string, args ...any) { - fmt.Fprintf(a.stderr, "warning: "+format+"\n", args...) + _, _ = fmt.Fprintf(a.stderr, "warning: "+format+"\n", args...) } // verdict is what the policy says about one original for the line. diff --git a/cmd/backport-batch/notify.go b/cmd/backport-batch/notify.go index fe301a2ff..3460e287d 100644 --- a/cmd/backport-batch/notify.go +++ b/cmd/backport-batch/notify.go @@ -135,11 +135,11 @@ func (a *app) notify(args []string) int { stalled := days >= stalledAfter var text []string if stalled { - fmt.Fprintf(a.stdout, "%s: batch pull request #%d has been open since %s, %d days\n", l.branch(), pr.Number, opened.Format(time.DateOnly), days) + _, _ = fmt.Fprintf(a.stdout, "%s: batch pull request #%d has been open since %s, %d days\n", l.branch(), pr.Number, opened.Format(time.DateOnly), days) text = append(text, fmt.Sprintf("This batch has been open since %s, %d days, through at least one scheduled run. Merge it, or close it and the next run starts a new one; until then each run adds what the rules take to it.", opened.Format(time.DateOnly), days)) } if len(partial) > 0 { - fmt.Fprintf(a.stdout, "%s: %d units on #%d carry only part of their originals\n", l.branch(), len(partial), pr.Number) + _, _ = fmt.Fprintf(a.stdout, "%s: %d units on #%d carry only part of their originals\n", l.branch(), len(partial), pr.Number) var b strings.Builder b.WriteString("These are on the batch but carry only part of their originals on this line; take each out with `backport-batch run --defer` before the cut, and port what applies as a pull request of its own:\n") for _, n := range partial { @@ -148,7 +148,7 @@ func (a *app) notify(args []string) int { text = append(text, b.String()) } if len(conflicts) > 0 { - fmt.Fprintf(a.stdout, "%s: %d deferred conflicts on #%d wait for a maintainer\n", l.branch(), count(conflicts), pr.Number) + _, _ = fmt.Fprintf(a.stdout, "%s: %d deferred conflicts on #%d wait for a maintainer\n", l.branch(), count(conflicts), pr.Number) var b strings.Builder fmt.Fprintf(&b, "These conflicts wait for a maintainer, as [resolving a conflict](%sresolve-conflict.md) describes; unresolved, each goes to the next batch, and so does what waits on one:\n", tasksURL) for _, n := range conflicts { @@ -158,9 +158,9 @@ func (a *app) notify(args []string) int { } if len(waiting) > 0 { if count(waiting) == len(waiting) { - fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s and wait for a maintainer\n", l.branch(), len(waiting), pr.Number, workflowsDir) + _, _ = fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s and wait for a maintainer\n", l.branch(), len(waiting), pr.Number, workflowsDir) } else { - fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s or wait on one that does\n", l.branch(), len(waiting), pr.Number, workflowsDir) + _, _ = fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s or wait on one that does\n", l.branch(), len(waiting), pr.Number, workflowsDir) } var b strings.Builder fmt.Fprintf(&b, "These change `%s`, which the batch's token may not push, and wait for a maintainer to bring them over, as [changes to workflows](%s#changes-to-workflows) describes, or wait on one that does:\n", workflowsDir, policyURL) @@ -170,7 +170,7 @@ func (a *app) notify(args []string) int { text = append(text, b.String()) } if len(parks) > 0 { - fmt.Fprintf(a.stdout, "%s: %d parked originals on #%d, named once\n", l.branch(), len(parks), pr.Number) + _, _ = fmt.Fprintf(a.stdout, "%s: %d parked originals on #%d, named once\n", l.branch(), len(parks), pr.Number) var b strings.Builder fmt.Fprintf(&b, "These cannot land on this line whole, so the batch parked them; each is named here once. A maintainer ports what applies as a pull request of its own, as [fixes that exist only on a release line](%s#fixes-that-exist-only-on-a-release-line) describes, or drops the original with `backport-batch run --drop`:\n", policyURL) for _, p := range parks { @@ -354,7 +354,7 @@ func (a *app) lines(args []string) int { return a.fail(err) } for _, l := range slices.Backward(lines) { - fmt.Fprintln(a.stdout, l.branch()) + _, _ = fmt.Fprintln(a.stdout, l.branch()) } return 0 } diff --git a/cmd/backport-batch/output_test.go b/cmd/backport-batch/output_test.go new file mode 100644 index 000000000..0362f44b7 --- /dev/null +++ b/cmd/backport-batch/output_test.go @@ -0,0 +1,56 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "strings" + "testing" + "time" +) + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("broken pipe") } + +// Output that never reached its reader fails the command, and stderr says why. +func TestUnwritableOutputFailsTheCommand(t *testing.T) { + var stderr strings.Builder + a := &app{gh: &fakeGH{}, stdout: failingWriter{}, stderr: &stderr} + if code := a.main([]string{"help"}); code != 2 { + t.Fatalf("help with an unwritable stdout = %d, want 2", code) + } + if !strings.Contains(stderr.String(), "broken pipe") { + t.Errorf("stderr does not name the write failure:\n%s", stderr.String()) + } +} + +// A command that already failed keeps its own exit code when its output is +// lost as well: check still says something is missing. +func TestUnwritableOutputKeepsTheCommandsFailure(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + var stderr strings.Builder + a := &app{gh: &fakeGH{remoteHead: f.remoteHead}, stdout: failingWriter{}, stderr: &stderr, now: func() time.Time { return f.now }, tool: f.tool} + if code := a.main([]string{"check", "--line", "release-1.1"}); code != 1 { + t.Fatalf("check with #6 missing and an unwritable stdout = %d, want 1\n%s", code, stderr.String()) + } + if !strings.Contains(stderr.String(), "broken pipe") { + t.Errorf("stderr does not name the write failure:\n%s", stderr.String()) + } +} diff --git a/cmd/backport-batch/report.go b/cmd/backport-batch/report.go index d1ae4a4b5..ee072e332 100644 --- a/cmd/backport-batch/report.go +++ b/cmd/backport-batch/report.go @@ -708,7 +708,7 @@ func (a *app) report(args []string) int { return a.fail(fmt.Errorf("batch pull request #%d keeps no report %.12s", pr.Number, *id)) } if *asJSON { - fmt.Fprintln(a.stdout, s.raw) + _, _ = fmt.Fprintln(a.stdout, s.raw) return 0 } a.printReport(&pr, s, inBody, inComment) @@ -721,29 +721,29 @@ func (a *app) report(args []string) int { func (a *app) printReport(pr *pullRequest, s storedReport, inBody *storedReport, inComment bool) { d, b := s.doc, s.doc.Built safe := terminalSafe - fmt.Fprintf(a.stdout, "report %s for %s, made %s by a %s run", safe(d.ID), safe(d.Line), safe(b.At), safe(b.Options.Mode)) + _, _ = fmt.Fprintf(a.stdout, "report %s for %s, made %s by a %s run", safe(d.ID), safe(d.Line), safe(b.At), safe(b.Options.Mode)) if b.Options.NoWorkflowFiles { - fmt.Fprint(a.stdout, " with --no-workflow-files") + _, _ = fmt.Fprint(a.stdout, " with --no-workflow-files") } - fmt.Fprintln(a.stdout) - fmt.Fprintf(a.stdout, " main %s, line %s, newer line %s, policy %s\n batch %s, tool %s, run %s\n", safe(b.Main), safe(b.Line), safe(or(b.Newer, "none")), safe(b.Policy), safe(b.Batch), safe(or(b.Tool, "unknown")), safe(or(b.Run, "none"))) + _, _ = fmt.Fprintln(a.stdout) + _, _ = fmt.Fprintf(a.stdout, " main %s, line %s, newer line %s, policy %s\n batch %s, tool %s, run %s\n", safe(b.Main), safe(b.Line), safe(or(b.Newer, "none")), safe(b.Policy), safe(b.Batch), safe(or(b.Tool, "unknown")), safe(or(b.Run, "none"))) current := "none" if inBody != nil { current = safe(inBody.doc.ID) } switch { case inBody != nil && inBody.doc.ID == d.ID: - fmt.Fprintf(a.stdout, " #%d's body carries this report\n", pr.Number) + _, _ = fmt.Fprintf(a.stdout, " #%d's body carries this report\n", pr.Number) case pr.Head.SHA == b.Batch: - fmt.Fprintf(a.stdout, " #%d's head is %.12s, the batch this report describes, but its body carries report %.12s\n", pr.Number, safe(b.Batch), current) + _, _ = fmt.Fprintf(a.stdout, " #%d's head is %.12s, the batch this report describes, but its body carries report %.12s\n", pr.Number, safe(b.Batch), current) default: - fmt.Fprintf(a.stdout, " publication unconfirmed: #%d's head is %.12s, not %.12s, and its body carries report %.12s\n", pr.Number, safe(pr.Head.SHA), safe(b.Batch), current) + _, _ = fmt.Fprintf(a.stdout, " publication unconfirmed: #%d's head is %.12s, not %.12s, and its body carries report %.12s\n", pr.Number, safe(pr.Head.SHA), safe(b.Batch), current) } if !inComment { if prState(pr) == "open" { - fmt.Fprintln(a.stdout, " comment not posted; the next backport-batch run --push posts it") + _, _ = fmt.Fprintln(a.stdout, " comment not posted; the next backport-batch run --push posts it") } else { - fmt.Fprintf(a.stdout, " comment not posted; kept in the body only, since nothing is written to a %s pull request\n", prState(pr)) + _, _ = fmt.Fprintf(a.stdout, " comment not posted; kept in the body only, since nothing is written to a %s pull request\n", prState(pr)) } } type line struct { @@ -802,22 +802,22 @@ func (a *app) printReport(pr *pullRequest, s storedReport, inBody *storedReport, } slices.SortFunc(lines, func(x, y line) int { return cmp.Compare(x.n, y.n) }) for _, l := range lines { - fmt.Fprintf(a.stdout, "#%d %s\n", l.n, terminalSafe(l.text)) + _, _ = fmt.Fprintf(a.stdout, "#%d %s\n", l.n, terminalSafe(l.text)) } overlay := func(name string, list []int) { if len(list) > 0 { - fmt.Fprintf(a.stdout, "%s: %s\n", name, strings.Trim(fmt.Sprint(list), "[]")) + _, _ = fmt.Fprintf(a.stdout, "%s: %s\n", name, strings.Trim(fmt.Sprint(list), "[]")) } } overlay("stale, still on the batch", d.Overlays.Stale) overlay("only part of it applies, still on the batch", d.Overlays.Partial) for _, e := range d.Overlays.Dropped { - fmt.Fprintf(a.stdout, "dropped: #%d, merged as %.12s, on #%d: %s\n", e.Original, terminalSafe(e.Merge), e.On, terminalSafe(e.Reason)) + _, _ = fmt.Fprintf(a.stdout, "dropped: #%d, merged as %.12s, on #%d: %s\n", e.Original, terminalSafe(e.Merge), e.On, terminalSafe(e.Reason)) } overlay("changed in the pick", d.Overlays.Changed) for _, n := range slices.Sorted(maps.Keys(d.Diag)) { for _, ln := range d.Diag[n] { - fmt.Fprintf(a.stdout, "#%s: %s\n", terminalSafe(n), terminalSafe(ln)) + _, _ = fmt.Fprintf(a.stdout, "#%s: %s\n", terminalSafe(n), terminalSafe(ln)) } } if len(d.Other) > 0 { @@ -825,7 +825,7 @@ func (a *app) printReport(pr *pullRequest, s storedReport, inBody *storedReport, for i, sha := range d.Other { other[i] = safe(sha) } - fmt.Fprintf(a.stdout, "commits on main that merged no pull request: %s\n", strings.Join(other, ", ")) + _, _ = fmt.Fprintf(a.stdout, "commits on main that merged no pull request: %s\n", strings.Join(other, ", ")) } } diff --git a/cmd/backport-batch/verify.go b/cmd/backport-batch/verify.go index 0691df641..03b0e8955 100644 --- a/cmd/backport-batch/verify.go +++ b/cmd/backport-batch/verify.go @@ -66,7 +66,7 @@ func (a *app) verify(args []string) int { return a.fail(err) } if pr == nil { - fmt.Fprintf(a.stdout, "%s: no batch pull request is open; nothing to verify\n", l.branch()) + _, _ = fmt.Fprintf(a.stdout, "%s: no batch pull request is open; nothing to verify\n", l.branch()) return 0 } if head, err = batchHead(pr); err != nil { @@ -102,7 +102,7 @@ func (a *app) verify(args []string) int { } if why != "" { refused = append(refused, why) - fmt.Fprintf(a.stdout, "refused %.12s: %s\n", u.commit, why) + _, _ = fmt.Fprintf(a.stdout, "refused %.12s: %s\n", u.commit, why) continue } seen[v] = u.commit @@ -110,13 +110,13 @@ func (a *app) verify(args []string) int { if !u.merge() { n = 1 } - fmt.Fprintf(a.stdout, "ok %.12s #%d, %d %s\n", u.commit, v.o.number, n, plural(n, "commit")) + _, _ = fmt.Fprintf(a.stdout, "ok %.12s #%d, %d %s\n", u.commit, v.o.number, n, plural(n, "commit")) } if len(refused) > 0 { - fmt.Fprintf(a.stdout, "%s: %d of %d units at %.12s are not originals in the batch's shape\n", l.branch(), len(refused), len(units), head) + _, _ = fmt.Fprintf(a.stdout, "%s: %d of %d units at %.12s are not originals in the batch's shape\n", l.branch(), len(refused), len(units), head) return 1 } - fmt.Fprintf(a.stdout, "%s: %d %s at %.12s, each one original in the batch's shape\n", l.branch(), len(units), plural(len(units), "unit"), head) + _, _ = fmt.Fprintf(a.stdout, "%s: %d %s at %.12s, each one original in the batch's shape\n", l.branch(), len(units), plural(len(units), "unit"), head) return 0 } From 20ce6c691e50f03e30b31c012d3ba3dd63c62fa8 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 00:14:38 +0300 Subject: [PATCH 154/174] fix(sse): close a client's channel once when the server shuts down Shutdown closed every open client's channel, and the client's handler closed the same channel again when its request ended, so any stream still open at shutdown panicked with a close of a closed channel. Stopping a client is now idempotent. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- internal/sse/server.go | 11 ++++- internal/sse/server_test.go | 98 +++++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 internal/sse/server_test.go diff --git a/internal/sse/server.go b/internal/sse/server.go index 0c55032ec..f65817fb1 100644 --- a/internal/sse/server.go +++ b/internal/sse/server.go @@ -60,6 +60,7 @@ type Server struct { type client struct { ch chan string closeCh chan struct{} + closeOnce sync.Once flusher http.Flusher w http.ResponseWriter req *http.Request @@ -122,7 +123,7 @@ func (s *Server) ListenAndServe() error { func (s *Server) Shutdown(ctx context.Context) error { s.clientsMu.Lock() for c := range s.clients { - close(c.closeCh) + c.stop() } s.clientsMu.Unlock() return s.http.Shutdown(ctx) @@ -206,12 +207,18 @@ func (s *Server) handleStream(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() // Unregister client - close(c.closeCh) + c.stop() s.clientsMu.Lock() delete(s.clients, c) s.clientsMu.Unlock() } +// stop ends the client's pump. Shutdown and the client's own handler both +// stop it, in either order. +func (c *client) stop() { + c.closeOnce.Do(func() { close(c.closeCh) }) +} + func (c *client) pump() { t := time.NewTicker(c.heartbeat) defer t.Stop() diff --git a/internal/sse/server_test.go b/internal/sse/server_test.go new file mode 100644 index 000000000..be7c21e34 --- /dev/null +++ b/internal/sse/server_test.go @@ -0,0 +1,98 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sse + +import ( + "context" + "errors" + "io" + "log" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +// streamWriter accepts the first okWrites writes and fails every one after. +type streamWriter struct { + mu sync.Mutex + header http.Header + okWrites int + body strings.Builder +} + +func (w *streamWriter) Header() http.Header { return w.header } +func (w *streamWriter) WriteHeader(int) {} +func (w *streamWriter) Flush() {} + +func (w *streamWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.okWrites == 0 { + return 0, errors.New("connection reset") + } + w.okWrites-- + return w.body.Write(p) +} + +func (w *streamWriter) String() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.body.String() +} + +func (s *Server) snapshotClients() []*client { + s.clientsMu.RLock() + defer s.clientsMu.RUnlock() + var out []*client + for c := range s.clients { + out = append(out, c) + } + return out +} + +func newTestServer(heartbeat time.Duration) *Server { + return New(Options{Heartbeat: heartbeat, Logger: log.New(io.Discard, "", 0)}) +} + +// A stream still open when the server shuts down ends when its client +// disconnects, without closing the client a second time. +func TestShutdownThenDisconnect(t *testing.T) { + s := newTestServer(time.Hour) + ctx, cancel := context.WithCancel(context.Background()) + r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx) + done := make(chan struct{}) + go func() { + s.handleStream(&streamWriter{header: http.Header{}, okWrites: 100}, r) + close(done) + }() + deadline := time.Now().Add(time.Second) + for len(s.snapshotClients()) == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if err := s.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("the stream did not end after its client disconnected") + } +} From 41ad82842a395b238d98889f76378b7e611b1fe2 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 00:15:00 +0300 Subject: [PATCH 155/174] fix(sse): drop a client as soon as a write to it fails Write errors on the event stream were ignored, so a client whose connection broke kept its handler, pump and registration until the request context ended, and every event was written into the dead connection meanwhile. A failed write now ends the stream and unregisters the client, and the handler also returns once its pump ends, so Shutdown no longer leaves streams open. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- internal/sse/server.go | 52 +++++++++++++------- internal/sse/server_test.go | 97 ++++++++++++++++++++++++++++++++++--- 2 files changed, 126 insertions(+), 23 deletions(-) diff --git a/internal/sse/server.go b/internal/sse/server.go index f65817fb1..166b01f49 100644 --- a/internal/sse/server.go +++ b/internal/sse/server.go @@ -187,8 +187,16 @@ func (s *Server) handleStream(w http.ResponseWriter, r *http.Request) { s.clients[c] = struct{}{} s.clientsMu.Unlock() + defer func() { + s.clientsMu.Lock() + delete(s.clients, c) + s.clientsMu.Unlock() + }() + // Initial comment to open the stream for some proxies - fmt.Fprintf(w, ": connected %s\n\n", time.Now().Format(time.RFC3339)) + if _, err := fmt.Fprintf(w, ": connected %s\n\n", time.Now().Format(time.RFC3339)); err != nil { + return + } flusher.Flush() // Send latest if any @@ -196,21 +204,24 @@ func (s *Server) handleStream(w http.ResponseWriter, r *http.Request) { latest := s.latest s.latestMu.RUnlock() if latest != "" { - writeSSE(w, latest) + if err := writeSSE(w, latest); err != nil { + return + } flusher.Flush() } - // Start pump - go c.pump() - - // Block until client disconnects - <-r.Context().Done() - - // Unregister client + // A write the client no longer takes ends the pump; the handler then + // gives the client up without waiting for the request to end. + pumped := make(chan struct{}) + go func() { + c.pump() + close(pumped) + }() + select { + case <-r.Context().Done(): + case <-pumped: + } c.stop() - s.clientsMu.Lock() - delete(s.clients, c) - s.clientsMu.Unlock() } // stop ends the client's pump. Shutdown and the client's own handler both @@ -227,23 +238,30 @@ func (c *client) pump() { case <-c.closeCh: return case msg := <-c.ch: - writeSSE(c.w, msg) + if err := writeSSE(c.w, msg); err != nil { + return + } c.flusher.Flush() case <-t.C: // heartbeat comment (keeps connections alive through proxies) - fmt.Fprint(c.w, ": hb\n\n") + if _, err := fmt.Fprint(c.w, ": hb\n\n"); err != nil { + return + } c.flusher.Flush() } } } -func writeSSE(w http.ResponseWriter, msg string) { +func writeSSE(w http.ResponseWriter, msg string) error { // Split on lines; each needs its own "data:" field per the SSE spec lines := strings.SplitSeq(strings.TrimRight(msg, "\n"), "\n") for ln := range lines { - fmt.Fprintf(w, "data: %s\n", ln) + if _, err := fmt.Fprintf(w, "data: %s\n", ln); err != nil { + return err + } } - fmt.Fprint(w, "\n") + _, err := fmt.Fprint(w, "\n") + return err } // Minimal index page with live updates diff --git a/internal/sse/server_test.go b/internal/sse/server_test.go index be7c21e34..5ae5ad966 100644 --- a/internal/sse/server_test.go +++ b/internal/sse/server_test.go @@ -57,6 +57,37 @@ func (w *streamWriter) String() string { return w.body.String() } +// serve runs the stream handler for a client that never disconnects and +// reports whether the handler gave the client up within a second. +func serve(t *testing.T, s *Server, w *streamWriter, publish string) bool { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx) + done := make(chan struct{}) + go func() { + s.handleStream(w, r) + close(done) + }() + if publish != "" { + deadline := time.Now().Add(time.Second) + for len(s.snapshotClients()) == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + // Onto the client's queue, which only the pump reads, so the failing + // write is the pump's and not the handler's write of latest. + for _, c := range s.snapshotClients() { + c.ch <- publish + } + } + select { + case <-done: + return len(s.snapshotClients()) == 0 + case <-time.After(time.Second): + return false + } +} + func (s *Server) snapshotClients() []*client { s.clientsMu.RLock() defer s.clientsMu.RUnlock() @@ -71,15 +102,70 @@ func newTestServer(heartbeat time.Duration) *Server { return New(Options{Heartbeat: heartbeat, Logger: log.New(io.Discard, "", 0)}) } -// A stream still open when the server shuts down ends when its client -// disconnects, without closing the client a second time. -func TestShutdownThenDisconnect(t *testing.T) { +// A client whose connection stops taking writes is dropped at once, at +// whichever write first fails, instead of holding its handler until the +// request context ends. +func TestStreamDropsClientOnWriteFailure(t *testing.T) { + for _, tc := range []struct { + name string + okWrites int + latest string + publish string + heartbeat time.Duration + }{ + {name: "connect comment", okWrites: 0, heartbeat: time.Hour}, + {name: "latest payload", okWrites: 1, latest: "state", heartbeat: time.Hour}, + {name: "published event", okWrites: 1, publish: "event", heartbeat: time.Hour}, + {name: "heartbeat", okWrites: 1, heartbeat: 10 * time.Millisecond}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newTestServer(tc.heartbeat) + if tc.latest != "" { + s.Publish(tc.latest) + } + if !serve(t, s, &streamWriter{header: http.Header{}, okWrites: tc.okWrites}, tc.publish) { + t.Fatal("the handler kept a client whose writes fail") + } + }) + } +} + +// Each line of a payload is its own data field, and the event ends with an +// empty line. +func TestStreamFramesMultilinePayload(t *testing.T) { s := newTestServer(time.Hour) + s.Publish("one\ntwo\n") + w := &streamWriter{header: http.Header{}, okWrites: 100} ctx, cancel := context.WithCancel(context.Background()) r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx) done := make(chan struct{}) go func() { - s.handleStream(&streamWriter{header: http.Header{}, okWrites: 100}, r) + s.handleStream(w, r) + close(done) + }() + deadline := time.Now().Add(time.Second) + for !strings.Contains(w.String(), "data: two\n\n") && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + cancel() + <-done + if got := w.String(); !strings.Contains(got, "data: one\ndata: two\n\n") { + t.Fatalf("stream = %q, want both lines framed as one event", got) + } + if len(s.snapshotClients()) != 0 { + t.Fatal("a disconnected client stayed registered") + } +} + +// Shutdown ends every open stream, and the handler that then returns does +// not close the client a second time. +func TestShutdownEndsOpenStreams(t *testing.T) { + s := newTestServer(time.Hour) + w := &streamWriter{header: http.Header{}, okWrites: 100} + r := httptest.NewRequest(http.MethodGet, "/stream", nil) + done := make(chan struct{}) + go func() { + s.handleStream(w, r) close(done) }() deadline := time.Now().Add(time.Second) @@ -89,10 +175,9 @@ func TestShutdownThenDisconnect(t *testing.T) { if err := s.Shutdown(context.Background()); err != nil { t.Fatal(err) } - cancel() select { case <-done: case <-time.After(time.Second): - t.Fatal("the stream did not end after its client disconnected") + t.Fatal("Shutdown left the stream open") } } From bd998d9653e1f1b4ab7045599044ccaa744314a8 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 12 Nov 2026 23:46:11 +0300 Subject: [PATCH 156/174] fix(kube-ovn): fail the plunger when its status server cannot start The status stream was started in a bare goroutine whose error was dropped, and the runnable's registration error was discarded too, so a port that could not be bound left the plunger running with no status endpoint and nothing in its logs. The server now runs inside its runnable: a listen error fails the manager, as a metrics server bind failure does, and stopping the manager is not reported as an error. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../kubeovnplunger/kubeovn_plunger.go | 30 ++++++++--- .../kubeovnplunger/status_server_test.go | 51 +++++++++++++++++++ 2 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 internal/controller/kubeovnplunger/status_server_test.go diff --git a/internal/controller/kubeovnplunger/kubeovn_plunger.go b/internal/controller/kubeovnplunger/kubeovn_plunger.go index a45fba073..30aa42c00 100644 --- a/internal/controller/kubeovnplunger/kubeovn_plunger.go +++ b/internal/controller/kubeovnplunger/kubeovn_plunger.go @@ -3,8 +3,10 @@ package kubeovnplunger import ( "bytes" "context" + "errors" "fmt" "io" + "net/http" "strings" "time" @@ -153,12 +155,9 @@ func (r *KubeOVNPlunger) SetupWithManager(mgr ctrl.Manager, kubeOVNNamespace, ap Complete(r); err != nil { return err } - _ = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - go srv.ListenAndServe() - <-ctx.Done() - _ = srv.Shutdown(context.Background()) - return nil - })) + if err := mgr.Add(serveStatus(srv)); err != nil { + return err + } return mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { ticker := time.NewTicker(rescanInterval) defer ticker.Stop() @@ -180,6 +179,25 @@ func (r *KubeOVNPlunger) SetupWithManager(mgr ctrl.Manager, kubeOVNNamespace, ap })) } +// serveStatus runs the status stream for as long as the manager does. A server +// that cannot start fails the manager, as the metrics server does. +func serveStatus(s *sse.Server) manager.RunnableFunc { + return func(ctx context.Context) error { + served := make(chan error, 1) + go func() { served <- s.ListenAndServe() }() + select { + case err := <-served: + return err + case <-ctx.Done(): + } + _ = s.Shutdown(context.Background()) + if err := <-served; !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil + } +} + type ExecOptions struct { Namespace string Pod string diff --git a/internal/controller/kubeovnplunger/status_server_test.go b/internal/controller/kubeovnplunger/status_server_test.go new file mode 100644 index 000000000..c894ecf74 --- /dev/null +++ b/internal/controller/kubeovnplunger/status_server_test.go @@ -0,0 +1,51 @@ +package kubeovnplunger + +import ( + "context" + "io" + "log" + "net" + "testing" + "time" + + "github.com/cozystack/cozystack/internal/sse" +) + +func runStatusServer(ctx context.Context, t *testing.T, addr string) (bool, error) { + t.Helper() + ctx, cancel := context.WithCancel(ctx) + t.Cleanup(cancel) + s := sse.New(sse.Options{Addr: addr, Logger: log.New(io.Discard, "", 0)}) + done := make(chan error, 1) + go func() { done <- serveStatus(s)(ctx) }() + select { + case err := <-done: + return true, err + case <-time.After(2 * time.Second): + return false, nil + } +} + +// A status server that cannot listen fails its runnable, and with it the +// manager, rather than leaving the plunger running without it. +func TestStatusServerReportsListenFailure(t *testing.T) { + taken, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = taken.Close() }) + returned, err := runStatusServer(context.Background(), t, taken.Addr().String()) + if !returned || err == nil { + t.Fatalf("serveStatus on an occupied address = %v (returned %v), want an error", err, returned) + } +} + +// Stopping the manager stops the server, and that is not an error. +func TestStatusServerStopsWithoutError(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(100*time.Millisecond, cancel) + returned, err := runStatusServer(ctx, t, "127.0.0.1:0") + if !returned || err != nil { + t.Fatalf("serveStatus after cancel = %v (returned %v), want nil", err, returned) + } +} From 87b794a9925e2f9c4813f5da6425f8344f408c03 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 12 Nov 2026 23:53:11 +0300 Subject: [PATCH 157/174] chore: handle errors dropped in cleanup paths and tests The drain tests now assert that RunThenDrain returns the run's nil error, the idle-tail test fails on any error other than the stream ending or being cut, and a test fixture file fails the test when it cannot be closed. Removing a temporary directory and closing a response body that has been read are best-effort: a failure leaves nothing for the caller to act on and does not change any result, so those errors are discarded explicitly. The same holds for the stub servers' response writes, whose failure surfaces on the client side of the test. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../cnpg_artifact_size_test.go | 4 ++-- .../mariadb_artifact_size_test.go | 2 +- .../fluxplunger/drain_outcome_test.go | 18 ++++++++++++------ internal/crdinstall/install.go | 2 +- internal/fluxinstall/install.go | 2 +- internal/observabilityproxy/server_test.go | 8 +++++++- internal/telemetry/collector.go | 2 +- internal/telemetry/operator_collector.go | 2 +- pkg/cmd/server/watch_initial_events_test.go | 2 +- pkg/reservation/instancetype_test.go | 4 +++- 10 files changed, 30 insertions(+), 16 deletions(-) diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go index bdf9c5d7b..4afef7194 100644 --- a/internal/backupcontroller/cnpg_artifact_size_test.go +++ b/internal/backupcontroller/cnpg_artifact_size_test.go @@ -222,13 +222,13 @@ func cnpgListStub(prefix string, seen func(*http.Request)) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/xml") if _, ok := req.URL.Query()["location"]; ok { - fmt.Fprint(w, ``) + _, _ = fmt.Fprint(w, ``) return } if seen != nil { seen(req) } - fmt.Fprintf(w, ` + _, _ = fmt.Fprintf(w, ` bkt%[1]s21000false %[1]sbackup.info7"a"2026-10-21T10:01:00.000Z diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index 9d649c15c..ed4a2b7bc 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -464,7 +464,7 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization") gotDelimiter = req.URL.Query().Get("delimiter") w.Header().Set("Content-Type", "application/xml") - fmt.Fprintf(w, ` + _, _ = fmt.Fprintf(w, ` bkttenant/db/11000false %s31337"x"2026-10-21T10:01:00.000Z diff --git a/internal/controller/fluxplunger/drain_outcome_test.go b/internal/controller/fluxplunger/drain_outcome_test.go index cb74e0177..152091dec 100644 --- a/internal/controller/fluxplunger/drain_outcome_test.go +++ b/internal/controller/fluxplunger/drain_outcome_test.go @@ -65,12 +65,14 @@ func (s captureSink) WithName(string) logr.LogSink { return s } func TestRunThenDrain_FencesLeftBehindAreAnErrorNamingThem(t *testing.T) { logger, entries := newCaptureLogger() ctx := log.IntoContext(context.Background(), logger) - RunThenDrain(ctx, + if err := RunThenDrain(ctx, func(context.Context) error { return nil }, func(context.Context) (int, error) { return 1, errors.Join(context.DeadlineExceeded, errors.New("still fenced: tenant-root/vm-disk-0")) }, - time.Minute) + time.Minute); err != nil { + t.Fatalf("RunThenDrain = %v, want the run's nil error", err) + } for _, e := range *entries { if e.err != nil && strings.Contains(e.err.Error(), "tenant-root/vm-disk-0") { return @@ -84,10 +86,12 @@ func TestRunThenDrain_FencesLeftBehindAreAnErrorNamingThem(t *testing.T) { func TestRunThenDrain_AnUnansweredListingIsNotACountOfMinusOne(t *testing.T) { logger, entries := newCaptureLogger() ctx := log.IntoContext(context.Background(), logger) - RunThenDrain(ctx, + if err := RunThenDrain(ctx, func(context.Context) error { return nil }, func(context.Context) (int, error) { return -1, context.DeadlineExceeded }, - time.Minute) + time.Minute); err != nil { + t.Fatalf("RunThenDrain = %v, want the run's nil error", err) + } found := false for _, e := range *entries { for i := 0; i+1 < len(e.kv); i += 2 { @@ -108,10 +112,12 @@ func TestRunThenDrain_AnUnansweredListingIsNotACountOfMinusOne(t *testing.T) { func TestRunThenDrain_ACompleteDrainSaysSo(t *testing.T) { logger, entries := newCaptureLogger() ctx := log.IntoContext(context.Background(), logger) - RunThenDrain(ctx, + if err := RunThenDrain(ctx, func(context.Context) error { return nil }, func(context.Context) (int, error) { return 0, nil }, - time.Minute) + time.Minute); err != nil { + t.Fatalf("RunThenDrain = %v, want the run's nil error", err) + } for _, e := range *entries { if e.err == nil && strings.Contains(e.msg, "Released every uninstall fence") { return diff --git a/internal/crdinstall/install.go b/internal/crdinstall/install.go index 5760beb6b..12706060b 100644 --- a/internal/crdinstall/install.go +++ b/internal/crdinstall/install.go @@ -40,7 +40,7 @@ func Install(ctx context.Context, k8sClient client.Client, writeEmbeddedManifest if err != nil { return fmt.Errorf("failed to create temp directory: %w", err) } - defer os.RemoveAll(tmpDir) + defer func() { _ = os.RemoveAll(tmpDir) }() manifestsDir := filepath.Join(tmpDir, "manifests") if err := os.MkdirAll(manifestsDir, 0755); err != nil { diff --git a/internal/fluxinstall/install.go b/internal/fluxinstall/install.go index 042bdfd5e..63c977803 100644 --- a/internal/fluxinstall/install.go +++ b/internal/fluxinstall/install.go @@ -41,7 +41,7 @@ func Install(ctx context.Context, k8sClient client.Client, writeEmbeddedManifest if err != nil { return fmt.Errorf("failed to create temp directory: %w", err) } - defer os.RemoveAll(tmpDir) + defer func() { _ = os.RemoveAll(tmpDir) }() // Extract embedded manifests (generated by cozypkg) manifestsDir := filepath.Join(tmpDir, "manifests") diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index df3e23bc4..2117b1551 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -18,6 +18,7 @@ package observabilityproxy import ( "bufio" + "errors" "io" "net/http" "net/http/httptest" @@ -413,7 +414,12 @@ func TestServerTailCapsAndIdle(t *testing.T) { closed <- err }() select { - case <-closed: + case err := <-closed: + // The server cuts an idle tail off mid-stream rather than ending the + // response, so the client reads an unexpected EOF; a clean end is fine too. + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatalf("idle tail ended with %v, want the stream to end or be cut", err) + } case <-time.After(3 * time.Second): t.Fatal("idle tail not closed") } diff --git a/internal/telemetry/collector.go b/internal/telemetry/collector.go index a948c7495..636bbc85a 100644 --- a/internal/telemetry/collector.go +++ b/internal/telemetry/collector.go @@ -147,7 +147,7 @@ func (c *Collector) sendMetrics(clusterID, metrics string) error { if err != nil { return fmt.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return fmt.Errorf("unexpected status code: %d", resp.StatusCode) diff --git a/internal/telemetry/operator_collector.go b/internal/telemetry/operator_collector.go index 2c0e1b9bb..9f2f9fcbe 100644 --- a/internal/telemetry/operator_collector.go +++ b/internal/telemetry/operator_collector.go @@ -272,7 +272,7 @@ func (c *OperatorCollector) sendMetrics(clusterID, metrics string) error { if err != nil { return fmt.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return fmt.Errorf("unexpected status code: %d", resp.StatusCode) diff --git a/pkg/cmd/server/watch_initial_events_test.go b/pkg/cmd/server/watch_initial_events_test.go index 331417545..75d43c3e1 100644 --- a/pkg/cmd/server/watch_initial_events_test.go +++ b/pkg/cmd/server/watch_initial_events_test.go @@ -104,7 +104,7 @@ func watchStream(t *testing.T, ts *httptest.Server, cli client.Client, query str if err != nil { t.Fatalf("watch: %v", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { t.Fatalf("watch: status %d", resp.StatusCode) } diff --git a/pkg/reservation/instancetype_test.go b/pkg/reservation/instancetype_test.go index 0d072b548..f837af8e3 100644 --- a/pkg/reservation/instancetype_test.go +++ b/pkg/reservation/instancetype_test.go @@ -56,7 +56,9 @@ func shippedInstancetypes(t *testing.T) []map[string]any { out = append(out, obj) } } - fh.Close() + if err := fh.Close(); err != nil { + t.Fatalf("%s: %v", f, err) + } } return out } From 7a481c47656c041ef2369ef1ca798367c0713629 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 12 Nov 2026 23:53:12 +0300 Subject: [PATCH 158/174] chore(workloadmonitor): discard best-effort close errors explicitly The Prometheus response body is closed after it has been read, and the test stubs' writes fail on the client side of the test, so neither error changes a result. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- internal/controller/workloadmonitor_controller.go | 2 +- internal/controller/workloadmonitor_controller_test.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 217fd0bbf..b8b0bac13 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -525,7 +525,7 @@ func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, p if err != nil { return nil, fmt.Errorf("querying Prometheus: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("unexpected status %d from Prometheus", resp.StatusCode) diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index 353066c4b..87f9dd269 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -870,7 +870,7 @@ func TestReconcileNoBucketClaimSkips(t *testing.T) { func TestQueryAllBucketMetrics(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[ + _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[ {"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"bucket-aaa"},"value":[1713000000,"10485864"]}, {"metric":{"__name__":"SeaweedFS_s3_bucket_physical_size_bytes","bucket":"bucket-aaa"},"value":[1713000000,"20971728"]}, {"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"bucket-bbb"},"value":[1713000000,"0"]} @@ -909,7 +909,7 @@ func TestQueryAllBucketMetrics(t *testing.T) { func TestQueryAllBucketMetricsEmpty(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) + _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) })) defer srv.Close() @@ -953,7 +953,7 @@ func TestQueryAllBucketMetricsPathPrefixAndUserinfo(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { gotPath = r.URL.Path gotUser, gotPass, gotOK = r.BasicAuth() - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) + _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) })) defer srv.Close() @@ -1219,7 +1219,7 @@ func TestReconcileBucketMetricsFromEndpointOverride(t *testing.T) { if r.URL.Path != "/vm/select/0/prometheus/api/v1/query" { t.Errorf("unexpected query path %q", r.URL.Path) } - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[ + _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[ {"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"cosi-abc123"},"value":[1713000000,"4096"]} ]}}`) })) @@ -1331,7 +1331,7 @@ func TestReconcileRetainsLastKnownSizesOnQueryFailure(t *testing.T) { func TestReconcileRetainsLastKnownSizesWhenBucketMissingFromResult(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) + _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) })) defer srv.Close() From 50959ccca20fa212c836cc2ccccf7583f1bc279a Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Thu, 12 Nov 2026 23:53:20 +0300 Subject: [PATCH 159/174] chore(backport): discard temp-file cleanup errors explicitly A temporary file or directory that cannot be removed, or closed after its write already failed, leaves nothing to act on: the run's result is the same, and the primary error is the one returned. The test that unsets an environment variable now fails if it cannot. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- cmd/backport-batch/apisurface.go | 2 +- cmd/backport-batch/assemble.go | 4 ++-- cmd/backport-batch/effects.go | 2 +- cmd/backport-batch/notify.go | 6 +++--- cmd/backport-batch/report.go | 8 ++++---- cmd/backport-batch/report_test.go | 4 +++- cmd/backport-batch/shape.go | 4 ++-- 7 files changed, 16 insertions(+), 14 deletions(-) diff --git a/cmd/backport-batch/apisurface.go b/cmd/backport-batch/apisurface.go index 706e4be62..6966bce74 100644 --- a/cmd/backport-batch/apisurface.go +++ b/cmd/backport-batch/apisurface.go @@ -99,7 +99,7 @@ func apiSnapshot(commit string) (apigate.Snapshot, error) { if err != nil { return nil, err } - defer os.RemoveAll(dir) + defer func() { _ = os.RemoveAll(dir) }() for name, blob := range paths { file := filepath.Join(dir, filepath.FromSlash(name)) if err := os.MkdirAll(filepath.Dir(file), 0o700); err != nil { diff --git a/cmd/backport-batch/assemble.go b/cmd/backport-batch/assemble.go index a0aab98db..2ebca3483 100644 --- a/cmd/backport-batch/assemble.go +++ b/cmd/backport-batch/assemble.go @@ -727,7 +727,7 @@ func (a *app) publish(b *batch, p publication, rebuilt bool) error { if err != nil { return err } - defer os.Remove(name) + defer func() { _ = os.Remove(name) }() if b.pr == nil { n, url, err := a.gh.createPull(b.line.branch(), p.branch, p.title, name) if err != nil { @@ -828,7 +828,7 @@ func (a *app) closeBatch(b *batch, p publication) error { if err != nil { return err } - defer os.Remove(name) + defer func() { _ = os.Remove(name) }() if err := a.stillOpen(c.PR); err != nil { return err } diff --git a/cmd/backport-batch/effects.go b/cmd/backport-batch/effects.go index 350f9a961..6388c42ae 100644 --- a/cmd/backport-batch/effects.go +++ b/cmd/backport-batch/effects.go @@ -367,7 +367,7 @@ func (b *batch) alreadyApplied(pre, cur, post string) (applied, unsupported bool if err != nil { return false, false, err } - defer os.RemoveAll(dir) + defer func() { _ = os.RemoveAll(dir) }() names := map[string]string{} for _, f := range []struct{ name, oid string }{{"cur", cur}, {"pre", pre}, {"post", post}} { names[f.name] = filepath.Join(dir, f.name) diff --git a/cmd/backport-batch/notify.go b/cmd/backport-batch/notify.go index 3460e287d..e9ee23234 100644 --- a/cmd/backport-batch/notify.go +++ b/cmd/backport-batch/notify.go @@ -229,7 +229,7 @@ func (a *app) announced(pr *pullRequest, rec *lineRecord, parks []parkedEntry) e if err != nil { return err } - defer os.Remove(name) + defer func() { _ = os.Remove(name) }() return a.gh.editBody(pr.Number, name) } @@ -309,9 +309,9 @@ func (a *app) comment(pr *pullRequest, mention string, today time.Time, text []s if err != nil { return a.fail(err) } - defer os.Remove(f.Name()) + defer func() { _ = os.Remove(f.Name()) }() if _, err := f.WriteString(b.String()); err != nil { - f.Close() + _ = f.Close() return a.fail(err) } if err := f.Close(); err != nil { diff --git a/cmd/backport-batch/report.go b/cmd/backport-batch/report.go index ee072e332..fbfc4d569 100644 --- a/cmd/backport-batch/report.go +++ b/cmd/backport-batch/report.go @@ -850,7 +850,7 @@ func (a *app) postReport(n int, s storedReport) error { if err != nil { return err } - defer os.Remove(name) + defer func() { _ = os.Remove(name) }() return a.gh.createComment(n, name) } @@ -873,12 +873,12 @@ func writeTemp(pattern, content string) (string, error) { return "", err } if _, err := f.WriteString(content); err != nil { - f.Close() - os.Remove(f.Name()) + _ = f.Close() + _ = os.Remove(f.Name()) return "", err } if err := f.Close(); err != nil { - os.Remove(f.Name()) + _ = os.Remove(f.Name()) return "", err } return f.Name(), nil diff --git a/cmd/backport-batch/report_test.go b/cmd/backport-batch/report_test.go index a496f68e0..754b599bb 100644 --- a/cmd/backport-batch/report_test.go +++ b/cmd/backport-batch/report_test.go @@ -224,7 +224,9 @@ func TestReportPartition(t *testing.T) { } t.Setenv("GIT_COMMITTER_DATE", "2026-01-01T00:00:00Z") f.pr(4, "fix(app): four, before the start", "", map[string]string{"app/four.txt": "4\n"}) - os.Unsetenv("GIT_COMMITTER_DATE") + if err := os.Unsetenv("GIT_COMMITTER_DATE"); err != nil { + t.Fatal(err) + } f.setPolicy("release-1.0: 2026-06-01\n") f.onLine("release-1.0", "fix(app): a line-only eleven", map[string]string{"app/eleven.txt": "line\n"}) pr(5, "fix(app): five", map[string]string{"app/five.txt": "5\n"}, true) diff --git a/cmd/backport-batch/shape.go b/cmd/backport-batch/shape.go index ee0a6940d..31d2da501 100644 --- a/cmd/backport-batch/shape.go +++ b/cmd/backport-batch/shape.go @@ -121,9 +121,9 @@ func commitMerge(base, tip, message string) (string, error) { if err != nil { return "", err } - defer os.Remove(f.Name()) + defer func() { _ = os.Remove(f.Name()) }() if _, err := f.WriteString(message); err != nil { - f.Close() + _ = f.Close() return "", err } if err := f.Close(); err != nil { From 939a1ec44b940566cc1267c5b43ffee4f6d32007 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 00:23:41 +0300 Subject: [PATCH 160/174] refactor: read managed fields through FieldsV1 accessors Direct access to FieldsV1.Raw is deprecated; GetRawBytes returns the same bytes and is nil-safe, and NewFieldsV1 builds the test fixtures. The write-options test still covers a raw-only orphanDependents delete option, now decoded from its wire form, and asserts that the converted options keep it. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- internal/controller/fluxplunger/flux_plunger.go | 5 +++-- .../controller/fluxplunger/flux_plunger_test.go | 4 ++-- pkg/registry/write_options_test.go | 16 +++++++++++++--- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/internal/controller/fluxplunger/flux_plunger.go b/internal/controller/fluxplunger/flux_plunger.go index d53ec3eef..22775837b 100644 --- a/internal/controller/fluxplunger/flux_plunger.go +++ b/internal/controller/fluxplunger/flux_plunger.go @@ -486,11 +486,12 @@ func ownsSuspensionManagedFields(hr *helmv2.HelmRelease) bool { // fieldsManageSuspend reports whether a managed-fields set includes spec.suspend, // encoded by the apiserver as {"f:spec":{"f:suspend":{}}}. func fieldsManageSuspend(fs *metav1.FieldsV1) bool { - if fs == nil || len(fs.Raw) == 0 { + raw := fs.GetRawBytes() + if len(raw) == 0 { return false } var top map[string]json.RawMessage - if err := json.Unmarshal(fs.Raw, &top); err != nil { + if err := json.Unmarshal(raw, &top); err != nil { return false } specRaw, ok := top["f:spec"] diff --git a/internal/controller/fluxplunger/flux_plunger_test.go b/internal/controller/fluxplunger/flux_plunger_test.go index 365250621..fc1f8adad 100644 --- a/internal/controller/fluxplunger/flux_plunger_test.go +++ b/internal/controller/fluxplunger/flux_plunger_test.go @@ -83,7 +83,7 @@ func suspendManagedFields(manager string, op metav1.ManagedFieldsOperationType) Operation: op, APIVersion: helmv2.GroupVersion.String(), FieldsType: "FieldsV1", - FieldsV1: &metav1.FieldsV1{Raw: []byte(`{"f:spec":{"f:suspend":{}}}`)}, + FieldsV1: metav1.NewFieldsV1(`{"f:spec":{"f:suspend":{}}}`), }} } @@ -286,7 +286,7 @@ func TestOwnsSuspensionManagedFields(t *testing.T) { {"no managed fields", nil, false}, {"plunger apply of other fields", []metav1.ManagedFieldsEntry{{ Manager: suspendFieldManager, Operation: metav1.ManagedFieldsOperationApply, - FieldsType: "FieldsV1", FieldsV1: &metav1.FieldsV1{Raw: []byte(`{"f:spec":{"f:chart":{}}}`)}, + FieldsType: "FieldsV1", FieldsV1: metav1.NewFieldsV1(`{"f:spec":{"f:chart":{}}}`), }}, false}, } for _, tc := range cases { diff --git a/pkg/registry/write_options_test.go b/pkg/registry/write_options_test.go index 488d7a51a..8d6b375f9 100644 --- a/pkg/registry/write_options_test.go +++ b/pkg/registry/write_options_test.go @@ -17,7 +17,9 @@ limitations under the License. package registry import ( + "encoding/json" "reflect" + "strings" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,7 +28,6 @@ import ( func TestClientWriteOptionsPreserveAPISemantics(t *testing.T) { grace := int64(7) - orphan := true propagation := metav1.DeletePropagationForeground uid := types.UID("expected-uid") rv := "42" @@ -74,11 +75,20 @@ func TestClientWriteOptionsPreserveAPISemantics(t *testing.T) { t.Fatalf("conversion mutated API options: create=%#v update=%#v delete=%#v", create, update, deleteOptions) } - orphanDelete := &metav1.DeleteOptions{OrphanDependents: &orphan, DryRun: []string{metav1.DryRunAll}} + // orphanDependents is deprecated in Go but still accepted on the wire, + // which is how a raw-only option reaches the registry. + orphanDelete := &metav1.DeleteOptions{} + if err := json.Unmarshal([]byte(`{"orphanDependents":true,"dryRun":["All"]}`), orphanDelete); err != nil { + t.Fatal(err) + } orphanExpected := orphanDelete.DeepCopy() - if got := ClientDeleteOptions(orphanDelete).AsDeleteOptions(); !reflect.DeepEqual(got, orphanExpected) { + got := ClientDeleteOptions(orphanDelete).AsDeleteOptions() + if !reflect.DeepEqual(got, orphanExpected) { t.Fatalf("raw-only delete option lost: got %#v, want %#v", got, orphanExpected) } + if wire, err := json.Marshal(got); err != nil || !strings.Contains(string(wire), `"orphanDependents":true`) { + t.Fatalf("converted delete options = %s (%v), want orphanDependents kept", wire, err) + } if !reflect.DeepEqual(orphanDelete, orphanExpected) { t.Fatalf("raw-only delete conversion mutated input: got %#v, want %#v", orphanDelete, orphanExpected) } From 648928f27299118af5d2d1ee1b559cce007e3a18 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 00:23:43 +0300 Subject: [PATCH 161/174] refactor(workloadmonitor): use k8s.io/utils/ptr instead of pointer The pointer package is deprecated; its Bool and BoolPtr are aliases of ptr.To, so the values written are unchanged. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- internal/controller/workloadmonitor_controller.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index b8b0bac13..120ba2533 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -28,7 +28,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "k8s.io/client-go/util/workqueue" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" @@ -423,8 +423,8 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) { Name: monitor.GetName(), UID: monitor.GetUID(), // Set Controller to false to avoid conflict as multiple controllers are not allowed - Controller: pointer.BoolPtr(false), - BlockOwnerDeletion: pointer.BoolPtr(true), + Controller: ptr.To(false), + BlockOwnerDeletion: ptr.To(true), } owners = append(owners, newOwnerRef) } @@ -1174,9 +1174,9 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ // Default to operational = true, but check MinReplicas if set. // Use fresh.Spec to avoid making decisions based on a stale cached copy // when the spec was updated between the initial read and this retry. - fresh.Status.Operational = pointer.Bool(true) + fresh.Status.Operational = ptr.To(true) if fresh.Spec.MinReplicas != nil && availableReplicas < *fresh.Spec.MinReplicas { - fresh.Status.Operational = pointer.Bool(false) + fresh.Status.Operational = ptr.To(false) } // Only the DataVolume check writes Message and Reason, so the stored ones // are the last DataVolume verdict: kept while no reader is installed yet, so a @@ -1189,7 +1189,7 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } if fresh.Status.Message != "" { - fresh.Status.Operational = pointer.Bool(false) + fresh.Status.Operational = ptr.To(false) } if equality.Semantic.DeepEqual(stored, &fresh.Status) { return nil From 86da6c7a5bbfd11aa5a0f5c61481ddfb475e6c63 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 00:26:47 +0300 Subject: [PATCH 162/174] style(lineage-webhook): prefix sentinel error names with Err Go names sentinel error variables Err...; the package is internal and the webhook is their only user. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- internal/lineagecontrollerwebhook/webhook.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/lineagecontrollerwebhook/webhook.go b/internal/lineagecontrollerwebhook/webhook.go index 7b29e1f76..c3efd84bd 100644 --- a/internal/lineagecontrollerwebhook/webhook.go +++ b/internal/lineagecontrollerwebhook/webhook.go @@ -30,8 +30,8 @@ import ( ) var ( - NoAncestors = errors.New("no managed apps found in lineage") - AncestryAmbiguous = errors.New("object ancestry is ambiguous") + ErrNoAncestors = errors.New("no managed apps found in lineage") + ErrAncestryAmbiguous = errors.New("object ancestry is ambiguous") ) const ( @@ -115,9 +115,9 @@ func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Req owner, err := h.getOwner(ctx, obj) switch { - case err != nil && errors.Is(err, AncestryAmbiguous): + case err != nil && errors.Is(err, ErrAncestryAmbiguous): warn = append(warn, "object ancestry ambiguous, using first ancestor found") - case err != nil && errors.Is(err, NoAncestors): + case err != nil && errors.Is(err, ErrNoAncestors): // not a problem, mark object as unmanaged case err != nil: logger.Error(err, "error computing lineage labels") @@ -147,14 +147,14 @@ func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Req func (h *LineageControllerWebhook) getOwner(ctx context.Context, o *unstructured.Unstructured) (*unstructured.Unstructured, error) { owners := lineage.WalkOwnershipGraphWithCache(ctx, h.dynClient, h.mapper, h, h.ownerCache, o) if len(owners) == 0 { - return nil, NoAncestors + return nil, ErrNoAncestors } obj, err := owners[0].GetUnstructuredCached(ctx, h.dynClient, h.mapper, h.ownerCache) if err != nil { return nil, err } if len(owners) > 1 { - err = AncestryAmbiguous + err = ErrAncestryAmbiguous } return obj, err } From 40d340a3fc8147e5d07e6026f75c628db849e91f Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 00:56:58 +0300 Subject: [PATCH 163/174] fix(sse): wait for the pump before returning When a client disconnected, the handler stopped its pump and returned at once, while the pump could still be writing an event or a heartbeat to the response. net/http does not allow a write to a response after ServeHTTP has returned. The handler now waits for the pump to exit; a pump blocked in a write returns once the connection is gone. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- internal/sse/server.go | 1 + internal/sse/server_test.go | 63 +++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/internal/sse/server.go b/internal/sse/server.go index 166b01f49..67e19cbc6 100644 --- a/internal/sse/server.go +++ b/internal/sse/server.go @@ -222,6 +222,7 @@ func (s *Server) handleStream(w http.ResponseWriter, r *http.Request) { case <-pumped: } c.stop() + <-pumped } // stop ends the client's pump. Shutdown and the client's own handler both diff --git a/internal/sse/server_test.go b/internal/sse/server_test.go index 5ae5ad966..ecb0bcd54 100644 --- a/internal/sse/server_test.go +++ b/internal/sse/server_test.go @@ -181,3 +181,66 @@ func TestShutdownEndsOpenStreams(t *testing.T) { t.Fatal("Shutdown left the stream open") } } + +// blockingWriter holds every write after the first until release is closed. +type blockingWriter struct { + streamWriter + writing chan struct{} + release chan struct{} + returned chan struct{} +} + +func (w *blockingWriter) Write(p []byte) (int, error) { + if w.String() == "" { + return w.streamWriter.Write(p) + } + select { + case w.writing <- struct{}{}: + default: + } + <-w.release + return w.streamWriter.Write(p) +} + +// The handler does not return while its pump is still writing: net/http does +// not allow a write to a response after ServeHTTP has returned. +func TestStreamWaitsForPumpBeforeReturning(t *testing.T) { + s := newTestServer(time.Hour) + w := &blockingWriter{ + streamWriter: streamWriter{header: http.Header{}, okWrites: 100}, + writing: make(chan struct{}, 1), + release: make(chan struct{}), + returned: make(chan struct{}), + } + ctx, cancel := context.WithCancel(context.Background()) + r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx) + go func() { + s.handleStream(w, r) + close(w.returned) + }() + deadline := time.Now().Add(time.Second) + for len(s.snapshotClients()) == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + // Straight onto the client's queue, which only the pump reads: Publish + // would also set latest, which the handler itself may still write. + s.snapshotClients()[0].ch <- "event" + select { + case <-w.writing: + case <-time.After(time.Second): + t.Fatal("the pump never wrote the event") + } + cancel() + select { + case <-w.returned: + close(w.release) + t.Fatal("the handler returned while its pump was writing") + case <-time.After(100 * time.Millisecond): + } + close(w.release) + select { + case <-w.returned: + case <-time.After(time.Second): + t.Fatal("the handler did not return once its pump finished") + } +} From 6f2d6ed17b7a712c91f33c49f771db7a2f352af1 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 01:46:20 +0300 Subject: [PATCH 164/174] fix(sse): refuse streams that open while the server shuts down Shutdown stopped the clients registered at that moment and then shut the HTTP server down. A stream accepted in between registered after the stop loop, so nothing ended it, and http.Server.Shutdown waited for its handler. The kube-ovn plunger calls Shutdown without a deadline, so such a stream delayed stopping until the manager's graceful-shutdown timeout and then failed it. Shutdown now marks the server as closing under the same lock, and a stream that opens after that is answered 503 without being registered. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- internal/sse/server.go | 9 +++++++++ internal/sse/server_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/internal/sse/server.go b/internal/sse/server.go index 67e19cbc6..3c121a08a 100644 --- a/internal/sse/server.go +++ b/internal/sse/server.go @@ -51,6 +51,9 @@ type Server struct { clientsMu sync.RWMutex clients map[*client]struct{} + // closing is set by Shutdown under clientsMu, so no stream registers + // after the clients it stops. + closing bool // latest holds the most recent payload (sent to new clients on connect). latestMu sync.RWMutex @@ -122,6 +125,7 @@ func (s *Server) ListenAndServe() error { // Shutdown gracefully stops the server. func (s *Server) Shutdown(ctx context.Context) error { s.clientsMu.Lock() + s.closing = true for c := range s.clients { c.stop() } @@ -184,6 +188,11 @@ func (s *Server) handleStream(w http.ResponseWriter, r *http.Request) { // Register client s.clientsMu.Lock() + if s.closing { + s.clientsMu.Unlock() + http.Error(w, "server is shutting down", http.StatusServiceUnavailable) + return + } s.clients[c] = struct{}{} s.clientsMu.Unlock() diff --git a/internal/sse/server_test.go b/internal/sse/server_test.go index ecb0bcd54..577d19308 100644 --- a/internal/sse/server_test.go +++ b/internal/sse/server_test.go @@ -244,3 +244,32 @@ func TestStreamWaitsForPumpBeforeReturning(t *testing.T) { t.Fatal("the handler did not return once its pump finished") } } + +// A stream that opens once Shutdown has run is refused instead of being +// registered after the clients were stopped, where nothing would end it. +func TestStreamOpenedDuringShutdownIsRefused(t *testing.T) { + s := newTestServer(time.Hour) + if err := s.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + rec := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx) + done := make(chan struct{}) + go func() { + s.handleStream(rec, r) + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("a stream opened after Shutdown stayed open") + } + if rec.Code != http.StatusServiceUnavailable { + t.Errorf("status = %d, want %d", rec.Code, http.StatusServiceUnavailable) + } + if len(s.snapshotClients()) != 0 { + t.Error("a stream opened after Shutdown was registered") + } +} From a3305e8dd3cd1401f844bc31eb495057fb886831 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 03:00:27 +0300 Subject: [PATCH 165/174] fix(backport): push and publish nothing once output is lost backport-batch writes its body, summary, findings and warnings before it pushes, opens or edits a pull request, closes a batch or posts a comment. A run that kept going after that output was lost acted on what nobody read, and a closed pipe was worse: with Go's default SIGPIPE handling the process died silently mid-run. Lines that confirm an action after it happened, such as the URL of an opened pull request, go to stderr, so a lost stderr matters as much as a lost stdout. A failed write to either stream is kept, and from then on every push and every GitHub write is refused at the point it would be made: the GitHub client is wrapped so that each write checks first, and each push checks before it runs. Once output is lost, the run changes nothing more: no push, no GitHub write, no local branch move. No new unit starts after output is lost; a unit already in flight finishes its rollback. A reset whose own hint, the only way back to the commits it drops, cannot be written does not run either. The run exits 2 saying the output was lost; a confirmation lost after an action fails the run too. Such a confirmation is lost only when stderr itself fails, so a lost confirmation can only be reported by the exit code. A test holds every method of the GitHub interface to being a known read or a guarded write. The tool catches SIGPIPE with signal.Notify, so a closed pipe is a write error like any other. It does not use signal.Ignore, whose ignored disposition the git and gh children would inherit across exec; a test checks that a child still dies of SIGPIPE. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- cmd/backport-batch/README.md | 2 +- cmd/backport-batch/assemble.go | 17 +- cmd/backport-batch/defer.go | 2 +- cmd/backport-batch/github.go | 48 +++++ cmd/backport-batch/main.go | 33 ++- cmd/backport-batch/output_test.go | 329 ++++++++++++++++++++++++++++++ cmd/backport-batch/shape.go | 17 ++ 7 files changed, 439 insertions(+), 9 deletions(-) diff --git a/cmd/backport-batch/README.md b/cmd/backport-batch/README.md index 4919035c0..7137bdaf0 100644 --- a/cmd/backport-batch/README.md +++ b/cmd/backport-batch/README.md @@ -27,7 +27,7 @@ Requires `git` and an authenticated `gh` on PATH, and a remote called `origin`. Build the binary wherever an exit code matters: `go run` turns any non-zero exit into its own `1`, so under it a check that could not be completed looks the same as one that found something missing. -A command whose output on stdout could not be written exits 2 where it would have exited 0, and says on stderr that the output was lost; anything the run did before that, a push or a published pull request included, has still happened. +Output is written before a push or a publish. Once a write to stdout or stderr fails, a closed pipe included, the run changes nothing more: no push, no GitHub write, no local branch move, and no new pick; a pick already in flight still finishes its rollback. It exits 2 where it would have exited 0, saying on stderr, while it still can, that the output was lost. A confirmation lost after a push or a publish fails the run the same way. ### run diff --git a/cmd/backport-batch/assemble.go b/cmd/backport-batch/assemble.go index 2ebca3483..7f5529bcd 100644 --- a/cmd/backport-batch/assemble.go +++ b/cmd/backport-batch/assemble.go @@ -273,7 +273,7 @@ func (a *app) run(args []string) int { continue } } - p, err := pickOriginal(v, func(base, head string) ([]string, error) { + p, err := a.pickUnit(v, func(base, head string) ([]string, error) { omitted, diag, err := b.omittedEffects(v, base, head) v.addDiag(diag...) return omitted, err @@ -529,6 +529,9 @@ func cleanTree() error { // the move drops commits of and how to get them back: a clean tree says // nothing about commits only a local branch has. func (a *app) warnReset(branch, start string) error { + if err := a.mayAct(); err != nil { + return err + } old, err := lookup("refs/heads/" + branch) if err != nil || old == "" || old == start { return err @@ -537,7 +540,11 @@ func (a *app) warnReset(branch, start string) error { if err != nil || kept { return err } - _, _ = fmt.Fprintf(a.stderr, "resetting the local %s from %s, dropping commits %.12s does not carry; git branch %s-%.12s %s restores them\n", branch, old, start, branch, old, old) + // The hint is the only way back to the dropped commits, so the reset does + // not run when it could not be written. + if _, err := fmt.Fprintf(a.stderr, "resetting the local %s from %s, dropping commits %.12s does not carry; git branch %s-%.12s %s restores them\n", branch, old, start, branch, old, old); err != nil { + return fmt.Errorf("not resetting the local %s: %w", branch, err) + } return nil } @@ -719,6 +726,9 @@ func (a *app) publish(b *batch, p publication, rebuilt bool) error { } } if pushed != p.head { + if err := a.mayAct(); err != nil { + return err + } if _, err := git(pushArgs(p.branch, p.head, lease)...); err != nil { return err } @@ -878,6 +888,9 @@ func (a *app) deleteBranch(l line, c closing) error { case h != c.Head: return movedBranch(c, h) } + if err := a.mayAct(); err != nil { + return err + } if _, err := git(deleteArgs(c.Branch, c.Head)...); err != nil { return fmt.Errorf("deleting %s, left by the close of #%d: %w; backport-batch run --line %s --push deletes it", c.Branch, c.PR, err, l.branch()) } diff --git a/cmd/backport-batch/defer.go b/cmd/backport-batch/defer.go index de77d5d5c..84e05d5e8 100644 --- a/cmd/backport-batch/defer.go +++ b/cmd/backport-batch/defer.go @@ -102,7 +102,7 @@ func (a *app) reassemble(b *batch, out, hold map[int]string, drops []droppedEntr return a.fail(err) } for _, u := range replay { - failed, files, err := replayUnit(u) + failed, files, err := a.replayNext(u) switch { case err != nil: return a.fail(err) diff --git a/cmd/backport-batch/github.go b/cmd/backport-batch/github.go index 7161ef15b..0e549b1f6 100644 --- a/cmd/backport-batch/github.go +++ b/cmd/backport-batch/github.go @@ -137,6 +137,54 @@ type github interface { batchBot() string } +// guardedGitHub refuses every GitHub write once the run's output was lost. +type guardedGitHub struct { + github + a *app +} + +func (g guardedGitHub) createPull(base, head, title, bodyFile string) (int, string, error) { + if err := g.a.mayAct(); err != nil { + return 0, "", err + } + return g.github.createPull(base, head, title, bodyFile) +} + +func (g guardedGitHub) editPull(number int, title, bodyFile string) error { + if err := g.a.mayAct(); err != nil { + return err + } + return g.github.editPull(number, title, bodyFile) +} + +func (g guardedGitHub) editBody(number int, bodyFile string) error { + if err := g.a.mayAct(); err != nil { + return err + } + return g.github.editBody(number, bodyFile) +} + +func (g guardedGitHub) closePull(number int) error { + if err := g.a.mayAct(); err != nil { + return err + } + return g.github.closePull(number) +} + +func (g guardedGitHub) createComment(number int, bodyFile string) error { + if err := g.a.mayAct(); err != nil { + return err + } + return g.github.createComment(number, bodyFile) +} + +func (g guardedGitHub) editComment(id int64, bodyFile string) error { + if err := g.a.mayAct(); err != nil { + return err + } + return g.github.editComment(id, bodyFile) +} + type ghCLI struct{ bot string } func (g ghCLI) batchBot() string { return g.bot } diff --git a/cmd/backport-batch/main.go b/cmd/backport-batch/main.go index 19102e57f..c49fa951f 100644 --- a/cmd/backport-batch/main.go +++ b/cmd/backport-batch/main.go @@ -24,7 +24,9 @@ import ( "fmt" "io" "os" + "os/signal" "strings" + "syscall" "time" ) @@ -111,21 +113,31 @@ type app struct { policyRef string // tool is the commit backport-batch was built from, for the record. tool string - // unwritten is the first error writing to stdout, which fails the run: - // what it printed never reached its reader. + // unwritten is the first error writing to stdout or stderr. Output goes + // out before a push or a publish, so a run that lost it acts on nothing, + // and one that lost a confirmation afterwards fails. unwritten error } func main() { + catchSIGPIPE() a := &app{gh: ghCLI{bot: batchBotLogin(os.LookupEnv)}, stdout: os.Stdout, stderr: os.Stderr, now: time.Now, tool: toolVersion()} os.Exit(a.main(os.Args[1:])) } +// catchSIGPIPE makes a write to a closed stdout or stderr fail with EPIPE like +// any other write, so the run stops before it pushes or publishes and says +// why, instead of dying silently. signal.Ignore would do the same here but +// would also leave SIGPIPE ignored in the git and gh children, across exec. +func catchSIGPIPE() { + signal.Notify(make(chan os.Signal, 1), syscall.SIGPIPE) +} + func (a *app) main(args []string) int { - a.stdout, a.stderr = textWriter{w: a.stdout, failed: &a.unwritten}, textWriter{w: a.stderr} + a.stdout, a.stderr = textWriter{w: a.stdout, failed: &a.unwritten}, textWriter{w: a.stderr, failed: &a.unwritten} code := a.command(args) if a.unwritten != nil { - _, _ = fmt.Fprintf(a.stderr, "backport-batch: output was lost: %v; anything the run did, such as a push or a publish, still happened\n", a.unwritten) + _, _ = fmt.Fprintf(a.stderr, "backport-batch: output was lost: %v\n", a.unwritten) if code == 0 { code = 2 } @@ -134,7 +146,7 @@ func (a *app) main(args []string) int { } func (a *app) command(args []string) int { - a.gh = &permCache{github: a.gh, perms: map[string]string{}} + a.gh = guardedGitHub{github: &permCache{github: a.gh, perms: map[string]string{}}, a: a} if len(args) == 0 { _, _ = fmt.Fprint(a.stderr, usageText) return 2 @@ -208,6 +220,17 @@ func parse(fs *flag.FlagSet, args []string) error { return nil } +// mayAct refuses any further change once output was lost: the run would act +// on what it printed while nobody read it. Every push and every move of a +// local batch branch go through it, and every GitHub write through +// guardedGitHub. +func (a *app) mayAct() error { + if a.unwritten != nil { + return fmt.Errorf("output was lost: %w; nothing more is pushed or written to GitHub", a.unwritten) + } + return nil +} + func (a *app) fail(err error) int { _, _ = fmt.Fprintf(a.stderr, "backport-batch: %v\n", err) return 2 diff --git a/cmd/backport-batch/output_test.go b/cmd/backport-batch/output_test.go index 0362f44b7..a094d4b75 100644 --- a/cmd/backport-batch/output_test.go +++ b/cmd/backport-batch/output_test.go @@ -18,6 +18,10 @@ package main import ( "errors" + "io" + "os" + "os/exec" + "reflect" "strings" "testing" "time" @@ -54,3 +58,328 @@ func TestUnwritableOutputKeepsTheCommandsFailure(t *testing.T) { t.Errorf("stderr does not name the write failure:\n%s", stderr.String()) } } + +// A closed pipe on stdout does not kill the run before it can say what +// already happened: the write fails like any other, and the run exits 2. +func TestClosedStdoutPipeIsReported(t *testing.T) { + if os.Getenv("BACKPORT_BATCH_CLOSED_STDOUT_HELPER") == "1" { + os.Args = []string{"backport-batch", "help"} + main() + return + } + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + _ = r.Close() + t.Cleanup(func() { _ = w.Close() }) + cmd := exec.Command(os.Args[0], "-test.run=^TestClosedStdoutPipeIsReported$") + cmd.Env = append(os.Environ(), "BACKPORT_BATCH_CLOSED_STDOUT_HELPER=1") + var stderr strings.Builder + cmd.Stdout, cmd.Stderr = w, &stderr + err = cmd.Run() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 2 { + t.Fatalf("help into a closed pipe = %v, want exit 2\nstderr:\n%s", err, stderr.String()) + } + if !strings.Contains(stderr.String(), "output was lost") { + t.Errorf("stderr does not say the output was lost:\n%s", stderr.String()) + } +} + +// failOn fails every write that contains want, and takes every other one. +type failOn struct { + want string + buf strings.Builder +} + +func (w *failOn) Write(p []byte) (int, error) { + if strings.Contains(string(p), w.want) { + return 0, errors.New("broken pipe") + } + return w.buf.Write(p) +} + +func runWith(f *fixture, gh *fakeGH, stdout, stderr io.Writer, args ...string) int { + f.t.Helper() + if gh.remoteHead == nil { + gh.remoteHead = f.remoteHead + } + a := &app{gh: gh, stdout: stdout, stderr: stderr, now: func() time.Time { return f.now }, tool: f.tool} + return a.main(args) +} + +// Output is written before the push; when it is lost, on either stream, +// nothing is pushed and no pull request is opened. +func TestLostOutputPushesNothing(t *testing.T) { + for _, stream := range []string{"stdout", "stderr"} { + t.Run(stream, func(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{} + var stdout, stderr io.Writer = &strings.Builder{}, &strings.Builder{} + if stream == "stdout" { + stdout = failingWriter{} + } else { + stderr = failingWriter{} + } + if code := runWith(f, gh, stdout, stderr, "run", "--line", "release-1.1", "--push"); code != 2 { + t.Fatalf("run --push with a failing %s = %d, want 2", stream, code) + } + if head := f.remoteHead(batchBranch1); head != "" { + t.Errorf("the batch was pushed at %.12s", head) + } + if len(gh.prs) != 0 { + t.Errorf("a pull request was opened: %+v", gh.prs) + } + }) + } +} + +// A confirmation lost after the push fails the run: the pull request was +// opened, and the run does not exit 0 without having said so. +func TestLostConfirmationFailsTheRun(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{} + if code := runWith(f, gh, &strings.Builder{}, &failOn{want: "opened draft"}, "run", "--line", "release-1.1", "--push"); code != 2 { + t.Fatalf("run --push losing its confirmation = %d, want 2", code) + } + if len(gh.prs) != 1 { + t.Errorf("%d pull requests, want the one the run opened", len(gh.prs)) + } + if len(gh.issueComments) != 0 { + t.Errorf("the report was posted after the confirmation was lost: %+v", gh.issueComments) + } +} + +// notify writes what it found before it comments; when that is lost, it +// does not comment. +func TestLostOutputCommentsNothing(t *testing.T) { + gh := &fakeGH{prs: []pullRequest{notifyPR("20261019", "")}} + var stderr strings.Builder + a := &app{gh: gh, stdout: failingWriter{}, stderr: &stderr, now: func() time.Time { return time.Date(2026, 10, 26, 6, 23, 0, 0, time.UTC) }} + if code := a.main([]string{"notify", "--line", "release-1.1", "--push"}); code != 2 { + t.Fatalf("notify --push with a failing stdout = %d, want 2\n%s", code, stderr.String()) + } + if len(gh.issueComments) != 0 { + t.Errorf("notify commented: %+v", gh.issueComments) + } +} + +// writesGitHub records the GitHub writes that reach it. +type writesGitHub struct { + github + writes []string +} + +func (g *writesGitHub) createPull(string, string, string, string) (int, string, error) { + g.writes = append(g.writes, "createPull") + return 1, "", nil +} +func (g *writesGitHub) editPull(int, string, string) error { + g.writes = append(g.writes, "editPull") + return nil +} +func (g *writesGitHub) editBody(int, string) error { + g.writes = append(g.writes, "editBody") + return nil +} +func (g *writesGitHub) closePull(int) error { + g.writes = append(g.writes, "closePull") + return nil +} +func (g *writesGitHub) createComment(int, string) error { + g.writes = append(g.writes, "createComment") + return nil +} +func (g *writesGitHub) editComment(int64, string) error { + g.writes = append(g.writes, "editComment") + return nil +} + +// Every GitHub write is refused once output was lost. Every method of +// github is either a read or in this table, so a write added later without +// a guard fails here. +func TestEveryGitHubWriteIsGuarded(t *testing.T) { + reads := map[string]bool{"openPulls": true, "recentPulls": true, "permission": true, "labelled": true, "pull": true, "comments": true, "batchBot": true} + inner := &writesGitHub{} + g := guardedGitHub{github: inner, a: &app{unwritten: errors.New("broken pipe")}} + writes := map[string]func() error{ + "createPull": func() error { _, _, err := g.createPull("b", "h", "t", "f"); return err }, + "editPull": func() error { return g.editPull(1, "t", "f") }, + "editBody": func() error { return g.editBody(1, "f") }, + "closePull": func() error { return g.closePull(1) }, + "createComment": func() error { return g.createComment(1, "f") }, + "editComment": func() error { return g.editComment(1, "f") }, + } + iface := reflect.TypeFor[github]() + for i := range iface.NumMethod() { + name := iface.Method(i).Name + if _, ok := writes[name]; !ok && !reads[name] { + t.Errorf("github.%s is neither a known read nor a guarded write", name) + } + } + for name, write := range writes { + if err := write(); err == nil || !strings.Contains(err.Error(), "output was lost") { + t.Errorf("%s with output lost = %v, want a refusal", name, err) + } + } + if len(inner.writes) != 0 { + t.Errorf("writes reached GitHub after output was lost: %q", inner.writes) + } +} + +// A warning lost while the open batch is being updated stops the update: +// nothing is pushed, posted or edited. +func TestLostWarningLeavesTheOpenBatchAlone(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{} + mustRun(t, f, gh, "run", "--line", "release-1.1", "--push") + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + bad := issueComment{ID: 8, Body: reportMarker + "names none\n"} + bad.User.Login, bad.User.Type = "maintainer", "User" + gh.issueComments = append(gh.issueComments, bad) + head, comments, body := f.remoteHead(batchBranch1), len(gh.issueComments), gh.find(90).Body + f.now = f.now.AddDate(0, 0, 1) + if code := runWith(f, gh, &strings.Builder{}, &failOn{want: "does not count as a report"}, "run", "--line", "release-1.1", "--continue", "--push"); code != 2 { + t.Fatalf("run --continue --push losing a warning = %d, want 2", code) + } + if got := f.remoteHead(batchBranch1); got != head { + t.Errorf("the batch moved to %.12s", got) + } + if len(gh.issueComments) != comments { + t.Errorf("a comment was posted: %d comments, want %d", len(gh.issueComments), comments) + } + if gh.find(90).Body != body { + t.Error("the batch pull request was edited") + } +} + +// A warning lost while a batch is being closed leaves it open, with no +// closing record half written. +func TestLostWarningLeavesTheBatchUnclosed(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{} + mustRun(t, f, gh, "run", "--line", "release-1.1", "--push") + bad := issueComment{ID: 8, Body: reportMarker + "names none\n"} + bad.User.Login, bad.User.Type = "maintainer", "User" + gh.issueComments = append(gh.issueComments, bad) + body := gh.find(90).Body + if code := runWith(f, gh, &strings.Builder{}, &failOn{want: "does not count as a report"}, "run", "--line", "release-1.1", "--defer", "5", "--reason", "x", "--push"); code != 2 { + t.Fatalf("run --defer losing a warning = %d, want 2", code) + } + if gh.find(90).Body != body { + t.Error("the batch pull request was edited") + } + if !gh.isOpen(90) { + t.Error("the batch pull request was closed") + } +} + +// The git and gh children keep the default SIGPIPE: catching it here must not +// leave it ignored in them, as signal.Ignore would through exec. +func TestChildrenKeepTheDefaultSIGPIPE(t *testing.T) { + catchSIGPIPE() + out, err := exec.Command("sh", "-c", "kill -PIPE $$; echo survived").Output() + if err == nil || strings.Contains(string(out), "survived") { + t.Fatalf("a child survived SIGPIPE: %q, %v", out, err) + } +} + +// The branch a close left is not deleted once output was lost: a deletion +// is a push like any other. +func TestLostOutputDeletesNoBranch(t *testing.T) { + f, gh, h1 := closeWorld(t) + f.git("switch", "-q", "main") + l, err := parseLine("release-1.1") + if err != nil { + t.Fatal(err) + } + a := &app{gh: gh, unwritten: errors.New("broken pipe")} + if err := a.deleteBranch(l, closing{PR: 90, Branch: batchBranch1, Head: h1}); err == nil || !strings.Contains(err.Error(), "output was lost") { + t.Fatalf("deleteBranch with output lost = %v, want a refusal", err) + } + if got := f.remoteHead(batchBranch1); got != h1 { + t.Errorf("the branch is at %q, want it left at %.12s", got, h1) + } +} + +// A park line lost on stdout stops notify before it writes: no comment, and +// the record is not marked as announced. +func TestLostParkLineWritesNothing(t *testing.T) { + park := parkedEntry{Original: 5, Merge: strings.Repeat("c", 40), Kind: parkDependency, Needs: []int{3}} + gh := &fakeGH{prs: []pullRequest{notifyPR("20261026", parkedBody(park))}} + body := gh.prs[0].Body + var stderr strings.Builder + a := &app{gh: gh, stdout: &failOn{want: "parked originals"}, stderr: &stderr, now: func() time.Time { return time.Date(2026, 10, 26, 6, 23, 0, 0, time.UTC) }} + if code := a.main([]string{"notify", "--line", "release-1.1", "--push"}); code != 2 { + t.Fatalf("notify --push losing the park line = %d, want 2\n%s", code, stderr.String()) + } + if len(gh.issueComments) != 0 { + t.Errorf("notify commented: %+v", gh.issueComments) + } + if gh.prs[0].Body != body { + t.Error("notify marked the park as announced") + } +} + +// The hint naming the commits a reset drops is the only way back to them, so +// the local branch is not reset when the hint is lost. +func TestLostResetHintKeepsTheBranch(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.git("switch", "-q", "-C", batchBranch1, "origin/release-1.1") + f.write("app/mine.txt", "unpushed\n") + f.git("add", "-A") + f.git("commit", "-q", "-m", "an unpushed resolution") + tip := f.git("rev-parse", "HEAD") + f.git("switch", "-q", "main") + if code := runWith(f, &fakeGH{}, &strings.Builder{}, &failOn{want: "resetting the local"}, "run", "--line", "release-1.1"); code != 2 { + t.Fatalf("run losing the reset hint = %d, want 2", code) + } + if got := f.git("rev-parse", batchBranch1); got != tip { + t.Errorf("the local %s moved to %.12s, want it kept at %.12s", batchBranch1, got, tip) + } +} + +// Once output is lost the run moves no local branch, even one a reset would +// only fast-forward. +func TestLostOutputMovesNoLocalBranch(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.git("branch", "-f", batchBranch1, "origin/release-1.1^") + before := f.git("rev-parse", batchBranch1) + if code := runWith(f, &fakeGH{}, &strings.Builder{}, &failOn{want: "prepared"}, "run", "--line", "release-1.1"); code != 2 { + t.Fatalf("run losing an early line = %d, want 2", code) + } + if got := f.git("rev-parse", batchBranch1); got != before { + t.Errorf("the local %s moved to %.12s, want it kept at %.12s", batchBranch1, got, before) + } +} + +// No new unit starts once output is lost: neither a pick nor a replay touches +// the branch. +func TestLostOutputStartsNoUnit(t *testing.T) { + f := newFixture(t) + f.lines() + before := f.git("rev-parse", "HEAD") + a := &app{unwritten: errors.New("broken pipe")} + if _, err := a.pickUnit(&verdict{}, nil); err == nil || !strings.Contains(err.Error(), "output was lost") { + t.Errorf("pickUnit with output lost = %v, want a refusal", err) + } + if _, _, err := a.replayNext(unit{}); err == nil || !strings.Contains(err.Error(), "output was lost") { + t.Errorf("replayNext with output lost = %v, want a refusal", err) + } + if got := f.git("rev-parse", "HEAD"); got != before { + t.Errorf("HEAD moved to %.12s", got) + } +} diff --git a/cmd/backport-batch/shape.go b/cmd/backport-batch/shape.go index 31d2da501..b4a1e759c 100644 --- a/cmd/backport-batch/shape.go +++ b/cmd/backport-batch/shape.go @@ -114,6 +114,23 @@ func mergeMessage(message, merge string) string { return strings.TrimRight(message, " \t\r\n") + "\n\n" + batchTrailer + ": " + merge + "\n" } +// pickUnit and replayNext start the next unit only while the run's output is +// intact: once it was lost no new unit starts, and a unit already in flight +// still finishes its rollback. +func (a *app) pickUnit(v *verdict, validate func(base, head string) ([]string, error)) (pick, error) { + if err := a.mayAct(); err != nil { + return pick{}, err + } + return pickOriginal(v, validate) +} + +func (a *app) replayNext(u unit) (string, []string, error) { + if err := a.mayAct(); err != nil { + return "", nil, err + } + return replayUnit(u) +} + // commitMerge makes the merge of tip onto base, whose tree is tip's since tip // descends from base, and moves the current branch from tip to it. func commitMerge(base, tip, message string) (string, error) { From 456ea6b98b7d90e25a120f4b1d4afd24f1993261 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 11:58:30 +0300 Subject: [PATCH 166/174] ci: run the standalone pkg and cmd unit tests in go-unit-tests go-unit-tests ran only pkg/registry, pkg/config and pkg/cmd/server, so the tests in pkg/lineage, pkg/ovnstatus, pkg/reservation, cmd/cozypkg and cmd/check-readiness ran on no CI job. None of them needs a cluster or the network. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- Makefile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 2feea00e9..adc652dd7 100644 --- a/Makefile +++ b/Makefile @@ -139,13 +139,14 @@ rd-presets-check: migrations-target-check: hack/check-migrations-target.sh -# Scoped go test over the cozystack-api surface that this repo owns. Kept -# narrow intentionally - running `go test ./...` pulls in generated code -# round-trip suites whose behavior depends on tool versions outside this -# repo's control (kubebuilder, openapi-gen, etc.) and is better exercised -# from their generator workflows. +# Go tests of the cozystack-api surface and of the standalone packages and +# commands listed here, whose tests need no cluster. Kept narrow +# intentionally - running `go test ./...` pulls in generated code round-trip +# suites whose behavior depends on tool versions outside this repo's control +# (kubebuilder, openapi-gen, etc.) and is better exercised from their +# generator workflows. go-unit-tests: - go test ./pkg/registry/... ./pkg/config/... ./pkg/cmd/server/... + go test ./pkg/registry/... ./pkg/config/... ./pkg/cmd/server/... ./pkg/lineage/... ./pkg/ovnstatus/... ./pkg/reservation/... ./cmd/cozypkg/... ./cmd/check-readiness/... # The nested Go modules (image sources and the published API types) are # outside ./... of the root module, so no other target reaches their tests. @@ -158,9 +159,7 @@ go-module-tests: done # Go tests for the controllers and supporting packages under ./internal. -# Excludes ./pkg/... and ./cmd/... — those are run separately by -# go-unit-tests above (pkg subset) and skipped (cmd) until their tests -# stabilise. CI schedules this target in the same four-slot make invocation as +# CI schedules this target in the same four-slot make invocation as # unit-tests; locally invoke it directly or chain the two targets. test-controllers: go test ./internal/... -count=1 @@ -195,7 +194,8 @@ test-controllers-envtest: $(SETUP_ENVTEST) test-check-readiness: go test ./test/check-readiness/ -count=1 -# ./cmd/... is excluded from go-unit-tests, so the batch tool needs its own target. +# The batch tool's suite drives real git repositories for minutes, so it is +# kept out of go-unit-tests in a target of its own. test-backport-batch: go test ./cmd/backport-batch/... -count=1 From 6cab97f0107c641638b884278e24eff7e5a54022 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 12:03:19 +0300 Subject: [PATCH 167/174] style: use tagged switches and drop unused test assignments Use tagged switches where staticcheck suggests them and drop two test assignments whose values are never read. Behaviour is unchanged. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- cmd/backport-batch/effects_test.go | 4 ++-- cmd/backport-batch/notify.go | 12 ++++++------ cmd/backport-batch/report.go | 4 ++-- .../backupcontroller/cnpgstrategy_controller_test.go | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/backport-batch/effects_test.go b/cmd/backport-batch/effects_test.go index f96f5a596..9dfe0de95 100644 --- a/cmd/backport-batch/effects_test.go +++ b/cmd/backport-batch/effects_test.go @@ -53,7 +53,7 @@ func emptyBatch(f *fixture, gh *fakeGH, body string) *foldWorld { // and every check of the batch's shape takes the old name as the same file: // verify, a hand fold, and a rebuild by --defer. func TestRenamePickThroughFoldAndVerify(t *testing.T) { - f, mc5 := renameWorld(t) + f, _ := renameWorld(t) f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) gh := &fakeGH{} head := openBatch(t, f, gh) @@ -74,7 +74,7 @@ func TestRenamePickThroughFoldAndVerify(t *testing.T) { t.Errorf("verify after the rebuild = %d\n%s\n%s", code, out, stderr) } - f, mc5 = renameWorld(t) + f, mc5 := renameWorld(t) w := emptyBatch(f, &fakeGH{}, "") pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/old.txt": strings.Replace(appFile, "three", "THREE", 1)}) fold := w.merge(w.head, pick, "fix(app): fix the app file (#5)\n\n"+batchTrailer+": "+mc5) diff --git a/cmd/backport-batch/notify.go b/cmd/backport-batch/notify.go index e9ee23234..b2a8f7c18 100644 --- a/cmd/backport-batch/notify.go +++ b/cmd/backport-batch/notify.go @@ -102,10 +102,10 @@ func (a *app) notify(args []string) int { var conflicts, waiting []int after := map[int][]int{} for n, e := range deferred { - switch place := placement(deferred, n, map[int]bool{}); { - case place == "workflow": + switch placement(deferred, n, map[int]bool{}) { + case "workflow": waiting = append(waiting, n) - case place == "conflict": + case "conflict": conflicts = append(conflicts, n) } if len(e.Needs) > 0 { @@ -259,10 +259,10 @@ func placement(deferred map[int]deferredEntry, n int, seen map[int]bool) string seen[n] = true place := "" for _, f := range e.Needs { - switch p := placement(deferred, f, seen); { - case p == "workflow": + switch p := placement(deferred, f, seen); p { + case "workflow": return p - case p == "conflict": + case "conflict": place = p } } diff --git a/cmd/backport-batch/report.go b/cmd/backport-batch/report.go index fbfc4d569..27fd8860c 100644 --- a/cmd/backport-batch/report.go +++ b/cmd/backport-batch/report.go @@ -780,8 +780,8 @@ func (a *app) printReport(pr *pullRequest, s storedReport, inBody *storedReport, } for _, e := range d.Primary.Parked { text := "parked: " - switch { - case e.Kind == parkPartial: + switch e.Kind { + case parkPartial: text += "only part of it applies, it would leave " + strings.Join(e.Files, ", ") + " without effect" default: var needs []string diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 5cd990f18..d88408100 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -2830,7 +2830,7 @@ func TestReconcileCNPG_ObjectStoreApplyRecovers(t *testing.T) { } failing = false - j, _ = f.pass() + _, _ = f.pass() f.settleDestination() j, backups := f.pass() if len(backups) != 1 { From 21fd2b7b6909811045d40aa5c8140b1757fe6672 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 9 Nov 2026 13:57:59 +0100 Subject: [PATCH 168/174] feat(bucket): add publicRead to serve a bucket without credentials A bucket cannot serve its objects without a signature: every BucketAccess issues a key, and the keys grant neither WriteAcp nor PutBucketPolicy, so a tenant cannot open the bucket themselves with put-bucket-acl. Static assets then need a proxy, presigned URLs or an operator running weed shell by hand, which no reconcile tracks and which outlives the bucket. publicRead renders one more BucketAccess on a new -public access class whose accessPolicy is public-read. The COSI driver handles that policy by adding Read: to the gateway's anonymous identity and removing only that action on revoke, so deleting the bucket or turning the flag off closes it again without touching other public buckets. Anonymous callers can read objects but cannot list the bucket or write to it. The access is named -public-read, the name a user called public-read would take, so that combination fails to render instead of fighting over one BucketAccess. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- api/apps/v1alpha1/bucket/types.go | 3 + packages/apps/bucket/README.md | 15 ++-- .../apps/bucket/templates/bucketclaim.yaml | 15 ++++ .../apps/bucket/tests/public_read_test.yaml | 82 +++++++++++++++++++ packages/apps/bucket/values.schema.json | 5 ++ packages/apps/bucket/values.yaml | 3 + .../templates/client/cosi-bucket-class.yaml | 9 ++ .../storage-pool-bucket-classes.yaml | 9 ++ .../tests/public_access_class_test.yaml | 50 +++++++++++ packages/system/bucket-rd/cozyrds/bucket.yaml | 4 +- .../templates/cosi/cosi-bucket-class.yaml | 9 ++ .../cosi-bucket-class-lock-readonly.patch | 13 ++- .../tests/cosi_public_class_test.yaml | 25 ++++++ 13 files changed, 231 insertions(+), 11 deletions(-) create mode 100644 packages/apps/bucket/tests/public_read_test.yaml create mode 100644 packages/extra/seaweedfs/tests/public_access_class_test.yaml create mode 100644 packages/system/seaweedfs/tests/cosi_public_class_test.yaml diff --git a/api/apps/v1alpha1/bucket/types.go b/api/apps/v1alpha1/bucket/types.go index 6632ca530..8a4f331d5 100644 --- a/api/apps/v1alpha1/bucket/types.go +++ b/api/apps/v1alpha1/bucket/types.go @@ -25,6 +25,9 @@ type ConfigSpec struct { // Deploys the web UI (s3manager) with its Service and its Ingress or HTTPRoute at `.`. Set to `false` to run the bucket without them. // +kubebuilder:default:=true Ui bool `json:"ui"` + // Serves the bucket's objects to anyone without credentials at `//`. Anonymous callers can read objects but cannot list the bucket or write to it. + // +kubebuilder:default:=false + PublicRead bool `json:"publicRead"` // Users configuration map. // +kubebuilder:default:={} Users map[string]User `json:"users,omitempty"` diff --git a/packages/apps/bucket/README.md b/packages/apps/bucket/README.md index 13e52d108..4db481810 100644 --- a/packages/apps/bucket/README.md +++ b/packages/apps/bucket/README.md @@ -4,11 +4,12 @@ ### Parameters -| Name | Description | Type | Value | -| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ------- | -| `locking` | Provisions bucket from the `-lock` BucketClass (with object lock enabled). | `bool` | `false` | -| `storagePool` | Selects a specific BucketClass by storage pool name. | `string` | `""` | -| `ui` | Deploys the web UI (s3manager) with its Service and its Ingress or HTTPRoute at `.`. Set to `false` to run the bucket without them. | `bool` | `true` | -| `users` | Users configuration map. | `map[string]object` | `{}` | -| `users[name].readonly` | Whether the user has read-only access. | `bool` | `false` | +| Name | Description | Type | Value | +| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ------- | +| `locking` | Provisions bucket from the `-lock` BucketClass (with object lock enabled). | `bool` | `false` | +| `storagePool` | Selects a specific BucketClass by storage pool name. | `string` | `""` | +| `ui` | Deploys the web UI (s3manager) with its Service and its Ingress or HTTPRoute at `.`. Set to `false` to run the bucket without them. | `bool` | `true` | +| `publicRead` | Serves the bucket's objects to anyone without credentials at `//`. Anonymous callers can read objects but cannot list the bucket or write to it. | `bool` | `false` | +| `users` | Users configuration map. | `map[string]object` | `{}` | +| `users[name].readonly` | Whether the user has read-only access. | `bool` | `false` | diff --git a/packages/apps/bucket/templates/bucketclaim.yaml b/packages/apps/bucket/templates/bucketclaim.yaml index 0639916f0..8ce03d454 100644 --- a/packages/apps/bucket/templates/bucketclaim.yaml +++ b/packages/apps/bucket/templates/bucketclaim.yaml @@ -22,3 +22,18 @@ spec: credentialsSecretName: {{ $.Release.Name }}-{{ $name }} protocol: s3 {{- end }} +{{- if .Values.publicRead }} +{{- if hasKey .Values.users "public-read" }} +{{- fail "publicRead creates the BucketAccess -public-read, which the user \"public-read\" already owns: rename that user" }} +{{- end }} +--- +apiVersion: objectstorage.k8s.io/v1alpha1 +kind: BucketAccess +metadata: + name: {{ .Release.Name }}-public-read +spec: + bucketAccessClassName: {{ $seaweedfs }}{{- if $pool }}-{{ $pool }}{{- end }}-public + bucketClaimName: {{ .Release.Name }} + credentialsSecretName: {{ .Release.Name }}-public-read + protocol: s3 +{{- end }} diff --git a/packages/apps/bucket/tests/public_read_test.yaml b/packages/apps/bucket/tests/public_read_test.yaml new file mode 100644 index 000000000..1bdf5d82b --- /dev/null +++ b/packages/apps/bucket/tests/public_read_test.yaml @@ -0,0 +1,82 @@ +suite: bucket publicRead +templates: + - templates/bucketclaim.yaml + +release: + name: bucket-my-bucket + namespace: tenant-root + +set: + _namespace: + seaweedfs: tenant-root + +tests: + - it: renders no public BucketAccess by default + asserts: + - hasDocuments: + count: 1 + - isKind: + of: BucketClaim + + - it: renders a public-read BucketAccess on the -public class + set: + publicRead: true + documentSelector: + path: metadata.name + value: bucket-my-bucket-public-read + asserts: + - isKind: + of: BucketAccess + - equal: + path: spec.bucketAccessClassName + value: tenant-root-public + - equal: + path: spec.bucketClaimName + value: bucket-my-bucket + - equal: + path: spec.credentialsSecretName + value: bucket-my-bucket-public-read + + - it: uses the storage pool's -public class + set: + publicRead: true + storagePool: fast + documentSelector: + path: metadata.name + value: bucket-my-bucket-public-read + asserts: + - equal: + path: spec.bucketAccessClassName + value: tenant-root-fast-public + + - it: keeps the access class free of -lock on a locked bucket + set: + publicRead: true + locking: true + documentSelector: + path: metadata.name + value: bucket-my-bucket-public-read + asserts: + - equal: + path: spec.bucketAccessClassName + value: tenant-root-public + + - it: keeps the users' BucketAccesses next to the public one + set: + publicRead: true + users: + app: {} + reader: + readonly: true + asserts: + - hasDocuments: + count: 4 + + - it: refuses a user that would own the public BucketAccess name + set: + publicRead: true + users: + public-read: {} + asserts: + - failedTemplate: + errorPattern: user "public-read" already owns diff --git a/packages/apps/bucket/values.schema.json b/packages/apps/bucket/values.schema.json index 249169395..641822c51 100644 --- a/packages/apps/bucket/values.schema.json +++ b/packages/apps/bucket/values.schema.json @@ -20,6 +20,11 @@ "type": "boolean", "default": true }, + "publicRead": { + "description": "Serves the bucket's objects to anyone without credentials at `\u003cendpoint\u003e/\u003cbucket\u003e/\u003ckey\u003e`. Anonymous callers can read objects but cannot list the bucket or write to it.", + "type": "boolean", + "default": false + }, "users": { "description": "Users configuration map.", "type": "object", diff --git a/packages/apps/bucket/values.yaml b/packages/apps/bucket/values.yaml index 3b79909fc..a5d87b9b3 100644 --- a/packages/apps/bucket/values.yaml +++ b/packages/apps/bucket/values.yaml @@ -8,6 +8,9 @@ storagePool: "" ## @param {bool} ui=true - Deploys the web UI (s3manager) with its Service and its Ingress or HTTPRoute at `.`. Set to `false` to run the bucket without them. ui: true +## @param {bool} publicRead=false - Serves the bucket's objects to anyone without credentials at `//`. Anonymous callers can read objects but cannot list the bucket or write to it. +publicRead: false + ## @typedef {struct} User - Bucket user configuration. ## @field {bool} [readonly] - Whether the user has read-only access. diff --git a/packages/extra/seaweedfs/templates/client/cosi-bucket-class.yaml b/packages/extra/seaweedfs/templates/client/cosi-bucket-class.yaml index ce049b316..0b262f47d 100644 --- a/packages/extra/seaweedfs/templates/client/cosi-bucket-class.yaml +++ b/packages/extra/seaweedfs/templates/client/cosi-bucket-class.yaml @@ -35,4 +35,13 @@ driverName: {{ .Release.Namespace }}.seaweedfs.objectstorage.k8s.io authenticationType: KEY parameters: accessPolicy: readonly +--- +kind: BucketAccessClass +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: {{ .Release.Namespace }}-public +driverName: {{ .Release.Namespace }}.seaweedfs.objectstorage.k8s.io +authenticationType: KEY +parameters: + accessPolicy: public-read {{- end }} diff --git a/packages/extra/seaweedfs/templates/storage-pool-bucket-classes.yaml b/packages/extra/seaweedfs/templates/storage-pool-bucket-classes.yaml index bf58340a2..faa99f137 100644 --- a/packages/extra/seaweedfs/templates/storage-pool-bucket-classes.yaml +++ b/packages/extra/seaweedfs/templates/storage-pool-bucket-classes.yaml @@ -51,5 +51,14 @@ driverName: {{ $.Release.Namespace }}.seaweedfs.objectstorage.k8s.io authenticationType: KEY parameters: accessPolicy: readonly +--- +kind: BucketAccessClass +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: {{ $.Release.Namespace }}-{{ $poolName }}-public +driverName: {{ $.Release.Namespace }}.seaweedfs.objectstorage.k8s.io +authenticationType: KEY +parameters: + accessPolicy: public-read {{- end }} {{- end }} diff --git a/packages/extra/seaweedfs/tests/public_access_class_test.yaml b/packages/extra/seaweedfs/tests/public_access_class_test.yaml new file mode 100644 index 000000000..67ea3239c --- /dev/null +++ b/packages/extra/seaweedfs/tests/public_access_class_test.yaml @@ -0,0 +1,50 @@ +suite: seaweedfs public-read BucketAccessClass +release: + name: seaweedfs + namespace: tenant-root + +set: + filer: + grpcHost: seaweedfs-filer.tenant-other.svc + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + +tests: + - it: offers a -public access class to a Client instance + template: templates/client/cosi-bucket-class.yaml + set: + topology: Client + documentSelector: + path: metadata.name + value: tenant-root-public + asserts: + - isKind: + of: BucketAccessClass + - equal: + path: parameters.accessPolicy + value: public-read + - equal: + path: driverName + value: tenant-root.seaweedfs.objectstorage.k8s.io + + - it: offers a -public access class per storage pool + template: templates/storage-pool-bucket-classes.yaml + set: + topology: Simple + volume: + pools: + fast: + diskType: ssd + documentSelector: + path: metadata.name + value: tenant-root-fast-public + asserts: + - isKind: + of: BucketAccessClass + - equal: + path: parameters.accessPolicy + value: public-read diff --git a/packages/system/bucket-rd/cozyrds/bucket.yaml b/packages/system/bucket-rd/cozyrds/bucket.yaml index f75c736bd..0f05f4dfe 100644 --- a/packages/system/bucket-rd/cozyrds/bucket.yaml +++ b/packages/system/bucket-rd/cozyrds/bucket.yaml @@ -8,7 +8,7 @@ spec: plural: buckets singular: bucket openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"locking":{"description":"Provisions bucket from the `-lock` BucketClass (with object lock enabled).","type":"boolean","default":false},"storagePool":{"description":"Selects a specific BucketClass by storage pool name.","type":"string","default":"","x-cozystack-options":{"source":"storagepool"}},"ui":{"description":"Deploys the web UI (s3manager) with its Service and its Ingress or HTTPRoute at `.`. Set to `false` to run the bucket without them.","type":"boolean","default":true},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"readonly":{"description":"Whether the user has read-only access.","type":"boolean"}}}}}} + {"title":"Chart Values","type":"object","properties":{"locking":{"description":"Provisions bucket from the `-lock` BucketClass (with object lock enabled).","type":"boolean","default":false},"storagePool":{"description":"Selects a specific BucketClass by storage pool name.","type":"string","default":"","x-cozystack-options":{"source":"storagepool"}},"ui":{"description":"Deploys the web UI (s3manager) with its Service and its Ingress or HTTPRoute at `.`. Set to `false` to run the bucket without them.","type":"boolean","default":true},"publicRead":{"description":"Serves the bucket's objects to anyone without credentials at `//`. Anonymous callers can read objects but cannot list the bucket or write to it.","type":"boolean","default":false},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"readonly":{"description":"Whether the user has read-only access.","type":"boolean"}}}}}} release: prefix: bucket- labels: @@ -26,7 +26,7 @@ spec: tags: - storage icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODNfMzA5MSkiLz4KPHBhdGggZmlsbC1ydWxlPSJldmVub2RkIiBjbGlwLXJ1bGU9ImV2ZW5vZGQiIGQ9Ik03MiAzMC4xNjQxTDExNy45ODMgMzYuNzc4OVY0MC42NzM5QzExNy45ODMgNDYuNDY1MyA5Ny4zODYyIDUxLjEzMzIgNzEuOTgyNyA1MS4xMzMyQzQ2LjU3OTIgNTEuMTMzMiAyNiA0Ni40NjUzIDI2IDQwLjY3MzlWMzYuNDQzMUw3MiAzMC4xNjQxWk03MiA1OC4yNjc4QzkxLjIwODQgNTguMjY3OCAxMDcuNjU4IDU1LjU5ODYgMTE0LjU0NyA1MS44MDQ4TDExNi44MDMgNDguMTExTDExNy43MjMgNDQuNzUzVjQ4LjkxNzFMMTAyLjY3OSAxMTEuMDMzQzEwMi42NzkgMTE0Ljg5NSA4OC45NTMzIDExOCA3Mi4wMTcyIDExOEM1NS4wODEyIDExOCA0MS4zNzQzIDExNC44OTUgNDEuMzc0MyAxMTEuMDMzTDI2LjMzIDQ4LjkxNzFWNDQuODM2OUwyOS44MDA3IDUxLjkzODJDMzYuNzA2NSA1NS42NjUzIDUyLjk5OTcgNTguMjY3OCA3MiA1OC4yNjc4WiIgZmlsbD0iIzhDMzEyMyIvPgo8cGF0aCBmaWxsLXJ1bGU9ImV2ZW5vZGQiIGNsaXAtcnVsZT0iZXZlbm9kZCIgZD0iTTcyLjAwMDMgMjZDOTcuNDAzOCAyNiAxMTggMzAuNjgzOSAxMTggMzYuNDQyQzExOCA0Mi4yIDk3LjM4NjYgNDYuODUwNyA3Mi4wMDAzIDQ2Ljg1MDdDNDYuNjE0MSA0Ni44NTA3IDI2LjAxNzYgNDIuMjM0NSAyNi4wMTc2IDM2LjQ0MkMyNi4wMTc2IDMwLjY0OTQgNDYuNTk2OCAyNiA3Mi4wMDAzIDI2Wk03Mi4wMDAzIDU0LjEwMzdDOTUuNjg1NyA1NC4xMDM3IDExNS4xNzIgNTAuMDU4IDExNy43MDYgNDQuODE5N0wxMDIuNjYyIDEwNi45MzdDMTAyLjY2MiAxMTAuNzk5IDg4LjkzNjQgMTEzLjkwNSA3Mi4wMDAzIDExMy45MDVDNTUuMDY0MyAxMTMuOTA1IDQxLjMzOSAxMTAuODE2IDQxLjMzOSAxMDYuOTU0TDI2LjI5NTkgNDQuODM3QzI4Ljg0NjYgNTAuMDU4IDQ4LjMzMzMgNTQuMTAzNyA3Mi4wMDAzIDU0LjEwMzdaIiBmaWxsPSIjRTA1MjQzIi8+CjxwYXRoIGZpbGwtcnVsZT0iZXZlbm9kZCIgY2xpcC1ydWxlPSJldmVub2RkIiBkPSJNNjEuMTcyNSA2MC4wMjkzSDgxLjA5MjhWNzkuMTY3Nkg2MS4xNzI1VjYwLjAyOTNaTTQ1LjMzMDEgOTUuMzY4OEM0NS4zMzAxIDkwLjE0MiA0OS43MTA0IDg1LjkzNDIgNTUuMTUxMSA4NS45MzQyQzYwLjU5MTcgODUuOTM0MiA2NC45NzIxIDkwLjE0MiA2NC45NzIxIDk1LjM2ODhDNjQuOTcyMSAxMDAuNTk2IDYwLjU5MTcgMTA0LjgwMyA1NS4xNTExIDEwNC44MDNDNDkuNzEwNCAxMDQuODAzIDQ1LjMzMDEgMTAwLjU5NiA0NS4zMzAxIDk1LjM2ODhaTTk2LjQ0ODcgMTA0LjM2OEg3Ni43NzIyTDg2LjYxMDUgODYuNzczN0w5Ni40NDg3IDEwNC4zNjhaIiBmaWxsPSJ3aGl0ZSIvPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGlkPSJwYWludDBfbGluZWFyXzY4M18zMDkxIiB4MT0iMCIgeTE9IjAiIHgyPSIxNTEiIHkyPSIxODAiIGdyYWRpZW50VW5pdHM9InVzZXJTcGFjZU9uVXNlIj4KPHN0b3Agc3RvcC1jb2xvcj0iI0ZGRjBFRSIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiNFQzg4N0QiLz4KPC9saW5lYXJHcmFkaWVudD4KPC9kZWZzPgo8L3N2Zz4K - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "locking"], ["spec", "storagePool"], ["spec", "ui"], ["spec", "users"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "locking"], ["spec", "storagePool"], ["spec", "ui"], ["spec", "publicRead"], ["spec", "users"]] secrets: exclude: [] include: diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml index 1ff0e7768..11d7fb779 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml @@ -39,4 +39,13 @@ driverName: {{ .Values.cosi.driverName }} authenticationType: KEY parameters: accessPolicy: readonly +--- +kind: BucketAccessClass +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: {{ .Values.cosi.bucketClassName }}-public +driverName: {{ .Values.cosi.driverName }} +authenticationType: KEY +parameters: + accessPolicy: public-read {{- end }} diff --git a/packages/system/seaweedfs/patches/cosi-bucket-class-lock-readonly.patch b/packages/system/seaweedfs/patches/cosi-bucket-class-lock-readonly.patch index ddca7873e..84ca4fcf2 100644 --- a/packages/system/seaweedfs/patches/cosi-bucket-class-lock-readonly.patch +++ b/packages/system/seaweedfs/patches/cosi-bucket-class-lock-readonly.patch @@ -1,8 +1,8 @@ diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml -index 4d19cc128..1ff0e7768 100644 +index 4d19cc128..11d7fb779 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml -@@ -11,10 +11,32 @@ parameters: +@@ -11,10 +11,41 @@ {{- toYaml . | nindent 2 }} {{- end }} --- @@ -34,4 +34,13 @@ index 4d19cc128..1ff0e7768 100644 +authenticationType: KEY +parameters: + accessPolicy: readonly ++--- ++kind: BucketAccessClass ++apiVersion: objectstorage.k8s.io/v1alpha1 ++metadata: ++ name: {{ .Values.cosi.bucketClassName }}-public ++driverName: {{ .Values.cosi.driverName }} ++authenticationType: KEY ++parameters: ++ accessPolicy: public-read {{- end }} diff --git a/packages/system/seaweedfs/tests/cosi_public_class_test.yaml b/packages/system/seaweedfs/tests/cosi_public_class_test.yaml new file mode 100644 index 000000000..72c815e0d --- /dev/null +++ b/packages/system/seaweedfs/tests/cosi_public_class_test.yaml @@ -0,0 +1,25 @@ +suite: vendored COSI classes include public-read +templates: + - charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + - it: renders a -public BucketAccessClass with the public-read policy + set: + seaweedfs: + cosi: + enabled: true + bucketClassName: tenant-root + driverName: tenant-root.seaweedfs.objectstorage.k8s.io + documentSelector: + path: metadata.name + value: tenant-root-public + asserts: + - isKind: + of: BucketAccessClass + - equal: + path: parameters.accessPolicy + value: public-read From c2d370e95c621dab9910658bacb7368ccf62dda0 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 13:02:13 +0300 Subject: [PATCH 169/174] fix(seaweedfs): pin the COSI driver release with public-read access The public BucketAccessClass relies on the public-read accessPolicy, which the driver supports from v0.5.0; v0.4.0 would hand out keys instead of anonymous read access. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag | 2 +- packages/system/seaweedfs/tests/cosi_image_test.yaml | 8 ++++---- packages/system/seaweedfs/values.yaml | 5 +++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag b/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag index bf4084277..b84ea7484 100644 --- a/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag +++ b/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag @@ -1 +1 @@ -ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.4.0 +ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.5.0 diff --git a/packages/system/seaweedfs/tests/cosi_image_test.yaml b/packages/system/seaweedfs/tests/cosi_image_test.yaml index d43aa176e..4057c989e 100644 --- a/packages/system/seaweedfs/tests/cosi_image_test.yaml +++ b/packages/system/seaweedfs/tests/cosi_image_test.yaml @@ -7,8 +7,8 @@ suite: seaweedfs COSI driver image pin # BucketAccess and ignores the accessPolicy parameter. The chart ships a # -readonly BucketAccessClass with parameters.accessPolicy: readonly, so on # v0.1.2 a credential meant to be read-only is silently granted read-write. -# The package values.yaml overrides cosi.image to v0.4.0, which honours -# accessPolicy: readonly. +# The package values.yaml overrides cosi.image to v0.5.0, which honours +# accessPolicy: readonly and public-read. # # This test does not set the image itself; it relies on the package values.yaml # override, so it goes red if that override is removed and the image falls back @@ -25,7 +25,7 @@ release: namespace: cozy-seaweedfs tests: - - it: pins the cosi-driver container to v0.4.0 (read-only access enforcement) + - it: pins the cosi-driver container to v0.5.0 (read-only and public-read access policies) template: charts/seaweedfs/templates/cosi/cosi-deployment.yaml set: seaweedfs.cosi.enabled: true @@ -35,4 +35,4 @@ tests: value: seaweedfs-cosi-driver - equal: path: spec.template.spec.containers[0].image - value: ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.4.0 + value: ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.5.0 diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 64e2562ba..24881e6f8 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -313,8 +313,9 @@ seaweedfs: # actually issues read-only (Read,List) credentials instead of read-write. # v0.4.0 also writes each grant to the per-identity IAM file: v0.3.1 # rewrites only the legacy identity.json, which the S3 gateways migrate - # concurrently, so a grant could be lost for good. - image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.4.0" + # concurrently, so a grant could be lost for good. v0.5.0 adds the + # public-read accessPolicy the -public BucketAccessClass relies on. + image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.5.0" podLabels: policy.cozystack.io/allow-to-apiserver: "true" driverName: "seaweedfs.objectstorage.k8s.io" From fa1cf776708758aac18f5daf5a64462a2d478b9f Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 13 Nov 2026 11:04:57 +0100 Subject: [PATCH 170/174] fix(seaweedfs): reserve the -public suffix in pool names A pool's classes take its name as a prefix, so a pool named -public renders a readwrite BucketAccessClass --public with the same name as pool 's public-read class, and whichever object wins gives one of the two pools the wrong policy. A pool named public takes -public, the instance's own public-read class. The pool name check already reserves -lock and -readonly for the same reason. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../extra/seaweedfs/templates/seaweedfs.yaml | 8 +-- .../tests/public_access_class_test.yaml | 63 +++++++++++++++++++ 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index 92b40ef4d..ed24b8fe4 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -32,8 +32,8 @@ {{- if not (regexMatch "^[a-z0-9]([a-z0-9-]*[a-z0-9])?$" $poolName) }} {{- fail (printf "volume.pools key '%s' must be a valid DNS label (lowercase alphanumeric and hyphens, no dots)." $poolName) }} {{- end }} -{{- if or (hasSuffix "-lock" $poolName) (hasSuffix "-readonly" $poolName) }} -{{- fail (printf "volume.pools key '%s' must not end with '-lock' or '-readonly' (reserved suffixes for COSI resources)." $poolName) }} +{{- if or (hasSuffix "-lock" $poolName) (hasSuffix "-readonly" $poolName) (hasSuffix "-public" $poolName) (eq "public" $poolName) }} +{{- fail (printf "volume.pools key '%s' must not be 'public' or end with '-lock', '-readonly' or '-public' (reserved for COSI resources)." $poolName) }} {{- end }} {{- if not $pool.diskType }} {{- fail (printf "volume.pools.%s.diskType is required." $poolName) }} @@ -52,8 +52,8 @@ {{- if not (regexMatch "^[a-z0-9]([a-z0-9-]*[a-z0-9])?$" $poolName) }} {{- fail (printf "volume.zones.%s.pools key '%s' must be a valid DNS label." $zoneName $poolName) }} {{- end }} -{{- if or (hasSuffix "-lock" $poolName) (hasSuffix "-readonly" $poolName) }} -{{- fail (printf "volume.zones.%s.pools key '%s' must not end with '-lock' or '-readonly' (reserved suffixes for COSI resources)." $zoneName $poolName) }} +{{- if or (hasSuffix "-lock" $poolName) (hasSuffix "-readonly" $poolName) (hasSuffix "-public" $poolName) (eq "public" $poolName) }} +{{- fail (printf "volume.zones.%s.pools key '%s' must not be 'public' or end with '-lock', '-readonly' or '-public' (reserved for COSI resources)." $zoneName $poolName) }} {{- end }} {{- if not $pool.diskType }} {{- fail (printf "volume.zones.%s.pools.%s.diskType is required." $zoneName $poolName) }} diff --git a/packages/extra/seaweedfs/tests/public_access_class_test.yaml b/packages/extra/seaweedfs/tests/public_access_class_test.yaml index 67ea3239c..dbbcc84e7 100644 --- a/packages/extra/seaweedfs/tests/public_access_class_test.yaml +++ b/packages/extra/seaweedfs/tests/public_access_class_test.yaml @@ -48,3 +48,66 @@ tests: - equal: path: parameters.accessPolicy value: public-read + + # A pool named -public would render a readwrite BucketAccessClass + # --public, the same name as pool 's public-read class; a pool named + # public would take -public, the instance's own public-read class. + - it: refuses a pool name ending in -public + template: templates/seaweedfs.yaml + set: + volume: + pools: + fast: + diskType: ssd + fast-public: + diskType: nvme + asserts: + - failedTemplate: + errorPattern: volume.pools key 'fast-public' must not be 'public' or end with + + - it: refuses a pool named public + template: templates/seaweedfs.yaml + set: + volume: + pools: + public: + diskType: ssd + asserts: + - failedTemplate: + errorPattern: volume.pools key 'public' must not be 'public' or end with + + - it: refuses a zone pool name ending in -public + template: templates/seaweedfs.yaml + set: + topology: MultiZone + replicationFactor: 1 + volume: + zones: + a: + pools: + fast-public: + diskType: ssd + asserts: + - failedTemplate: + errorPattern: volume.zones.a.pools key 'fast-public' must not be 'public' or end with + + - it: still refuses a pool name ending in -readonly + template: templates/seaweedfs.yaml + set: + volume: + pools: + fast-readonly: + diskType: ssd + asserts: + - failedTemplate: + errorPattern: volume.pools key 'fast-readonly' must not be 'public' or end with + + - it: accepts a pool name that only contains public + template: templates/seaweedfs.yaml + set: + volume: + pools: + publicfast: + diskType: ssd + asserts: + - notFailedTemplate: {} From e3e20180752219d23aff6c1ac26b909b9085b965 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 14 Sep 2026 10:25:13 +0200 Subject: [PATCH 171/174] fix(seaweedfs, cozy-lib): cap the Go heap so components survive their memory limit Every SeaweedFS component is a Go binary running under a cgroup memory limit, and none of them set GOMEMLIMIT. The collector then works from GOGC alone, a target relative to the live heap with no absolute ceiling, so a burst of allocation grows the heap past the limit and the kernel kills the container. Seen on an S3 gateway holding 6.4 GiB of heap_sys against an 8 GiB limit while only 389 MiB was actually in use: ten kills in seven days, each one an interruption of the object store every tenant writes its backups to. Raising the limit does not help, since nothing bounds the growth; it only moves the ceiling the heap climbs to. Derive the value from the limit already computed for each component, at the same 75% the javaHeap helper uses for the JVM. The remaining quarter covers what GOMEMLIMIT does not account for, goroutine stacks and mmap'd regions among them. Volume pools and zones carry their own value: the subchart merges volume into every entry of volumes, so a pool would otherwise inherit the value sized for the default volume's limit, above its own. The lifecycle admin and worker run the same binary under limits the parent chart does not set, so their value is pinned next to those limits. A limit under 2Mi floors to 0MiB, which Go accepts and then collects almost continuously; the render fails instead. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../extra/seaweedfs/templates/seaweedfs.yaml | 40 ++++- .../seaweedfs/tests/gomemlimit_test.yaml | 162 ++++++++++++++++++ .../library/cozy-lib/templates/_resources.tpl | 20 +++ .../seaweedfs/tests/lifecycle_test.yaml | 30 ++++ packages/system/seaweedfs/values.yaml | 8 + .../templates/tests/resources-gomemlimit.yaml | 8 + .../tests/resources_gomemlimit_test.yaml | 61 +++++++ .../tests/resources_gomemlimit_values.yaml | 16 ++ 8 files changed, 338 insertions(+), 7 deletions(-) create mode 100644 packages/extra/seaweedfs/tests/gomemlimit_test.yaml create mode 100644 packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml create mode 100644 packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml create mode 100644 packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index 92b40ef4d..d69a3e493 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -248,13 +248,19 @@ spec: {{- /* An explicit null is dropped before schema validation, so neither bound sees it, and the master exits on the empty flag it renders. */}} volumeSizeLimitMB: {{ required "master.volumeSizeLimitMB must be set" .Values.master.volumeSizeLimitMB }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.master.resourcesPreset .Values.master.resources $) | nindent 10 }} +{{- $masterResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.master.resourcesPreset .Values.master.resources $) | fromYaml }} + resources: {{- toYaml $masterResources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $masterResources | quote }} volume: {{ if eq .Values.topology "MultiZone" }} enabled: false {{- end }} replicas: {{ .Values.volume.replicas }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.volume.resourcesPreset .Values.volume.resources $) | nindent 10 }} +{{- $volumeResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.volume.resourcesPreset .Values.volume.resources $) | fromYaml }} + resources: {{- toYaml $volumeResources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $volumeResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -273,7 +279,13 @@ spec: {{- range $poolName, $pool := .Values.volume.pools }} {{ $poolName }}: replicas: {{ ternary $pool.replicas $.Values.volume.replicas (hasKey $pool "replicas") }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | nindent 12 }} +{{- $poolResources := include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | fromYaml }} + resources: {{- toYaml $poolResources | nindent 12 }} + {{- /* The subchart merges volume into every entry of volumes, so a + pool that does not set its own value runs on the default + volume's, sized for a different limit. */}} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $poolResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -289,7 +301,9 @@ spec: {{- range $zoneName, $zone := .Values.volume.zones }} {{ $zoneName }}: replicas: {{ ternary $zone.replicas $.Values.volume.replicas (hasKey $zone "replicas") }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list $.Values.volume.resourcesPreset $.Values.volume.resources $) | nindent 12 }} + resources: {{- toYaml $volumeResources | nindent 12 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $volumeResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -314,7 +328,13 @@ spec: {{- range $poolName, $pool := (dig "pools" dict $zone) }} {{ $zoneName }}-{{ $poolName }}: replicas: {{ ternary $pool.replicas (ternary $zone.replicas $.Values.volume.replicas (hasKey $zone "replicas")) (hasKey $pool "replicas") }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | nindent 12 }} +{{- $poolResources := include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | fromYaml }} + resources: {{- toYaml $poolResources | nindent 12 }} + {{- /* The subchart merges volume into every entry of volumes, so a + pool that does not set its own value runs on the default + volume's, sized for a different limit. */}} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $poolResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -345,7 +365,10 @@ spec: replicas: {{ .Values.filer.replicas }} s3: domainName: {{ .Values.host | default (printf "s3.%s" $host) }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.filer.resourcesPreset .Values.filer.resources $) | nindent 10 }} +{{- $filerResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.filer.resourcesPreset .Values.filer.resources $) | fromYaml }} + resources: {{- toYaml $filerResources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $filerResources | quote }} s3: replicas: {{ .Values.s3.replicas }} ingress: @@ -381,7 +404,10 @@ spec: {{- if not $wildcardSecret }} secretName: {{ .Release.Name }}-s3-ingress-tls {{- end }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.s3.resourcesPreset .Values.s3.resources $) | nindent 10 }} +{{- $s3Resources := include "cozy-lib.resources.defaultingSanitize" (list .Values.s3.resourcesPreset .Values.s3.resources $) | fromYaml }} + resources: {{- toYaml $s3Resources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $s3Resources | quote }} admin: enabled: {{ .Values.lifecycle.enabled }} secret: diff --git a/packages/extra/seaweedfs/tests/gomemlimit_test.yaml b/packages/extra/seaweedfs/tests/gomemlimit_test.yaml new file mode 100644 index 000000000..2b298b10e --- /dev/null +++ b/packages/extra/seaweedfs/tests/gomemlimit_test.yaml @@ -0,0 +1,162 @@ +suite: seaweedfs components carry a GOMEMLIMIT derived from their memory limit +templates: + - templates/seaweedfs.yaml + +# Every SeaweedFS component is a Go binary. Without GOMEMLIMIT the runtime +# collects on a relative target and keeps no absolute ceiling, so a burst of +# allocation grows the heap past the cgroup limit and the kernel kills the +# container. Each component sizes its own value from its own memory limit, +# so the assertions below track the limit rather than a fixed string. + +tests: + - it: derives GOMEMLIMIT from the default preset on every component + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + asserts: + - equal: + path: spec.values.seaweedfs.master.resources.limits.memory + value: 512Mi + - equal: + path: spec.values.seaweedfs.master.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + - equal: + path: spec.values.seaweedfs.volume.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + - equal: + path: spec.values.seaweedfs.filer.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + - equal: + path: spec.values.seaweedfs.s3.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + + - it: follows the memory limit when a component is resized + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + s3: + resources: + memory: 8Gi + volume: + resources: + memory: 2Gi + asserts: + - equal: + path: spec.values.seaweedfs.s3.resources.limits.memory + value: 8Gi + - equal: + path: spec.values.seaweedfs.s3.extraEnvironmentVars.GOMEMLIMIT + value: 6144MiB + - equal: + path: spec.values.seaweedfs.volume.extraEnvironmentVars.GOMEMLIMIT + value: 1536MiB + - equal: + path: spec.values.seaweedfs.master.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + + - it: keeps GOMEMLIMIT below the cgroup limit it protects + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + s3: + resources: + memory: 1Gi + asserts: + - equal: + path: spec.values.seaweedfs.s3.resources.limits.memory + value: 1Gi + - equal: + path: spec.values.seaweedfs.s3.extraEnvironmentVars.GOMEMLIMIT + value: 768MiB + + # The subchart merges volume into every entry of volumes, so a pool that + # did not carry its own value would run on the default volume's, sized for + # a larger limit than its own. + - it: sizes a Simple pool from its own limit, not the default volume's + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + volume: + pools: + small: + diskType: ssd + resourcesPreset: t1.nano + asserts: + - equal: + path: spec.values.seaweedfs.volumes.small.resources.limits.memory + value: 128Mi + - equal: + path: spec.values.seaweedfs.volumes.small.extraEnvironmentVars.GOMEMLIMIT + value: 96MiB + - equal: + path: spec.values.seaweedfs.volume.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + + - it: sizes zones and zone pools from their own limits in MultiZone + documentSelector: + path: kind + value: HelmRelease + set: + topology: MultiZone + replicationFactor: 1 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + volume: + resources: + memory: 2Gi + zones: + a: + pools: + small: + diskType: ssd + resourcesPreset: t1.nano + asserts: + - equal: + path: spec.values.seaweedfs.volumes.a.extraEnvironmentVars.GOMEMLIMIT + value: 1536MiB + - equal: + path: spec.values.seaweedfs.volumes.a-small.resources.limits.memory + value: 128Mi + - equal: + path: spec.values.seaweedfs.volumes.a-small.extraEnvironmentVars.GOMEMLIMIT + value: 96MiB diff --git a/packages/library/cozy-lib/templates/_resources.tpl b/packages/library/cozy-lib/templates/_resources.tpl index 48b0718fd..9b621fe0d 100644 --- a/packages/library/cozy-lib/templates/_resources.tpl +++ b/packages/library/cozy-lib/templates/_resources.tpl @@ -202,6 +202,26 @@ {{- printf `-Xms%dm -Xmx%dm` $xmsMi $xmxMi }} {{- end }} +{{- /* + goMemLimit takes a sanitized .Values.resources and returns a GOMEMLIMIT + value set to 75% of the memory limit. Without it the Go runtime collects + on a relative target (GOGC) with no absolute ceiling, so a burst of + allocation grows the heap past the cgroup limit and the kernel kills the + container. The 25% headroom covers what GOMEMLIMIT does not account for: + goroutine stacks and any mmap'd region. Accepts only sanitized resource + maps. +*/}} +{{- define "cozy-lib.resources.goMemLimit" }} +{{- $memoryLimitInt := include "cozy-lib.resources.toFloat" .limits.memory | float64 | int64 }} +{{- /* 4194304 is 4Mi */}} +{{- $limitMiB := div (mul $memoryLimitInt 3) 4194304 }} +{{- /* Go accepts 0, and then collects almost without pause. */}} +{{- if lt $limitMiB 1 }} +{{- fail (printf "ERROR: a memory limit of %s leaves no room for a Go heap; GOMEMLIMIT needs at least 2Mi." .limits.memory) }} +{{- end }} +{{- printf `%dMiB` $limitMiB }} +{{- end }} + {{- define "cozy-lib.resources.flatten" -}} {{- $out := dict -}} {{- $res := include "cozy-lib.resources.sanitize" . | fromYaml -}} diff --git a/packages/system/seaweedfs/tests/lifecycle_test.yaml b/packages/system/seaweedfs/tests/lifecycle_test.yaml index 7f3cec8ff..0fd02b8a8 100644 --- a/packages/system/seaweedfs/tests/lifecycle_test.yaml +++ b/packages/system/seaweedfs/tests/lifecycle_test.yaml @@ -57,6 +57,36 @@ tests: path: spec.template.spec.containers[0].command[2] pattern: '-admin=seaweedfs-admin\.tenant-root:23646\.33646 ' + # Both are the weed binary under a fixed memory limit the parent chart does + # not set, so their GOMEMLIMIT is pinned here rather than derived: without + # one the heap has no ceiling and a large lifecycle pass grows it past the + # limit. + - it: caps the admin heap below its memory limit + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: *enabled + asserts: + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 256Mi + - contains: + path: spec.template.spec.containers[0].env + content: + name: GOMEMLIMIT + value: 192MiB + + - it: caps the worker heap below its memory limit + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + set: *enabled + asserts: + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 512Mi + - contains: + path: spec.template.spec.containers[0].env + content: + name: GOMEMLIMIT + value: 384MiB + # The worker executes every job type it registers for. Lifecycle is the only # one this release asks for: vacuum, balance and erasure coding move data and # are not something enabling lifecycle should start doing as a side effect. diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 64e2562ba..81b486002 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -349,6 +349,11 @@ seaweedfs: limits: cpu: "500m" memory: "256Mi" + # 75% of the limit above, as cozy-lib.resources.goMemLimit derives it for + # the components whose resources the parent chart sets. Without it the + # heap has no absolute ceiling and grows past the limit. + extraEnvironmentVars: + GOMEMLIMIT: "192MiB" worker: enabled: false # Vacuum, balance and erasure coding move data; enabling lifecycle must @@ -378,6 +383,9 @@ seaweedfs: limits: cpu: "1" memory: "512Mi" + # 75% of the limit above; see admin.extraEnvironmentVars. + extraEnvironmentVars: + GOMEMLIMIT: "384MiB" certificates: commonName: "SeaweedFS CA" ipAddresses: [] diff --git a/packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml b/packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml new file mode 100644 index 000000000..f9d3739b0 --- /dev/null +++ b/packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-gomemlimit +data: +{{- range $name, $resources := .Values.sanitizedResources }} + {{ $name }}: {{ include "cozy-lib.resources.goMemLimit" $resources | quote }} +{{- end }} diff --git a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml new file mode 100644 index 000000000..17a7d0b97 --- /dev/null +++ b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml @@ -0,0 +1,61 @@ +# ./tests/resources_gomemlimit_test.yaml +suite: resources goMemLimit helper + +# The helper turns a memory limit into a GOMEMLIMIT the Go runtime accepts, +# at 75% of the limit. Two properties matter and neither is visible from a +# single value: the result carries the MiB suffix Go parses, and it stays +# strictly under the limit it protects, whatever unit the quantity came in. +# A value that came out at or above the limit would leave the runtime with +# no headroom and the container would still be killed. + +templates: + - templates/tests/resources-gomemlimit.yaml + +tests: + - it: takes 75% of the limit and keeps the unit Go parses + values: + - resources_gomemlimit_values.yaml + + release: + name: myrelease + namespace: default + + asserts: + - equal: + path: data.binaryGi + value: "6144MiB" + + - equal: + path: data.binaryMi + value: "384MiB" + + # A decimal quantity is smaller than its binary lookalike: 1G is + # 1000000000 bytes, so 75% of it lands below 768MiB. + - equal: + path: data.decimalG + value: "715MiB" + + - equal: + path: data.bareBytes + value: "768MiB" + + # The smallest limit that still yields a usable value: 75% of 2Mi + # floors to 1MiB. + - equal: + path: data.twoMiB + value: "1MiB" + + # Integer division floors, so a limit under 2Mi comes out at 0MiB. Go + # accepts that and then collects almost without pause, slowing the + # component instead of protecting it, so the render stops instead. + - it: refuses a limit too small to leave a Go heap + values: + - resources_gomemlimit_values.yaml + set: + sanitizedResources: + belowOneMiB: + limits: + memory: "1Mi" + asserts: + - failedTemplate: + errorPattern: "leaves no room for a Go heap" diff --git a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml new file mode 100644 index 000000000..3e367cc21 --- /dev/null +++ b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml @@ -0,0 +1,16 @@ +sanitizedResources: + binaryGi: + limits: + memory: "8Gi" + binaryMi: + limits: + memory: "512Mi" + decimalG: + limits: + memory: "1G" + bareBytes: + limits: + memory: "1073741824" + twoMiB: + limits: + memory: "2Mi" From 3b893c626af984810b87acae19dd7377f5eb874f Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 13 Nov 2026 11:43:13 +0100 Subject: [PATCH 172/174] fix(cozy-lib): state the real lower bound in the goMemLimit error The limit that floors to 0MiB is anything under 4/3 Mi, not under 2Mi: 1536Ki still yields 1MiB. The message named the wrong bound and the tests pinned 2Mi and 1Mi, neither of which sits on the edge. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../library/cozy-lib/templates/_resources.tpl | 2 +- .../tests/resources_gomemlimit_test.yaml | 15 ++++++++------- .../tests/resources_gomemlimit_values.yaml | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/library/cozy-lib/templates/_resources.tpl b/packages/library/cozy-lib/templates/_resources.tpl index 9b621fe0d..5830e1400 100644 --- a/packages/library/cozy-lib/templates/_resources.tpl +++ b/packages/library/cozy-lib/templates/_resources.tpl @@ -217,7 +217,7 @@ {{- $limitMiB := div (mul $memoryLimitInt 3) 4194304 }} {{- /* Go accepts 0, and then collects almost without pause. */}} {{- if lt $limitMiB 1 }} -{{- fail (printf "ERROR: a memory limit of %s leaves no room for a Go heap; GOMEMLIMIT needs at least 2Mi." .limits.memory) }} +{{- fail (printf "ERROR: a memory limit of %s leaves no room for a Go heap; GOMEMLIMIT needs a limit of at least 1398102 bytes (4/3 Mi) to come out at 1MiB." .limits.memory) }} {{- end }} {{- printf `%dMiB` $limitMiB }} {{- end }} diff --git a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml index 17a7d0b97..1753f925f 100644 --- a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml +++ b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml @@ -39,15 +39,16 @@ tests: path: data.bareBytes value: "768MiB" - # The smallest limit that still yields a usable value: 75% of 2Mi - # floors to 1MiB. + # The smallest limit that still yields a usable value: 75% of + # 1398102 bytes (4/3 Mi, rounded up) floors to 1MiB. - equal: - path: data.twoMiB + path: data.smallestUsable value: "1MiB" - # Integer division floors, so a limit under 2Mi comes out at 0MiB. Go - # accepts that and then collects almost without pause, slowing the - # component instead of protecting it, so the render stops instead. + # Integer division floors, so a limit one byte under 4/3 Mi comes out + # at 0MiB. Go accepts that and then collects almost without pause, + # slowing the component instead of protecting it, so the render stops + # instead. - it: refuses a limit too small to leave a Go heap values: - resources_gomemlimit_values.yaml @@ -55,7 +56,7 @@ tests: sanitizedResources: belowOneMiB: limits: - memory: "1Mi" + memory: "1398101" asserts: - failedTemplate: errorPattern: "leaves no room for a Go heap" diff --git a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml index 3e367cc21..f6be917e0 100644 --- a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml +++ b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml @@ -11,6 +11,6 @@ sanitizedResources: bareBytes: limits: memory: "1073741824" - twoMiB: + smallestUsable: limits: - memory: "2Mi" + memory: "1398102" From f3bf92acc1562f0a33374838b6ea49c741cc2cb2 Mon Sep 17 00:00:00 2001 From: "cozystack-ci[bot]" <274107086+cozystack-ci[bot]@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:57:32 +0000 Subject: [PATCH 173/174] docs: add changelog for v1.7.1 Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com> --- docs/changelogs/v1.7.1.md | 82 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 docs/changelogs/v1.7.1.md diff --git a/docs/changelogs/v1.7.1.md b/docs/changelogs/v1.7.1.md new file mode 100644 index 000000000..500735d9f --- /dev/null +++ b/docs/changelogs/v1.7.1.md @@ -0,0 +1,82 @@ + + +# Cozystack v1.7.1 + +Cozystack v1.7.1 is a patch release on the v1.7 line. It fixes the COSI driver bug that could leave freshly granted S3 keys rejected for good, makes marketplace taps digest-pinned, makes MariaDB, PostgreSQL and ClickHouse backups report their real outcome, and repairs several tenant-facing issues: tenant cluster metrics, SecurityGroups on Kubernetes workers, Redis with a single replica, the Proxmox cloud-controller-manager and the tcp-balancer. Read the Migration and Upgrades section before upgrading. + +## Fixes + +* **SeaweedFS COSI driver v0.4.0**: bucket users granted together could get S3 keys that never work (`403 InvalidAccessKeyId`) because the v0.3.1 driver rewrote a legacy IAM file the S3 gateways were migrating. The driver is bumped to v0.4.0, which writes each grant to a per-identity IAM file. Keys already lost on running clusters stay broken: remove the user from the Bucket, wait until its BucketAccess is gone, and add it back ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5284, backport #5290). +* **Marketplace taps are pinned to a digest**: a tapped repository no longer follows a mutable tag, so moving the tag upstream cannot change what runs in the cluster. `tap connect` accepts `oci://...@sha256:...` references and an optional keyless cosign verify policy, anchored to an exact identity and preserved across re-connects; with the verify flags the CLI verifies the signature and pins the verified digest at tap time. Community index entries can record a digest too ([**@IvanHunters**](https://github.com/IvanHunters) in #5082, backport #5246). +* **Tenant cluster metrics reach the tenant's Monitoring**: the vmagent of tenant Kubernetes clusters now writes to the tenant's vminsert and labels series with the owning tenant instead of `cozy-monitoring`; before, no tenant cluster metric reached the tenant's Monitoring ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5024, backport #5246). +* **SecurityGroups apply to Kubernetes worker nodes**: the virt-launcher pods of Kubernetes worker nodes are now resolved to their Kubernetes application and marked as managed by Cozystack, so a SecurityGroup attached to a Kubernetes cluster applies to its workers, not only to its control plane ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5124, backport #5246). +* **Membership-only SecurityGroups**: a SecurityGroup with attachments and no rules no longer leaves an invalid CiliumNetworkPolicy behind; its backing policy is accepted by Cilium and still allows or denies nothing ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5133, backport #5246). +* **Proxmox CCM and CSI images**: the Proxmox cloud-controller-manager runs v0.16.1 and the Proxmox CCM and CSI driver images are pinned by digest. The v0.15.0 tag was re-pushed upstream to a build that panics on start, which left every new Proxmox worker with the uninitialized taint ([**@themoriarti**](https://github.com/themoriarti) in #4917, backport #5246). +* **Autoscaler capacity of KubernetesNodes pools**: on the kubevirt substrate the cluster-autoscaler capacity annotation is planned on the cores the VM actually gets, so a leading-zero `resources.cpu` such as `"010"` (decoded as YAML, eight cores) no longer yields a wrong capacity. No pool rolls ([**@themoriarti**](https://github.com/themoriarti) in #4974, backport #5246). +* **Tenant quota admission counts extended resources**: admission now reads the parent pool's usage of extended resources and hugepages under the `requests.*` key the controller counts, so a child tenant can no longer get a share the parent has already used ([**@lexfrei**](https://github.com/lexfrei) in #5184, backport #5246). +* **Postgres backup jobs fail as soon as the CloudNativePG Backup fails**: a Postgres `BackupJob` now fails with the recorded error instead of after 30 minutes, and reports a `WaitingForCNPGBackup` condition with the Backup's phase while it waits ([**@scooby87**](https://github.com/scooby87) in #5211, backport #5246). +* **MariaDB backups and restores report real failures**: a `BackupJob` or `RestoreJob` whose operator Job failed now ends Failed instead of Succeeded, and a failed backup no longer creates a Ready Backup without an archive ([**@scooby87**](https://github.com/scooby87) in #5207, backport #5246). +* **ClickHouse restores use the right archive**: a `RestoreJob` restores the archive its Backup was taken as, not the newest one. New Backups record the archive name; Backups taken before this release are restored best effort from the newest archive stamped no later than their `takenAt`, with a warning in the restore pod log. A ClickHouse `BackupJob` created under a name whose Backup still exists now fails instead of reporting that Backup as its result ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5208, backport #5246). +* **Backup S3 endpoint changes are followed**: the backup controller re-renders the Velero BSL, the default backup strategies and the platform bucket's credentials Secret when their S3 endpoint is stale, so after the platform bucket's BucketAccess is re-granted for a new `publishing.host`, backups move to the new endpoint without forcing releases by hand ([**@scooby87**](https://github.com/scooby87) in #5230, backport #5246). +* **Backup clients run on a dedicated image**: the ClickHouse (Altinity) and RabbitMQ backup strategy Pods now run a small `backup-http-client` image instead of the 349 MB `platform-migrations` image, so a platform migration change no longer re-digests the backup clients. Air-gapped mirrors need the new `ghcr.io/cozystack/cozystack/backup-http-client` repository ([**@androndo**](https://github.com/androndo) in #4756, backport #5246). +* **PostgreSQL system databases**: `databases` refuses `postgres`, `template0` and `template1`, and the init job never runs `DROP DATABASE` on them, which used to fail the init job on every upgrade and leave the HelmRelease in `UpgradeFailed` ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5155, backport #5246). +* **PostgreSQL init script is stable**: the declared databases render in a stable order, so an upgrade that leaves the databases unchanged no longer rewrites the init-script Secret ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5091, backport #5246). +* **Redis with a single replica**: an instance with `replicas: 1` runs one Sentinel instead of three, without a Sentinel PodDisruptionBudget, so node drains are no longer blocked by it; instances with two replicas or more are unchanged ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5151, backport #5246). +* **MariaDB operator on a custom cluster domain**: the operator uses the platform's `networking.clusterDomain` instead of a hard-coded `cozy.local`, so replication and User/Database/Grant reconciliation work on clusters with a non-default domain ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5123, backport #5246). +* **tcp-balancer no longer OOMs**: HAProxy takes its fd and thread caps from the replica's own memory and CPU limits instead of the node, so a `t1.nano` balancer is no longer OOMKilled on nodes with many CPUs ([**@yankawai**](https://github.com/yankawai) in #5005, backport #5246). +* **etcd alert noise**: `etcdHighNumberOfFailedGRPCRequests` counts only the gRPC codes the etcd mixin counts, so the Canceled watch streams of a stopping Kubernetes API server no longer fire it on every member ([**@yankawai**](https://github.com/yankawai) in #4985, backport #5246). +* **Cilium IPAM hook on larger clusters**: the hook passes the CiliumNode list to `jq` on stdin and no longer fails with "Argument list too long" on clusters with more than a few nodes, which blocked the cilium upgrade and every package depending on networking ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5212, backport #5246). +* **OIDC bootstrap Job with many users**: the Job no longer fails to start for a tenant cluster with about 240 or more `oidc.users`; the bindings now come from a ConfigMap the Job mounts ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5213, backport #5246). +* **Dashboard tenant list links to the tenant page**: the tenant name in the tenant list opens its detail page, where Delete lives, and top-level tenants get their Edit button back ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #4004, backport #5246). +* **Dead `registries` platform value removed**: the platform chart no longer has the `registries` value or the `patch-containerd` Secret, which nothing read. Mirrors for tenant workers are the per-pool `talos.registryMirrors` of `KubernetesNodes` ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5190, backport #5246). + +## Security + +* **Marketplace digest pinning and signature verification**: see the marketplace entry under Fixes (#5082). + +## Development, Testing, and CI/CD + +* **Chainsaw asserts survive transient API errors**: an e2e assert keeps polling through an error such as an etcd leader change instead of failing the suite early ([**@lexfrei**](https://github.com/lexfrei) in #4867, backport #5246). +* **Proxmox e2e hygiene**: the Proxmox tenant kubeconfig is kept per run and removed afterwards, the HelmRelease guard reads the `KubernetesNodes` pool release, and the leak allowlist is validated against every suite layout ([**@themoriarti**](https://github.com/themoriarti) in #4971, #4970 and #4972, backports #5246). +* **Frozen-line builds start with `GITHUB_TOKEN`**: the initial packages build dispatched when a release line is frozen now runs ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #4888, backport #5246). +* **Unit tests after the release image restamp**: two unit tests that read fields the release build rewrites stay green on `release-1.7`; this is a release-branch-only fix ([**@themoriarti**](https://github.com/themoriarti) in #5179). + +## Documentation + +* **[website] Documentation**: the v1.7 docs gain the guides that lived only in the cozystack repository, the pending v1.7 docs PRs, a DRBD note for the replicated storage class, a refreshed Chainsaw e2e guide, restored legacy URLs and a grouping of older doc versions ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#763, cozystack/website#782, cozystack/website#675, cozystack/website#611, cozystack/website#775 and cozystack/website#767). +* **[website] Blog and demo**: a full-stack observability post for v1.7, the Cozystack 1.7 announcement, a three-board Raspberry Pi 5 post, and a collapsible navbar with a closable hint on the live demo page ([**@tym83**](https://github.com/tym83) in cozystack/website#790, cozystack/website#801 and cozystack/website#802; [**@kvaps**](https://github.com/kvaps) in cozystack/website#789). +* **[website] Pages deploy**: the deploy workflow drops the Dart Sass install ([**@kvaps**](https://github.com/kvaps) in cozystack/website#798). +* **[ansible-cozystack] Installer v1.7.0**: tracks the Cozystack installer image at v1.7.0, released in ansible-cozystack v1.7.0 ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#89). + +## Migration and Upgrades + +Platform migrations do not change in this release: `migrations.targetVersion` stays at 61. + +* **Remove system databases from PostgreSQL `databases` before upgrading (#5155).** A release that declares `postgres`, `template0` or `template1` under `databases` is refused at render; remove the entry first. A system database a previous version took over is handed back to the superuser on the next upgrade: the event trigger is removed, ownership and the `public` schema are restored, and its `_admin` and `_readonly` roles are dropped. +* **Tenant quota updates can be rejected after upgrading (#5184).** Admission now counts the parent's usage of extended resources and hugepages, so an update to an existing tenant can be rejected when its quota for these resources exceeds the parent's remaining budget, including an update that leaves the quota unchanged. +* **Air-gapped mirrors need `backup-http-client` (#4756).** Mirror `ghcr.io/cozystack/cozystack/backup-http-client` before upgrading, or ClickHouse and RabbitMQ backup Pods cannot pull their image. +* **tcp-balancer pods roll once (#5005).** The new HAProxy `global` section changes the rendered config; `maxconn` follows the memory limit at about 16 connections per MiB (2035 on `t1.nano`). +* **Redis scaled to one replica loses its Sentinel PDB (#5151).** A post-upgrade hook deletes the Sentinel PodDisruptionBudget the operator created earlier for an instance with `replicas: 1`; scaling an instance between 1 and 2 replicas scales its Sentinel Deployment between 1 and 3. +* **Backups created by failed MariaDB runs are not cleaned up (#5207).** Backups made by failed runs before this fix are not removed; restoring from one can fail or select another dump from the shared prefix. +* **Keys lost to the COSI grant bug stay lost (#5284).** Re-add the affected user to the Bucket as described in the Fixes entry. +* **`registries` in the platform Package is ignored (#5190).** A Package that still sets it keeps installing; the `patch-containerd` Secret in `cozy-system` is deleted by the release. +* **SecurityGroups now reach Kubernetes workers (#5124).** A SecurityGroup attached to a `Kubernetes` application starts applying to its worker nodes after the upgrade, so review attached groups before upgrading. + +## Contributors + +We'd like to thank all contributors who made this release possible: + +* [**@androndo**](https://github.com/androndo) +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@kvaps**](https://github.com/kvaps) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@scooby87**](https://github.com/scooby87) +* [**@themoriarti**](https://github.com/themoriarti) +* [**@tym83**](https://github.com/tym83) +* [**@yankawai**](https://github.com/yankawai) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.7.0...v1.7.1 From 5d453c4d238463244e30d328ddbe862daadbb1c3 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Fri, 13 Nov 2026 16:33:57 +0300 Subject: [PATCH 174/174] test(backups): give the fake S3 bucket a StatObject main added StatObject to the object lister interface, so the fake bucket the MariaDB cleanup tests use has to implement it to compile. Assisted-by: LLM Signed-off-by: Aleksei Sviridkin --- .../backupcontroller/mariadbstrategy_cleanup_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go index 1d2f10cbe..7efb3046f 100644 --- a/internal/backupcontroller/mariadbstrategy_cleanup_test.go +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -1368,3 +1368,12 @@ func TestMinioObjectRemover_ReturnsTheServersError(t *testing.T) { t.Fatal("RemoveObject = nil, want the AccessDenied error") } } + +func (f *fakeS3Bucket) StatObject(_ context.Context, _, key string) (int64, error) { + for _, o := range f.objects { + if o.Key == key { + return o.Size, nil + } + } + return 0, errors.New("not found") +}