From 2405f0f3fd637345a2b99cd0d7039c4d0c892957 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:06:42 +0300 Subject: [PATCH 01/28] feat(monitoring): take an Alertmanager template file for alert emails The email receiver added in #3800 sends the default Alertmanager subject and HTML body, a subject that strings together every common label of the group and a table of labels and annotations. Subject and body templates were left out of the values on purpose, because Alertmanager parses inline templates only when it sends, so a broken one drops mail with nothing but a log line to show for it. alerta.alerts.email.template takes the text of a template file instead. The chart stores it in the alertmanager-email-template ConfigMap and lists it in VMAlertmanager spec.templates. The operator mounts it, adds it to the configuration templates and has the config reloader watch the mounted directory, so Alertmanager parses the file while loading its configuration. Defining email.default.subject and email.default.html in it replaces the subject and the body of every email. The text has to contain a define action, a value that is not text fails the render, and template counts as a set field for the check that rejects a partial email configuration. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- packages/extra/monitoring/README.md | 1 + packages/extra/monitoring/values.schema.json | 5 ++ packages/extra/monitoring/values.yaml | 2 + .../monitoring-rd/cozyrds/monitoring.yaml | 4 +- .../monitoring/templates/alerta/alerta.yaml | 26 ++++++- .../tests/alertmanager_email_test.yaml | 67 +++++++++++++++++++ .../alertmanager_email_validation_test.yaml | 25 +++++++ packages/system/monitoring/values.yaml | 1 + 8 files changed, 128 insertions(+), 3 deletions(-) diff --git a/packages/extra/monitoring/README.md b/packages/extra/monitoring/README.md index 95e3a5db0..01d189ab2 100644 --- a/packages/extra/monitoring/README.md +++ b/packages/extra/monitoring/README.md @@ -115,6 +115,7 @@ | `alerta.alerts.email.repeatInterval` | Minimum time before a still-firing alert is emailed again, as an Alertmanager duration with units `y`, `w`, `d`, `h`, `m`, `s`, `ms` in descending order, each at most once, such as `5m`, `12h` or `1h30m`. Delivery is evaluated every 5m, so shorter intervals take effect at 5m and other intervals round up to a multiple of 5m. Zero, a unit out of order and a total beyond what Alertmanager accepts fail the render. Empty keeps the 12h default. | `string` | `""` | | `alerta.alerts.email.alertnames` | Alert names emailed whether or not the rule carries the notify_email label, with no severity filter. Earlier platform routes still take precedence. Alerts with `event="Heartbeat"`, including `Watchdog`, go only to Alerta, and alerts with severity `none`, `CDIDefaultStorageClassDegraded` and `CDINoDefaultStorageClass` are blackholed and never reach email. This is the way to opt in alerts the platform ships, since their helm-owned rule definitions cannot be labelled per cluster. Entries must be plain Prometheus alert names such as `KubeAPIDown`, and anything else fails the render. Empty adds no extra route. | `[]string` | `[]` | | `alerta.alerts.email.severities` | Severities the notify_email route accepts (e.g. ["critical","major","warning"]). Empty keeps the default of critical and major. List every severity in use to email opted-in alerts regardless of severity. Entries must be plain severity names, and anything else fails the render. | `[]string` | `[]` | +| `alerta.alerts.email.template` | Text of an Alertmanager template file that changes how the emails look. Define `email.default.subject` and `email.default.html` in it to replace the subject and the HTML body Alertmanager uses for every email, for example to mark the severity or to link to Alerta. The chart stores the text in a ConfigMap named `alertmanager-email-template` and lists it among the Alertmanager templates, so Alertmanager parses it together with its configuration. A template that does not parse fails that load, a running Alertmanager keeps the configuration it already has and a pod that starts with it does not come up. A template that parses but fails on a notification, for example on a missing key, drops that email. The text must contain at least one `define` action. Empty keeps the Alertmanager defaults. | `string` | `""` | ### Grafana configuration diff --git a/packages/extra/monitoring/values.schema.json b/packages/extra/monitoring/values.schema.json index e65feb743..d69046c74 100644 --- a/packages/extra/monitoring/values.schema.json +++ b/packages/extra/monitoring/values.schema.json @@ -557,6 +557,11 @@ "type": "string", "default": "" }, + "template": { + "description": "Text of an Alertmanager template file that changes how the emails look. Define `email.default.subject` and `email.default.html` in it to replace the subject and the HTML body Alertmanager uses for every email, for example to mark the severity or to link to Alerta. The chart stores the text in a ConfigMap named `alertmanager-email-template` and lists it among the Alertmanager templates, so Alertmanager parses it together with its configuration. A template that does not parse fails that load, a running Alertmanager keeps the configuration it already has and a pod that starts with it does not come up. A template that parses but fails on a notification, for example on a missing key, drops that email. The text must contain at least one `define` action. Empty keeps the Alertmanager defaults.", + "type": "string", + "default": "" + }, "to": { "description": "Recipient address, or several separated by commas. A display name is accepted, such as `Oncall \u003concall@example.org\u003e`, as long as it carries no comma. Quote display names containing punctuation such as `:`, `@`, parentheses or `?`. RFC 2047 encoded words are not accepted. IP domain literals accept dotted-quad IPv4 only. IPv6 literals are rejected because Go mail parsers disagree on their syntax across versions. ASCII control characters are rejected. Addresses must be literal, so Alertmanager template actions are rejected. The accepted form is deliberately narrower than RFC 5322. The local part must be a dot-atom, with no leading, trailing or doubled dot, and quoted local parts, comments and a `mailto:` prefix are rejected. An address outside that form fails the render, because Alertmanager parses addresses only when it sends and drops the notification with no record a tenant can read.", "type": "string", diff --git a/packages/extra/monitoring/values.yaml b/packages/extra/monitoring/values.yaml index cd2716acd..f0d16ffac 100644 --- a/packages/extra/monitoring/values.yaml +++ b/packages/extra/monitoring/values.yaml @@ -163,6 +163,7 @@ tracingCollector: ## @field {string} [repeatInterval] - Minimum time before a still-firing alert is emailed again, as an Alertmanager duration with units `y`, `w`, `d`, `h`, `m`, `s`, `ms` in descending order, each at most once, such as `5m`, `12h` or `1h30m`. Delivery is evaluated every 5m, so shorter intervals take effect at 5m and other intervals round up to a multiple of 5m. Zero, a unit out of order and a total beyond what Alertmanager accepts fail the render. Empty keeps the 12h default. ## @field {[]string} [alertnames] - Alert names emailed whether or not the rule carries the notify_email label, with no severity filter. Earlier platform routes still take precedence. Alerts with `event="Heartbeat"`, including `Watchdog`, go only to Alerta, and alerts with severity `none`, `CDIDefaultStorageClassDegraded` and `CDINoDefaultStorageClass` are blackholed and never reach email. This is the way to opt in alerts the platform ships, since their helm-owned rule definitions cannot be labelled per cluster. Entries must be plain Prometheus alert names such as `KubeAPIDown`, and anything else fails the render. Empty adds no extra route. ## @field {[]string} [severities] - Severities the notify_email route accepts (e.g. ["critical","major","warning"]). Empty keeps the default of critical and major. List every severity in use to email opted-in alerts regardless of severity. Entries must be plain severity names, and anything else fails the render. +## @field {string} [template] - Text of an Alertmanager template file that changes how the emails look. Define `email.default.subject` and `email.default.html` in it to replace the subject and the HTML body Alertmanager uses for every email, for example to mark the severity or to link to Alerta. The chart stores the text in a ConfigMap named `alertmanager-email-template` and lists it among the Alertmanager templates, so Alertmanager parses it together with its configuration. A template that does not parse fails that load, a running Alertmanager keeps the configuration it already has and a pod that starts with it does not come up. A template that parses but fails on a notification, for example on a missing key, drops that email. The text must contain at least one `define` action. Empty keeps the Alertmanager defaults. ## @typedef {struct} Alerts - Alert routing configuration. ## @field {TelegramAlerts} [telegram] - Configuration for Telegram alerts. @@ -202,6 +203,7 @@ alerta: repeatInterval: "" alertnames: [] severities: [] + template: "" ## ## @section Grafana configuration ## diff --git a/packages/system/monitoring-rd/cozyrds/monitoring.yaml b/packages/system/monitoring-rd/cozyrds/monitoring.yaml index 4e02ea731..51b1a2dd7 100644 --- a/packages/system/monitoring-rd/cozyrds/monitoring.yaml +++ b/packages/system/monitoring-rd/cozyrds/monitoring.yaml @@ -20,7 +20,7 @@ spec: singular: monitoring plural: monitorings openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access Grafana externally (defaults to 'grafana' subdomain for the tenant host).","type":"string","default":""},"metricsStorages":{"description":"Configuration of metrics storage instances.","type":"array","default":[{"deduplicationInterval":"15s","name":"shortterm","retentionPeriod":"3d","storage":"10Gi","storageClassName":""},{"deduplicationInterval":"5m","name":"longterm","retentionPeriod":"14d","storage":"10Gi","storageClassName":""}],"items":{"type":"object","required":["deduplicationInterval","name","retentionPeriod","storage"],"properties":{"deduplicationInterval":{"description":"Deduplication interval for metrics.","type":"string"},"name":{"description":"Name of the storage instance.","type":"string"},"retentionPeriod":{"description":"Retention period for metrics.","type":"string"},"storage":{"description":"Persistent volume size.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used for the data.","type":"string"},"vminsert":{"description":"Configuration for vminsert.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"vmselect":{"description":"Configuration for vmselect.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"vmstorage":{"description":"Configuration for vmstorage.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}}}}},"logsStorages":{"description":"Configuration of logs storage instances.","type":"array","default":[{"name":"generic","retentionPeriod":"1","storage":"10Gi","storageClassName":"replicated"}],"items":{"type":"object","required":["name","retentionPeriod","storage","storageClassName"],"properties":{"name":{"description":"Name of the storage instance.","type":"string"},"retentionPeriod":{"description":"Retention period for logs.","type":"string","default":"1"},"storage":{"description":"Persistent volume size.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"}}}},"tracingStorages":{"description":"Configuration of traces storage instances. Empty by default (tracing off): no backend and no trace collector are rendered. Adding an entry provisions a VictoriaTraces backend and, while tracingCollector.enabled is true, deploys the OTLP collector in front of it. A tenant with shared-central tracing (the Tenant's tracingCentral) gets the collector, while tracingCollector.enabled is true, without an entry, which then exports to the shared store; entries still listed keep their store and datasource, so traces stored before the switch stay readable until the entry is removed.","type":"array","default":[],"items":{"type":"object","required":["mode","name","retentionPeriod","storage","storageClassName"],"properties":{"mode":{"description":"Deployment mode: \"cluster\" renders a VTCluster; \"single\" renders a single-binary VTSingle for edge/dev/small clusters. Treat as immutable once a backend exists: mode selects which Kind is rendered under the same name, so changing it on a running backend re-identifies the object rather than reconfiguring it. Switching from \"single\" to \"cluster\" prunes the VTSingle and the operator garbage-collects its PVC (stored traces are lost); switching from \"cluster\" to \"single\" (or removing the entry) leaves the vtstorage PVCs Bound and billed until the whole Monitoring release is deleted. The API cannot guard this (transition rules are not available on an array-item field), so this note is the only guard: migrate deliberately, accepting the loss.","type":"string","default":"cluster","enum":["cluster","single"]},"name":{"description":"Name of the storage instance: a lowercase RFC 1123 label of at most 42 characters. Treat as immutable once a backend exists: the VTCluster/VTSingle, its Services, PVCs and Grafana datasource are all named after it, so renaming an entry deploys a new, empty backend and removes the old one, the same way changing mode does (single mode loses the old PVC, cluster mode leaves the old vtstorage PVCs Bound until the whole Monitoring release is deleted).","type":"string"},"retentionDiskUsageBytes":{"description":"Optional disk-usage cap as a VictoriaMetrics BytesString (e.g. \"8GB\" or \"8GiB\"), bounding disk use on top of the age-based retentionPeriod. Uses byte-suffix grammar (kb/mb/gb/tb, KiB/MiB/GiB/TiB), NOT the Kubernetes-quantity grammar of the sibling storage field, so \"8Gi\" is rejected by the operator. In cluster mode the cap is per vtstorage replica (replicaCount 2), so the effective total is twice the value. Unset, the store drops its oldest day once its filesystem is 80% full instead, so every store is disk-bounded. Either way VictoriaTraces keeps the last two days, so spans arriving faster than the cap allows for two days can still fill the disk.","type":"string"},"retentionPeriod":{"description":"Retention period for traces.","type":"string","default":"14d"},"storage":{"description":"Persistent volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClassName":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"}}}},"tracingCentralHost":{"description":"tenant-root only: accept spans from tenants that turned on shared-central tracing (the Tenant's tracingCentral) into this Monitoring's tracingStorages entry named `generic`, which must be in `cluster` mode. Every central tenant then shares that store's disk and retention with tenant-root, and nothing reserves space per tenant: once the store's disk cap is reached (retentionDiskUsageBytes, or 80% of its filesystem) the oldest spans of all of them are evicted together. Each tenant is held to the platform's monitoring.tracingCentralTenantBytesPerSecond, which by default lets one tenant write at most the cap of a default 10Gi store, counted uncompressed and before the tenant's sampling, over the two days VictoriaTraces always keeps, so raise it with the store; several tenants writing at their limit together still can fill it. That bound holds for a tenant whose collector keeps running: each collector pod starts with a full bucket, up to 5 MiB or one second of its share of the rate when that is larger, and any change to the tenant's Monitoring rolls every collector pod, so a tenant that keeps changing it writes that much more on each rollout, for up to ten replicas; closing that needs a limiter with shared state, which the collector does not run. Ignored outside tenant-root. The tenant-root namespace is inside the trust boundary of the shared store, as it is for the metrics of tenants that rely on a parent's monitoring: vmauth pins each central tenant to its own account, but workloads in tenant-root reach the store directly, and one that learns a tenant's account ID (a hash of its namespace name and UID, which its VMUsers carry) can read and write that account. Whoever tenant-root admits to its Grafana is inside it too, since tenant-root's own datasource for the store reaches vtselect without vmauth; who that is follows its Grafana's oidc mode (with System, tenant-root's own groups, which already hold the same access level in every tenant under tenant-root). The render refuses to stop hosting while central tenants remain, but removing the tenant-root Monitoring altogether is not a render and is not refused: turn tracingCentral off on the central tenants first, since until then every Monitoring render of theirs fails, their metrics, logs and alerting changes included. The refusals read each other's objects when they render, so a tenant turning tracingCentral on while tenant-root stops hosting can pass both: that tenant's collector then exports to a vmauth that is gone while every release reports Ready, until its Monitoring renders again. Recover by hosting again; turning that tenant's tracingCentral off instead ends the silent drop, and turning it on again is then refused until tenant-root hosts.","type":"boolean","default":false},"tracingCollector":{"description":"OpenTelemetry Collector for trace ingestion. With shared-central tracing it refuses spans above the platform's per-tenant write rate: OTLP/HTTP senders retry them, OTLP/gRPC senders drop them, so export over HTTP. It also refuses a request larger than 3 MiB (decompressed) with a 400, which senders do not retry, so keep export batches below that.","type":"object","default":{},"required":["enabled","replicas","samplingPercentage"],"properties":{"enabled":{"description":"Deploy the OTLP collector. When disabled, applications must send OTLP straight to the traces backend (no central sampling/redaction); with shared-central tracing they have no path to it, so no new spans arrive, while the traces already stored stay readable. The collector fronts a single backend, and the limit holds with shared-central tracing too, since turning that off makes it export to the local entry again: while it is enabled, tracingStorages must hold at most one entry, and adding a second one fails the whole Monitoring render until the extra entry is removed or the collector is disabled.","type":"boolean","default":true},"redactAttributes":{"description":"Span attribute keys to drop before export (the PII/secret control point, e.g. db.statement).","type":"array","default":[],"items":{"type":"string"}},"replicas":{"description":"Number of collector replicas. Head sampling is trace-id-deterministic, so replicas>1 are safe and remove the single-ingest-path SPOF. With shared-central tracing each replica sends up to two requests at a time against this tenant's write limit of four per vmauth replica, so more replicas do not raise the tenant's throughput; the tenant's write rate (the platform's monitoring.tracingCentralTenantBytesPerSecond) is split across them as well, so a sender whose connection stays on one replica gets that replica's share, and each replica that starts lets through above the rate up to 5 MiB, or one second of its share when that is larger, which is why shared-central tracing allows at most 10 replicas and fails the render above that.","type":"integer","default":1,"minimum":1},"resources":{"description":"Resource configuration for the collector.","type":"object","default":{},"properties":{"limits":{"description":"Resource limits.","type":"object","default":{},"properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"requests":{"description":"Resource requests.","type":"object","default":{},"properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"samplingPercentage":{"description":"Head-sampling percentage applied at the collector before export.","type":"integer","default":10,"maximum":100,"minimum":0}}},"alerta":{"description":"Configuration for the Alerta service.","type":"object","default":{},"properties":{"alerts":{"description":"Alert routing configuration.","type":"object","default":{},"properties":{"email":{"description":"Configuration for email alerts. Alerts carrying the label notify_email with the string value true are emailed when their severity is in the severities list (default critical and major). A bare YAML boolean does not match. Alerts listed in alertnames are emailed unless matched by an earlier platform route.","type":"object","default":{},"required":["from","smarthost","to"],"properties":{"alertnames":{"description":"Alert names emailed whether or not the rule carries the notify_email label, with no severity filter. Earlier platform routes still take precedence. Alerts with `event=\"Heartbeat\"`, including `Watchdog`, go only to Alerta, and alerts with severity `none`, `CDIDefaultStorageClassDegraded` and `CDINoDefaultStorageClass` are blackholed and never reach email. This is the way to opt in alerts the platform ships, since their helm-owned rule definitions cannot be labelled per cluster. Entries must be plain Prometheus alert names such as `KubeAPIDown`, and anything else fails the render. Empty adds no extra route.","type":"array","default":[],"items":{"type":"string"}},"authUsername":{"description":"Username for SMTP LOGIN/PLAIN authentication. The password is read from the `password` key of a Secret named `alertmanager-email-password` in the release namespace, which the platform operator creates. The name is fixed and the Secret is mounted into Alertmanager, so a missing Secret blocks the whole Alertmanager rollout, not only email delivery. The credential in it must belong to this tenant alone. `smarthost` is chosen by whoever writes the Monitoring, so one relay credential provisioned under the same name in every namespace can be collected by pointing `smarthost` at a host the tenant controls.","type":"string","default":""},"from":{"description":"Sender address, one only. A display name is accepted. It takes the same narrower form as `to`, and an address outside it fails the render.","type":"string","default":""},"repeatInterval":{"description":"Minimum time before a still-firing alert is emailed again, as an Alertmanager duration with units `y`, `w`, `d`, `h`, `m`, `s`, `ms` in descending order, each at most once, such as `5m`, `12h` or `1h30m`. Delivery is evaluated every 5m, so shorter intervals take effect at 5m and other intervals round up to a multiple of 5m. Zero, a unit out of order and a total beyond what Alertmanager accepts fail the render. Empty keeps the 12h default.","type":"string","default":""},"severities":{"description":"Severities the notify_email route accepts (e.g. [\"critical\",\"major\",\"warning\"]). Empty keeps the default of critical and major. List every severity in use to email opted-in alerts regardless of severity. Entries must be plain severity names, and anything else fails the render.","type":"array","default":[],"items":{"type":"string"}},"smarthost":{"description":"SMTP server as `host:port` (e.g. \"smtp.example.com:465\"). A value without a port, an invalid bracketed IP address or a port with leading zeros fails the render. Port 465 uses implicit TLS, other ports use STARTTLS.","type":"string","default":""},"to":{"description":"Recipient address, or several separated by commas. A display name is accepted, such as `Oncall `, as long as it carries no comma. Quote display names containing punctuation such as `:`, `@`, parentheses or `?`. RFC 2047 encoded words are not accepted. IP domain literals accept dotted-quad IPv4 only. IPv6 literals are rejected because Go mail parsers disagree on their syntax across versions. ASCII control characters are rejected. Addresses must be literal, so Alertmanager template actions are rejected. The accepted form is deliberately narrower than RFC 5322. The local part must be a dot-atom, with no leading, trailing or doubled dot, and quoted local parts, comments and a `mailto:` prefix are rejected. An address outside that form fails the render, because Alertmanager parses addresses only when it sends and drops the notification with no record a tenant can read.","type":"string","default":""}}},"slack":{"description":"Configuration for Slack alerts.","type":"object","default":{},"required":["url"],"properties":{"disabledSeverity":{"description":"List of severities without alerts (e.g. [\"informational\",\"warning\"]).","type":"array","default":[],"items":{"type":"string"}},"url":{"description":"Configuration uri for Slack alerts.","type":"string","default":""}}},"telegram":{"description":"Configuration for Telegram alerts.","type":"object","default":{},"required":["chatID","token"],"properties":{"chatID":{"description":"Telegram chat ID(s), separated by commas.","type":"string","default":""},"disabledSeverity":{"description":"List of severities without alerts (e.g. [\"informational\",\"warning\"]).","type":"array","default":[],"items":{"type":"string"}},"token":{"description":"Telegram bot token.","type":"string","default":""}}}}},"resources":{"description":"Resource configuration.","type":"object","default":{},"properties":{"limits":{"description":"Resource limits.","type":"object","default":{},"properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"requests":{"description":"Resource requests.","type":"object","default":{},"properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"storage":{"description":"Persistent volume size for the database.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used for the database.","type":"string","default":""}}},"grafana":{"description":"Configuration for Grafana.","type":"object","default":{},"properties":{"db":{"description":"Database configuration.","type":"object","default":{},"properties":{"size":{"description":"Persistent volume size for the database.","type":"string","default":"10Gi"}}},"resources":{"description":"Resource configuration.","type":"object","default":{},"properties":{"limits":{"description":"Resource limits.","type":"object","default":{},"properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"requests":{"description":"Resource requests.","type":"object","default":{},"properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}}}},"vmagent":{"description":"Configuration for VictoriaMetrics Agent.","type":"object","default":{},"properties":{"externalLabels":{"description":"External labels applied to all metrics.","type":"object","default":{"cluster":"cozystack"}},"remoteWrite":{"description":"Remote write configuration.","type":"object","default":{"urls":["http://vminsert-shortterm:8480/insert/0/prometheus","http://vminsert-longterm:8480/insert/0/prometheus"]}}}},"oidc":{"description":"OIDC authentication for the Grafana instance. See https://cozystack.io/docs/v1.6/operations/services/monitoring/oidc-authentication/ for the operator guide.","type":"object","default":{},"required":["mode"],"properties":{"customConfig":{"description":"Tenant-supplied `[auth.generic_oauth]` payload; consumed only when `mode: CustomConfig`.","type":"object","default":{},"properties":{"config":{"description":"Inline map of `[auth.generic_oauth]` keys. Written verbatim into the Grafana CR `spec.config`. Mutually exclusive with `secretRef.name`.","type":"object","default":{}},"secretRef":{"description":"Reference to a Secret holding an `auth.ini` fragment; mounted under `/etc/grafana/oidc` and loaded via `GF_PATHS_CUSTOM_INI`. With shared-central tracing only its `auth.ini` key is mounted, so the fragment cannot read another key of the Secret with `$__file{}`.","type":"object","default":{},"properties":{"name":{"description":"Name of an existing Secret in the release namespace whose `auth.ini` key holds a ready-made `[auth.generic_oauth]` block. Mutually exclusive with `customConfig.config`.","type":"string","default":""}}}}},"mode":{"description":"Identity mode. `None`: no OIDC, only the `grafana-admin-password` Secret works. `System`: trust the platform `cozy` realm via a per-instance confidential client and audience binding; zero-config default. `CustomConfig`: trust a tenant-supplied issuer directly (BYO); `cozy` is not in the path.","type":"string","default":"None","enum":["None","System","CustomConfig"]},"users":{"description":"Users granted access to this Grafana instance's org. Each entry is reconciled into Grafana org membership + role by a chart-owned post-install/post-upgrade Job. Works for both `System` and `CustomConfig` modes; ignored when `mode: None`. Users not listed here get nothing (no default org role from group mapping).","type":"array","default":[],"items":{"type":"object","required":["email","role"],"properties":{"email":{"description":"Email address matched against the `email` claim from the issuer. The chart pre-provisions a Grafana account with this email so the users-Job can set the org role before the operator's first login (`oauth_allow_insecure_email_lookup = true` glues the two together). Must match a conservative pattern: local part is `[A-Za-z0-9._%+-]+`, domain is `[A-Za-z0-9.-]+\\.[A-Za-z]{2,}`. Whitespace, quoted literals, and unusual URL-reserved characters are rejected on schema level so the reconcile-Job's shell handling never has to defend against malformed input.","type":"string"},"role":{"description":"Grafana org role to bind: `Admin`, `Editor`, or `Viewer`. Applied to the release's Grafana org (`Main Org.`); server-level `GrafanaAdmin` promotion is out of scope for Phase 1. Required — the users-Job POSTs `{loginOrEmail, role}` to `/api/orgs/1/users` and PATCHes `{role}` to converge across re-runs; omitting the field would render `role: \"null\"` into the body and fail with HTTP 400.","type":"string","enum":["Admin","Editor","Viewer"]}}}}}}}} + {"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access Grafana externally (defaults to 'grafana' subdomain for the tenant host).","type":"string","default":""},"metricsStorages":{"description":"Configuration of metrics storage instances.","type":"array","default":[{"deduplicationInterval":"15s","name":"shortterm","retentionPeriod":"3d","storage":"10Gi","storageClassName":""},{"deduplicationInterval":"5m","name":"longterm","retentionPeriod":"14d","storage":"10Gi","storageClassName":""}],"items":{"type":"object","required":["deduplicationInterval","name","retentionPeriod","storage"],"properties":{"deduplicationInterval":{"description":"Deduplication interval for metrics.","type":"string"},"name":{"description":"Name of the storage instance.","type":"string"},"retentionPeriod":{"description":"Retention period for metrics.","type":"string"},"storage":{"description":"Persistent volume size.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used for the data.","type":"string"},"vminsert":{"description":"Configuration for vminsert.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"vmselect":{"description":"Configuration for vmselect.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"vmstorage":{"description":"Configuration for vmstorage.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}}}}},"logsStorages":{"description":"Configuration of logs storage instances.","type":"array","default":[{"name":"generic","retentionPeriod":"1","storage":"10Gi","storageClassName":"replicated"}],"items":{"type":"object","required":["name","retentionPeriod","storage","storageClassName"],"properties":{"name":{"description":"Name of the storage instance.","type":"string"},"retentionPeriod":{"description":"Retention period for logs.","type":"string","default":"1"},"storage":{"description":"Persistent volume size.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"}}}},"tracingStorages":{"description":"Configuration of traces storage instances. Empty by default (tracing off): no backend and no trace collector are rendered. Adding an entry provisions a VictoriaTraces backend and, while tracingCollector.enabled is true, deploys the OTLP collector in front of it. A tenant with shared-central tracing (the Tenant's tracingCentral) gets the collector, while tracingCollector.enabled is true, without an entry, which then exports to the shared store; entries still listed keep their store and datasource, so traces stored before the switch stay readable until the entry is removed.","type":"array","default":[],"items":{"type":"object","required":["mode","name","retentionPeriod","storage","storageClassName"],"properties":{"mode":{"description":"Deployment mode: \"cluster\" renders a VTCluster; \"single\" renders a single-binary VTSingle for edge/dev/small clusters. Treat as immutable once a backend exists: mode selects which Kind is rendered under the same name, so changing it on a running backend re-identifies the object rather than reconfiguring it. Switching from \"single\" to \"cluster\" prunes the VTSingle and the operator garbage-collects its PVC (stored traces are lost); switching from \"cluster\" to \"single\" (or removing the entry) leaves the vtstorage PVCs Bound and billed until the whole Monitoring release is deleted. The API cannot guard this (transition rules are not available on an array-item field), so this note is the only guard: migrate deliberately, accepting the loss.","type":"string","default":"cluster","enum":["cluster","single"]},"name":{"description":"Name of the storage instance: a lowercase RFC 1123 label of at most 42 characters. Treat as immutable once a backend exists: the VTCluster/VTSingle, its Services, PVCs and Grafana datasource are all named after it, so renaming an entry deploys a new, empty backend and removes the old one, the same way changing mode does (single mode loses the old PVC, cluster mode leaves the old vtstorage PVCs Bound until the whole Monitoring release is deleted).","type":"string"},"retentionDiskUsageBytes":{"description":"Optional disk-usage cap as a VictoriaMetrics BytesString (e.g. \"8GB\" or \"8GiB\"), bounding disk use on top of the age-based retentionPeriod. Uses byte-suffix grammar (kb/mb/gb/tb, KiB/MiB/GiB/TiB), NOT the Kubernetes-quantity grammar of the sibling storage field, so \"8Gi\" is rejected by the operator. In cluster mode the cap is per vtstorage replica (replicaCount 2), so the effective total is twice the value. Unset, the store drops its oldest day once its filesystem is 80% full instead, so every store is disk-bounded. Either way VictoriaTraces keeps the last two days, so spans arriving faster than the cap allows for two days can still fill the disk.","type":"string"},"retentionPeriod":{"description":"Retention period for traces.","type":"string","default":"14d"},"storage":{"description":"Persistent volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClassName":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"}}}},"tracingCentralHost":{"description":"tenant-root only: accept spans from tenants that turned on shared-central tracing (the Tenant's tracingCentral) into this Monitoring's tracingStorages entry named `generic`, which must be in `cluster` mode. Every central tenant then shares that store's disk and retention with tenant-root, and nothing reserves space per tenant: once the store's disk cap is reached (retentionDiskUsageBytes, or 80% of its filesystem) the oldest spans of all of them are evicted together. Each tenant is held to the platform's monitoring.tracingCentralTenantBytesPerSecond, which by default lets one tenant write at most the cap of a default 10Gi store, counted uncompressed and before the tenant's sampling, over the two days VictoriaTraces always keeps, so raise it with the store; several tenants writing at their limit together still can fill it. That bound holds for a tenant whose collector keeps running: each collector pod starts with a full bucket, up to 5 MiB or one second of its share of the rate when that is larger, and any change to the tenant's Monitoring rolls every collector pod, so a tenant that keeps changing it writes that much more on each rollout, for up to ten replicas; closing that needs a limiter with shared state, which the collector does not run. Ignored outside tenant-root. The tenant-root namespace is inside the trust boundary of the shared store, as it is for the metrics of tenants that rely on a parent's monitoring: vmauth pins each central tenant to its own account, but workloads in tenant-root reach the store directly, and one that learns a tenant's account ID (a hash of its namespace name and UID, which its VMUsers carry) can read and write that account. Whoever tenant-root admits to its Grafana is inside it too, since tenant-root's own datasource for the store reaches vtselect without vmauth; who that is follows its Grafana's oidc mode (with System, tenant-root's own groups, which already hold the same access level in every tenant under tenant-root). The render refuses to stop hosting while central tenants remain, but removing the tenant-root Monitoring altogether is not a render and is not refused: turn tracingCentral off on the central tenants first, since until then every Monitoring render of theirs fails, their metrics, logs and alerting changes included. The refusals read each other's objects when they render, so a tenant turning tracingCentral on while tenant-root stops hosting can pass both: that tenant's collector then exports to a vmauth that is gone while every release reports Ready, until its Monitoring renders again. Recover by hosting again; turning that tenant's tracingCentral off instead ends the silent drop, and turning it on again is then refused until tenant-root hosts.","type":"boolean","default":false},"tracingCollector":{"description":"OpenTelemetry Collector for trace ingestion. With shared-central tracing it refuses spans above the platform's per-tenant write rate: OTLP/HTTP senders retry them, OTLP/gRPC senders drop them, so export over HTTP. It also refuses a request larger than 3 MiB (decompressed) with a 400, which senders do not retry, so keep export batches below that.","type":"object","default":{},"required":["enabled","replicas","samplingPercentage"],"properties":{"enabled":{"description":"Deploy the OTLP collector. When disabled, applications must send OTLP straight to the traces backend (no central sampling/redaction); with shared-central tracing they have no path to it, so no new spans arrive, while the traces already stored stay readable. The collector fronts a single backend, and the limit holds with shared-central tracing too, since turning that off makes it export to the local entry again: while it is enabled, tracingStorages must hold at most one entry, and adding a second one fails the whole Monitoring render until the extra entry is removed or the collector is disabled.","type":"boolean","default":true},"redactAttributes":{"description":"Span attribute keys to drop before export (the PII/secret control point, e.g. db.statement).","type":"array","default":[],"items":{"type":"string"}},"replicas":{"description":"Number of collector replicas. Head sampling is trace-id-deterministic, so replicas>1 are safe and remove the single-ingest-path SPOF. With shared-central tracing each replica sends up to two requests at a time against this tenant's write limit of four per vmauth replica, so more replicas do not raise the tenant's throughput; the tenant's write rate (the platform's monitoring.tracingCentralTenantBytesPerSecond) is split across them as well, so a sender whose connection stays on one replica gets that replica's share, and each replica that starts lets through above the rate up to 5 MiB, or one second of its share when that is larger, which is why shared-central tracing allows at most 10 replicas and fails the render above that.","type":"integer","default":1,"minimum":1},"resources":{"description":"Resource configuration for the collector.","type":"object","default":{},"properties":{"limits":{"description":"Resource limits.","type":"object","default":{},"properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"requests":{"description":"Resource requests.","type":"object","default":{},"properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"samplingPercentage":{"description":"Head-sampling percentage applied at the collector before export.","type":"integer","default":10,"maximum":100,"minimum":0}}},"alerta":{"description":"Configuration for the Alerta service.","type":"object","default":{},"properties":{"alerts":{"description":"Alert routing configuration.","type":"object","default":{},"properties":{"email":{"description":"Configuration for email alerts. Alerts carrying the label notify_email with the string value true are emailed when their severity is in the severities list (default critical and major). A bare YAML boolean does not match. Alerts listed in alertnames are emailed unless matched by an earlier platform route.","type":"object","default":{},"required":["from","smarthost","to"],"properties":{"alertnames":{"description":"Alert names emailed whether or not the rule carries the notify_email label, with no severity filter. Earlier platform routes still take precedence. Alerts with `event=\"Heartbeat\"`, including `Watchdog`, go only to Alerta, and alerts with severity `none`, `CDIDefaultStorageClassDegraded` and `CDINoDefaultStorageClass` are blackholed and never reach email. This is the way to opt in alerts the platform ships, since their helm-owned rule definitions cannot be labelled per cluster. Entries must be plain Prometheus alert names such as `KubeAPIDown`, and anything else fails the render. Empty adds no extra route.","type":"array","default":[],"items":{"type":"string"}},"authUsername":{"description":"Username for SMTP LOGIN/PLAIN authentication. The password is read from the `password` key of a Secret named `alertmanager-email-password` in the release namespace, which the platform operator creates. The name is fixed and the Secret is mounted into Alertmanager, so a missing Secret blocks the whole Alertmanager rollout, not only email delivery. The credential in it must belong to this tenant alone. `smarthost` is chosen by whoever writes the Monitoring, so one relay credential provisioned under the same name in every namespace can be collected by pointing `smarthost` at a host the tenant controls.","type":"string","default":""},"from":{"description":"Sender address, one only. A display name is accepted. It takes the same narrower form as `to`, and an address outside it fails the render.","type":"string","default":""},"repeatInterval":{"description":"Minimum time before a still-firing alert is emailed again, as an Alertmanager duration with units `y`, `w`, `d`, `h`, `m`, `s`, `ms` in descending order, each at most once, such as `5m`, `12h` or `1h30m`. Delivery is evaluated every 5m, so shorter intervals take effect at 5m and other intervals round up to a multiple of 5m. Zero, a unit out of order and a total beyond what Alertmanager accepts fail the render. Empty keeps the 12h default.","type":"string","default":""},"severities":{"description":"Severities the notify_email route accepts (e.g. [\"critical\",\"major\",\"warning\"]). Empty keeps the default of critical and major. List every severity in use to email opted-in alerts regardless of severity. Entries must be plain severity names, and anything else fails the render.","type":"array","default":[],"items":{"type":"string"}},"smarthost":{"description":"SMTP server as `host:port` (e.g. \"smtp.example.com:465\"). A value without a port, an invalid bracketed IP address or a port with leading zeros fails the render. Port 465 uses implicit TLS, other ports use STARTTLS.","type":"string","default":""},"template":{"description":"Text of an Alertmanager template file that changes how the emails look. Define `email.default.subject` and `email.default.html` in it to replace the subject and the HTML body Alertmanager uses for every email, for example to mark the severity or to link to Alerta. The chart stores the text in a ConfigMap named `alertmanager-email-template` and lists it among the Alertmanager templates, so Alertmanager parses it together with its configuration. A template that does not parse fails that load, a running Alertmanager keeps the configuration it already has and a pod that starts with it does not come up. A template that parses but fails on a notification, for example on a missing key, drops that email. The text must contain at least one `define` action. Empty keeps the Alertmanager defaults.","type":"string","default":""},"to":{"description":"Recipient address, or several separated by commas. A display name is accepted, such as `Oncall `, as long as it carries no comma. Quote display names containing punctuation such as `:`, `@`, parentheses or `?`. RFC 2047 encoded words are not accepted. IP domain literals accept dotted-quad IPv4 only. IPv6 literals are rejected because Go mail parsers disagree on their syntax across versions. ASCII control characters are rejected. Addresses must be literal, so Alertmanager template actions are rejected. The accepted form is deliberately narrower than RFC 5322. The local part must be a dot-atom, with no leading, trailing or doubled dot, and quoted local parts, comments and a `mailto:` prefix are rejected. An address outside that form fails the render, because Alertmanager parses addresses only when it sends and drops the notification with no record a tenant can read.","type":"string","default":""}}},"slack":{"description":"Configuration for Slack alerts.","type":"object","default":{},"required":["url"],"properties":{"disabledSeverity":{"description":"List of severities without alerts (e.g. [\"informational\",\"warning\"]).","type":"array","default":[],"items":{"type":"string"}},"url":{"description":"Configuration uri for Slack alerts.","type":"string","default":""}}},"telegram":{"description":"Configuration for Telegram alerts.","type":"object","default":{},"required":["chatID","token"],"properties":{"chatID":{"description":"Telegram chat ID(s), separated by commas.","type":"string","default":""},"disabledSeverity":{"description":"List of severities without alerts (e.g. [\"informational\",\"warning\"]).","type":"array","default":[],"items":{"type":"string"}},"token":{"description":"Telegram bot token.","type":"string","default":""}}}}},"resources":{"description":"Resource configuration.","type":"object","default":{},"properties":{"limits":{"description":"Resource limits.","type":"object","default":{},"properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"requests":{"description":"Resource requests.","type":"object","default":{},"properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"storage":{"description":"Persistent volume size for the database.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used for the database.","type":"string","default":""}}},"grafana":{"description":"Configuration for Grafana.","type":"object","default":{},"properties":{"db":{"description":"Database configuration.","type":"object","default":{},"properties":{"size":{"description":"Persistent volume size for the database.","type":"string","default":"10Gi"}}},"resources":{"description":"Resource configuration.","type":"object","default":{},"properties":{"limits":{"description":"Resource limits.","type":"object","default":{},"properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"requests":{"description":"Resource requests.","type":"object","default":{},"properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}}}},"vmagent":{"description":"Configuration for VictoriaMetrics Agent.","type":"object","default":{},"properties":{"externalLabels":{"description":"External labels applied to all metrics.","type":"object","default":{"cluster":"cozystack"}},"remoteWrite":{"description":"Remote write configuration.","type":"object","default":{"urls":["http://vminsert-shortterm:8480/insert/0/prometheus","http://vminsert-longterm:8480/insert/0/prometheus"]}}}},"oidc":{"description":"OIDC authentication for the Grafana instance. See https://cozystack.io/docs/v1.6/operations/services/monitoring/oidc-authentication/ for the operator guide.","type":"object","default":{},"required":["mode"],"properties":{"customConfig":{"description":"Tenant-supplied `[auth.generic_oauth]` payload; consumed only when `mode: CustomConfig`.","type":"object","default":{},"properties":{"config":{"description":"Inline map of `[auth.generic_oauth]` keys. Written verbatim into the Grafana CR `spec.config`. Mutually exclusive with `secretRef.name`.","type":"object","default":{}},"secretRef":{"description":"Reference to a Secret holding an `auth.ini` fragment; mounted under `/etc/grafana/oidc` and loaded via `GF_PATHS_CUSTOM_INI`. With shared-central tracing only its `auth.ini` key is mounted, so the fragment cannot read another key of the Secret with `$__file{}`.","type":"object","default":{},"properties":{"name":{"description":"Name of an existing Secret in the release namespace whose `auth.ini` key holds a ready-made `[auth.generic_oauth]` block. Mutually exclusive with `customConfig.config`.","type":"string","default":""}}}}},"mode":{"description":"Identity mode. `None`: no OIDC, only the `grafana-admin-password` Secret works. `System`: trust the platform `cozy` realm via a per-instance confidential client and audience binding; zero-config default. `CustomConfig`: trust a tenant-supplied issuer directly (BYO); `cozy` is not in the path.","type":"string","default":"None","enum":["None","System","CustomConfig"]},"users":{"description":"Users granted access to this Grafana instance's org. Each entry is reconciled into Grafana org membership + role by a chart-owned post-install/post-upgrade Job. Works for both `System` and `CustomConfig` modes; ignored when `mode: None`. Users not listed here get nothing (no default org role from group mapping).","type":"array","default":[],"items":{"type":"object","required":["email","role"],"properties":{"email":{"description":"Email address matched against the `email` claim from the issuer. The chart pre-provisions a Grafana account with this email so the users-Job can set the org role before the operator's first login (`oauth_allow_insecure_email_lookup = true` glues the two together). Must match a conservative pattern: local part is `[A-Za-z0-9._%+-]+`, domain is `[A-Za-z0-9.-]+\\.[A-Za-z]{2,}`. Whitespace, quoted literals, and unusual URL-reserved characters are rejected on schema level so the reconcile-Job's shell handling never has to defend against malformed input.","type":"string"},"role":{"description":"Grafana org role to bind: `Admin`, `Editor`, or `Viewer`. Applied to the release's Grafana org (`Main Org.`); server-level `GrafanaAdmin` promotion is out of scope for Phase 1. Required — the users-Job POSTs `{loginOrEmail, role}` to `/api/orgs/1/users` and PATCHes `{role}` to converge across re-runs; omitting the field would render `role: \"null\"` into the body and fail with HTTP 400.","type":"string","enum":["Admin","Editor","Viewer"]}}}}}}}} release: prefix: "" labels: @@ -38,7 +38,7 @@ spec: description: Monitoring and observability stack module: true icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODdfMzI2OCkiLz4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzY4N18zMjY4KSI+CjxwYXRoIGQ9Ik04OS41MDM5IDExMS43MDdINTQuNDk3QzU0LjE3MjcgMTExLjcwNyA1NC4wMTA4IDExMS4yMjEgNTQuMzM0OSAxMTEuMDU5TDU3LjI1MjIgMTA4Ljk1MkM2MC4zMzE0IDEwNi42ODMgNjEuOTUyMiAxMDIuNjMxIDYwLjk3OTcgOTguNzQxMkg4My4wMjFDODIuMDQ4NSAxMDIuNjMxIDgzLjY2OTMgMTA2LjY4MyA4Ni43NDg1IDEwOC45NTJMODkuNjY1OCAxMTEuMDU5Qzg5Ljk5IDExMS4yMjEgODkuODI3OSAxMTEuNzA3IDg5LjUwMzkgMTExLjcwN1oiIGZpbGw9IiNCMEI2QkIiLz4KPHBhdGggZD0iTTExMy4zMjggOTguNzQxSDMwLjY3MjVDMjcuNTkzMSA5OC43NDEgMjUgOTYuMTQ4IDI1IDkzLjA2ODdWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJWOTMuMDY4N0MxMTkgOTYuMTQ4IDExNi40MDcgOTguNzQxIDExMy4zMjggOTguNzQxWiIgZmlsbD0iI0U4RURFRSIvPgo8cGF0aCBkPSJNMTE5IDg0LjE1NDlIMjVWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJMMTE5IDg0LjE1NDlaIiBmaWxsPSIjMzg0NTRGIi8+CjxwYXRoIGQ9Ik05MC42Mzc0IDExNi41NjlINTMuMzYxNkM1Mi4wNjUxIDExNi41NjkgNTAuOTMwNyAxMTUuNDM1IDUwLjkzMDcgMTE0LjEzOEM1MC45MzA3IDExMi44NDEgNTIuMDY1MSAxMTEuNzA3IDUzLjM2MTYgMTExLjcwN0g5MC42Mzc0QzkxLjkzMzkgMTExLjcwNyA5My4wNjg0IDExMi44NDEgOTMuMDY4NCAxMTQuMTM4QzkzLjA2ODQgMTE1LjQzNSA5MS45MzM5IDExNi41NjkgOTAuNjM3NCAxMTYuNTY5WiIgZmlsbD0iI0U4RURFRSIvPgo8cGF0aCBkPSJNNTQuMTcyMiAzOC43NzU3SDMzLjEwMzJDMzIuMTMwNyAzOC43NzU3IDMxLjQ4MjQgMzguMTI3NCAzMS40ODI0IDM3LjE1NDlDMzEuNDgyNCAzNi4xODI0IDMyLjEzMDcgMzUuNTM0MiAzMy4xMDMyIDM1LjUzNDJINTQuMTcyMkM1NS4xNDQ3IDM1LjUzNDIgNTUuNzkzIDM2LjE4MjQgNTUuNzkzIDM3LjE1NDlDNTUuNzkyOCAzOC4xMjc0IDU1LjE0NDUgMzguNzc1NyA1NC4xNzIyIDM4Ljc3NTdaIiBmaWxsPSIjREQzNDJFIi8+CjxwYXRoIGQ9Ik02My44OTYzIDQ1LjI1OTFINDEuMjA2N0M0MC4yMzQyIDQ1LjI1OTEgMzkuNTg1OSA0NC42MTA4IDM5LjU4NTkgNDMuNjM4M0MzOS41ODU5IDQyLjY2NTggNDAuMjM0MiA0Mi4wMTc2IDQxLjIwNjcgNDIuMDE3Nkg2My44OTYzQzY0Ljg2ODggNDIuMDE3NiA2NS41MTcxIDQyLjY2NTggNjUuNTE3MSA0My42MzgzQzY1LjUxNzEgNDQuNjEwOCA2NC44Njg4IDQ1LjI1OTEgNjMuODk2MyA0NS4yNTkxWiIgZmlsbD0iIzczODNCRiIvPgo8cGF0aCBkPSJNMzQuNzI0IDQ1LjI1OTFIMzMuMTAzMkMzMi4xMzA3IDQ1LjI1OTEgMzEuNDgyNCA0NC42MTA4IDMxLjQ4MjQgNDMuNjM4M0MzMS40ODI0IDQyLjY2NTggMzIuMTMwNyA0Mi4wMTc2IDMzLjEwMzIgNDIuMDE3NkgzNC43MjRDMzUuNjk2NCA0Mi4wMTc2IDM2LjM0NDcgNDIuNjY1OCAzNi4zNDQ3IDQzLjYzODNDMzYuMzQ0NyA0NC42MTA4IDM1LjY5NjMgNDUuMjU5MSAzNC43MjQgNDUuMjU5MVoiIGZpbGw9IiM0MkIwNUMiLz4KPHBhdGggZD0iTTYzLjg5NjMgMzguNzc1N0g2MC42NTQ5QzU5LjY4MjQgMzguNzc1NyA1OS4wMzQyIDM4LjEyNzQgNTkuMDM0MiAzNy4xNTQ5QzU5LjAzNDIgMzYuMTgyNCA1OS42ODI0IDM1LjUzNDIgNjAuNjU0OSAzNS41MzQySDYzLjg5NjNDNjQuODY4OCAzNS41MzQyIDY1LjUxNzEgMzYuMTgyNCA2NS41MTcxIDM3LjE1NDlDNjUuNTE3MSAzOC4xMjc0IDY0Ljg2ODggMzguNzc1NyA2My44OTYzIDM4Ljc3NTdaIiBmaWxsPSIjRUNCQTE2Ii8+CjxwYXRoIGQ9Ik00Ny42ODkzIDUxLjc0MTNIMzMuMTAzMkMzMi4xMzA3IDUxLjc0MTMgMzEuNDgyNCA1MS4wOTMxIDMxLjQ4MjQgNTAuMTIwNkMzMS40ODI0IDQ5LjE0ODEgMzIuMTMwNyA0OC41IDMzLjEwMzIgNDguNUg0Ny42ODkzQzQ4LjY2MTggNDguNSA0OS4zMTAxIDQ5LjE0ODMgNDkuMzEwMSA1MC4xMjA4QzQ5LjMxMDEgNTEuMDkzMyA0OC42NjE4IDUxLjc0MTMgNDcuNjg5MyA1MS43NDEzWiIgZmlsbD0iI0REMzQyRSIvPgo8cGF0aCBkPSJNNjMuODk2OCA1MS43NDEzSDU0LjE3MjVDNTMuMiA1MS43NDEzIDUyLjU1MTggNTEuMDkzMSA1Mi41NTE4IDUwLjEyMDZDNTIuNTUxOCA0OS4xNDgxIDUzLjIwMDIgNDguNSA1NC4xNzI3IDQ4LjVINjMuODk2OUM2NC44Njk0IDQ4LjUgNjUuNTE3NyA0OS4xNDgzIDY1LjUxNzcgNTAuMTIwOEM2NS41MTc3IDUxLjA5MzMgNjQuODY5MiA1MS43NDEzIDYzLjg5NjggNTEuNzQxM1oiIGZpbGw9IiNFQ0JBMTYiLz4KPHBhdGggZD0iTTU0LjE3MjIgNTguMjI0SDMzLjEwMzJDMzIuMTMwNyA1OC4yMjQgMzEuNDgyNCA1Ny41NzU3IDMxLjQ4MjQgNTYuNjAzMkMzMS40ODI0IDU1LjYzMDcgMzIuMTMwNyA1NC45ODI0IDMzLjEwMzIgNTQuOTgyNEg1NC4xNzIyQzU1LjE0NDcgNTQuOTgyNCA1NS43OTMgNTUuNjMwNyA1NS43OTMgNTYuNjAzMkM1NS43OTMgNTcuNTc1NyA1NS4xNDQ1IDU4LjIyNCA1NC4xNzIyIDU4LjIyNFoiIGZpbGw9IiM0MkIwNUMiLz4KPHBhdGggZD0iTTYzLjg5NjMgNjQuNzA3NEg0MS4yMDY3QzQwLjIzNDIgNjQuNzA3NCAzOS41ODU5IDY0LjA1OTEgMzkuNTg1OSA2My4wODY2QzM5LjU4NTkgNjIuMTE0MSA0MC4yMzQyIDYxLjQ2NTggNDEuMjA2NyA2MS40NjU4SDYzLjg5NjNDNjQuODY4OCA2MS40NjU4IDY1LjUxNzEgNjIuMTE0MSA2NS41MTcxIDYzLjA4NjZDNjUuNTE3MSA2NC4wNTkxIDY0Ljg2ODggNjQuNzA3NCA2My44OTYzIDY0LjcwNzRaIiBmaWxsPSIjRUNCQTE2Ii8+CjxwYXRoIGQ9Ik0zNC43MjQgNjQuNzA3NEgzMy4xMDMyQzMyLjEzMDcgNjQuNzA3NCAzMS40ODI0IDY0LjA1OTEgMzEuNDgyNCA2My4wODY2QzMxLjQ4MjQgNjIuMTE0MSAzMi4xMzA3IDYxLjQ2NTggMzMuMTAzMiA2MS40NjU4SDM0LjcyNEMzNS42OTY0IDYxLjQ2NTggMzYuMzQ0NyA2Mi4xMTQxIDM2LjM0NDcgNjMuMDg2NkMzNi4zNDQ3IDY0LjA1OTEgMzUuNjk2MyA2NC43MDc0IDM0LjcyNCA2NC43MDc0WiIgZmlsbD0iI0REMzQyRSIvPgo8cGF0aCBkPSJNNDcuNjg5MyA3MS4xODk4SDMzLjEwMzJDMzIuMTMwNyA3MS4xODk4IDMxLjQ4MjQgNzAuNTQxNSAzMS40ODI0IDY5LjU2OUMzMS40ODI0IDY4LjU5NjUgMzIuMTMwNyA2Ny45NDgyIDMzLjEwMzIgNjcuOTQ4Mkg0Ny42ODkzQzQ4LjY2MTggNjcuOTQ4MiA0OS4zMTAxIDY4LjU5NjUgNDkuMzEwMSA2OS41NjlDNDkuMzEwMSA3MC41NDE1IDQ4LjY2MTggNzEuMTg5OCA0Ny42ODkzIDcxLjE4OThaIiBmaWxsPSIjNDJCMDVDIi8+CjxwYXRoIGQ9Ik02My44OTY4IDcxLjE4OThINTQuMTcyNUM1My4yIDcxLjE4OTggNTIuNTUxOCA3MC41NDE1IDUyLjU1MTggNjkuNTY5QzUyLjU1MTggNjguNTk2NSA1My4yIDY3Ljk0ODIgNTQuMTcyNSA2Ny45NDgySDYzLjg5NjhDNjQuODY5MiA2Ny45NDgyIDY1LjUxNzUgNjguNTk2NSA2NS41MTc1IDY5LjU2OUM2NS41MTc1IDcwLjU0MTUgNjQuODY5MiA3MS4xODk4IDYzLjg5NjggNzEuMTg5OFoiIGZpbGw9IiM3MzgzQkYiLz4KPHBhdGggZD0iTTU0LjE3MjIgNzcuNjcyMkgzMy4xMDMyQzMyLjEzMDcgNzcuNjcyMiAzMS40ODI0IDc3LjAyMzkgMzEuNDgyNCA3Ni4wNTE0QzMxLjQ4MjQgNzUuMDc4OSAzMi4xMzA3IDc0LjQzMDcgMzMuMTAzMiA3NC40MzA3SDU0LjE3MjJDNTUuMTQ0NyA3NC40MzA3IDU1Ljc5MyA3NS4wNzg5IDU1Ljc5MyA3Ni4wNTE0QzU1Ljc5MjggNzcuMDIzOSA1NS4xNDQ1IDc3LjY3MjIgNTQuMTcyMiA3Ny42NzIyWiIgZmlsbD0iI0VDQkExNiIvPgo8cGF0aCBkPSJNNjMuODk2MyA3Ny42NzIySDYwLjY1NDlDNTkuNjgyNCA3Ny42NzIyIDU5LjAzNDIgNzcuMDIzOSA1OS4wMzQyIDc2LjA1MTRDNTkuMDM0MiA3NS4wNzg5IDU5LjY4MjQgNzQuNDMwNyA2MC42NTQ5IDc0LjQzMDdINjMuODk2M0M2NC44Njg4IDc0LjQzMDcgNjUuNTE3MSA3NS4wNzg5IDY1LjUxNzEgNzYuMDUxNEM2NS41MTcxIDc3LjAyMzkgNjQuODY4OCA3Ny42NzIyIDYzLjg5NjMgNzcuNjcyMloiIGZpbGw9IiM0MkIwNUMiLz4KPHBhdGggZD0iTTEwMS4xNzIgNzcuNjcyMkg4MC4xMDMyQzc5LjEzMDcgNzcuNjcyMiA3OC40ODI0IDc3LjAyMzkgNzguNDgyNCA3Ni4wNTE0Qzc4LjQ4MjQgNzUuMDc4OSA3OS4xMzA3IDc0LjQzMDcgODAuMTAzMiA3NC40MzA3SDEwMS4xNzJDMTAyLjE0NSA3NC40MzA3IDEwMi43OTMgNzUuMDc4OSAxMDIuNzkzIDc2LjA1MTRDMTAyLjc5MyA3Ny4wMjM5IDEwMi4xNDUgNzcuNjcyMiAxMDEuMTcyIDc3LjY3MjJaIiBmaWxsPSIjREQzNDJFIi8+CjxwYXRoIGQ9Ik0xMTAuODk2IDc3LjY3MjJIMTA3LjY1NUMxMDYuNjgyIDc3LjY3MjIgMTA2LjAzNCA3Ny4wMjM5IDEwNi4wMzQgNzYuMDUxNEMxMDYuMDM0IDc1LjA3ODkgMTA2LjY4MiA3NC40MzA3IDEwNy42NTUgNzQuNDMwN0gxMTAuODk2QzExMS44NjkgNzQuNDMwNyAxMTIuNTE3IDc1LjA3ODkgMTEyLjUxNyA3Ni4wNTE0QzExMi41MTcgNzcuMDIzOSAxMTEuODY5IDc3LjY3MjIgMTEwLjg5NiA3Ny42NzIyWiIgZmlsbD0iIzQyQjA1QyIvPgo8cGF0aCBkPSJNNjMuODk2MyA1OC4yMjRINjAuNjU0OUM1OS42ODI0IDU4LjIyNCA1OS4wMzQyIDU3LjU3NTcgNTkuMDM0MiA1Ni42MDMyQzU5LjAzNDIgNTUuNjMwNyA1OS42ODI0IDU0Ljk4MjQgNjAuNjU0OSA1NC45ODI0SDYzLjg5NjNDNjQuODY4OCA1NC45ODI0IDY1LjUxNzEgNTUuNjMwNyA2NS41MTcxIDU2LjYwMzJDNjUuNTE3MSA1Ny41NzU3IDY0Ljg2ODggNTguMjI0IDYzLjg5NjMgNTguMjI0WiIgZmlsbD0iIzczODNCRiIvPgo8cGF0aCBkPSJNMTEyLjUxNyA1MS43NDExQzExMi41MTcgNjAuNjU0OSAxMDUuMjI0IDY3Ljk0OCA5Ni4zMTA0IDY3Ljk0OEM4Ny4zOTY2IDY3Ljk0OCA4MC4xMDM1IDYwLjY1NDkgODAuMTAzNSA1MS43NDExQzgwLjEwMzUgNDIuODI3MyA4Ny4zOTY2IDM1LjUzNDIgOTYuMzEwNCAzNS41MzQyQzEwNS4yMjQgMzUuNTM0MiAxMTIuNTE3IDQyLjgyNzMgMTEyLjUxNyA1MS43NDExWiIgZmlsbD0iI0VDQkExNiIvPgo8cGF0aCBkPSJNODAuMTAzNSA1MS43NDExQzgwLjEwMzUgNTIuMjI3MyA4MC4xMDM1IDUyLjg3NTUgODAuMTAzNSA1My4zNjE5SDk2LjMxMDRWMzUuNTM0MkM4Ny4zOTY2IDM1LjUzNDIgODAuMTAzNSA0Mi44MjczIDgwLjEwMzUgNTEuNzQxMVoiIGZpbGw9IiM0MkIwNUMiLz4KPC9nPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGlkPSJwYWludDBfbGluZWFyXzY4N18zMjY4IiB4MT0iMS4yMzIzOWUtMDYiIHkxPSItOS41MDAwMSIgeDI9IjE2OCIgeTI9IjE2MiIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjOEZEREZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzAwNzVGRiIvPgo8L2xpbmVhckdyYWRpZW50Pgo8Y2xpcFBhdGggaWQ9ImNsaXAwXzY4N18zMjY4Ij4KPHJlY3Qgd2lkdGg9Ijk0IiBoZWlnaHQ9Ijk0IiBmaWxsPSJ3aGl0ZSIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoMjUgMjUpIi8+CjwvY2xpcFBhdGg+CjwvZGVmcz4KPC9zdmc+Cg== - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "metricsStorages"], ["spec", "logsStorages"], ["spec", "tracingStorages"], ["spec", "tracingCentralHost"], ["spec", "tracingCollector"], ["spec", "tracingCollector", "enabled"], ["spec", "tracingCollector", "replicas"], ["spec", "tracingCollector", "samplingPercentage"], ["spec", "tracingCollector", "redactAttributes"], ["spec", "tracingCollector", "resources"], ["spec", "tracingCollector", "resources", "limits"], ["spec", "tracingCollector", "resources", "limits", "cpu"], ["spec", "tracingCollector", "resources", "limits", "memory"], ["spec", "tracingCollector", "resources", "requests"], ["spec", "tracingCollector", "resources", "requests", "cpu"], ["spec", "tracingCollector", "resources", "requests", "memory"], ["spec", "alerta"], ["spec", "alerta", "storage"], ["spec", "alerta", "storageClassName"], ["spec", "alerta", "resources"], ["spec", "alerta", "resources", "limits"], ["spec", "alerta", "resources", "limits", "cpu"], ["spec", "alerta", "resources", "limits", "memory"], ["spec", "alerta", "resources", "requests"], ["spec", "alerta", "resources", "requests", "cpu"], ["spec", "alerta", "resources", "requests", "memory"], ["spec", "alerta", "alerts"], ["spec", "alerta", "alerts", "telegram"], ["spec", "alerta", "alerts", "telegram", "token"], ["spec", "alerta", "alerts", "telegram", "chatID"], ["spec", "alerta", "alerts", "telegram", "disabledSeverity"], ["spec", "alerta", "alerts", "slack"], ["spec", "alerta", "alerts", "slack", "url"], ["spec", "alerta", "alerts", "slack", "disabledSeverity"], ["spec", "alerta", "alerts", "email"], ["spec", "alerta", "alerts", "email", "to"], ["spec", "alerta", "alerts", "email", "from"], ["spec", "alerta", "alerts", "email", "smarthost"], ["spec", "alerta", "alerts", "email", "authUsername"], ["spec", "alerta", "alerts", "email", "repeatInterval"], ["spec", "alerta", "alerts", "email", "alertnames"], ["spec", "alerta", "alerts", "email", "severities"], ["spec", "grafana"], ["spec", "grafana", "db"], ["spec", "grafana", "db", "size"], ["spec", "grafana", "resources"], ["spec", "grafana", "resources", "limits"], ["spec", "grafana", "resources", "limits", "cpu"], ["spec", "grafana", "resources", "limits", "memory"], ["spec", "grafana", "resources", "requests"], ["spec", "grafana", "resources", "requests", "cpu"], ["spec", "grafana", "resources", "requests", "memory"], ["spec", "vmagent"], ["spec", "vmagent", "externalLabels"], ["spec", "vmagent", "externalLabels", "cluster"], ["spec", "vmagent", "remoteWrite"], ["spec", "vmagent", "remoteWrite", "urls"], ["spec", "oidc"], ["spec", "oidc", "mode"], ["spec", "oidc", "customConfig"], ["spec", "oidc", "customConfig", "config"], ["spec", "oidc", "customConfig", "secretRef"], ["spec", "oidc", "customConfig", "secretRef", "name"], ["spec", "oidc", "users"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "metricsStorages"], ["spec", "logsStorages"], ["spec", "tracingStorages"], ["spec", "tracingCentralHost"], ["spec", "tracingCollector"], ["spec", "tracingCollector", "enabled"], ["spec", "tracingCollector", "replicas"], ["spec", "tracingCollector", "samplingPercentage"], ["spec", "tracingCollector", "redactAttributes"], ["spec", "tracingCollector", "resources"], ["spec", "tracingCollector", "resources", "limits"], ["spec", "tracingCollector", "resources", "limits", "cpu"], ["spec", "tracingCollector", "resources", "limits", "memory"], ["spec", "tracingCollector", "resources", "requests"], ["spec", "tracingCollector", "resources", "requests", "cpu"], ["spec", "tracingCollector", "resources", "requests", "memory"], ["spec", "alerta"], ["spec", "alerta", "storage"], ["spec", "alerta", "storageClassName"], ["spec", "alerta", "resources"], ["spec", "alerta", "resources", "limits"], ["spec", "alerta", "resources", "limits", "cpu"], ["spec", "alerta", "resources", "limits", "memory"], ["spec", "alerta", "resources", "requests"], ["spec", "alerta", "resources", "requests", "cpu"], ["spec", "alerta", "resources", "requests", "memory"], ["spec", "alerta", "alerts"], ["spec", "alerta", "alerts", "telegram"], ["spec", "alerta", "alerts", "telegram", "token"], ["spec", "alerta", "alerts", "telegram", "chatID"], ["spec", "alerta", "alerts", "telegram", "disabledSeverity"], ["spec", "alerta", "alerts", "slack"], ["spec", "alerta", "alerts", "slack", "url"], ["spec", "alerta", "alerts", "slack", "disabledSeverity"], ["spec", "alerta", "alerts", "email"], ["spec", "alerta", "alerts", "email", "to"], ["spec", "alerta", "alerts", "email", "from"], ["spec", "alerta", "alerts", "email", "smarthost"], ["spec", "alerta", "alerts", "email", "authUsername"], ["spec", "alerta", "alerts", "email", "repeatInterval"], ["spec", "alerta", "alerts", "email", "alertnames"], ["spec", "alerta", "alerts", "email", "severities"], ["spec", "alerta", "alerts", "email", "template"], ["spec", "grafana"], ["spec", "grafana", "db"], ["spec", "grafana", "db", "size"], ["spec", "grafana", "resources"], ["spec", "grafana", "resources", "limits"], ["spec", "grafana", "resources", "limits", "cpu"], ["spec", "grafana", "resources", "limits", "memory"], ["spec", "grafana", "resources", "requests"], ["spec", "grafana", "resources", "requests", "cpu"], ["spec", "grafana", "resources", "requests", "memory"], ["spec", "vmagent"], ["spec", "vmagent", "externalLabels"], ["spec", "vmagent", "externalLabels", "cluster"], ["spec", "vmagent", "remoteWrite"], ["spec", "vmagent", "remoteWrite", "urls"], ["spec", "oidc"], ["spec", "oidc", "mode"], ["spec", "oidc", "customConfig"], ["spec", "oidc", "customConfig", "config"], ["spec", "oidc", "customConfig", "secretRef"], ["spec", "oidc", "customConfig", "secretRef", "name"], ["spec", "oidc", "users"]] secrets: exclude: [] include: diff --git a/packages/system/monitoring/templates/alerta/alerta.yaml b/packages/system/monitoring/templates/alerta/alerta.yaml index 6376eaa9e..33c8c5b77 100644 --- a/packages/system/monitoring/templates/alerta/alerta.yaml +++ b/packages/system/monitoring/templates/alerta/alerta.yaml @@ -10,8 +10,10 @@ {{- $emailAlertnames := $email.alertnames | default list }} {{- $emailSeverities := $email.severities | default (list "critical" "major") }} {{- $emailPasswordSecret := "alertmanager-email-password" }} +{{- $emailTemplateConfigMap := "alertmanager-email-template" }} +{{- $emailTemplate := $email.template | default "" }} {{- $emailSet := list }} -{{- range $field := list "to" "from" "smarthost" "authUsername" "repeatInterval" "alertnames" "severities" }} +{{- range $field := list "to" "from" "smarthost" "authUsername" "repeatInterval" "alertnames" "severities" "template" }} {{- if index $email $field }}{{ $emailSet = append $emailSet $field }}{{ end }} {{- end }} {{- $emailEnabled := gt (len $emailSet) 0 }} @@ -101,6 +103,14 @@ {{- fail (printf "alerta.alerts.email.severities entries must be plain severity names, got %q" (toString $severity)) }} {{- end }} {{- end }} +{{- if $emailTemplate }} +{{- if not (kindIs "string" $emailTemplate) }} +{{- fail (printf "alerta.alerts.email.template must be the text of an Alertmanager template file, got a %s" (kindOf $emailTemplate)) }} +{{- end }} +{{- if not (regexMatch "\\{\\{-?\\s*define\\s" $emailTemplate) }} +{{- fail "alerta.alerts.email.template must define named templates, such as email.default.subject and email.default.html, because Alertmanager only uses the file through the names it defines" }} +{{- end }} +{{- end }} {{- end }} {{- $apiKey := randAlphaNum 32 }} @@ -408,6 +418,15 @@ stringData: {{- end }} send_resolved: true {{- end }} +{{- if and $emailEnabled $emailTemplate }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $emailTemplateConfigMap }} +data: + email.tmpl: {{ $emailTemplate | quote }} +{{- end }} --- apiVersion: operator.victoriametrics.com/v1beta1 kind: VMAlertmanager @@ -425,6 +444,11 @@ spec: secrets: - {{ $emailPasswordSecret | quote }} {{- end }} + {{- if and $emailEnabled $emailTemplate }} + templates: + - name: {{ $emailTemplateConfigMap }} + key: email.tmpl + {{- end }} podMetadata: labels: policy.cozystack.io/allow-to-apiserver: "true" diff --git a/packages/system/monitoring/tests/alertmanager_email_test.yaml b/packages/system/monitoring/tests/alertmanager_email_test.yaml index 0d92cd2ed..c84d5479c 100644 --- a/packages/system/monitoring/tests/alertmanager_email_test.yaml +++ b/packages/system/monitoring/tests/alertmanager_email_test.yaml @@ -1149,3 +1149,70 @@ tests: asserts: - failedTemplate: errorMessage: "alerta.alerts.email.from must be one address, such as alerts@example.org or Alerts , got \"alerts@[IPv6:2001:db8::25]\"" + + # alerta.alerts.email.template carries an Alertmanager template file. It is + # stored in a ConfigMap and listed in VMAlertmanager spec.templates, so + # Alertmanager parses it while loading its configuration, and the definitions + # it holds can replace email.default.subject and email.default.html. + - it: mounts no email template when template is empty + set: + alerta: + alerts: + email: + to: "oncall@example.org" + from: "alerts@example.org" + smarthost: "smtp.example.org:465" + asserts: + - containsDocument: + apiVersion: v1 + kind: ConfigMap + name: alertmanager-email-template + any: true + not: true + - documentSelector: + path: kind + value: VMAlertmanager + notExists: + path: spec.templates + + - it: stores the template text verbatim and lists it among the Alertmanager templates + set: + alerta: + alerts: + email: + to: "oncall@example.org" + from: "alerts@example.org" + smarthost: "smtp.example.org:465" + authUsername: "mailer@example.org" + template: | + {{ define "email.default.subject" }}[prod] {{ if eq .Status "resolved" }}✅ resolved{{ else }}🔴 {{ .CommonLabels.alertname }}{{ end }}{{ end }} + {{ define "email.default.html" }}

{{ .CommonAnnotations.summary }}, проверка "кавычек" и \ слеша

{{ end }} + asserts: + - containsDocument: + apiVersion: v1 + kind: ConfigMap + name: alertmanager-email-template + any: true + - documentSelector: + path: metadata.name + value: alertmanager-email-template + equal: + path: data["email.tmpl"] + value: | + {{ define "email.default.subject" }}[prod] {{ if eq .Status "resolved" }}✅ resolved{{ else }}🔴 {{ .CommonLabels.alertname }}{{ end }}{{ end }} + {{ define "email.default.html" }}

{{ .CommonAnnotations.summary }}, проверка "кавычек" и \ слеша

{{ end }} + - documentSelector: + path: kind + value: VMAlertmanager + equal: + path: spec.templates + value: + - name: alertmanager-email-template + key: email.tmpl + - documentSelector: + path: kind + value: VMAlertmanager + equal: + path: spec.secrets + value: + - alertmanager-email-password diff --git a/packages/system/monitoring/tests/alertmanager_email_validation_test.yaml b/packages/system/monitoring/tests/alertmanager_email_validation_test.yaml index 9691a5746..5c2271155 100644 --- a/packages/system/monitoring/tests/alertmanager_email_validation_test.yaml +++ b/packages/system/monitoring/tests/alertmanager_email_validation_test.yaml @@ -226,3 +226,28 @@ tests: asserts: - failedTemplate: errorMessage: "alerta.alerts.email.to has an empty entry, got \" \"" + + - it: rejects a template without a define action + set: + alerta.alerts.email.template: "plain text {{ .Status }}" + asserts: + - failedTemplate: + errorMessage: "alerta.alerts.email.template must define named templates, such as email.default.subject and email.default.html, because Alertmanager only uses the file through the names it defines" + + - it: rejects a template that is not text + set: + alerta.alerts.email.template: + subject: "[prod]" + asserts: + - failedTemplate: + errorMessage: "alerta.alerts.email.template must be the text of an Alertmanager template file, got a map" + + - it: names template among the fields still set when the addresses are cleared + set: + alerta.alerts.email.to: "" + alerta.alerts.email.from: "" + alerta.alerts.email.smarthost: "" + alerta.alerts.email.template: '{{ define "email.default.subject" }}[prod]{{ end }}' + asserts: + - failedTemplate: + errorMessage: "alerta.alerts.email to, from and smarthost must all be set to enable email alerts. To turn email off, clear every field under alerta.alerts.email. Fields still set are template" diff --git a/packages/system/monitoring/values.yaml b/packages/system/monitoring/values.yaml index 2abaed368..aba63aab8 100644 --- a/packages/system/monitoring/values.yaml +++ b/packages/system/monitoring/values.yaml @@ -95,6 +95,7 @@ alerta: repeatInterval: "" alertnames: [] severities: [] + template: "" grafana: # Upstream Grafana image, pinned by digest and routed through # cozy-lib.image (so mirrored-registry installs work). Plugins (VictoriaLogs From fc1fef92913c071573d4f4b4e785c20e7c953447 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:49:28 +0300 Subject: [PATCH 02/28] fix(monitoring): reject a non-text email template before the default alerta.alerts.email.template went through default "" before its type was checked, so false, 0 or an empty list or map turned into an empty string and the chart rendered without a template instead of failing. The type is now checked on the value as written, and anything other than text or an absent key fails the render. Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- packages/system/monitoring/templates/alerta/alerta.yaml | 9 +++++---- .../tests/alertmanager_email_validation_test.yaml | 7 +++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/system/monitoring/templates/alerta/alerta.yaml b/packages/system/monitoring/templates/alerta/alerta.yaml index 33c8c5b77..0938f7e0b 100644 --- a/packages/system/monitoring/templates/alerta/alerta.yaml +++ b/packages/system/monitoring/templates/alerta/alerta.yaml @@ -11,7 +11,11 @@ {{- $emailSeverities := $email.severities | default (list "critical" "major") }} {{- $emailPasswordSecret := "alertmanager-email-password" }} {{- $emailTemplateConfigMap := "alertmanager-email-template" }} -{{- $emailTemplate := $email.template | default "" }} +{{- $emailTemplateRaw := index $email "template" }} +{{- if not (or (kindIs "string" $emailTemplateRaw) (kindIs "invalid" $emailTemplateRaw)) }} +{{- fail (printf "alerta.alerts.email.template must be the text of an Alertmanager template file, got a %s" (kindOf $emailTemplateRaw)) }} +{{- end }} +{{- $emailTemplate := $emailTemplateRaw | default "" }} {{- $emailSet := list }} {{- range $field := list "to" "from" "smarthost" "authUsername" "repeatInterval" "alertnames" "severities" "template" }} {{- if index $email $field }}{{ $emailSet = append $emailSet $field }}{{ end }} @@ -104,9 +108,6 @@ {{- end }} {{- end }} {{- if $emailTemplate }} -{{- if not (kindIs "string" $emailTemplate) }} -{{- fail (printf "alerta.alerts.email.template must be the text of an Alertmanager template file, got a %s" (kindOf $emailTemplate)) }} -{{- end }} {{- if not (regexMatch "\\{\\{-?\\s*define\\s" $emailTemplate) }} {{- fail "alerta.alerts.email.template must define named templates, such as email.default.subject and email.default.html, because Alertmanager only uses the file through the names it defines" }} {{- end }} diff --git a/packages/system/monitoring/tests/alertmanager_email_validation_test.yaml b/packages/system/monitoring/tests/alertmanager_email_validation_test.yaml index 5c2271155..5edc54242 100644 --- a/packages/system/monitoring/tests/alertmanager_email_validation_test.yaml +++ b/packages/system/monitoring/tests/alertmanager_email_validation_test.yaml @@ -251,3 +251,10 @@ tests: asserts: - failedTemplate: errorMessage: "alerta.alerts.email to, from and smarthost must all be set to enable email alerts. To turn email off, clear every field under alerta.alerts.email. Fields still set are template" + + - it: rejects a template given as a boolean instead of being dropped by the default + set: + alerta.alerts.email.template: false + asserts: + - failedTemplate: + errorMessage: "alerta.alerts.email.template must be the text of an Alertmanager template file, got a bool" From 55066b981d3a93aeb395a529506d765aae6278da Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Thu, 29 Oct 2026 10:06:40 +0100 Subject: [PATCH 03/28] fix(kubernetes): bring up a replacement worker before draining the old one A worker pool rollout (instance type, image or version change) ran with maxUnavailable 1, so CAPI drained and deleted an old Machine as soon as its replacement had been created rather than once it had joined. A one-node pool was left with no node for the few minutes the new VM took to boot and register, and every pool lost one node of capacity for each step of the rollout. With maxUnavailable 0 a node is only drained once its replacement is up. The surge stays at the pool ceiling, with a floor of 1 because CAPI refuses a rolling update where both values are 0 and maxReplicas may be 0. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../kubernetes-nodes/templates/nodegroup.yaml | 9 ++- .../render_snapshot_test.yaml.snap | 6 +- .../tests/rollout_strategy_test.yaml | 67 +++++++++++++++++++ 3 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml diff --git a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml index 18ec0e16b..534b93170 100644 --- a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml +++ b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml @@ -1024,10 +1024,15 @@ spec: # action until the autoscaler notices unschedulable Pods and # scales back up — a real disruption window on every routine # platform bump. + {{- /* maxUnavailable 0 makes a rollout (instance type, image, version) + wait for each replacement Machine to be up before draining an old + one; with 1, a one-node pool lost its only node until the new one + joined. CAPI refuses maxSurge and maxUnavailable both at 0, and + maxReplicas may be 0, hence the floor of 1. */}} strategy: rollingUpdate: - maxSurge: {{ $group.maxReplicas }} - maxUnavailable: 1 + maxSurge: {{ max 1 ($group.maxReplicas | int) }} + maxUnavailable: 0 type: RollingUpdate selector: matchLabels: diff --git a/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap b/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap index 281fc1238..8c7183b22 100644 --- a/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap +++ b/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap @@ -97,7 +97,7 @@ matches the golden render of a GPU pool: strategy: rollingUpdate: maxSurge: 3 - maxUnavailable: 1 + maxUnavailable: 0 type: RollingUpdate template: metadata: @@ -250,7 +250,7 @@ matches the golden render of a kubelet-override pool: strategy: rollingUpdate: maxSurge: 3 - maxUnavailable: 1 + maxUnavailable: 0 type: RollingUpdate template: metadata: @@ -405,7 +405,7 @@ matches the golden render of the pool objects: strategy: rollingUpdate: maxSurge: 3 - maxUnavailable: 1 + maxUnavailable: 0 type: RollingUpdate template: metadata: diff --git a/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml new file mode 100644 index 000000000..1741d77e1 --- /dev/null +++ b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml @@ -0,0 +1,67 @@ +suite: worker pool rollout keeps capacity +templates: + - templates/nodegroup.yaml +release: + name: kubernetes-nodes-myk8s-md0 + namespace: tenant-test +set: + cluster: myk8s + _cluster: + cluster-domain: cozy.local + version: "v1.35" + minReplicas: 1 + maxReplicas: 1 + instanceType: "" + diskSize: 20Gi + storageClass: replicated + resources: + cpu: "2" + memory: 4Gi +tests: + # A rollout replaces every Machine. With maxUnavailable 1 a one-node pool + # drained and deleted its only node as soon as the replacement Machine was + # created, leaving the pool empty until the new node joined. + - it: never drains a node before its replacement is ready + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.type + value: RollingUpdate + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 1 + + - it: surges up to the pool ceiling + set: + maxReplicas: 5 + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 5 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 + + # CAPI rejects a RollingUpdate with maxSurge and maxUnavailable both 0. + - it: keeps a surge of one on a pool capped at zero + set: + minReplicas: 0 + maxReplicas: 0 + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 1 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 From 392bbb3c96e278c3056ab98f5887d74771173374 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Thu, 29 Oct 2026 10:45:21 +0100 Subject: [PATCH 04/28] fix(kubernetes): keep maxUnavailable 1 on GPU worker pools A replacement VM in a pool with passed-through GPUs may need the very GPU the old node holds. With maxUnavailable 0 it would stay Pending and the rollout would never finish, so those pools keep removing a node first. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../kubernetes-nodes/templates/nodegroup.yaml | 6 ++++-- .../__snapshot__/render_snapshot_test.yaml.snap | 2 +- .../tests/rollout_strategy_test.yaml | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml index 534b93170..45ef30cc6 100644 --- a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml +++ b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml @@ -1027,12 +1027,14 @@ spec: {{- /* maxUnavailable 0 makes a rollout (instance type, image, version) wait for each replacement Machine to be up before draining an old one; with 1, a one-node pool lost its only node until the new one - joined. CAPI refuses maxSurge and maxUnavailable both at 0, and + joined. A pool with passed-through GPUs keeps 1: its replacement VM + may need the very GPU the old node holds, and would wait for it + forever. CAPI refuses maxSurge and maxUnavailable both at 0, and maxReplicas may be 0, hence the floor of 1. */}} strategy: rollingUpdate: maxSurge: {{ max 1 ($group.maxReplicas | int) }} - maxUnavailable: 0 + maxUnavailable: {{ ternary 1 0 (not (empty $group.gpus)) }} type: RollingUpdate selector: matchLabels: diff --git a/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap b/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap index 8c7183b22..c8e990a9c 100644 --- a/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap +++ b/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap @@ -97,7 +97,7 @@ matches the golden render of a GPU pool: strategy: rollingUpdate: maxSurge: 3 - maxUnavailable: 0 + maxUnavailable: 1 type: RollingUpdate template: metadata: diff --git a/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml index 1741d77e1..ae44bb03d 100644 --- a/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml @@ -65,3 +65,20 @@ tests: - equal: path: spec.strategy.rollingUpdate.maxUnavailable value: 0 + + # A replacement VM in a pool with passed-through GPUs may need the very GPU + # the old node holds; with maxUnavailable 0 it would wait for it forever. + - it: lets a GPU pool remove a node before its replacement is up + set: + gpus: + - name: nvidia.com/AD102GL_L40S + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 1 + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 1 From a8c4640bea007c19d012bbc332b3803051f31a2a Mon Sep 17 00:00:00 2001 From: Ahmed Kamal Date: Sat, 31 Oct 2026 23:07:29 +0200 Subject: [PATCH 05/28] test(tests): select resources in absence-only Helm checks Signed-off-by: Ahmed Kamal Assisted-by: LLM --- .github/workflows/pull-requests.yaml | 4 + hack/check-helm-document-indices.bats | 141 ++++++++++++++++++ hack/check-helm-document-indices.py | 79 ++++++++++ .../tests/dashboard-resourcemap_test.yaml | 4 +- .../tests/kamaji_managed_args_test.yaml | 8 +- packages/apps/mariadb/tests/tls_test.yaml | 27 +++- packages/apps/mongodb/tests/tls_test.yaml | 32 +++- packages/apps/redis/tests/tls_test.yaml | 8 +- .../tests/solver_test.yaml | 8 +- .../tests/volume_resize_hook_test.yaml | 6 +- 10 files changed, 294 insertions(+), 23 deletions(-) create mode 100644 hack/check-helm-document-indices.bats create mode 100644 hack/check-helm-document-indices.py diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index b8b06fedf..96dce2056 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -364,6 +364,10 @@ jobs: with: path: _out/envtest key: envtest-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }} + - name: Check new Helm document indices + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: python3 hack/check-helm-document-indices.py "$(git merge-base HEAD "origin/$BASE_REF")" # -k is required, not cosmetic: unit-tests and test-controllers are two # independent goals on the same command line, and without -k make stops # handing out new work the moment any unit-tests prerequisite fails — diff --git a/hack/check-helm-document-indices.bats b/hack/check-helm-document-indices.bats new file mode 100644 index 000000000..f797d0b7f --- /dev/null +++ b/hack/check-helm-document-indices.bats @@ -0,0 +1,141 @@ +#!/usr/bin/env bats + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" + +setup() { + strict_setup + tmp=$(mktemp -d) + cd "$tmp" + git init -q + mkdir -p packages/apps/example/templates packages/apps/example/tests + cat > packages/apps/example/templates/multi.yaml <<'EOF' +apiVersion: v1 +kind: ConfigMap +metadata: + name: first +--- +apiVersion: v1 +kind: Secret +metadata: + name: second +EOF + head -n 4 packages/apps/example/templates/multi.yaml > packages/apps/example/templates/single.yaml + git add . + git -c user.name=Test -c user.email=test@example.org commit -qm fixture + base=$(git rev-parse HEAD) +} + +teardown() { + cd "$REPO_ROOT" + rm -rf "$tmp" +} + +write_suite() { + cat > packages/apps/example/tests/resource_test.yaml < output 2>&1; then + echo "accepted a positional selector on a multi-document template" >&2 + false + fi + grep -q 'resource_test.yaml:6: use documentSelector' output +} + +@test "allows an index on a single-document template and ignores comments" { + write_suite single + printf '\n# documentIndex: 1\n' >> packages/apps/example/tests/resource_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "leaves existing indices alone when another assertion changes" { + write_suite multi + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + sed -i 's/spec.missing/spec.other/' packages/apps/example/tests/resource_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "follows assertion template overrides in later YAML suites" { + write_suite single + cat >> packages/apps/example/tests/resource_test.yaml <<'EOF' +--- +suite: overridden template +templates: + - templates/single.yaml +tests: + - it: checks another resource + asserts: + - notExists: + path: spec.missing + template: templates/multi.yaml + documentIndex: 1 +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted an assertion-level positional selector" >&2 + false + fi + grep -q 'use documentSelector.*templates/multi.yaml' output +} + +@test "rejects an invalid base rather than passing an empty diff" { + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" missing-base > output 2>&1; then + echo "accepted an invalid base" >&2 + false + fi +} + +@test "test-level indices follow each assertion's template override" { + write_suite multi + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: assertion override +templates: [templates/single.yaml, templates/multi.yaml] +tests: + - it: checks only the single-document template + documentIndex: 0 + asserts: + - template: templates/single.yaml + notExists: + path: spec.missing +EOF + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "conditional kinds inside one document are not multiple documents" { + write_suite single + cat > packages/apps/example/templates/single.yaml <<'EOF' +apiVersion: v1 +{{ if .Values.secret }} +kind: Secret +{{ else }} +kind: ConfigMap +{{ end }} +metadata: + name: single +EOF + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "rejects a positional selector whose value is on a separate line" { + write_suite multi + sed -i 's/documentIndex: 0/documentIndex:\n 0/' packages/apps/example/tests/resource_test.yaml + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a multiline positional selector" >&2 + false + fi + grep -q 'resource_test.yaml:6: use documentSelector' output +} diff --git a/hack/check-helm-document-indices.py b/hack/check-helm-document-indices.py new file mode 100644 index 000000000..bf9c56f47 --- /dev/null +++ b/hack/check-helm-document-indices.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Reject added positional selectors on multi-resource Helm templates. + +Usage: hack/check-helm-document-indices.py +Requires mikefarah/yq, like the chart test tooling. Existing indices are left +alone. This static check counts --- separated blocks with a top-level kind; +it does not evaluate conditional documents or resources emitted by helpers/ranges. +""" + +import json +from pathlib import Path +import re +import subprocess +import sys + + +def git(*args): + return subprocess.check_output(["git", *args], text=True) + + +def check(base): + root = Path(git("rev-parse", "--show-toplevel").strip()) + paths = git("diff", "--name-only", "--diff-filter=ACMR", base, "--", + "packages/*/*/tests/*_test.yaml").splitlines() + errors = [] + for path in paths: + added = set() + for line in git("diff", "--no-ext-diff", "--unified=0", base, "--", path).splitlines(): + if line.startswith("@@"): + number = int(re.search(r"\+(\d+)", line).group(1)) + elif line.startswith(("+++", "---", "diff ", "index ", "\\")): + continue + elif line.startswith("+"): + if "documentIndex" in line: + added.add(number) + number += 1 + elif line.startswith(" "): + number += 1 + if not added: + continue + + # Preserve the key's source line while yq resolves multi-document YAML, + # flow mappings and anchors; a text-only suite parser misses those. + expression = ('[. | (.. | select(tag == "!!map" and has("documentIndex")) ' + '| .documentIndex) |= {"line": key | line}]') + suites = json.loads(subprocess.check_output( + ["yq", "eval-all", "-o=json", expression, str(root / path)], text=True)) + chart = (root / path).parent.parent + for suite in suites: + if not suite: + continue + for test in suite.get("tests", []): + templates = test.get("templates", suite.get("templates", ["templates/*"])) + for assertion in test.get("asserts", []): + index = assertion.get("documentIndex", test.get("documentIndex", {})) + number = index.get("line") + if number not in added: + continue + selected = [assertion["template"]] if "template" in assertion else templates + for pattern in selected: + matches = list(chart.glob(pattern)) + if not matches: + matches = list((chart / "templates").glob(pattern)) + for template in matches: + if not template.is_file(): + continue + documents = re.split(r"(?m)^---[ \t]*(?:#.*)?$", template.read_text()) + if sum(bool(re.search(r"(?m)^kind\s*:", doc)) for doc in documents) > 1: + errors.append(f"{path}:{number}: use documentSelector, not " + f"documentIndex, for {template.relative_to(root)}") + for error in sorted(set(errors)): + print(error, file=sys.stderr) + return bool(errors) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + sys.exit("usage: check-helm-document-indices.py ") + sys.exit(check(sys.argv[1])) diff --git a/packages/apps/kafka/tests/dashboard-resourcemap_test.yaml b/packages/apps/kafka/tests/dashboard-resourcemap_test.yaml index ace053c58..88da0d69a 100644 --- a/packages/apps/kafka/tests/dashboard-resourcemap_test.yaml +++ b/packages/apps/kafka/tests/dashboard-resourcemap_test.yaml @@ -137,11 +137,13 @@ tests: release: name: test-kafka namespace: tenant-test + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[2].resourceNames content: test-kafka-zookeeper - documentIndex: 0 # External bootstrap service absent when external=false, regardless of tls.enabled - it: does not grant access to external bootstrap service when tls.enabled=true and external=false diff --git a/packages/apps/kubernetes/tests/kamaji_managed_args_test.yaml b/packages/apps/kubernetes/tests/kamaji_managed_args_test.yaml index b09343cf5..85045c9a1 100644 --- a/packages/apps/kubernetes/tests/kamaji_managed_args_test.yaml +++ b/packages/apps/kubernetes/tests/kamaji_managed_args_test.yaml @@ -241,18 +241,22 @@ tests: errorMessage: 'controlPlane.apiServer.extraArgs contains "--secure-port=6444" — the tenant control plane sets --secure-port itself and drops every entry naming it before the kube-apiserver sees the list, so this entry has no effect, and written without =value it leaves its value behind as an argument the apiserver refuses. Remove the entry; the description of controlPlane.apiServer.extraArgs lists every flag the control plane owns.' - it: leaves spec.admissionControllers out by default, so the control plane keeps its own default set + documentSelector: + path: kind + value: KamajiControlPlane asserts: - notExists: path: spec.admissionControllers - documentIndex: 1 - it: leaves spec.admissionControllers out for an explicitly empty list set: controlPlane.apiServer.admissionControllers: [] + documentSelector: + path: kind + value: KamajiControlPlane asserts: - notExists: path: spec.admissionControllers - documentIndex: 1 - it: renders controlPlane.apiServer.admissionControllers into spec.admissionControllers verbatim set: diff --git a/packages/apps/mariadb/tests/tls_test.yaml b/packages/apps/mariadb/tests/tls_test.yaml index 2f5f4548a..ac7b8ae79 100644 --- a/packages/apps/mariadb/tests/tls_test.yaml +++ b/packages/apps/mariadb/tests/tls_test.yaml @@ -530,7 +530,9 @@ tests: - templates/dashboard-resourcemap.yaml set: external: false - documentIndex: 0 + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[1].resourceNames @@ -544,7 +546,9 @@ tests: - templates/dashboard-resourcemap.yaml set: external: true - documentIndex: 0 + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[1].resourceNames @@ -560,7 +564,9 @@ tests: external: true tls: issuer: operator - documentIndex: 0 + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[1].resourceNames @@ -576,7 +582,9 @@ tests: external: false tls: issuer: cert-manager - documentIndex: 0 + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[1].resourceNames @@ -695,7 +703,9 @@ tests: set: tls: issuer: cert-manager - documentIndex: 3 + documentSelector: + path: metadata.name + value: my-mariadb-tls asserts: - notContains: path: spec.dnsNames @@ -1187,7 +1197,9 @@ tests: - it: "dashboard Role never grants the CA bundle even when TLS is unmanaged" templates: - templates/dashboard-resourcemap.yaml - documentIndex: 0 + documentSelector: + path: kind + value: Role set: tls: issuer: operator @@ -1436,6 +1448,9 @@ tests: enabled: true s3Bucket: s3.example.org/test cleanupStrategy: "--keep-last=3" + documentSelector: + path: kind + value: Secret asserts: - notMatchRegex: path: stringData["backup.sh"] diff --git a/packages/apps/mongodb/tests/tls_test.yaml b/packages/apps/mongodb/tests/tls_test.yaml index 0ddb38e46..8cf64b8cc 100644 --- a/packages/apps/mongodb/tests/tls_test.yaml +++ b/packages/apps/mongodb/tests/tls_test.yaml @@ -53,7 +53,9 @@ tests: value: mg-test-ssl - it: spec.tls has no issuerConf (the operator-managed secret is passed directly) - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: - notExists: path: spec.tls.issuerConf @@ -127,13 +129,17 @@ tests: # test would pass even with the external check removed. _namespace: host: example.com - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: - notExists: path: spec.replsets[0].splitHorizons - it: chart does not set allowInvalidCertificates, leaving the operator default - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # The README states the permissive default is inherited rather than # chosen here. Pin that, so the claim stays falsifiable. @@ -141,7 +147,9 @@ tests: path: spec.tls.allowInvalidCertificates - it: chart does not set certValidityDuration, leaving the operator default - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # Same class as allowInvalidCertificates above: the leaf lifetime is the # operator's to choose, not this chart's. Deliberately left unpinned. @@ -177,7 +185,9 @@ tests: # the CRD and notExists on it can never fail. - it: no TLS-disabling escape hatches with a default replica count - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: - notExists: path: spec.allowUnsafeConfigurations @@ -235,7 +245,9 @@ tests: - name: rs0 replicas: 3 size: 10Gi - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # splitHorizons is replica-set scoped and rendered only for the # non-sharded topology. In sharded mode external access goes through the @@ -255,7 +267,9 @@ tests: replicas: 0 _namespace: host: example.com - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # With replicas: 0 the range body renders nothing. Emitting the key with # a null value would be rejected by the API server: the CRD types @@ -269,7 +283,9 @@ tests: external: true _namespace: host: "" - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # The controller may create _namespace before populating host, so the # bootstrap window presents host as an empty string rather than as an diff --git a/packages/apps/redis/tests/tls_test.yaml b/packages/apps/redis/tests/tls_test.yaml index 9b1b3ac0b..c8f55ee91 100644 --- a/packages/apps/redis/tests/tls_test.yaml +++ b/packages/apps/redis/tests/tls_test.yaml @@ -557,7 +557,9 @@ tests: external: false tls: enabled: true - documentIndex: 3 + documentSelector: + path: metadata.name + value: my-redis-tls asserts: - notContains: path: spec.dnsNames @@ -840,7 +842,9 @@ tests: external: false tls: enabled: true - documentIndex: 3 + documentSelector: + path: metadata.name + value: my-redis-tls asserts: - notContains: path: spec.dnsNames diff --git a/packages/system/cert-manager-issuers/tests/solver_test.yaml b/packages/system/cert-manager-issuers/tests/solver_test.yaml index 953d978bc..2a9faebbf 100644 --- a/packages/system/cert-manager-issuers/tests/solver_test.yaml +++ b/packages/system/cert-manager-issuers/tests/solver_test.yaml @@ -228,10 +228,14 @@ tests: gateway-class-name: " cloudflare-tunnel " gateway-edge-classes: cloudflare-tunnel asserts: - - documentIndex: 0 + - documentSelector: + path: metadata.name + value: letsencrypt-prod notExists: path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].sectionName - - documentIndex: 1 + - documentSelector: + path: metadata.name + value: letsencrypt-stage notExists: path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].sectionName diff --git a/packages/system/seaweedfs/tests/volume_resize_hook_test.yaml b/packages/system/seaweedfs/tests/volume_resize_hook_test.yaml index 402748312..011a3de94 100644 --- a/packages/system/seaweedfs/tests/volume_resize_hook_test.yaml +++ b/packages/system/seaweedfs/tests/volume_resize_hook_test.yaml @@ -156,9 +156,11 @@ tests: resources: requests: storage: 10Gi + documentSelector: + path: kind + value: Job asserts: - - documentIndex: 0 - notMatchRegex: + - notMatchRegex: path: spec.template.spec.containers[0].args[0] pattern: kubectl patch pvc From f12887636c430b0f7ab0b871a968f94205c3aa66 Mon Sep 17 00:00:00 2001 From: Ahmed Kamal Date: Sat, 31 Oct 2026 23:14:29 +0200 Subject: [PATCH 06/28] fix(tests): reject matcher failures in negative file assertions Signed-off-by: Ahmed Kamal Assisted-by: LLM --- hack/run-kubernetes-block-io_test.bats | 30 ------- hack/run-kubernetes-cpu-throttle_test.bats | 39 --------- hack/run-kubernetes-kvm-exits_test.bats | 30 ------- .../run-kubernetes-network-counters_test.bats | 30 ------- hack/run-kubernetes-node-join_test.bats | 30 ------- hack/run-kubernetes-runner-canary_test.bats | 31 ------- hack/run-kubernetes-runner-cpu_test.bats | 31 ------- .../run-kubernetes-sandbox-cpu-time_test.bats | 30 ------- hack/run-kubernetes-serial-console_test.bats | 17 ---- ...run-kubernetes-talos-diagnostics_test.bats | 10 --- hack/run-kubernetes-thread-cpu_test.bats | 31 ------- hack/runner-identity.bats | 31 ------- hack/test-helper_test.bats | 85 +++++++++++++++++++ hack/test_helper.bash | 29 +++++++ 14 files changed, 114 insertions(+), 340 deletions(-) create mode 100644 hack/test-helper_test.bats diff --git a/hack/run-kubernetes-block-io_test.bats b/hack/run-kubernetes-block-io_test.bats index b083f9e4f..18b12d253 100644 --- a/hack/run-kubernetes-block-io_test.bats +++ b/hack/run-kubernetes-block-io_test.bats @@ -151,36 +151,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded into an `if`, both land in the passing - # branch and the assertion "this is absent" is satisfied by the matcher giving - # up. Every negative claim in this file rests on the three-way split. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command lands on the very stderr being diff --git a/hack/run-kubernetes-cpu-throttle_test.bats b/hack/run-kubernetes-cpu-throttle_test.bats index d254c7d49..c4ed0dfa4 100644 --- a/hack/run-kubernetes-cpu-throttle_test.bats +++ b/hack/run-kubernetes-cpu-throttle_test.bats @@ -94,45 +94,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". awk exits 2 on an unreadable path, - # which folded into an if reads the same way, so the file is checked first. - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status, not on truthiness. awk exits 1 for "no line - # matched" and 2 for "I could not evaluate this" -- an unparseable regex, most - # of all. Folded into `if ... then fail; fi`, both land in the passing branch, - # so the assertion "this pattern is absent" is satisfied by the matcher giving - # up. That is the wrong direction for a negative assertion: a positive one - # fails loudly when its matcher breaks and gets fixed, this one goes green and - # stays green. Every claim in this file about something NOT appearing rests on - # the three-way split below. - # `|| _rc=$?` rather than a bare call: both runners set -e, and awk exiting 1 - # for "no match" -- the ordinary success case here -- would otherwise abort the - # test before the branch below ever runs. The `if` form this replaces was - # exempt only because conditions are exempt. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # COZY_REPORT_DIR is set inline in each test below rather than through a helper, # which is the opposite of what the node-join suite does. The difference is # deliberate and shrinks to one line: there the assignment carries a ten-line diff --git a/hack/run-kubernetes-kvm-exits_test.bats b/hack/run-kubernetes-kvm-exits_test.bats index 0473e642f..47ff07d9e 100644 --- a/hack/run-kubernetes-kvm-exits_test.bats +++ b/hack/run-kubernetes-kvm-exits_test.bats @@ -81,36 +81,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded together, a negative assertion is - # satisfied by its own matcher giving up, which is the direction that goes - # green and stays green. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command lands on the very stderr being diff --git a/hack/run-kubernetes-network-counters_test.bats b/hack/run-kubernetes-network-counters_test.bats index 73da4c2b2..42f986821 100644 --- a/hack/run-kubernetes-network-counters_test.bats +++ b/hack/run-kubernetes-network-counters_test.bats @@ -148,36 +148,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded into an `if`, both land in the passing - # branch and the assertion "this is absent" is satisfied by the matcher giving - # up. Every negative claim in this file rests on the three-way split. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command lands on the very stderr being diff --git a/hack/run-kubernetes-node-join_test.bats b/hack/run-kubernetes-node-join_test.bats index 42dbe2f11..27bf9bdb9 100644 --- a/hack/run-kubernetes-node-join_test.bats +++ b/hack/run-kubernetes-node-join_test.bats @@ -300,36 +300,6 @@ assert_file_contains() { return 1 } -# The sibling suites' three-way split, and here for the same reason. A bare -# `! grep -q` succeeds on a file that does not exist, which is -# indistinguishable from the file existing without the pattern; and awk exits 1 -# for "no line matched" and 2 for "I could not evaluate this", so folding those -# together lets a broken matcher satisfy a negative assertion. A positive -# assertion fails loudly when its matcher breaks and gets fixed; this one would -# go green and stay green. -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - local _rc=0 - - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The ceilings both budget guards below price their sums from, read out of the # helper rather than restated in each. They are the same numbers held against # two different bounds -- the phase budget from below, the operation window from diff --git a/hack/run-kubernetes-runner-canary_test.bats b/hack/run-kubernetes-runner-canary_test.bats index 7e2f85f90..65c0626f8 100644 --- a/hack/run-kubernetes-runner-canary_test.bats +++ b/hack/run-kubernetes-runner-canary_test.bats @@ -67,37 +67,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded together, a negative assertion is - # satisfied by its own matcher giving up, which is the direction that goes - # green and stays green. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - cat "${file}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The clock the arms are timed by, replaced by a scripted sequence. # # Real durations are the one thing this suite must not depend on: they are set diff --git a/hack/run-kubernetes-runner-cpu_test.bats b/hack/run-kubernetes-runner-cpu_test.bats index 4eca0a3fc..eee4da69a 100644 --- a/hack/run-kubernetes-runner-cpu_test.bats +++ b/hack/run-kubernetes-runner-cpu_test.bats @@ -58,37 +58,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded together, a negative assertion is - # satisfied by its own matcher giving up, which is the direction that goes - # green and stays green. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - cat "${file}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The functions under test redirect a command's stderr into a report file and # decide which artifact to write from it. cozytest.sh runs under `set -x`, so the # tracer's own line for that command lands on the very stderr being redirected, diff --git a/hack/run-kubernetes-sandbox-cpu-time_test.bats b/hack/run-kubernetes-sandbox-cpu-time_test.bats index 0cd1bace4..c6f3028ca 100644 --- a/hack/run-kubernetes-sandbox-cpu-time_test.bats +++ b/hack/run-kubernetes-sandbox-cpu-time_test.bats @@ -83,36 +83,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded together, a negative assertion is - # satisfied by its own matcher giving up, which is the direction that goes - # green and stays green. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command lands on the very stderr being diff --git a/hack/run-kubernetes-serial-console_test.bats b/hack/run-kubernetes-serial-console_test.bats index 3dd91773d..4c77bd269 100644 --- a/hack/run-kubernetes-serial-console_test.bats +++ b/hack/run-kubernetes-serial-console_test.bats @@ -129,23 +129,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: awk exits 2 on an - # unreadable path, which is indistinguishable from "no line matched" once the - # status is folded into an if. - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - if awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}"; then - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi -} - # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command is emitted on the very stderr being diff --git a/hack/run-kubernetes-talos-diagnostics_test.bats b/hack/run-kubernetes-talos-diagnostics_test.bats index c5411837f..5ce8eef66 100644 --- a/hack/run-kubernetes-talos-diagnostics_test.bats +++ b/hack/run-kubernetes-talos-diagnostics_test.bats @@ -114,16 +114,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - if awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}"; then - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi -} - @test "worker VMI rows retain names whose default interface has no IP" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh rows=$(printf '%s\n' '{"items":[{"metadata":{"name":"worker-a"},"status":{"interfaces":[{"name":"default","ipAddress":"10.244.1.92"}]}},{"metadata":{"name":"worker-b"},"status":{"interfaces":[{"name":"other","ipAddress":"192.0.2.1"}]}}]}' | cozy_tenant_worker_vmi_rows) diff --git a/hack/run-kubernetes-thread-cpu_test.bats b/hack/run-kubernetes-thread-cpu_test.bats index 4d9f7c90b..b37c749e5 100644 --- a/hack/run-kubernetes-thread-cpu_test.bats +++ b/hack/run-kubernetes-thread-cpu_test.bats @@ -85,37 +85,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded into an `if`, both land in the passing - # branch and the assertion "this pattern is absent" is satisfied by the - # matcher giving up. Every claim in this file about something NOT appearing - # rests on the three-way split below. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command lands on the very stderr being diff --git a/hack/runner-identity.bats b/hack/runner-identity.bats index acfa36e32..ea42444f5 100644 --- a/hack/runner-identity.bats +++ b/hack/runner-identity.bats @@ -83,37 +83,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the pattern". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded together, a negative assertion is - # satisfied by its own matcher giving up, which is the direction that goes - # green and stays green. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - cat "${file}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The shape sizing object, as the `shapeConfig` subpath serves it: a flat object of # sizing numbers with one string in it, pretty-printed because the service is free # to answer either way and the reader buffers its whole input for that reason. diff --git a/hack/test-helper_test.bats b/hack/test-helper_test.bats new file mode 100644 index 000000000..d02580099 --- /dev/null +++ b/hack/test-helper_test.bats @@ -0,0 +1,85 @@ +#!/usr/bin/env bats + +load test_helper + +# A separate shell keeps errexit active inside the assertion, unlike calling +# the function directly through Bats' run helper. +run_lacks_pattern() { + run bash -eu -c ' + . hack/test_helper.bash + assert_file_lacks_pattern "$1" "$2" + printf "checked\n" + ' _ "$@" +} + +@test "a valid non-matching pattern passes under errexit" { + printf 'safe content\n' >"$BATS_TEST_TMPDIR/artifact" + + run_lacks_pattern 'secret:|secretName:' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 0 ] + [ "$output" = checked ] +} + +@test "an empty readable file passes for a valid pattern" { + : >"$BATS_TEST_TMPDIR/artifact" + + run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 0 ] + [ "$output" = checked ] +} + +@test "a matching pattern fails the assertion" { + printf 'safe content\nsecretName: credential\n' >"$BATS_TEST_TMPDIR/artifact" + + run_lacks_pattern 'secret:|secretName:' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 1 ] + [[ "$output" == *'not to match: secret:|secretName:'* ]] +} + +@test "an invalid pattern fails on a populated file" { + printf 'safe content\n' >"$BATS_TEST_TMPDIR/artifact" + + run_lacks_pattern '[' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 1 ] + [[ "$output" == *'could not evaluate pattern ['* ]] +} + +@test "an invalid pattern fails even on an empty file" { + : >"$BATS_TEST_TMPDIR/artifact" + + run_lacks_pattern '[' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 1 ] + [[ "$output" == *'could not evaluate pattern ['* ]] +} + +@test "a missing artifact fails rather than passing as no match" { + run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR/missing" + + [ "$status" -eq 1 ] + [[ "$output" == *'to exist so it could be checked'* ]] +} + +@test "a directory is not an artifact file" { + run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR" + + [ "$status" -eq 1 ] + [[ "$output" == *'to exist so it could be checked'* ]] +} + +@test "an unreadable artifact fails rather than passing as no match" { + printf 'safe content\n' >"$BATS_TEST_TMPDIR/artifact" + chmod 000 "$BATS_TEST_TMPDIR/artifact" + if [ -r "$BATS_TEST_TMPDIR/artifact" ]; then + skip 'this user can read files without read permission' + fi + + run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 1 ] + [[ "$output" == *'could not evaluate pattern secret:'* ]] +} diff --git a/hack/test_helper.bash b/hack/test_helper.bash index f3944c9c0..f5f74a8b1 100644 --- a/hack/test_helper.bash +++ b/hack/test_helper.bash @@ -9,3 +9,32 @@ strict_setup() { setup() { strict_setup } + +assert_file_lacks_pattern() { + local pattern="$1" + local file="$2" + local _rc=0 + + if [ ! -f "${file}" ]; then + printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 + return 1 + fi + # Validate the regex even for an empty file. Only status 1 means no match; + # keeping awk in an OR-list lets that normal result survive errexit. + awk -v pattern="${pattern}" ' + BEGIN { match("", pattern) } + $0 ~ pattern { found = 1 } + END { exit found ? 0 : 1 } + ' "${file}" || _rc=$? + case "${_rc}" in + 0) + printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 + return 1 + ;; + 1) return 0 ;; + *) + printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 + return 1 + ;; + esac +} From 8d40fc8e00ef099fd827d643d2485cfa7d316b18 Mon Sep 17 00:00:00 2001 From: Ahmed Kamal Date: Sat, 31 Oct 2026 23:36:26 +0200 Subject: [PATCH 07/28] fix(tests): honor singular Helm test template overrides Signed-off-by: Ahmed Kamal Assisted-by: LLM --- hack/check-helm-document-indices.bats | 33 +++++++++++++++++++++++++++ hack/check-helm-document-indices.py | 4 +++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/hack/check-helm-document-indices.bats b/hack/check-helm-document-indices.bats index f797d0b7f..ca84af677 100644 --- a/hack/check-helm-document-indices.bats +++ b/hack/check-helm-document-indices.bats @@ -106,6 +106,7 @@ suite: assertion override templates: [templates/single.yaml, templates/multi.yaml] tests: - it: checks only the single-document template + template: templates/multi.yaml documentIndex: 0 asserts: - template: templates/single.yaml @@ -139,3 +140,35 @@ EOF fi grep -q 'resource_test.yaml:6: use documentSelector' output } + +@test "rejects a singular test template overriding a single-document suite" { + write_suite single + sed -i '/ - it:/a\ template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a positional selector on the test template override" >&2 + false + fi + grep -q 'use documentSelector.*templates/multi.yaml' output +} + +@test "allows a singular test template selecting one document" { + write_suite multi + sed -i '/ - it:/a\ template: templates/single.yaml' packages/apps/example/tests/resource_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "plural test templates take precedence over the singular test template" { + write_suite multi + sed -i '/ - it:/a\ templates: [templates/single.yaml]\n template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "an empty plural test override falls back to the singular template" { + write_suite single + sed -i '/ - it:/a\ templates: []\n template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a positional selector after an empty plural override" >&2 + false + fi + grep -q 'use documentSelector.*templates/multi.yaml' output +} diff --git a/hack/check-helm-document-indices.py b/hack/check-helm-document-indices.py index bf9c56f47..212a50964 100644 --- a/hack/check-helm-document-indices.py +++ b/hack/check-helm-document-indices.py @@ -50,7 +50,9 @@ def check(base): if not suite: continue for test in suite.get("tests", []): - templates = test.get("templates", suite.get("templates", ["templates/*"])) + templates = test.get("templates") or ( + [test["template"]] if test.get("template") + else suite.get("templates", ["templates/*"])) for assertion in test.get("asserts", []): index = assertion.get("documentIndex", test.get("documentIndex", {})) number = index.get("line") From e6651ce222c43b84749998257ff3ab7cf148b2ea Mon Sep 17 00:00:00 2001 From: Ahmed Kamal Date: Sat, 31 Oct 2026 23:40:51 +0200 Subject: [PATCH 08/28] fix(tests): separate awk errors from match results Signed-off-by: Ahmed Kamal Assisted-by: LLM --- hack/test-helper_test.bats | 21 +++++++++++++++++++-- hack/test_helper.bash | 31 ++++++++++++++----------------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/hack/test-helper_test.bats b/hack/test-helper_test.bats index d02580099..902ed8af9 100644 --- a/hack/test-helper_test.bats +++ b/hack/test-helper_test.bats @@ -6,10 +6,10 @@ load test_helper # the function directly through Bats' run helper. run_lacks_pattern() { run bash -eu -c ' - . hack/test_helper.bash + . "$3" assert_file_lacks_pattern "$1" "$2" printf "checked\n" - ' _ "$@" + ' _ "$@" "$BATS_TEST_DIRNAME/test_helper.bash" } @test "a valid non-matching pattern passes under errexit" { @@ -83,3 +83,20 @@ run_lacks_pattern() { [ "$status" -eq 1 ] [[ "$output" == *'could not evaluate pattern secret:'* ]] } + +@test "a matcher error with status one cannot pass as no match" { + printf 'safe content\n' >"$BATS_TEST_TMPDIR/artifact" + mkdir "$BATS_TEST_TMPDIR/bin" + cat >"$BATS_TEST_TMPDIR/bin/awk" <<'SH' +#!/bin/sh +printf '0\n' +printf 'awk: evaluation failed\n' >&2 +exit 1 +SH + chmod +x "$BATS_TEST_TMPDIR/bin/awk" + + PATH="$BATS_TEST_TMPDIR/bin:$PATH" run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 1 ] + [[ "$output" == *'could not evaluate pattern secret:'* ]] +} diff --git a/hack/test_helper.bash b/hack/test_helper.bash index f5f74a8b1..592676b8c 100644 --- a/hack/test_helper.bash +++ b/hack/test_helper.bash @@ -13,28 +13,25 @@ setup() { assert_file_lacks_pattern() { local pattern="$1" local file="$2" - local _rc=0 + local found if [ ! -f "${file}" ]; then printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 return 1 fi - # Validate the regex even for an empty file. Only status 1 means no match; - # keeping awk in an OR-list lets that normal result survive errexit. - awk -v pattern="${pattern}" ' + # Validate even an empty file, and keep the match result separate from awk's + # status: some implementations also use status 1 for evaluation errors. + if ! found=$(awk -v pattern="${pattern}" ' BEGIN { match("", pattern) } $0 ~ pattern { found = 1 } - END { exit found ? 0 : 1 } - ' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac + END { print found ? 1 : 0 } + ' "${file}"); then + printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 + return 1 + fi + if [ "${found}" != 0 ]; then + printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 + return 1 + fi + return 0 } From 2757e2c60ad35e40d6e378a6049b8a20f6242ea5 Mon Sep 17 00:00:00 2001 From: Ahmed Kamal Date: Sun, 1 Nov 2026 01:23:55 +0200 Subject: [PATCH 09/28] test(tests): tighten Helm selector guard scope and coverage Signed-off-by: Ahmed Kamal Assisted-by: LLM --- .github/workflows/pull-requests.yaml | 9 +- hack/check-helm-document-indices.bats | 174 +++++++++++++++++- hack/check-helm-document-indices.py | 41 +++-- .../apps/kafka/tests/jvm_sizing_test.yaml | 4 +- 4 files changed, 197 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 96dce2056..1f44fc287 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -364,10 +364,6 @@ jobs: with: path: _out/envtest key: envtest-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('Makefile') }} - - name: Check new Helm document indices - env: - BASE_REF: ${{ github.event.pull_request.base.ref }} - run: python3 hack/check-helm-document-indices.py "$(git merge-base HEAD "origin/$BASE_REF")" # -k is required, not cosmetic: unit-tests and test-controllers are two # independent goals on the same command line, and without -k make stops # handing out new work the moment any unit-tests prerequisite fails — @@ -380,6 +376,11 @@ jobs: - name: Run Bats unit tests for docs-only changes if: needs.plan.outputs.code == 'false' run: make bats-unit-tests + - name: Check new Helm document indices + if: ${{ !cancelled() }} + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: python3 hack/check-helm-document-indices.py "$(git merge-base HEAD "origin/$BASE_REF")" - name: Upload unit test report if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 diff --git a/hack/check-helm-document-indices.bats b/hack/check-helm-document-indices.bats index ca84af677..5188f5b26 100644 --- a/hack/check-helm-document-indices.bats +++ b/hack/check-helm-document-indices.bats @@ -66,7 +66,8 @@ EOF write_suite multi git -c user.name=Test -c user.email=test@example.org commit -qm existing base=$(git rev-parse HEAD) - sed -i 's/spec.missing/spec.other/' packages/apps/example/tests/resource_test.yaml + sed 's/spec.missing/spec.other/' packages/apps/example/tests/resource_test.yaml > changed.yaml + mv changed.yaml packages/apps/example/tests/resource_test.yaml python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" } @@ -133,17 +134,37 @@ EOF @test "rejects a positional selector whose value is on a separate line" { write_suite multi - sed -i 's/documentIndex: 0/documentIndex:\n 0/' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: multiline index +templates: [templates/multi.yaml] +tests: + - it: checks a resource + documentIndex: + 0 + asserts: + - notExists: + path: spec.missing +EOF if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then echo "accepted a multiline positional selector" >&2 false fi - grep -q 'resource_test.yaml:6: use documentSelector' output + grep -q 'resource_test.yaml:5: use documentSelector' output } @test "rejects a singular test template overriding a single-document suite" { write_suite single - sed -i '/ - it:/a\ template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/single.yaml] +tests: + - it: checks a resource + template: templates/multi.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then echo "accepted a positional selector on the test template override" >&2 false @@ -153,22 +174,161 @@ EOF @test "allows a singular test template selecting one document" { write_suite multi - sed -i '/ - it:/a\ template: templates/single.yaml' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/multi.yaml] +tests: + - it: checks a resource + template: templates/single.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" } @test "plural test templates take precedence over the singular test template" { write_suite multi - sed -i '/ - it:/a\ templates: [templates/single.yaml]\n template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/multi.yaml] +tests: + - it: checks a resource + templates: [templates/single.yaml] + template: templates/multi.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" } @test "an empty plural test override falls back to the singular template" { write_suite single - sed -i '/ - it:/a\ templates: []\n template: templates/multi.yaml' packages/apps/example/tests/resource_test.yaml + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/single.yaml] +tests: + - it: checks a resource + templates: [] + template: templates/multi.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then echo "accepted a positional selector after an empty plural override" >&2 false fi grep -q 'use documentSelector.*templates/multi.yaml' output } + +@test "does not check vendored chart suites" { + mkdir -p packages/apps/example/charts/sub/templates packages/apps/example/charts/sub/tests + cp packages/apps/example/templates/multi.yaml packages/apps/example/charts/sub/templates/ + cat > packages/apps/example/charts/sub/tests/resource_test.yaml <<'EOF' +suite: upstream resource selection +templates: [templates/multi.yaml] +tests: + - it: checks an upstream resource + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF + git add . + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "a new test-level index overrides existing assertion indices" { + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: index inheritance +templates: [templates/multi.yaml] +tests: + - it: checks a resource + asserts: + - notExists: + path: spec.missing + documentIndex: 0 + - notExists: + path: spec.other + documentIndex: 0 +EOF + git add . + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: index inheritance +templates: [templates/multi.yaml] +tests: + - it: checks a resource + documentIndex: 1 + asserts: + - notExists: + path: spec.missing + documentIndex: 0 + - notExists: + path: spec.other + documentIndex: 0 +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a new inherited positional selector" >&2 + false + fi + grep -q 'resource_test.yaml:5: use documentSelector' output +} + +@test "allows a pure suite rename with existing indices" { + write_suite multi + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + git mv packages/apps/example/tests/resource_test.yaml packages/apps/example/tests/renamed_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "checks added index lines in a renamed suite" { + write_suite multi + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + git mv packages/apps/example/tests/resource_test.yaml packages/apps/example/tests/renamed_test.yaml + cat > packages/apps/example/tests/renamed_test.yaml <<'EOF' +suite: resource selection +templates: + - templates/multi.yaml +tests: + - it: checks a resource + documentIndex: 1 + asserts: + - notExists: + path: spec.missing +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a new positional selector in a renamed suite" >&2 + false + fi + grep -q 'renamed_test.yaml:6: use documentSelector' output +} + +@test "an empty suite template list includes nested chart templates" { + mkdir packages/apps/example/templates/nested + mv packages/apps/example/templates/multi.yaml packages/apps/example/templates/nested/ + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: default templates +templates: [] +tests: + - it: checks a resource + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF + git add . + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a positional selector with default templates" >&2 + false + fi + grep -q 'use documentSelector.*templates/nested/multi.yaml' output +} diff --git a/hack/check-helm-document-indices.py b/hack/check-helm-document-indices.py index 212a50964..4fd36b06f 100644 --- a/hack/check-helm-document-indices.py +++ b/hack/check-helm-document-indices.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Reject added positional selectors on multi-resource Helm templates. -Usage: hack/check-helm-document-indices.py +Usage: python3 hack/check-helm-document-indices.py Requires mikefarah/yq, like the chart test tooling. Existing indices are left alone. This static check counts --- separated blocks with a top-level kind; it does not evaluate conditional documents or resources emitted by helpers/ranges. @@ -20,22 +20,25 @@ def git(*args): def check(base): root = Path(git("rev-parse", "--show-toplevel").strip()) - paths = git("diff", "--name-only", "--diff-filter=ACMR", base, "--", - "packages/*/*/tests/*_test.yaml").splitlines() + additions = {} + path = None + for line in git("diff", "--no-ext-diff", "-M", "--unified=0", base, "--", + ":(glob)packages/*/*/tests/*_test.yaml").splitlines(): + if line.startswith("diff --git "): + path = None + elif line.startswith("+++ b/"): + path = line[6:] + additions[path] = set() + elif line.startswith("@@"): + number = int(re.search(r"\+(\d+)", line).group(1)) + elif path and line.startswith("+"): + if "documentIndex" in line: + additions[path].add(number) + number += 1 + elif line.startswith(" "): + number += 1 errors = [] - for path in paths: - added = set() - for line in git("diff", "--no-ext-diff", "--unified=0", base, "--", path).splitlines(): - if line.startswith("@@"): - number = int(re.search(r"\+(\d+)", line).group(1)) - elif line.startswith(("+++", "---", "diff ", "index ", "\\")): - continue - elif line.startswith("+"): - if "documentIndex" in line: - added.add(number) - number += 1 - elif line.startswith(" "): - number += 1 + for path, added in additions.items(): if not added: continue @@ -52,9 +55,9 @@ def check(base): for test in suite.get("tests", []): templates = test.get("templates") or ( [test["template"]] if test.get("template") - else suite.get("templates", ["templates/*"])) + else suite.get("templates") or ["templates/**/*"]) for assertion in test.get("asserts", []): - index = assertion.get("documentIndex", test.get("documentIndex", {})) + index = test.get("documentIndex") or assertion.get("documentIndex", {}) number = index.get("line") if number not in added: continue @@ -77,5 +80,5 @@ def check(base): if __name__ == "__main__": if len(sys.argv) != 2: - sys.exit("usage: check-helm-document-indices.py ") + sys.exit("usage: python3 hack/check-helm-document-indices.py ") sys.exit(check(sys.argv[1])) diff --git a/packages/apps/kafka/tests/jvm_sizing_test.yaml b/packages/apps/kafka/tests/jvm_sizing_test.yaml index 26a065d17..f4569a1e2 100644 --- a/packages/apps/kafka/tests/jvm_sizing_test.yaml +++ b/packages/apps/kafka/tests/jvm_sizing_test.yaml @@ -27,7 +27,9 @@ tests: # kafka-run-class.sh then drops its G1 settings. - it: leaves the node pool JVM options to Strimzi and Kafka template: templates/migration-hook.yaml - documentIndex: 3 + documentSelector: + path: kind + value: Job asserts: - notMatchRegex: path: spec.template.spec.containers[0].args[0] From 342283cfc781e343760148fb85775f914d114bcc Mon Sep 17 00:00:00 2001 From: Ahmed Kamal Date: Sun, 1 Nov 2026 02:17:57 +0200 Subject: [PATCH 10/28] docs(tests): document Helm selector guard limits Signed-off-by: Ahmed Kamal Assisted-by: LLM --- hack/check-helm-document-indices.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hack/check-helm-document-indices.py b/hack/check-helm-document-indices.py index 4fd36b06f..dd75e799a 100644 --- a/hack/check-helm-document-indices.py +++ b/hack/check-helm-document-indices.py @@ -2,9 +2,7 @@ """Reject added positional selectors on multi-resource Helm templates. Usage: python3 hack/check-helm-document-indices.py -Requires mikefarah/yq, like the chart test tooling. Existing indices are left -alone. This static check counts --- separated blocks with a top-level kind; -it does not evaluate conditional documents or resources emitted by helpers/ranges. +Requires mikefarah/yq, like the chart test tooling. """ import json @@ -19,6 +17,11 @@ def git(*args): def check(base): + """Check added indices, leaving existing ones alone. + + This static check counts --- separated blocks with a top-level kind; it does + not evaluate conditional documents or resources emitted by helpers/ranges. + """ root = Path(git("rev-parse", "--show-toplevel").strip()) additions = {} path = None From 21fe9d4195848d992fdcb1d67f03a263d1fc716b Mon Sep 17 00:00:00 2001 From: Ahmed Kamal Date: Sun, 1 Nov 2026 02:18:13 +0200 Subject: [PATCH 11/28] docs(tests): document negative file assertion contract Signed-off-by: Ahmed Kamal Assisted-by: LLM --- hack/test_helper.bash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hack/test_helper.bash b/hack/test_helper.bash index 592676b8c..bbbef8de8 100644 --- a/hack/test_helper.bash +++ b/hack/test_helper.bash @@ -10,6 +10,8 @@ setup() { strict_setup } +# Only a successful matcher explicitly reporting no match passes; missing, +# non-regular or unreadable files and matcher errors fail closed. assert_file_lacks_pattern() { local pattern="$1" local file="$2" From 670f33d68cac9a3a731bb3baaea89c48f3d3911e Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 2 Nov 2026 11:25:12 +0100 Subject: [PATCH 12/28] feat(kubernetes): let a worker pool set its rollout maxUnavailable A Proxmox template that passes a GPU through leaves gpus empty, so the chart renders maxUnavailable 0 for that pool and its replacement VM waits for the GPU the old worker still holds. The pool can now say so itself with maxUnavailable, which overrides the 0, or 1 with gpus, that the chart derives otherwise. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- api/apps/v1alpha1/kubernetesnodes/types.go | 3 ++ .../kubernetesnodes/zz_generated.deepcopy.go | 5 +++ packages/apps/kubernetes-nodes/README.md | 1 + .../kubernetes-nodes/templates/nodegroup.yaml | 11 +++++-- .../tests/rollout_strategy_test.yaml | 32 +++++++++++++++++++ .../apps/kubernetes-nodes/values.schema.json | 5 +++ packages/apps/kubernetes-nodes/values.yaml | 3 ++ .../cozyrds/kubernetes-nodes.yaml | 2 +- 8 files changed, 58 insertions(+), 4 deletions(-) diff --git a/api/apps/v1alpha1/kubernetesnodes/types.go b/api/apps/v1alpha1/kubernetesnodes/types.go index 59bbe59c6..c5c6dc15b 100644 --- a/api/apps/v1alpha1/kubernetesnodes/types.go +++ b/api/apps/v1alpha1/kubernetesnodes/types.go @@ -51,6 +51,9 @@ type ConfigSpec struct { // List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). // +kubebuilder:default:={} Gpus []GPU `json:"gpus,omitempty"` + // Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. + // +kubebuilder:validation:Minimum=0 + MaxUnavailable *int `json:"maxUnavailable,omitempty"` // Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls. KernelModules []KernelModule `json:"kernelModules,omitempty"` // Kubelet resource reservations for this pool. diff --git a/api/apps/v1alpha1/kubernetesnodes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetesnodes/zz_generated.deepcopy.go index 185d2fc22..15f0a8b6a 100644 --- a/api/apps/v1alpha1/kubernetesnodes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetesnodes/zz_generated.deepcopy.go @@ -67,6 +67,11 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { *out = make([]GPU, len(*in)) copy(*out, *in) } + if in.MaxUnavailable != nil { + in, out := &in.MaxUnavailable, &out.MaxUnavailable + *out = new(int) + **out = **in + } if in.KernelModules != nil { in, out := &in.KernelModules, &out.KernelModules *out = make([]KernelModule, len(*in)) diff --git a/packages/apps/kubernetes-nodes/README.md b/packages/apps/kubernetes-nodes/README.md index 900a19a7a..dd2872b93 100644 --- a/packages/apps/kubernetes-nodes/README.md +++ b/packages/apps/kubernetes-nodes/README.md @@ -43,6 +43,7 @@ the parent cluster `kubernetes-` in the same namespace, plus a | `podCpuRequest` | CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart. | `quantity` | `""` | | `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | | `gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` | +| `maxUnavailable` | Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. | `*int` | `null` | | `kernelModules` | Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls. | `[]object` | `[]` | | `kernelModules[i].name` | Module name as `modprobe` takes it, e.g. `nvidia_uvm`. | `string` | `""` | | `kernelModules[i].parameters` | Module parameters, each as a bare `key=value` string. | `[]string` | `[]` | diff --git a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml index 45ef30cc6..0581f02f9 100644 --- a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml +++ b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml @@ -1029,12 +1029,17 @@ spec: one; with 1, a one-node pool lost its only node until the new one joined. A pool with passed-through GPUs keeps 1: its replacement VM may need the very GPU the old node holds, and would wait for it - forever. CAPI refuses maxSurge and maxUnavailable both at 0, and - maxReplicas may be 0, hence the floor of 1. */}} + forever. A Proxmox template's GPU is invisible here, hence the + maxUnavailable override. CAPI refuses maxSurge and maxUnavailable both + at 0, and maxReplicas may be 0, hence the floor of 1. */}} + {{- $maxUnavailable := ternary 1 0 (not (empty $group.gpus)) }} + {{- if not (kindIs "invalid" $.Values.maxUnavailable) }} + {{- $maxUnavailable = $.Values.maxUnavailable | int }} + {{- end }} strategy: rollingUpdate: maxSurge: {{ max 1 ($group.maxReplicas | int) }} - maxUnavailable: {{ ternary 1 0 (not (empty $group.gpus)) }} + maxUnavailable: {{ $maxUnavailable }} type: RollingUpdate selector: matchLabels: diff --git a/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml index ae44bb03d..210afb068 100644 --- a/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml @@ -82,3 +82,35 @@ tests: - equal: path: spec.strategy.rollingUpdate.maxSurge value: 1 + + # A Proxmox template that passes a GPU through leaves gpus empty, so the + # chart cannot tell; the pool says so itself. + - it: lets a pool set maxUnavailable explicitly + set: + maxUnavailable: 1 + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 1 + + - it: honours an explicit 0 on a GPU pool + set: + maxUnavailable: 0 + gpus: + - name: nvidia.com/AD102GL_L40S + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 + + - it: refuses a negative maxUnavailable (schema minimum 0) + set: + maxUnavailable: -1 + asserts: + - failedTemplate: {} diff --git a/packages/apps/kubernetes-nodes/values.schema.json b/packages/apps/kubernetes-nodes/values.schema.json index 0e99096d5..9fbd9e7cd 100644 --- a/packages/apps/kubernetes-nodes/values.schema.json +++ b/packages/apps/kubernetes-nodes/values.schema.json @@ -146,6 +146,11 @@ } } }, + "maxUnavailable": { + "description": "Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.", + "type": "integer", + "minimum": 0 + }, "kernelModules": { "description": "Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls.", "type": "array", diff --git a/packages/apps/kubernetes-nodes/values.yaml b/packages/apps/kubernetes-nodes/values.yaml index c930b0433..f7a6d0fec 100644 --- a/packages/apps/kubernetes-nodes/values.yaml +++ b/packages/apps/kubernetes-nodes/values.yaml @@ -77,6 +77,9 @@ resources: {} ## @param {[]GPU} gpus - List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). gpus: [] +## @param {*int} [maxUnavailable] - Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. +## @minimum 0 + ## @param {[]KernelModule} [kernelModules] - Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls. ## @param {Kubelet} [kubelet] - Kubelet resource reservations for this pool. diff --git a/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml b/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml index dffa00956..739d5480a 100644 --- a/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml +++ b/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml @@ -21,7 +21,7 @@ spec: singular: kubernetesnodes plural: kubernetesnodeses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool name (the part after `-`) must not collide with a nodeGroup still managed by the parent kubernetes chart (e.g. the default `md0`), or the render fails on ownership conflict.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `\"010\"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `\"010\"` reserves what `\"10\"` does.","type":"object","default":{},"properties":{"cpu":{"description":"Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"podCpuLimit":{"description":"CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"podCpuRequest":{"description":"CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"kernelModules":{"description":"Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls.","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Module name as `modprobe` takes it, e.g. `nvidia_uvm`.","type":"string"},"parameters":{"description":"Module parameters, each as a bare `key=value` string.","type":"array","items":{"type":"string"}}}}},"kubelet":{"description":"Kubelet resource reservations for this pool.","type":"object","default":{},"properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"logSerialConsole":{"description":"Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it.","type":"boolean","default":false},"maxUnhealthy":{"description":"Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is a percentage. Drop to \"0%\" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At \"50%\" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two.","type":"string","default":"50%"},"nodeStartupTimeout":{"description":"Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.","type":"string","default":"20m"},"readyUnknownTimeout":{"description":"How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"5m"},"readyFalseTimeout":{"description":"How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"10m"},"maxNodeProvisionTime":{"description":"How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted. Keep in sync with the parent kubernetes chart.","type":"string","default":"30m"},"version":{"description":"Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool -- and must satisfy the Talos<->Kubernetes support matrix against `talos.version`.","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"osImage":{"description":"Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool.","type":"object","default":{},"properties":{"builtin":{"description":"Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled.","type":"object","properties":{"schematicID":{"description":"Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version of the golden to clone. Defaults to `talos.version`.","type":"string"}}},"factory":{"description":"Import from an Image Factory / mirror (default).","type":"object","properties":{"imageFactoryURL":{"description":"Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. NOTE: this redirects only the worker boot-disk import; the in-guest upgrade installer image is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.","type":"string"},"schematicID":{"description":"Talos schematic ID. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version. Defaults to `talos.version`.","type":"string"}}}}},"talos":{"description":"Talos worker image configuration. Keep in sync with the parent cluster's `talos`.","type":"object","default":{},"required":["imageFactoryURL","installerRepository","registryMirrors","schematicID","version"],"properties":{"imageFactoryCA":{"description":"Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there.","type":"string","default":""},"imageFactoryURL":{"description":"Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal.","type":"string","default":"https://factory.talos.dev"},"installerRepository":{"description":"OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.","type":"string","default":"factory.talos.dev/installer"},"registryMirrors":{"description":"Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true},"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"version":{"description":"Default Talos release for this pool's worker OS image and in-guest installer. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix rejects a pairing it lists as unsupported; it lists the Talos minors this chart ships and passes any other minor unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.","type":"string","default":"v1.13.6"}}},"substrate":{"description":"Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["network"],"properties":{"dnsServers":{"description":"Nameservers written into the worker machineconfig. On kubevirt the workers use the management cluster's CoreDNS, which they reach over the pod network; an off-cluster Proxmox worker cannot, so it needs reachable resolvers of its own. Required when `substrate` is `proxmox`. Keep in sync with the parent kubernetes chart's `proxmox.dnsServers`.","type":"array","default":[],"items":{"type":"string"}},"full":{"description":"Full clone rather than linked. Default is a linked clone: it costs kilobytes at creation instead of the whole disk, because a ZFS-backed linked clone writes only its own increment (measured: 8K against 10.2G for the same worker full-cloned onto a `sparse 0` pool). The cost is that the clone holds the template's base snapshot, so the template cannot be rotated while any linked clone still references it — which is what the consolidation CronJob resolves 24h after creation. Set true for a pool that must be independent of the template from the first second.","type":"boolean","default":false},"network":{"description":"NIC configuration.","type":"object","default":{},"required":["bridge"],"properties":{"bridge":{"description":"Proxmox bridge the VM NIC attaches to (e.g. `vmbr0`). Required when `substrate` is `proxmox`.","type":"string","default":""},"mtu":{"description":"NIC MTU. Omitted when unset.","type":"integer"},"vlan":{"description":"L2 VLAN tag. Omitted when unset.","type":"integer"}}},"pool":{"description":"Proxmox resource pool the worker VMs are added to. Set the same pool on every node pool of a tenant cluster and grant the tenant's tokens their privileges on `/pool/`: that is the one ACL path that follows every VM capmox creates, replacements included. The pool must exist on the hypervisor; capmox does not create it. Empty adds the workers to no pool.","type":"string","default":""},"storage":{"description":"Proxmox storage for the cloned disk; needs `full: true`. A target storage is a full-clone parameter — a linked clone always lives on the template's storage, and Proxmox refuses the pair — so the render refuses it too. Empty keeps the template's storage.","type":"string","default":""},"templateTags":{"description":"Tags identifying the Talos VM template to clone. capmox matches a template when its tag set is EQUAL to this list, not when it contains it, so every tag on the template must be listed here — a missing one yields `found 0 VM templates with tags ...` while the tags are plainly present. The template has to carry this pool's `talos.schematicID` at `talos.version` in the nocloud flavour, with the Proxmox guest agent enabled for the `qemu-guest-agent` extension of the default schematic; `hack/proxmox-talos-template.sh` builds it from these values and prints the tags to list here. Do not build it from the platform release's `nocloud-amd64.raw.xz`: that is the bare-metal node image, whose `zfs` extension keeps a VM from finishing its boot sequence and reboots it at the Talos boot timeout. Required when `substrate` is `proxmox`.","type":"array","default":[],"items":{"type":"string"}}}}}} + {"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool name (the part after `-`) must not collide with a nodeGroup still managed by the parent kubernetes chart (e.g. the default `md0`), or the render fails on ownership conflict.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `\"010\"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `\"010\"` reserves what `\"10\"` does.","type":"object","default":{},"properties":{"cpu":{"description":"Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"podCpuLimit":{"description":"CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"podCpuRequest":{"description":"CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"maxUnavailable":{"description":"Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.","type":"integer","minimum":0},"kernelModules":{"description":"Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls.","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Module name as `modprobe` takes it, e.g. `nvidia_uvm`.","type":"string"},"parameters":{"description":"Module parameters, each as a bare `key=value` string.","type":"array","items":{"type":"string"}}}}},"kubelet":{"description":"Kubelet resource reservations for this pool.","type":"object","default":{},"properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"logSerialConsole":{"description":"Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it.","type":"boolean","default":false},"maxUnhealthy":{"description":"Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is a percentage. Drop to \"0%\" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At \"50%\" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two.","type":"string","default":"50%"},"nodeStartupTimeout":{"description":"Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.","type":"string","default":"20m"},"readyUnknownTimeout":{"description":"How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"5m"},"readyFalseTimeout":{"description":"How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"10m"},"maxNodeProvisionTime":{"description":"How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted. Keep in sync with the parent kubernetes chart.","type":"string","default":"30m"},"version":{"description":"Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool -- and must satisfy the Talos<->Kubernetes support matrix against `talos.version`.","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"osImage":{"description":"Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool.","type":"object","default":{},"properties":{"builtin":{"description":"Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled.","type":"object","properties":{"schematicID":{"description":"Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version of the golden to clone. Defaults to `talos.version`.","type":"string"}}},"factory":{"description":"Import from an Image Factory / mirror (default).","type":"object","properties":{"imageFactoryURL":{"description":"Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. NOTE: this redirects only the worker boot-disk import; the in-guest upgrade installer image is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.","type":"string"},"schematicID":{"description":"Talos schematic ID. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version. Defaults to `talos.version`.","type":"string"}}}}},"talos":{"description":"Talos worker image configuration. Keep in sync with the parent cluster's `talos`.","type":"object","default":{},"required":["imageFactoryURL","installerRepository","registryMirrors","schematicID","version"],"properties":{"imageFactoryCA":{"description":"Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there.","type":"string","default":""},"imageFactoryURL":{"description":"Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal.","type":"string","default":"https://factory.talos.dev"},"installerRepository":{"description":"OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.","type":"string","default":"factory.talos.dev/installer"},"registryMirrors":{"description":"Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true},"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"version":{"description":"Default Talos release for this pool's worker OS image and in-guest installer. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix rejects a pairing it lists as unsupported; it lists the Talos minors this chart ships and passes any other minor unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.","type":"string","default":"v1.13.6"}}},"substrate":{"description":"Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["network"],"properties":{"dnsServers":{"description":"Nameservers written into the worker machineconfig. On kubevirt the workers use the management cluster's CoreDNS, which they reach over the pod network; an off-cluster Proxmox worker cannot, so it needs reachable resolvers of its own. Required when `substrate` is `proxmox`. Keep in sync with the parent kubernetes chart's `proxmox.dnsServers`.","type":"array","default":[],"items":{"type":"string"}},"full":{"description":"Full clone rather than linked. Default is a linked clone: it costs kilobytes at creation instead of the whole disk, because a ZFS-backed linked clone writes only its own increment (measured: 8K against 10.2G for the same worker full-cloned onto a `sparse 0` pool). The cost is that the clone holds the template's base snapshot, so the template cannot be rotated while any linked clone still references it — which is what the consolidation CronJob resolves 24h after creation. Set true for a pool that must be independent of the template from the first second.","type":"boolean","default":false},"network":{"description":"NIC configuration.","type":"object","default":{},"required":["bridge"],"properties":{"bridge":{"description":"Proxmox bridge the VM NIC attaches to (e.g. `vmbr0`). Required when `substrate` is `proxmox`.","type":"string","default":""},"mtu":{"description":"NIC MTU. Omitted when unset.","type":"integer"},"vlan":{"description":"L2 VLAN tag. Omitted when unset.","type":"integer"}}},"pool":{"description":"Proxmox resource pool the worker VMs are added to. Set the same pool on every node pool of a tenant cluster and grant the tenant's tokens their privileges on `/pool/`: that is the one ACL path that follows every VM capmox creates, replacements included. The pool must exist on the hypervisor; capmox does not create it. Empty adds the workers to no pool.","type":"string","default":""},"storage":{"description":"Proxmox storage for the cloned disk; needs `full: true`. A target storage is a full-clone parameter — a linked clone always lives on the template's storage, and Proxmox refuses the pair — so the render refuses it too. Empty keeps the template's storage.","type":"string","default":""},"templateTags":{"description":"Tags identifying the Talos VM template to clone. capmox matches a template when its tag set is EQUAL to this list, not when it contains it, so every tag on the template must be listed here — a missing one yields `found 0 VM templates with tags ...` while the tags are plainly present. The template has to carry this pool's `talos.schematicID` at `talos.version` in the nocloud flavour, with the Proxmox guest agent enabled for the `qemu-guest-agent` extension of the default schematic; `hack/proxmox-talos-template.sh` builds it from these values and prints the tags to list here. Do not build it from the platform release's `nocloud-amd64.raw.xz`: that is the bare-metal node image, whose `zfs` extension keeps a VM from finishing its boot sequence and reboots it at the Talos boot timeout. Required when `substrate` is `proxmox`.","type":"array","default":[],"items":{"type":"string"}}}}}} release: labels: sharding.fluxcd.io/key: tenants From ed40d8332306d7fde6583dbf55b0de36f820d2a1 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 2 Nov 2026 11:33:40 +0100 Subject: [PATCH 13/28] docs(kubernetes): say a worker rollout needs room for one extra VM With maxUnavailable 0 a pool short of quota or capacity for the replacement VM stalls its rollout instead of shrinking. The field description, which also feeds the API and dashboard documentation, now says so and names maxUnavailable: 1 as the way out. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- api/apps/v1alpha1/kubernetesnodes/types.go | 2 +- packages/apps/kubernetes-nodes/README.md | 2 +- packages/apps/kubernetes-nodes/values.schema.json | 2 +- packages/apps/kubernetes-nodes/values.yaml | 2 +- .../system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/apps/v1alpha1/kubernetesnodes/types.go b/api/apps/v1alpha1/kubernetesnodes/types.go index c5c6dc15b..81cca0bf9 100644 --- a/api/apps/v1alpha1/kubernetesnodes/types.go +++ b/api/apps/v1alpha1/kubernetesnodes/types.go @@ -51,7 +51,7 @@ type ConfigSpec struct { // List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). // +kubebuilder:default:={} Gpus []GPU `json:"gpus,omitempty"` - // Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. + // Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. // +kubebuilder:validation:Minimum=0 MaxUnavailable *int `json:"maxUnavailable,omitempty"` // Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls. diff --git a/packages/apps/kubernetes-nodes/README.md b/packages/apps/kubernetes-nodes/README.md index dd2872b93..348b21268 100644 --- a/packages/apps/kubernetes-nodes/README.md +++ b/packages/apps/kubernetes-nodes/README.md @@ -43,7 +43,7 @@ the parent cluster `kubernetes-` in the same namespace, plus a | `podCpuRequest` | CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart. | `quantity` | `""` | | `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | | `gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` | -| `maxUnavailable` | Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. | `*int` | `null` | +| `maxUnavailable` | Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. | `*int` | `null` | | `kernelModules` | Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls. | `[]object` | `[]` | | `kernelModules[i].name` | Module name as `modprobe` takes it, e.g. `nvidia_uvm`. | `string` | `""` | | `kernelModules[i].parameters` | Module parameters, each as a bare `key=value` string. | `[]string` | `[]` | diff --git a/packages/apps/kubernetes-nodes/values.schema.json b/packages/apps/kubernetes-nodes/values.schema.json index 9fbd9e7cd..53de10b95 100644 --- a/packages/apps/kubernetes-nodes/values.schema.json +++ b/packages/apps/kubernetes-nodes/values.schema.json @@ -147,7 +147,7 @@ } }, "maxUnavailable": { - "description": "Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.", + "description": "Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.", "type": "integer", "minimum": 0 }, diff --git a/packages/apps/kubernetes-nodes/values.yaml b/packages/apps/kubernetes-nodes/values.yaml index f7a6d0fec..f5d6c7dcf 100644 --- a/packages/apps/kubernetes-nodes/values.yaml +++ b/packages/apps/kubernetes-nodes/values.yaml @@ -77,7 +77,7 @@ resources: {} ## @param {[]GPU} gpus - List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). gpus: [] -## @param {*int} [maxUnavailable] - Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. +## @param {*int} [maxUnavailable] - Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. ## @minimum 0 ## @param {[]KernelModule} [kernelModules] - Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls. diff --git a/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml b/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml index 739d5480a..0e5a196fd 100644 --- a/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml +++ b/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml @@ -21,7 +21,7 @@ spec: singular: kubernetesnodes plural: kubernetesnodeses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool name (the part after `-`) must not collide with a nodeGroup still managed by the parent kubernetes chart (e.g. the default `md0`), or the render fails on ownership conflict.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `\"010\"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `\"010\"` reserves what `\"10\"` does.","type":"object","default":{},"properties":{"cpu":{"description":"Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"podCpuLimit":{"description":"CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"podCpuRequest":{"description":"CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"maxUnavailable":{"description":"Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.","type":"integer","minimum":0},"kernelModules":{"description":"Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls.","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Module name as `modprobe` takes it, e.g. `nvidia_uvm`.","type":"string"},"parameters":{"description":"Module parameters, each as a bare `key=value` string.","type":"array","items":{"type":"string"}}}}},"kubelet":{"description":"Kubelet resource reservations for this pool.","type":"object","default":{},"properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"logSerialConsole":{"description":"Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it.","type":"boolean","default":false},"maxUnhealthy":{"description":"Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is a percentage. Drop to \"0%\" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At \"50%\" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two.","type":"string","default":"50%"},"nodeStartupTimeout":{"description":"Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.","type":"string","default":"20m"},"readyUnknownTimeout":{"description":"How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"5m"},"readyFalseTimeout":{"description":"How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"10m"},"maxNodeProvisionTime":{"description":"How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted. Keep in sync with the parent kubernetes chart.","type":"string","default":"30m"},"version":{"description":"Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool -- and must satisfy the Talos<->Kubernetes support matrix against `talos.version`.","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"osImage":{"description":"Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool.","type":"object","default":{},"properties":{"builtin":{"description":"Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled.","type":"object","properties":{"schematicID":{"description":"Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version of the golden to clone. Defaults to `talos.version`.","type":"string"}}},"factory":{"description":"Import from an Image Factory / mirror (default).","type":"object","properties":{"imageFactoryURL":{"description":"Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. NOTE: this redirects only the worker boot-disk import; the in-guest upgrade installer image is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.","type":"string"},"schematicID":{"description":"Talos schematic ID. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version. Defaults to `talos.version`.","type":"string"}}}}},"talos":{"description":"Talos worker image configuration. Keep in sync with the parent cluster's `talos`.","type":"object","default":{},"required":["imageFactoryURL","installerRepository","registryMirrors","schematicID","version"],"properties":{"imageFactoryCA":{"description":"Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there.","type":"string","default":""},"imageFactoryURL":{"description":"Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal.","type":"string","default":"https://factory.talos.dev"},"installerRepository":{"description":"OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.","type":"string","default":"factory.talos.dev/installer"},"registryMirrors":{"description":"Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true},"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"version":{"description":"Default Talos release for this pool's worker OS image and in-guest installer. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix rejects a pairing it lists as unsupported; it lists the Talos minors this chart ships and passes any other minor unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.","type":"string","default":"v1.13.6"}}},"substrate":{"description":"Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["network"],"properties":{"dnsServers":{"description":"Nameservers written into the worker machineconfig. On kubevirt the workers use the management cluster's CoreDNS, which they reach over the pod network; an off-cluster Proxmox worker cannot, so it needs reachable resolvers of its own. Required when `substrate` is `proxmox`. Keep in sync with the parent kubernetes chart's `proxmox.dnsServers`.","type":"array","default":[],"items":{"type":"string"}},"full":{"description":"Full clone rather than linked. Default is a linked clone: it costs kilobytes at creation instead of the whole disk, because a ZFS-backed linked clone writes only its own increment (measured: 8K against 10.2G for the same worker full-cloned onto a `sparse 0` pool). The cost is that the clone holds the template's base snapshot, so the template cannot be rotated while any linked clone still references it — which is what the consolidation CronJob resolves 24h after creation. Set true for a pool that must be independent of the template from the first second.","type":"boolean","default":false},"network":{"description":"NIC configuration.","type":"object","default":{},"required":["bridge"],"properties":{"bridge":{"description":"Proxmox bridge the VM NIC attaches to (e.g. `vmbr0`). Required when `substrate` is `proxmox`.","type":"string","default":""},"mtu":{"description":"NIC MTU. Omitted when unset.","type":"integer"},"vlan":{"description":"L2 VLAN tag. Omitted when unset.","type":"integer"}}},"pool":{"description":"Proxmox resource pool the worker VMs are added to. Set the same pool on every node pool of a tenant cluster and grant the tenant's tokens their privileges on `/pool/`: that is the one ACL path that follows every VM capmox creates, replacements included. The pool must exist on the hypervisor; capmox does not create it. Empty adds the workers to no pool.","type":"string","default":""},"storage":{"description":"Proxmox storage for the cloned disk; needs `full: true`. A target storage is a full-clone parameter — a linked clone always lives on the template's storage, and Proxmox refuses the pair — so the render refuses it too. Empty keeps the template's storage.","type":"string","default":""},"templateTags":{"description":"Tags identifying the Talos VM template to clone. capmox matches a template when its tag set is EQUAL to this list, not when it contains it, so every tag on the template must be listed here — a missing one yields `found 0 VM templates with tags ...` while the tags are plainly present. The template has to carry this pool's `talos.schematicID` at `talos.version` in the nocloud flavour, with the Proxmox guest agent enabled for the `qemu-guest-agent` extension of the default schematic; `hack/proxmox-talos-template.sh` builds it from these values and prints the tags to list here. Do not build it from the platform release's `nocloud-amd64.raw.xz`: that is the bare-metal node image, whose `zfs` extension keeps a VM from finishing its boot sequence and reboots it at the Talos boot timeout. Required when `substrate` is `proxmox`.","type":"array","default":[],"items":{"type":"string"}}}}}} + {"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool name (the part after `-`) must not collide with a nodeGroup still managed by the parent kubernetes chart (e.g. the default `md0`), or the render fails on ownership conflict.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `\"010\"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `\"010\"` reserves what `\"10\"` does.","type":"object","default":{},"properties":{"cpu":{"description":"Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"podCpuLimit":{"description":"CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"podCpuRequest":{"description":"CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"maxUnavailable":{"description":"Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.","type":"integer","minimum":0},"kernelModules":{"description":"Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls.","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Module name as `modprobe` takes it, e.g. `nvidia_uvm`.","type":"string"},"parameters":{"description":"Module parameters, each as a bare `key=value` string.","type":"array","items":{"type":"string"}}}}},"kubelet":{"description":"Kubelet resource reservations for this pool.","type":"object","default":{},"properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"logSerialConsole":{"description":"Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it.","type":"boolean","default":false},"maxUnhealthy":{"description":"Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is a percentage. Drop to \"0%\" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At \"50%\" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two.","type":"string","default":"50%"},"nodeStartupTimeout":{"description":"Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.","type":"string","default":"20m"},"readyUnknownTimeout":{"description":"How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"5m"},"readyFalseTimeout":{"description":"How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"10m"},"maxNodeProvisionTime":{"description":"How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted. Keep in sync with the parent kubernetes chart.","type":"string","default":"30m"},"version":{"description":"Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool -- and must satisfy the Talos<->Kubernetes support matrix against `talos.version`.","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"osImage":{"description":"Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool.","type":"object","default":{},"properties":{"builtin":{"description":"Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled.","type":"object","properties":{"schematicID":{"description":"Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version of the golden to clone. Defaults to `talos.version`.","type":"string"}}},"factory":{"description":"Import from an Image Factory / mirror (default).","type":"object","properties":{"imageFactoryURL":{"description":"Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. NOTE: this redirects only the worker boot-disk import; the in-guest upgrade installer image is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.","type":"string"},"schematicID":{"description":"Talos schematic ID. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version. Defaults to `talos.version`.","type":"string"}}}}},"talos":{"description":"Talos worker image configuration. Keep in sync with the parent cluster's `talos`.","type":"object","default":{},"required":["imageFactoryURL","installerRepository","registryMirrors","schematicID","version"],"properties":{"imageFactoryCA":{"description":"Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there.","type":"string","default":""},"imageFactoryURL":{"description":"Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal.","type":"string","default":"https://factory.talos.dev"},"installerRepository":{"description":"OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.","type":"string","default":"factory.talos.dev/installer"},"registryMirrors":{"description":"Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true},"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"version":{"description":"Default Talos release for this pool's worker OS image and in-guest installer. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix rejects a pairing it lists as unsupported; it lists the Talos minors this chart ships and passes any other minor unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.","type":"string","default":"v1.13.6"}}},"substrate":{"description":"Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["network"],"properties":{"dnsServers":{"description":"Nameservers written into the worker machineconfig. On kubevirt the workers use the management cluster's CoreDNS, which they reach over the pod network; an off-cluster Proxmox worker cannot, so it needs reachable resolvers of its own. Required when `substrate` is `proxmox`. Keep in sync with the parent kubernetes chart's `proxmox.dnsServers`.","type":"array","default":[],"items":{"type":"string"}},"full":{"description":"Full clone rather than linked. Default is a linked clone: it costs kilobytes at creation instead of the whole disk, because a ZFS-backed linked clone writes only its own increment (measured: 8K against 10.2G for the same worker full-cloned onto a `sparse 0` pool). The cost is that the clone holds the template's base snapshot, so the template cannot be rotated while any linked clone still references it — which is what the consolidation CronJob resolves 24h after creation. Set true for a pool that must be independent of the template from the first second.","type":"boolean","default":false},"network":{"description":"NIC configuration.","type":"object","default":{},"required":["bridge"],"properties":{"bridge":{"description":"Proxmox bridge the VM NIC attaches to (e.g. `vmbr0`). Required when `substrate` is `proxmox`.","type":"string","default":""},"mtu":{"description":"NIC MTU. Omitted when unset.","type":"integer"},"vlan":{"description":"L2 VLAN tag. Omitted when unset.","type":"integer"}}},"pool":{"description":"Proxmox resource pool the worker VMs are added to. Set the same pool on every node pool of a tenant cluster and grant the tenant's tokens their privileges on `/pool/`: that is the one ACL path that follows every VM capmox creates, replacements included. The pool must exist on the hypervisor; capmox does not create it. Empty adds the workers to no pool.","type":"string","default":""},"storage":{"description":"Proxmox storage for the cloned disk; needs `full: true`. A target storage is a full-clone parameter — a linked clone always lives on the template's storage, and Proxmox refuses the pair — so the render refuses it too. Empty keeps the template's storage.","type":"string","default":""},"templateTags":{"description":"Tags identifying the Talos VM template to clone. capmox matches a template when its tag set is EQUAL to this list, not when it contains it, so every tag on the template must be listed here — a missing one yields `found 0 VM templates with tags ...` while the tags are plainly present. The template has to carry this pool's `talos.schematicID` at `talos.version` in the nocloud flavour, with the Proxmox guest agent enabled for the `qemu-guest-agent` extension of the default schematic; `hack/proxmox-talos-template.sh` builds it from these values and prints the tags to list here. Do not build it from the platform release's `nocloud-amd64.raw.xz`: that is the bare-metal node image, whose `zfs` extension keeps a VM from finishing its boot sequence and reboots it at the Talos boot timeout. Required when `substrate` is `proxmox`.","type":"array","default":[],"items":{"type":"string"}}}}}} release: labels: sharding.fluxcd.io/key: tenants From 81fd5fa872e92d7ef9dde526fe0c9206d9a394b6 Mon Sep 17 00:00:00 2001 From: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:55:11 +0100 Subject: [PATCH 14/28] fix(etcd): scope tenant alerts to their scrape job Assisted-by: LLM Signed-off-by: Yan Bondarenko <202671653+yankawai@users.noreply.github.com> --- .../etcd/templates/prometheus-rules.yaml | 30 +++++++++---------- .../etcd/tests/namespace_rules_test.yaml | 18 +++++++++++ .../system/monitoring-agents/alerts/etcd.yaml | 3 +- .../tests/etcd_rules_test.yaml | 20 +++++++++++++ 4 files changed, 55 insertions(+), 16 deletions(-) create mode 100644 packages/extra/etcd/tests/namespace_rules_test.yaml create mode 100644 packages/system/monitoring-agents/tests/etcd_rules_test.yaml diff --git a/packages/extra/etcd/templates/prometheus-rules.yaml b/packages/extra/etcd/templates/prometheus-rules.yaml index a53706541..6f2be8590 100644 --- a/packages/extra/etcd/templates/prometheus-rules.yaml +++ b/packages/extra/etcd/templates/prometheus-rules.yaml @@ -10,7 +10,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': insufficient members '{{`{{ $value }}`}}'." expr: | - sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2) + sum(up{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == bool 1) by (job) < ((count(up{job="{{ .Release.Namespace }}/etcd-pod-scrape"}) by (job) + 1) / 2) for: 3m labels: severity: critical @@ -19,7 +19,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': member '{{`{{ $labels.instance }}`}}' has no leader." expr: | - etcd_server_has_leader{job=~".*etcd.*"} == 0 + etcd_server_has_leader{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == 0 for: 1m labels: severity: critical @@ -28,7 +28,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': instance '{{`{{ $labels.instance }}`}}' has seen '{{`{{ $value }}`}}' leader changes within the last hour." expr: | - rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3 + rate(etcd_server_leader_changes_seen_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[15m]) > 3 for: 15m labels: severity: warning @@ -37,9 +37,9 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method) + 100 * sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method) / - sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method) + sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (job, instance, grpc_service, grpc_method) > 1 for: 10m labels: @@ -49,9 +49,9 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method) + 100 * sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method) / - sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method) + sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (job, instance, grpc_service, grpc_method) > 5 for: 5m labels: @@ -61,7 +61,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': gRPC requests to '{{`{{ $labels.grpc_method }}`}}' are taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le)) + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le)) > 0.15 for: 10m labels: @@ -71,7 +71,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': member communication with '{{`{{ $labels.To }}`}}' is taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) > 0.15 for: 10m labels: @@ -81,7 +81,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' proposal failures within the last hour on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + rate(etcd_server_proposals_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[15m]) > 5 for: 15m labels: severity: warning @@ -90,7 +90,7 @@ spec: annotations: summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.01 + sum(rate(etcd_http_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (method) > 0.01 for: 10m labels: severity: warning @@ -99,7 +99,7 @@ spec: annotations: summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.05 + sum(rate(etcd_http_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (method) > 0.05 for: 10m labels: severity: critical @@ -108,7 +108,7 @@ spec: annotations: summary: "etcd instance '{{`{{ $labels.instance }}`}}' HTTP requests to '{{`{{ $labels.method }}`}}' are slow." expr: | - histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) + histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) > 0.15 for: 10m labels: @@ -120,10 +120,10 @@ spec: description: 'etcd cluster "{{`{{ $labels.job }}`}}": members are down {{`{{ $value }}`}}.' expr: | max without (endpoint) ( - sum without (instance, pod) (up{job=~".*etcd.*"} == bool 0) + sum without (instance, pod) (up{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == bool 0) or count without (To) ( - sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 + sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[120s])) > 0.01 ) ) > 0 diff --git a/packages/extra/etcd/tests/namespace_rules_test.yaml b/packages/extra/etcd/tests/namespace_rules_test.yaml new file mode 100644 index 000000000..e972f3cd0 --- /dev/null +++ b/packages/extra/etcd/tests/namespace_rules_test.yaml @@ -0,0 +1,18 @@ +suite: etcd alerts stay inside their release namespace + +release: + name: etcd + namespace: tenant-scope + +templates: + - templates/prometheus-rules.yaml + +tests: + - it: scopes every alert query to this tenant's scrape job + asserts: + - matchRegex: + path: spec.groups[0].rules[*].expr + pattern: 'job="tenant-scope/etcd-pod-scrape"' + - notMatchRegex: + path: spec.groups[0].rules[*].expr + pattern: 'job=~' diff --git a/packages/system/monitoring-agents/alerts/etcd.yaml b/packages/system/monitoring-agents/alerts/etcd.yaml index 72c5ef7b2..48cdc84d9 100644 --- a/packages/system/monitoring-agents/alerts/etcd.yaml +++ b/packages/system/monitoring-agents/alerts/etcd.yaml @@ -116,8 +116,9 @@ spec: {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.' summary: etcd cluster member communication is slow. + # Tenant peer RTT is covered by the tenant etcd rules. expr: |- - histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*", job!~"tenant-.*/etcd-pod-scrape"}[5m])) > 0.15 for: 10m labels: diff --git a/packages/system/monitoring-agents/tests/etcd_rules_test.yaml b/packages/system/monitoring-agents/tests/etcd_rules_test.yaml new file mode 100644 index 000000000..78651297c --- /dev/null +++ b/packages/system/monitoring-agents/tests/etcd_rules_test.yaml @@ -0,0 +1,20 @@ +suite: tenant peer RTT is evaluated once + +templates: + - templates/alerts.yaml + +tests: + - it: leaves tenant peer RTT to the tenant rules + documentSelector: + path: metadata.name + value: alerts-etcd + asserts: + - matchRegex: + path: spec.groups[0].rules[7].expr + pattern: 'job!~"tenant-\.\*/etcd-pod-scrape"' + - notMatchRegex: + path: spec.groups[0].rules[9].expr + pattern: 'job!~' + - notMatchRegex: + path: spec.groups[0].rules[10].expr + pattern: 'job!~' From 41765174c09650dfaab59152abfe54daa4f2899d Mon Sep 17 00:00:00 2001 From: Marian Koreniuk Date: Tue, 3 Nov 2026 21:00:09 +0100 Subject: [PATCH 15/28] docs: add CageForge to ADOPTERS.md Assisted-by: LLM Signed-off-by: Marian Koreniuk --- ADOPTERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ADOPTERS.md b/ADOPTERS.md index d6cacdc55..936bfe296 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -33,3 +33,4 @@ This list is sorted in chronological order, based on the submission date. | [Hidora](https://hikube.cloud) | @matthieu-robin | 2025-09-17 | Hidora is a Swiss cloud provider delivering managed services and infrastructure solutions through datacenters located in Switzerland, ensuring data sovereignty and reliability. Its sovereign cloud platform, Hikube, is designed to run workloads with high availability across multiple datacenters, providing enterprises with a secure and scalable foundation for their applications based on Cozystack. | | [QOSI](https://qosi.kz) | @tabu-a | 2025-10-04 | QOSI is a non-profit organization driving open-source adoption and digital sovereignty across Kazakhstan and Central Asia. We use Cozystack as a platform for deploying sovereign, GPU-enabled clouds and educational environments under the National AI Program. Our goal is to accelerate the region’s transition toward open, self-hosted cloud-native technologies | | [Cloupard](https://cloupard.kz/) | @serjiott | 2025-12-18 | Cloupard is a public cloud provider offering IaaS and PaaS services via datacenters in Kazakhstan and Uzbekistan. Uses CozyStack on bare metal to extend its managed PaaS offerings. | +| [CageForge](https://cageforge.com/) | @themoriarti | 2026-11-03 | CageForge is a short-cycle R&D company providing independent expertise in cloud native, Cozystack and security for customers in finance, telecom, hosting and cloud providers. We run Cozystack both as a bare-metal installation and on Proxmox VE through the Proxmox integration, and contribute the Proxmox-related work back upstream. | From 525e4161341a4fee333576d3f3adbb9747452bda Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Sun, 8 Nov 2026 19:39:07 +0100 Subject: [PATCH 16/28] fix(kubernetes-nodes): let ComputePlane pools set maxUnavailable and bound it to int32 The ComputePlane wrapper forwards only the pool fields it names, and maxUnavailable was not one of them. A ComputePlane pool without room for one extra VM therefore had no way back to the drain-first rollout, and its rollouts stalled. CAPI decodes maxUnavailable as an int32, so a value above 2147483647 passed the chart schema and was then refused by the MachineDeployment decoder. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- api/apps/v1alpha1/kubernetesnodes/types.go | 1 + .../tests/rollout_strategy_test.yaml | 20 +++++++++++ .../apps/kubernetes-nodes/values.schema.json | 1 + packages/apps/kubernetes-nodes/values.yaml | 1 + packages/extra/computeplane/README.md | 1 + .../extra/computeplane/templates/cluster.yaml | 2 +- .../computeplane/tests/cluster_test.yaml | 35 +++++++++++++++++++ .../extra/computeplane/values.schema.json | 6 ++++ packages/extra/computeplane/values.yaml | 3 ++ .../computeplane-rd/cozyrds/computeplane.yaml | 2 +- .../cozyrds/kubernetes-nodes.yaml | 2 +- 11 files changed, 71 insertions(+), 3 deletions(-) diff --git a/api/apps/v1alpha1/kubernetesnodes/types.go b/api/apps/v1alpha1/kubernetesnodes/types.go index 81cca0bf9..592cb5f4f 100644 --- a/api/apps/v1alpha1/kubernetesnodes/types.go +++ b/api/apps/v1alpha1/kubernetesnodes/types.go @@ -53,6 +53,7 @@ type ConfigSpec struct { Gpus []GPU `json:"gpus,omitempty"` // Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=2.147483647e+09 MaxUnavailable *int `json:"maxUnavailable,omitempty"` // Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls. KernelModules []KernelModule `json:"kernelModules,omitempty"` diff --git a/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml index 210afb068..17e732d33 100644 --- a/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml @@ -114,3 +114,23 @@ tests: maxUnavailable: -1 asserts: - failedTemplate: {} + + # CAPI decodes the field as an int32, so a larger value would render and + # then be refused by the MachineDeployment decoder. + - it: refuses a maxUnavailable above the int32 range (schema maximum) + set: + maxUnavailable: 2147483648 + asserts: + - failedTemplate: + errorPattern: "maxUnavailable" + + - it: accepts the int32 maximum + set: + maxUnavailable: 2147483647 + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 2147483647 diff --git a/packages/apps/kubernetes-nodes/values.schema.json b/packages/apps/kubernetes-nodes/values.schema.json index 53de10b95..8f1068dce 100644 --- a/packages/apps/kubernetes-nodes/values.schema.json +++ b/packages/apps/kubernetes-nodes/values.schema.json @@ -149,6 +149,7 @@ "maxUnavailable": { "description": "Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.", "type": "integer", + "maximum": 2147483647, "minimum": 0 }, "kernelModules": { diff --git a/packages/apps/kubernetes-nodes/values.yaml b/packages/apps/kubernetes-nodes/values.yaml index f5d6c7dcf..9c0ab999d 100644 --- a/packages/apps/kubernetes-nodes/values.yaml +++ b/packages/apps/kubernetes-nodes/values.yaml @@ -79,6 +79,7 @@ gpus: [] ## @param {*int} [maxUnavailable] - Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. ## @minimum 0 +## @maximum 2147483647 ## @param {[]KernelModule} [kernelModules] - Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls. diff --git a/packages/extra/computeplane/README.md b/packages/extra/computeplane/README.md index fe6c1af67..15779168f 100644 --- a/packages/extra/computeplane/README.md +++ b/packages/extra/computeplane/README.md @@ -59,4 +59,5 @@ See the full design in | `nodeGroups[name].kubelet.kubeReservedCpu` | CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | | `nodeGroups[name].kubelet.evictionHardMemory` | Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%). | `string` | `7%` | | `nodeGroups[name].kubelet.evictionSoftMemory` | Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%). | `string` | `10%` | +| `nodeGroups[name].maxUnavailable` | Workers a rollout may remove before their replacement is up. Unset, the `kubernetes-nodes` pool chart uses 0, so each replacement joins before an old worker is drained (1 on a pool with `gpus`). Set 1 on a pool without room for one extra VM, whose rollout would otherwise stall with the new Machine Pending. | `*int` | `null` | diff --git a/packages/extra/computeplane/templates/cluster.yaml b/packages/extra/computeplane/templates/cluster.yaml index 82ecdc452..9c598d1af 100644 --- a/packages/extra/computeplane/templates/cluster.yaml +++ b/packages/extra/computeplane/templates/cluster.yaml @@ -211,7 +211,7 @@ spec: clone or repoint itself at another Image Factory, both of which are operator decisions about the platform's own images. Keep this list in sync with the kubernetes-nodes values schema's pool-shape params. */}} - {{- with pick $group "minReplicas" "maxReplicas" "instanceType" "diskSize" "storageClass" "roles" "resources" "gpus" "kubelet" "logSerialConsole" "maxUnhealthy" "nodeStartupTimeout" "readyUnknownTimeout" "readyFalseTimeout" "podCpuLimit" "podCpuRequest" "maxNodeProvisionTime" }} + {{- with pick $group "minReplicas" "maxReplicas" "instanceType" "diskSize" "storageClass" "roles" "resources" "gpus" "kubelet" "logSerialConsole" "maxUnhealthy" "nodeStartupTimeout" "readyUnknownTimeout" "readyFalseTimeout" "podCpuLimit" "podCpuRequest" "maxNodeProvisionTime" "maxUnavailable" }} {{- toYaml . | nindent 4 }} {{- end }} valuesFrom: diff --git a/packages/extra/computeplane/tests/cluster_test.yaml b/packages/extra/computeplane/tests/cluster_test.yaml index 9d1a6768f..d56e21d30 100644 --- a/packages/extra/computeplane/tests/cluster_test.yaml +++ b/packages/extra/computeplane/tests/cluster_test.yaml @@ -280,6 +280,41 @@ tests: - notExists: path: spec.values.nameOverride + # kubernetes-nodes drains a worker only once its replacement is up; a pool + # without room for one extra VM needs maxUnavailable 1 to finish a rollout, + # so the allowlist must forward it. + - it: forwards maxUnavailable to the pool chart + documentSelector: + path: metadata.name + value: kubernetes-nodes-computeplane-cluster-md0 + set: + nodeGroups: + md0: + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.large + diskSize: 32Gi + roles: ["ingress-nginx"] + maxUnavailable: 1 + asserts: + - equal: + path: spec.values.maxUnavailable + value: 1 + + - it: refuses a maxUnavailable above the int32 range CAPI decodes + set: + nodeGroups: + md0: + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.large + diskSize: 32Gi + roles: ["ingress-nginx"] + maxUnavailable: 2147483648 + asserts: + - failedTemplate: + errorPattern: "maxUnavailable" + # Helm caps a HelmRelease name at 53 chars and each pool renders a 38-char fixed # prefix, so a pool key over 15 chars would create a release the apiserver # accepts but Flux never reconciles. The render must refuse instead (the same diff --git a/packages/extra/computeplane/values.schema.json b/packages/extra/computeplane/values.schema.json index 41db84758..fa640f3d9 100644 --- a/packages/extra/computeplane/values.schema.json +++ b/packages/extra/computeplane/values.schema.json @@ -93,6 +93,12 @@ "type": "integer", "default": 10 }, + "maxUnavailable": { + "description": "Workers a rollout may remove before their replacement is up. Unset, the `kubernetes-nodes` pool chart uses 0, so each replacement joins before an old worker is drained (1 on a pool with `gpus`). Set 1 on a pool without room for one extra VM, whose rollout would otherwise stall with the new Machine Pending.", + "type": "integer", + "maximum": 2147483647, + "minimum": 0 + }, "minReplicas": { "description": "Minimum number of replicas.", "type": "integer", diff --git a/packages/extra/computeplane/values.yaml b/packages/extra/computeplane/values.yaml index 8ab49ea95..2e568ddf3 100644 --- a/packages/extra/computeplane/values.yaml +++ b/packages/extra/computeplane/values.yaml @@ -30,6 +30,9 @@ ## @field {Resources} [resources] - Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Set both `cpu` and `memory` together or neither. ## @field {[]GPU} gpus - List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). ## @field {Kubelet} [kubelet] - Kubelet resource reservations for this node group. +## @field {*int} [maxUnavailable] - Workers a rollout may remove before their replacement is up. Unset, the `kubernetes-nodes` pool chart uses 0, so each replacement joins before an old worker is drained (1 on a pool with `gpus`). Set 1 on a pool without room for one extra VM, whose rollout would otherwise stall with the new Machine Pending. +## @minimum 0 +## @maximum 2147483647 ## @param {map[string]NodeGroup} nodeGroups - Worker node groups of the ComputePlane cluster (cluster shape only — the security posture is fixed by the module and cannot be overridden through this knob: the module forwards only the documented per-pool fields to the `kubernetes-nodes` pool chart, so any other `NodeGroup` field, e.g. `talos`/`version`/`images`, is silently dropped — accepted but inert). Same structure as the `kubernetes` application's `nodeGroups`. When left empty, the module itself materialises a default scale-from-zero `md0` pool with `roles: [ingress-nginx]` (the wrapped `kubernetes` chart no longer injects one after the Phase 2 split), so the cluster sits idle until a pending Pod (e.g. the ingress-nginx controller) triggers the cluster-autoscaler. Supplied groups are authoritative (not merged with the default), so at least one of them must carry `roles: [ingress-nginx]` — enforced at render time. Each supplied pool key must be at most 15 characters so its `kubernetes-nodes-computeplane-cluster-` HelmRelease name fits Helm's 53-character limit — also enforced at render time. nodeGroups: {} diff --git a/packages/system/computeplane-rd/cozyrds/computeplane.yaml b/packages/system/computeplane-rd/cozyrds/computeplane.yaml index c84d39b78..1a2b68a60 100644 --- a/packages/system/computeplane-rd/cozyrds/computeplane.yaml +++ b/packages/system/computeplane-rd/cozyrds/computeplane.yaml @@ -8,7 +8,7 @@ spec: singular: computeplane plural: computeplanes openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"nodeGroups":{"description":"Worker node groups of the ComputePlane cluster (cluster shape only — the security posture is fixed by the module and cannot be overridden through this knob: the module forwards only the documented per-pool fields to the `kubernetes-nodes` pool chart, so any other `NodeGroup` field, e.g. `talos`/`version`/`images`, is silently dropped — accepted but inert). Same structure as the `kubernetes` application's `nodeGroups`. When left empty, the module itself materialises a default scale-from-zero `md0` pool with `roles: [ingress-nginx]` (the wrapped `kubernetes` chart no longer injects one after the Phase 2 split), so the cluster sits idle until a pending Pod (e.g. the ingress-nginx controller) triggers the cluster-autoscaler. Supplied groups are authoritative (not merged with the default), so at least one of them must carry `roles: [ingress-nginx]` — enforced at render time. Each supplied pool key must be at most 15 characters so its `kubernetes-nodes-computeplane-cluster-` HelmRelease name fits Helm's 53-character limit — also enforced at render time.","type":"object","default":{},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas"],"properties":{"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Set both `cpu` and `memory` together or neither.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles. At least one group must carry `ingress-nginx` — the module pins the ingress-nginx addon on, and its controller only schedules onto nodes with that role (enforced at render time).","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, the `kubernetes-nodes` pool chart applies its own default (`replicated`).","type":"string","x-cozystack-options":{"source":"storageclass"}}}}}}} + {"title":"Chart Values","type":"object","properties":{"nodeGroups":{"description":"Worker node groups of the ComputePlane cluster (cluster shape only — the security posture is fixed by the module and cannot be overridden through this knob: the module forwards only the documented per-pool fields to the `kubernetes-nodes` pool chart, so any other `NodeGroup` field, e.g. `talos`/`version`/`images`, is silently dropped — accepted but inert). Same structure as the `kubernetes` application's `nodeGroups`. When left empty, the module itself materialises a default scale-from-zero `md0` pool with `roles: [ingress-nginx]` (the wrapped `kubernetes` chart no longer injects one after the Phase 2 split), so the cluster sits idle until a pending Pod (e.g. the ingress-nginx controller) triggers the cluster-autoscaler. Supplied groups are authoritative (not merged with the default), so at least one of them must carry `roles: [ingress-nginx]` — enforced at render time. Each supplied pool key must be at most 15 characters so its `kubernetes-nodes-computeplane-cluster-` HelmRelease name fits Helm's 53-character limit — also enforced at render time.","type":"object","default":{},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas"],"properties":{"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"maxUnavailable":{"description":"Workers a rollout may remove before their replacement is up. Unset, the `kubernetes-nodes` pool chart uses 0, so each replacement joins before an old worker is drained (1 on a pool with `gpus`). Set 1 on a pool without room for one extra VM, whose rollout would otherwise stall with the new Machine Pending.","type":"integer","maximum":2147483647,"minimum":0},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Set both `cpu` and `memory` together or neither.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles. At least one group must carry `ingress-nginx` — the module pins the ingress-nginx addon on, and its controller only schedules onto nodes with that role (enforced at render time).","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, the `kubernetes-nodes` pool chart applies its own default (`replicated`).","type":"string","x-cozystack-options":{"source":"storageclass"}}}}}}} release: prefix: "" labels: diff --git a/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml b/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml index 0e5a196fd..df883809c 100644 --- a/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml +++ b/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml @@ -21,7 +21,7 @@ spec: singular: kubernetesnodes plural: kubernetesnodeses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool name (the part after `-`) must not collide with a nodeGroup still managed by the parent kubernetes chart (e.g. the default `md0`), or the render fails on ownership conflict.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `\"010\"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `\"010\"` reserves what `\"10\"` does.","type":"object","default":{},"properties":{"cpu":{"description":"Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"podCpuLimit":{"description":"CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"podCpuRequest":{"description":"CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"maxUnavailable":{"description":"Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.","type":"integer","minimum":0},"kernelModules":{"description":"Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls.","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Module name as `modprobe` takes it, e.g. `nvidia_uvm`.","type":"string"},"parameters":{"description":"Module parameters, each as a bare `key=value` string.","type":"array","items":{"type":"string"}}}}},"kubelet":{"description":"Kubelet resource reservations for this pool.","type":"object","default":{},"properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"logSerialConsole":{"description":"Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it.","type":"boolean","default":false},"maxUnhealthy":{"description":"Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is a percentage. Drop to \"0%\" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At \"50%\" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two.","type":"string","default":"50%"},"nodeStartupTimeout":{"description":"Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.","type":"string","default":"20m"},"readyUnknownTimeout":{"description":"How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"5m"},"readyFalseTimeout":{"description":"How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"10m"},"maxNodeProvisionTime":{"description":"How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted. Keep in sync with the parent kubernetes chart.","type":"string","default":"30m"},"version":{"description":"Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool -- and must satisfy the Talos<->Kubernetes support matrix against `talos.version`.","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"osImage":{"description":"Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool.","type":"object","default":{},"properties":{"builtin":{"description":"Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled.","type":"object","properties":{"schematicID":{"description":"Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version of the golden to clone. Defaults to `talos.version`.","type":"string"}}},"factory":{"description":"Import from an Image Factory / mirror (default).","type":"object","properties":{"imageFactoryURL":{"description":"Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. NOTE: this redirects only the worker boot-disk import; the in-guest upgrade installer image is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.","type":"string"},"schematicID":{"description":"Talos schematic ID. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version. Defaults to `talos.version`.","type":"string"}}}}},"talos":{"description":"Talos worker image configuration. Keep in sync with the parent cluster's `talos`.","type":"object","default":{},"required":["imageFactoryURL","installerRepository","registryMirrors","schematicID","version"],"properties":{"imageFactoryCA":{"description":"Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there.","type":"string","default":""},"imageFactoryURL":{"description":"Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal.","type":"string","default":"https://factory.talos.dev"},"installerRepository":{"description":"OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.","type":"string","default":"factory.talos.dev/installer"},"registryMirrors":{"description":"Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true},"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"version":{"description":"Default Talos release for this pool's worker OS image and in-guest installer. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix rejects a pairing it lists as unsupported; it lists the Talos minors this chart ships and passes any other minor unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.","type":"string","default":"v1.13.6"}}},"substrate":{"description":"Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["network"],"properties":{"dnsServers":{"description":"Nameservers written into the worker machineconfig. On kubevirt the workers use the management cluster's CoreDNS, which they reach over the pod network; an off-cluster Proxmox worker cannot, so it needs reachable resolvers of its own. Required when `substrate` is `proxmox`. Keep in sync with the parent kubernetes chart's `proxmox.dnsServers`.","type":"array","default":[],"items":{"type":"string"}},"full":{"description":"Full clone rather than linked. Default is a linked clone: it costs kilobytes at creation instead of the whole disk, because a ZFS-backed linked clone writes only its own increment (measured: 8K against 10.2G for the same worker full-cloned onto a `sparse 0` pool). The cost is that the clone holds the template's base snapshot, so the template cannot be rotated while any linked clone still references it — which is what the consolidation CronJob resolves 24h after creation. Set true for a pool that must be independent of the template from the first second.","type":"boolean","default":false},"network":{"description":"NIC configuration.","type":"object","default":{},"required":["bridge"],"properties":{"bridge":{"description":"Proxmox bridge the VM NIC attaches to (e.g. `vmbr0`). Required when `substrate` is `proxmox`.","type":"string","default":""},"mtu":{"description":"NIC MTU. Omitted when unset.","type":"integer"},"vlan":{"description":"L2 VLAN tag. Omitted when unset.","type":"integer"}}},"pool":{"description":"Proxmox resource pool the worker VMs are added to. Set the same pool on every node pool of a tenant cluster and grant the tenant's tokens their privileges on `/pool/`: that is the one ACL path that follows every VM capmox creates, replacements included. The pool must exist on the hypervisor; capmox does not create it. Empty adds the workers to no pool.","type":"string","default":""},"storage":{"description":"Proxmox storage for the cloned disk; needs `full: true`. A target storage is a full-clone parameter — a linked clone always lives on the template's storage, and Proxmox refuses the pair — so the render refuses it too. Empty keeps the template's storage.","type":"string","default":""},"templateTags":{"description":"Tags identifying the Talos VM template to clone. capmox matches a template when its tag set is EQUAL to this list, not when it contains it, so every tag on the template must be listed here — a missing one yields `found 0 VM templates with tags ...` while the tags are plainly present. The template has to carry this pool's `talos.schematicID` at `talos.version` in the nocloud flavour, with the Proxmox guest agent enabled for the `qemu-guest-agent` extension of the default schematic; `hack/proxmox-talos-template.sh` builds it from these values and prints the tags to list here. Do not build it from the platform release's `nocloud-amd64.raw.xz`: that is the bare-metal node image, whose `zfs` extension keeps a VM from finishing its boot sequence and reboots it at the Talos boot timeout. Required when `substrate` is `proxmox`.","type":"array","default":[],"items":{"type":"string"}}}}}} + {"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool name (the part after `-`) must not collide with a nodeGroup still managed by the parent kubernetes chart (e.g. the default `md0`), or the render fails on ownership conflict.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `\"010\"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `\"010\"` reserves what `\"10\"` does.","type":"object","default":{},"properties":{"cpu":{"description":"Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"podCpuLimit":{"description":"CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"podCpuRequest":{"description":"CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"maxUnavailable":{"description":"Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.","type":"integer","maximum":2147483647,"minimum":0},"kernelModules":{"description":"Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls.","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Module name as `modprobe` takes it, e.g. `nvidia_uvm`.","type":"string"},"parameters":{"description":"Module parameters, each as a bare `key=value` string.","type":"array","items":{"type":"string"}}}}},"kubelet":{"description":"Kubelet resource reservations for this pool.","type":"object","default":{},"properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"logSerialConsole":{"description":"Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it.","type":"boolean","default":false},"maxUnhealthy":{"description":"Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is a percentage. Drop to \"0%\" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At \"50%\" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two.","type":"string","default":"50%"},"nodeStartupTimeout":{"description":"Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.","type":"string","default":"20m"},"readyUnknownTimeout":{"description":"How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"5m"},"readyFalseTimeout":{"description":"How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"10m"},"maxNodeProvisionTime":{"description":"How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted. Keep in sync with the parent kubernetes chart.","type":"string","default":"30m"},"version":{"description":"Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool -- and must satisfy the Talos<->Kubernetes support matrix against `talos.version`.","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"osImage":{"description":"Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool.","type":"object","default":{},"properties":{"builtin":{"description":"Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled.","type":"object","properties":{"schematicID":{"description":"Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version of the golden to clone. Defaults to `talos.version`.","type":"string"}}},"factory":{"description":"Import from an Image Factory / mirror (default).","type":"object","properties":{"imageFactoryURL":{"description":"Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. NOTE: this redirects only the worker boot-disk import; the in-guest upgrade installer image is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.","type":"string"},"schematicID":{"description":"Talos schematic ID. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version. Defaults to `talos.version`.","type":"string"}}}}},"talos":{"description":"Talos worker image configuration. Keep in sync with the parent cluster's `talos`.","type":"object","default":{},"required":["imageFactoryURL","installerRepository","registryMirrors","schematicID","version"],"properties":{"imageFactoryCA":{"description":"Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there.","type":"string","default":""},"imageFactoryURL":{"description":"Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal.","type":"string","default":"https://factory.talos.dev"},"installerRepository":{"description":"OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.","type":"string","default":"factory.talos.dev/installer"},"registryMirrors":{"description":"Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true},"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"version":{"description":"Default Talos release for this pool's worker OS image and in-guest installer. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix rejects a pairing it lists as unsupported; it lists the Talos minors this chart ships and passes any other minor unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.","type":"string","default":"v1.13.6"}}},"substrate":{"description":"Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["network"],"properties":{"dnsServers":{"description":"Nameservers written into the worker machineconfig. On kubevirt the workers use the management cluster's CoreDNS, which they reach over the pod network; an off-cluster Proxmox worker cannot, so it needs reachable resolvers of its own. Required when `substrate` is `proxmox`. Keep in sync with the parent kubernetes chart's `proxmox.dnsServers`.","type":"array","default":[],"items":{"type":"string"}},"full":{"description":"Full clone rather than linked. Default is a linked clone: it costs kilobytes at creation instead of the whole disk, because a ZFS-backed linked clone writes only its own increment (measured: 8K against 10.2G for the same worker full-cloned onto a `sparse 0` pool). The cost is that the clone holds the template's base snapshot, so the template cannot be rotated while any linked clone still references it — which is what the consolidation CronJob resolves 24h after creation. Set true for a pool that must be independent of the template from the first second.","type":"boolean","default":false},"network":{"description":"NIC configuration.","type":"object","default":{},"required":["bridge"],"properties":{"bridge":{"description":"Proxmox bridge the VM NIC attaches to (e.g. `vmbr0`). Required when `substrate` is `proxmox`.","type":"string","default":""},"mtu":{"description":"NIC MTU. Omitted when unset.","type":"integer"},"vlan":{"description":"L2 VLAN tag. Omitted when unset.","type":"integer"}}},"pool":{"description":"Proxmox resource pool the worker VMs are added to. Set the same pool on every node pool of a tenant cluster and grant the tenant's tokens their privileges on `/pool/`: that is the one ACL path that follows every VM capmox creates, replacements included. The pool must exist on the hypervisor; capmox does not create it. Empty adds the workers to no pool.","type":"string","default":""},"storage":{"description":"Proxmox storage for the cloned disk; needs `full: true`. A target storage is a full-clone parameter — a linked clone always lives on the template's storage, and Proxmox refuses the pair — so the render refuses it too. Empty keeps the template's storage.","type":"string","default":""},"templateTags":{"description":"Tags identifying the Talos VM template to clone. capmox matches a template when its tag set is EQUAL to this list, not when it contains it, so every tag on the template must be listed here — a missing one yields `found 0 VM templates with tags ...` while the tags are plainly present. The template has to carry this pool's `talos.schematicID` at `talos.version` in the nocloud flavour, with the Proxmox guest agent enabled for the `qemu-guest-agent` extension of the default schematic; `hack/proxmox-talos-template.sh` builds it from these values and prints the tags to list here. Do not build it from the platform release's `nocloud-amd64.raw.xz`: that is the bare-metal node image, whose `zfs` extension keeps a VM from finishing its boot sequence and reboots it at the Talos boot timeout. Required when `substrate` is `proxmox`.","type":"array","default":[],"items":{"type":"string"}}}}}} release: labels: sharding.fluxcd.io/key: tenants From 28b66bbc638366994a7654b912b9021e42c7a781 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 9 Nov 2026 10:08:04 +0100 Subject: [PATCH 17/28] test(computeplane): forward an explicit zero and the int32 maximum of maxUnavailable An explicit 0 is what overrides the pool chart's default of 1 for GPU pools, so it must reach the pool unchanged rather than be dropped as an empty value. The int32 maximum is the largest value the schema accepts and must be forwarded as is. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- .../computeplane/tests/cluster_test.yaml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/extra/computeplane/tests/cluster_test.yaml b/packages/extra/computeplane/tests/cluster_test.yaml index d56e21d30..6dc4b8835 100644 --- a/packages/extra/computeplane/tests/cluster_test.yaml +++ b/packages/extra/computeplane/tests/cluster_test.yaml @@ -301,6 +301,44 @@ tests: path: spec.values.maxUnavailable value: 1 + # An explicit 0 must reach the pool chart: there it overrides the 1 a GPU + # pool gets by default, so dropping it as empty would change the strategy. + - it: forwards an explicit maxUnavailable of 0 to the pool chart + documentSelector: + path: metadata.name + value: kubernetes-nodes-computeplane-cluster-md0 + set: + nodeGroups: + md0: + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.large + diskSize: 32Gi + roles: ["ingress-nginx"] + maxUnavailable: 0 + asserts: + - equal: + path: spec.values.maxUnavailable + value: 0 + + - it: forwards the int32 maximum to the pool chart + documentSelector: + path: metadata.name + value: kubernetes-nodes-computeplane-cluster-md0 + set: + nodeGroups: + md0: + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.large + diskSize: 32Gi + roles: ["ingress-nginx"] + maxUnavailable: 2147483647 + asserts: + - equal: + path: spec.values.maxUnavailable + value: 2147483647 + - it: refuses a maxUnavailable above the int32 range CAPI decodes set: nodeGroups: From f2ee3ec8ad742649c6359a4d50e94496aa967fce Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 9 Nov 2026 13:24:36 +0100 Subject: [PATCH 18/28] test(computeplane): pin the maxUnavailable bounds and zero through the API A zero dropped anywhere between the ComputePlane API and the pool chart would let a GPU pool fall back to the chart default of 1, and the render tests never cross the aggregated API. A negative value let through by the ComputePlane schema would only be refused by the pool chart, after the parent release had already been upgraded. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- hack/e2e-chainsaw/computeplane/chainsaw-test.yaml | 14 +++++++++----- .../extra/computeplane/tests/cluster_test.yaml | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml b/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml index 2a02dc06e..a81ff89ef 100644 --- a/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml @@ -102,7 +102,9 @@ spec: # # nameOverride is outside the module's per-pool allowlist. The ComputePlane # schema accepts it, so it reaches the module and the pool check below can - # see whether the module dropped it. + # see whether the module dropped it. maxUnavailable is set to 0 because a + # zero lost anywhere on the way would let a GPU pool fall back to the + # chart's default of 1; only this path crosses the aggregated API. - script: timeout: 3m content: | @@ -118,6 +120,7 @@ spec: diskSize: 20Gi instanceType: u1.medium logSerialConsole: true + maxUnavailable: 0 minReplicas: 2 maxReplicas: 2 podCpuLimit: 2 @@ -175,8 +178,9 @@ spec: enabled: true exposeMethod: Proxied - # The pool takes the patched shape, and the nameOverride the patch sent - # does not reach it, nor does any other key outside the allowlist. + # The pool takes the patched shape, the explicit maxUnavailable of 0 + # included, and the nameOverride the patch sent does not reach it, nor does + # any other key outside the allowlist. - script: timeout: 1m content: | @@ -186,8 +190,8 @@ spec: sc=$(cozy_e2e_storage_class) values=$(timeout -k 5 30 kubectl -n tenant-cplane get helmrelease kubernetes-nodes-computeplane-cluster-md0 -o json | jq -c '.spec.values') echo "pool values: $values" - echo "$values" | jq -e --arg sc "$sc" '.storageClass == $sc and .minReplicas == 2 and .cluster == "computeplane-cluster" and .clusterReleaseName == "computeplane-cluster"' - extra=$(echo "$values" | jq -r 'keys - ["cluster","clusterReleaseName","minReplicas","maxReplicas","instanceType","diskSize","storageClass","roles","resources","gpus","kubelet","logSerialConsole","maxUnhealthy","nodeStartupTimeout","readyUnknownTimeout","readyFalseTimeout","podCpuLimit","podCpuRequest","maxNodeProvisionTime"] | .[]') + echo "$values" | jq -e --arg sc "$sc" '.storageClass == $sc and .minReplicas == 2 and .maxUnavailable == 0 and .cluster == "computeplane-cluster" and .clusterReleaseName == "computeplane-cluster"' + extra=$(echo "$values" | jq -r 'keys - ["cluster","clusterReleaseName","minReplicas","maxReplicas","instanceType","diskSize","storageClass","roles","resources","gpus","kubelet","logSerialConsole","maxUnhealthy","nodeStartupTimeout","readyUnknownTimeout","readyFalseTimeout","podCpuLimit","podCpuRequest","maxNodeProvisionTime","maxUnavailable"] | .[]') if [ -n "$extra" ]; then echo "pool HelmRelease carries keys outside the module allowlist: $extra" >&2 exit 1 diff --git a/packages/extra/computeplane/tests/cluster_test.yaml b/packages/extra/computeplane/tests/cluster_test.yaml index 6dc4b8835..ad9af03fd 100644 --- a/packages/extra/computeplane/tests/cluster_test.yaml +++ b/packages/extra/computeplane/tests/cluster_test.yaml @@ -353,6 +353,20 @@ tests: - failedTemplate: errorPattern: "maxUnavailable" + - it: refuses a negative maxUnavailable + set: + nodeGroups: + md0: + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.large + diskSize: 32Gi + roles: ["ingress-nginx"] + maxUnavailable: -1 + asserts: + - failedTemplate: + errorPattern: "maxUnavailable" + # Helm caps a HelmRelease name at 53 chars and each pool renders a 38-char fixed # prefix, so a pool key over 15 chars would create a release the apiserver # accepts but Flux never reconciles. The render must refuse instead (the same From c368c48fd8a0dbb858fdf1c1136f20448f0facd8 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Tue, 10 Nov 2026 09:03:55 +0100 Subject: [PATCH 19/28] fix(controller, platform): give unbounded pool members the root's LimitRange The tenant chart renders the tenant-range-limits LimitRange only next to the tenant-quota of a tenant that declares resourceQuotas. Since quotas became hierarchical, an unbounded sub-tenant shares its parent's pool and gets a controller-owned tenant-quota-allocated ResourceQuota, but still no LimitRange. A ResourceQuota on limits.cpu or limits.memory makes admission reject any container that omits that limit, and with no LimitRange nothing fills it in. Pods that ran before the quota was introduced fail as soon as they are recreated; the barman-cloud plugin sidecar, which sets no CPU limit, takes every backed-up postgres instance in such a namespace down on its next restart. The controller already knows which namespaces it clamps, so it copies the pool root's chart LimitRange into each unbounded member, under its own name so that Helm can still create the chart object if the tenant later declares a quota of its own, and removes the copy once the namespace leaves the pool. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- internal/controller/tenantquota/reconciler.go | 85 ++++++++++- .../controller/tenantquota/reconciler_test.go | 136 ++++++++++++++++++ .../cozystack-controller/templates/rbac.yaml | 8 +- 3 files changed, 224 insertions(+), 5 deletions(-) diff --git a/internal/controller/tenantquota/reconciler.go b/internal/controller/tenantquota/reconciler.go index dddd0fbac..6c77eff6d 100644 --- a/internal/controller/tenantquota/reconciler.go +++ b/internal/controller/tenantquota/reconciler.go @@ -52,6 +52,17 @@ const ( // whenever it is below the chart quota, without the controller fighting Flux // over the chart-owned object. allocatedQuotaName = "tenant-quota-allocated" + // chartLimitRangeName is the LimitRange the tenant chart renders next to + // chartQuotaName, giving containers default requests and limits. The chart + // renders both only for a tenant that declares resourceQuotas. + chartLimitRangeName = "tenant-range-limits" + // allocatedLimitRangeName is the controller-owned copy of the pool root's + // chart LimitRange, written into every unbounded pool member. Those members + // render no chart LimitRange, yet carry an allocated quota; without + // container defaults, any pod omitting a limit the quota bounds is rejected + // at admission. A distinct name keeps the copy from colliding with the + // chart-owned object if the member later declares its own quota. + allocatedLimitRangeName = "tenant-range-limits-allocated" // managedByLabel marks the controller-owned ResourceQuotas so they can be // listed and garbage-collected. managedByLabel = "quota.cozystack.io/managed-by" @@ -102,6 +113,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result pools := ComputePools(tenants) desired := map[string]corev1.ResourceList{} + // limitRangeSource maps each unbounded pool member to the pool root whose + // chart LimitRange it must inherit. + limitRangeSource := map[string]string{} for _, p := range pools { if over := p.Overcommitted(); len(over) > 0 { logger.Info("tenant quota pool is overcommitted by sub-tenant carve-outs", "pool", p.Root, "overcommit", over) @@ -120,6 +134,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result for _, ns := range p.Members { if existing[ns] { desired[ns] = buffered.EnforcedHard(ns, usedByNS) + if ns != p.Root { + limitRangeSource[ns] = p.Root + } } } } @@ -134,6 +151,22 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result logger.Error(err, "failed to garbage-collect stale allocated quotas") } + written := map[string]bool{} + for ns, root := range limitRangeSource { + ok, err := r.upsertAllocatedLimitRange(ctx, ns, root) + if err != nil { + logger.Error(err, "failed to apply allocated limit range", "namespace", ns, "poolRoot", root) + } + // Keep the existing copy on a transient error rather than dropping the + // defaults the namespace's pods rely on. + if ok || err != nil { + written[ns] = true + } + } + if err := r.gcAllocatedLimitRanges(ctx, written); err != nil { + logger.Error(err, "failed to garbage-collect stale allocated limit ranges") + } + return ctrl.Result{RequeueAfter: resyncInterval}, nil } @@ -224,6 +257,50 @@ func (r *Reconciler) gcAllocatedQuotas(ctx context.Context, desired map[string]c return nil } +// upsertAllocatedLimitRange copies the pool root's chart LimitRange into an +// unbounded member namespace. It reports false, without error, when the root +// has no chart LimitRange to copy. +func (r *Reconciler) upsertAllocatedLimitRange(ctx context.Context, namespace, poolRoot string) (bool, error) { + source := &corev1.LimitRange{} + if err := r.Get(ctx, types.NamespacedName{Namespace: poolRoot, Name: chartLimitRangeName}, source); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + lr := &corev1.LimitRange{ObjectMeta: metav1.ObjectMeta{Name: allocatedLimitRangeName, Namespace: namespace}} + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, lr, func() error { + if lr.Labels == nil { + lr.Labels = map[string]string{} + } + lr.Labels[managedByLabel] = managedByValue + lr.Labels["internal.cozystack.io/managed-by-cozystack"] = "" + lr.Spec = *source.Spec.DeepCopy() + return nil + }) + return err == nil, err +} + +// gcAllocatedLimitRanges deletes controller-owned LimitRanges in namespaces +// that are no longer unbounded pool members (e.g. the tenant gained its own +// quota and now renders the chart LimitRange, or was deleted). +func (r *Reconciler) gcAllocatedLimitRanges(ctx context.Context, keep map[string]bool) error { + managed := &corev1.LimitRangeList{} + if err := r.List(ctx, managed, client.MatchingLabels{managedByLabel: managedByValue}); err != nil { + return err + } + for i := range managed.Items { + m := &managed.Items[i] + if keep[m.Namespace] { + continue + } + if err := r.Delete(ctx, m); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + return nil +} + func (r *Reconciler) recordOvercommit(ctx context.Context, rootNamespace string, over corev1.ResourceList) { if r.Recorder == nil { return @@ -237,8 +314,8 @@ func (r *Reconciler) recordOvercommit(ctx context.Context, rootNamespace string, } // SetupWithManager wires the controller. Every relevant change (a tenant -// HelmRelease, a namespace, or a tenant ResourceQuota whose usage moved) -// coalesces into one full-tree sweep. +// HelmRelease, a namespace, a tenant ResourceQuota whose usage moved, or a +// tenant LimitRange) coalesces into one full-tree sweep. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { toSweep := handler.EnqueueRequestsFromMapFunc(func(context.Context, client.Object) []reconcile.Request { return []reconcile.Request{sweepKey} @@ -249,11 +326,15 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { tenantQuotas := predicate.NewPredicateFuncs(func(o client.Object) bool { return o.GetName() == chartQuotaName || o.GetName() == allocatedQuotaName }) + tenantLimitRanges := predicate.NewPredicateFuncs(func(o client.Object) bool { + return o.GetName() == chartLimitRangeName || o.GetName() == allocatedLimitRangeName + }) return ctrl.NewControllerManagedBy(mgr). Named("tenantquota-controller"). Watches(&helmv2.HelmRelease{}, toSweep, builder.WithPredicates(tenantReleases)). Watches(&corev1.Namespace{}, toSweep). Watches(&corev1.ResourceQuota{}, toSweep, builder.WithPredicates(tenantQuotas)). + Watches(&corev1.LimitRange{}, toSweep, builder.WithPredicates(tenantLimitRanges)). Complete(r) } diff --git a/internal/controller/tenantquota/reconciler_test.go b/internal/controller/tenantquota/reconciler_test.go index 5f19ed1b1..4624b2e36 100644 --- a/internal/controller/tenantquota/reconciler_test.go +++ b/internal/controller/tenantquota/reconciler_test.go @@ -227,3 +227,139 @@ func TestReconcile_GarbageCollect(t *testing.T) { t.Fatalf("stale allocated quota in tenant-gone should have been garbage-collected") } } + +// chartLimitRange builds the chart-rendered "tenant-range-limits" LimitRange, +// which the tenant chart emits alongside its "tenant-quota". +func chartLimitRange(namespace, defaultCPU string) *corev1.LimitRange { + return &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{Name: chartLimitRangeName, Namespace: namespace}, + Spec: corev1.LimitRangeSpec{Limits: []corev1.LimitRangeItem{{ + Type: corev1.LimitTypeContainer, + Default: resourceList(map[string]string{"cpu": defaultCPU, "memory": "128Mi"}), + DefaultRequest: resourceList(map[string]string{"cpu": "25m", "memory": "128Mi"}), + }}}, + } +} + +func getLimitRange(c client.Client, namespace, name string) (*corev1.LimitRange, bool) { + lr := &corev1.LimitRange{} + if err := c.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: name}, lr); err != nil { + return nil, false + } + return lr, true +} + +// TestReconcile_UnboundedMemberGetsPoolLimitRange: an unbounded child renders +// no chart quota and therefore no chart LimitRange, yet the controller clamps +// it with an allocated quota. Without container defaults, every pod that omits +// a limit the quota bounds (e.g. a sidecar without a CPU limit) is rejected at +// admission, so the controller must copy the pool root's LimitRange into it. +func TestReconcile_UnboundedMemberGetsPoolLimitRange(t *testing.T) { + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + chartLimitRange("tenant-foo", "250m"), + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + lr, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName) + if !ok { + t.Fatalf("unbounded member tenant-foo-bar has an allocated quota but no LimitRange") + } + if len(lr.Spec.Limits) != 1 { + t.Fatalf("tenant-foo-bar LimitRange limits = %d, want 1", len(lr.Spec.Limits)) + } + if got := lr.Spec.Limits[0].Default[corev1.ResourceCPU]; got.String() != "250m" { + t.Fatalf("tenant-foo-bar default cpu = %q, want 250m copied from the pool root", got.String()) + } + if lr.Labels[managedByLabel] != managedByValue { + t.Fatalf("tenant-foo-bar LimitRange is not marked as controller-owned") + } + if _, ok := getLimitRange(c, "tenant-foo", allocatedLimitRangeName); ok { + t.Fatalf("pool root tenant-foo already has the chart LimitRange and must not get a second one") + } +} + +// TestReconcile_PoolLimitRangeFollowsRoot: a change to the root's chart +// LimitRange propagates to the members on the next sweep. +func TestReconcile_PoolLimitRangeFollowsRoot(t *testing.T) { + root := chartLimitRange("tenant-foo", "250m") + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + root, + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + root.Spec.Limits[0].Default[corev1.ResourceCPU] = resource.MustParse("500m") + if err := c.Update(context.Background(), root); err != nil { + t.Fatalf("update root LimitRange: %v", err) + } + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + lr, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName) + if !ok { + t.Fatalf("tenant-foo-bar lost its LimitRange") + } + if got := lr.Spec.Limits[0].Default[corev1.ResourceCPU]; got.String() != "500m" { + t.Fatalf("tenant-foo-bar default cpu = %q, want 500m after the root changed", got.String()) + } +} + +// TestReconcile_PoolLimitRangeGarbageCollected: once the child declares its own +// quota, the chart renders its LimitRange and the controller-owned copy goes. +func TestReconcile_PoolLimitRangeGarbageCollected(t *testing.T) { + stale := &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: allocatedLimitRangeName, + Namespace: "tenant-foo-bar", + Labels: map[string]string{managedByLabel: managedByValue}, + }, + } + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + chartLimitRange("tenant-foo", "250m"), + chartQuota("tenant-foo-bar", map[string]string{"limits.cpu": "4"}, nil), + chartLimitRange("tenant-foo-bar", "250m"), + stale, + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { + t.Fatalf("bounded child tenant-foo-bar renders its own LimitRange; the controller copy must be removed") + } + if _, ok := getLimitRange(c, "tenant-foo-bar", chartLimitRangeName); !ok { + t.Fatalf("the chart-owned LimitRange must never be touched") + } +} + +// TestReconcile_PoolRootWithoutLimitRange: if the root has no chart LimitRange +// there is nothing to copy; the quota is still enforced and the sweep succeeds. +func TestReconcile_PoolRootWithoutLimitRange(t *testing.T) { + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"cpu": "10"}, nil), + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, ok := allocatedHard(t, c, "tenant-foo-bar", "cpu"); !ok { + t.Fatalf("tenant-foo-bar must still get its allocated quota") + } + if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { + t.Fatalf("no LimitRange can be copied from a root that has none") + } +} diff --git a/packages/system/cozystack-controller/templates/rbac.yaml b/packages/system/cozystack-controller/templates/rbac.yaml index e40ab9080..da8f71f49 100644 --- a/packages/system/cozystack-controller/templates/rbac.yaml +++ b/packages/system/cozystack-controller/templates/rbac.yaml @@ -49,10 +49,12 @@ rules: resources: ["namespaces"] verbs: ["get", "list", "watch", "patch", "update"] # TenantQuotaReconciler manages the per-namespace "tenant-quota-allocated" -# ResourceQuota that enforces each tenant's share of its parent pool, and emits -# events on the tenant namespace when sub-tenant quotas overcommit the budget. +# ResourceQuota that enforces each tenant's share of its parent pool, copies the +# pool root's "tenant-range-limits" LimitRange into unbounded members as +# "tenant-range-limits-allocated", and emits events on the tenant namespace when +# sub-tenant quotas overcommit the budget. - apiGroups: [""] - resources: ["resourcequotas"] + resources: ["resourcequotas", "limitranges"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: [""] resources: ["events"] From 9bd8d12cb8816a8c5e5f4bd3229be0206f5b3c54 Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Thu, 12 Nov 2026 18:24:23 +0300 Subject: [PATCH 20/28] fix(postgres): hold the WAL archive destination for backups no job tracks A BackupJob rewrote the shared WAL archive destination while a cnpg.io/Backup of the Cluster was running if nothing tracked that Backup through a running BackupJob: one created by hand or by a ScheduledBackup, or one whose BackupJob was deleted or had already finished. The base backup then stayed where it started while the WAL it needs went to the new location, and the backup could not be restored. Such a Backup now holds the destination while CloudNativePG executes it. Only the phases in which CloudNativePG itself starts no other Backup of the Cluster count, so the hold never outlasts the queue the operator already imposes: a Backup that waits in pending, or was sent back to it, starts its base backup over against the destination current then. Assisted-by: LLM Signed-off-by: Alexey Artamonov --- .../cnpgstrategy_controller.go | 70 +++++++++- .../cnpgstrategy_controller_test.go | 132 +++++++++++++++++- 2 files changed, 190 insertions(+), 12 deletions(-) diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index e43deff33..3dae071cb 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -44,6 +44,8 @@ const ( cnpgClusterLabel = "cnpg.io/cluster" cnpgBackupPhaseComplete = "completed" cnpgBackupPhaseFailed = "failed" + cnpgBackupPhaseStarted = "started" + cnpgBackupPhaseRunning = "running" // barmanObjectNameParam / barmanServerNameParam are the barman-cloud plugin // parameter keys on a Cluster's spec.plugins entry. barmanObjectName points @@ -388,9 +390,10 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp // The ObjectStore is the Cluster's WAL archive destination, shared by // every backup of the app, so it is written only before this job's - // Backup exists, and only while no other job's Backup is in flight and - // no other running job without a Backup yet holds the destination it - // last wrote. + // Backup exists, and only while no other job's Backup is in flight, no + // other running job without a Backup yet holds the destination it last + // wrote, and CloudNativePG runs no Backup of the Cluster that no running + // job tracks. writeDestination := existingBackup == nil if writeDestination { // The gate in Reconcile sees only a job that has not started, and @@ -421,6 +424,17 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp return r.waitCNPGBackupJob(ctx, j, "WaitingForOtherBackup", fmt.Sprintf( "waiting for BackupJob %s of the same application to finish its backup, which writes to the shared WAL archive destination", holder)) } + untracked, err := r.untrackedCNPGBackupExecuting(ctx, j, clusterName) + if err != nil { + return ctrl.Result{}, err + } + if untracked != "" { + apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied) + apimeta.SetStatusCondition(&j.Status.Conditions, cnpgDestinationWriteHeld("WaitingForUntrackedBackup")) + return r.waitCNPGBackupJob(ctx, j, "WaitingForUntrackedBackup", fmt.Sprintf( + "waiting for cnpg.io/Backup %s/%s of cnpg.io/Cluster %s/%s, which no running BackupJob tracks, to finish: CloudNativePG is running it against the shared WAL archive destination", + j.Namespace, untracked, j.Namespace, clusterName)) + } if apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld) != nil { apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ Type: backupCondDestinationWriteHeld, @@ -1162,8 +1176,9 @@ func (r *BackupJobReconciler) findCNPGBackupForJob(ctx context.Context, j *backu // otherCNPGBackupInFlight returns the name of another BackupJob that is still // running and whose cnpg.io/Backup of clusterName has not finished, or "" when -// there is none. A Backup left behind by a finished or deleted job does not -// hold the queue. +// there is none. A Backup left behind by a finished or deleted job is not +// counted here; untrackedCNPGBackupExecuting holds for it while +// CloudNativePG executes it. // // The Backups are read from the apiserver, not the cache, which can still // miss a Backup another job's earlier pass created. Only the ones this @@ -1206,6 +1221,51 @@ func (r *BackupJobReconciler) otherCNPGBackupInFlight(ctx context.Context, j *ba return holders[0], nil } +// untrackedCNPGBackupExecuting returns the name of a cnpg.io/Backup of +// clusterName that CloudNativePG is running, whoever created it. It is +// asked after otherCNPGBackupInFlight and destinationClaimHolder found no +// running BackupJob to wait for, so what it finds is a Backup no running +// job tracks: one created by hand or by a ScheduledBackup, or one whose +// BackupJob was deleted or finished first. Its base backup goes to the +// destination it started with, so a rewrite now would send the WAL it +// needs elsewhere. +// +// Only the phases BackupStatus.IsExecuting reports hold, the ones in which +// CloudNativePG's BackupList.CanExecuteBackup lets no other Backup of the +// Cluster start, so the wait never outlasts CloudNativePG's own queue. A pending +// Backup, even one that ran before and was sent back to pending, starts +// its base backup over against the destination current then. +// +// A Backup created by hand need carry no label at all, and CloudNativePG +// labels only the ones a ScheduledBackup creates, so they are matched by +// spec.cluster.name in a list of the whole namespace. That list comes from +// the cache rather than the apiserver, which every waiting job would +// otherwise ask for the namespace's whole Backup history on each poll. The +// cache can miss a Backup that started moments ago, the same race that +// already lies between this check and the write that follows it, which a +// read from the apiserver would not close either. +func (r *BackupJobReconciler) untrackedCNPGBackupExecuting(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string) (string, error) { + list := &cnpgtypes.BackupList{} + if err := r.List(ctx, list, client.InNamespace(j.Namespace)); err != nil { + return "", fmt.Errorf("list cnpg.io/Backups in %s: %w", j.Namespace, err) + } + var executing []string + for i := range list.Items { + b := &list.Items[i] + if b.Spec.Cluster.Name != clusterName { + continue + } + if b.Status.Phase == cnpgBackupPhaseStarted || b.Status.Phase == cnpgBackupPhaseRunning { + executing = append(executing, b.Name) + } + } + if len(executing) == 0 { + return "", nil + } + sort.Strings(executing) + return executing[0], nil +} + func cnpgDestinationWriteHeld(reason string) metav1.Condition { return metav1.Condition{ Type: backupCondDestinationWriteHeld, diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 5cd990f18..ab74ecc07 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -2643,6 +2643,9 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) { owner string cluster string wantWait bool + // wantUntracked expects the hold to name the cnpg.io/Backup rather + // than a running BackupJob: nothing the controller tracks owns it. + wantUntracked bool }{ {name: "another running job with a running Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", wantWait: true}, {name: "another running job with a pending Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "pending", wantWait: true}, @@ -2652,7 +2655,14 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) { {name: "a Backup of a succeeded job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseSucceeded, otherBackup: "pending"}, {name: "a Backup whose job is gone does not", otherBackup: "pending"}, {name: "a Backup of another Cluster does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", cluster: "postgres-other"}, - {name: "a Backup created by hand does not", otherBackup: "running", owner: "-"}, + {name: "a Backup created by hand that has not started does not", otherBackup: "pending", owner: "-"}, + {name: "a Backup created by hand that is running holds it", otherBackup: "running", owner: "-", wantWait: true, wantUntracked: true}, + {name: "a Backup created by hand that has started holds it", otherBackup: "started", owner: "-", wantWait: true, wantUntracked: true}, + {name: "a Backup created by hand of another Cluster does not", otherBackup: "started", owner: "-", cluster: "postgres-other"}, + {name: "a running Backup whose job is gone holds it", otherBackup: "started", wantWait: true, wantUntracked: true}, + {name: "a running Backup of a failed job holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseFailed, otherBackup: "started", wantWait: true, wantUntracked: true}, + {name: "a running Backup of a succeeded job holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseSucceeded, otherBackup: "running", wantWait: true, wantUntracked: true}, + {name: "a Backup created by hand that is finalizing does not", otherBackup: "finalizing", owner: "-"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -2702,8 +2712,15 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) { if ours != 0 { t.Errorf("the waiting job created %d cnpg.io/Backup(s)", ours) } - if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "bj-other") { - t.Errorf("Ready = %+v, want False/WaitingForOtherBackup naming bj-other", ready) + wantReason, wantNamed := "WaitingForOtherBackup", "BackupJob bj-other " + if tc.wantUntracked { + wantReason, wantNamed = "WaitingForUntrackedBackup", "cnpg.io/Backup "+rolloutNS+"/bj-other-cnpg " + } + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != wantReason || !strings.Contains(ready.Message, wantNamed) { + t.Errorf("Ready = %+v, want False/%s naming %q", ready, wantReason, wantNamed) + } + if held := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); held == nil || held.Status != metav1.ConditionTrue || held.Reason != wantReason { + t.Errorf("%s = %+v, want True/%s", backupCondDestinationWriteHeld, held, wantReason) } if res.RequeueAfter == 0 { t.Errorf("a queued job is not requeued, and nothing else wakes it when the holder finishes: %+v", res) @@ -2728,7 +2745,7 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) { if got := f.liveDestination(); got != "s3://bucket/" { t.Errorf("once the holder finished, destination = %q, want the job's own s3://bucket/", got) } - if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" { + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && (ready.Reason == "WaitingForOtherBackup" || ready.Reason == "WaitingForUntrackedBackup") { t.Errorf("Ready still names the finished holder: %+v", ready) } return @@ -2736,7 +2753,7 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) { if got := f.liveDestination(); got != "s3://bucket/" { t.Errorf("destination = %q, want the job's own s3://bucket/", got) } - if ready != nil && ready.Reason == "WaitingForOtherBackup" { + if ready != nil && (ready.Reason == "WaitingForOtherBackup" || ready.Reason == "WaitingForUntrackedBackup") { t.Errorf("Ready = %+v, the job was held", ready) } f.settleDestination() @@ -3878,16 +3895,29 @@ func TestReconcileCNPG_PluginAttachedByAnotherWriterKeepsTheLatch(t *testing.T) // Waiting for another backup resets the retry budget of the destination // write, so the queue does not spend it. func TestReconcileCNPG_QueueResetsDestinationRetryBudget(t *testing.T) { + for name, owner := range map[string]string{"a running BackupJob's Backup": "bj-other", "an untracked Backup": ""} { + t.Run(name, func(t *testing.T) { + testQueueResetsDestinationRetryBudget(t, owner) + }) + } +} + +func testQueueResetsDestinationRetryBudget(t *testing.T, owner string) { expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) f := newAttachedObjectStoreFixture(t, expired, "s3://other/") f.setJobCondition(metav1.Condition{ Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired), }) - f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) - f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + if owner != "" { + f.seedOtherBackupJob(owner, backupsv1alpha1.BackupJobPhaseRunning) + } + f.seedCNPGBackup("bj-other-cnpg", owner, rolloutCluster, "running") j, _ := f.pass() + if held := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); held == nil || held.Status != metav1.ConditionTrue { + t.Fatalf("%s = %+v, want the job held", backupCondDestinationWriteHeld, held) + } if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil { t.Fatalf("%s = %+v kept while waiting for another backup", backupCondDestinationApplied, cond) } @@ -6774,3 +6804,91 @@ func TestReconcileCNPG_LooksUpItsOwnBackupOncePerPass(t *testing.T) { t.Fatalf("got %d cnpg.io/Backups, want 1", len(backups)) } } + +// With two untracked Backups running the wait names the same one on every +// pass, whatever order the cache lists them in, so a waiting job does not +// rewrite its status each poll. +func TestReconcileCNPG_UntrackedHoldNamesOneBackupAcrossPasses(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedCNPGBackup("a-cnpg", "", rolloutCluster, "running") + f.seedCNPGBackup("z-cnpg", "", rolloutCluster, "started") + statusWrites := 0 + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := cl.List(ctx, list, opts...); err != nil { + return err + } + if backups, ok := list.(*cnpgtypes.BackupList); ok { + slices.Reverse(backups.Items) + } + return nil + }, + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + statusWrites++ + return cl.SubResource(sub).Update(ctx, obj, opts...) + }, + }) + + j, _ := f.pass() + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForUntrackedBackup" || !strings.Contains(ready.Message, "/a-cnpg ") { + t.Fatalf("Ready = %+v, want WaitingForUntrackedBackup naming a-cnpg", ready) + } + statusWrites = 0 + f.pass() + if statusWrites != 0 { + t.Errorf("a second pass with the same two untracked Backups wrote the BackupJob status %d time(s)", statusWrites) + } +} + +// An error listing the namespace's Backups is not taken as "nothing runs". +func TestReconcileCNPG_UntrackedBackupListErrorIsReturned(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + lo := &client.ListOptions{} + lo.ApplyOptions(opts) + if _, ok := list.(*cnpgtypes.BackupList); ok && (lo.LabelSelector == nil || lo.LabelSelector.Empty()) { + return apierrors.NewServiceUnavailable("cache not synced") + } + return cl.List(ctx, list, opts...) + }, + }) + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("reconcileCNPG swallowed the error listing untracked Backups") + } + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination written although the running Backups could not be listed: %q", got) + } +} + +// A ScheduledBackup labels its Backups with the Cluster but with no +// BackupJob; such a Backup holds the destination while it runs. +func TestReconcileCNPG_ScheduledBackupHoldsTheDestination(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + b := &cnpgtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "nightly-20261112", Labels: map[string]string{ + cnpgClusterLabel: rolloutCluster, + "cnpg.io/scheduled-backup": "nightly", + "cnpg.io/immediateBackup": "false", + }}, + Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: rolloutCluster}}, + Status: cnpgtypes.BackupStatus{Phase: cnpgBackupPhaseStarted}, + } + if err := f.c.Create(context.Background(), b); err != nil { + t.Fatalf("seed scheduled cnpg.io/Backup: %v", err) + } + j, backups := f.pass() + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination rewritten under a running scheduled backup: %q", got) + } + if len(backups) != 1 { + t.Errorf("got %d cnpg.io/Backups, want only the scheduled one", len(backups)) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForUntrackedBackup" || !strings.Contains(ready.Message, "/nightly-20261112 ") { + t.Errorf("Ready = %+v, want WaitingForUntrackedBackup naming nightly-20261112", ready) + } +} From e6127f7e423f4839d5d22262f98e2b91fbc8865f Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 13 Nov 2026 10:42:20 +0100 Subject: [PATCH 21/28] fix(controller, platform): copy the LimitRange only under a compute-limit quota The tenant chart renders tenant-range-limits under any non-empty resourceQuotas, including one that bounds no compute, such as services.loadbalancers alone. Copying it into an unbounded member whose allocated quota bounds neither limits.cpu nor limits.memory imposes a 128Mi default memory limit the quota never asked for: a container that requests more and sets no limit, virt-launcher among them, is refused where it was admitted before. Only a quota on limits.cpu or limits.memory makes admission reject a container that omits a limit, so the copy is now limited to that case, and removed otherwise. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- internal/controller/tenantquota/reconciler.go | 18 ++++++++-- .../controller/tenantquota/reconciler_test.go | 33 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/internal/controller/tenantquota/reconciler.go b/internal/controller/tenantquota/reconciler.go index 6c77eff6d..ed18499ce 100644 --- a/internal/controller/tenantquota/reconciler.go +++ b/internal/controller/tenantquota/reconciler.go @@ -113,8 +113,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result pools := ComputePools(tenants) desired := map[string]corev1.ResourceList{} - // limitRangeSource maps each unbounded pool member to the pool root whose - // chart LimitRange it must inherit. + // limitRangeSource maps each unbounded pool member whose quota bounds a + // compute limit to the pool root whose chart LimitRange it must inherit. limitRangeSource := map[string]string{} for _, p := range pools { if over := p.Overcommitted(); len(over) > 0 { @@ -134,7 +134,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result for _, ns := range p.Members { if existing[ns] { desired[ns] = buffered.EnforcedHard(ns, usedByNS) - if ns != p.Root { + if ns != p.Root && boundsComputeLimits(desired[ns]) { limitRangeSource[ns] = p.Root } } @@ -352,6 +352,18 @@ func ownedNamespace(hrNamespace, appName string) string { } } +// boundsComputeLimits reports whether a quota bounds limits.cpu or +// limits.memory, the only case in which admission rejects a container that +// omits a limit. The chart renders its LimitRange under any non-empty +// resourceQuotas, so the root may carry one under a quota that bounds no +// compute at all; copying it then would impose default limits the quota +// never asked for. +func boundsComputeLimits(hard corev1.ResourceList) bool { + _, cpu := hard[corev1.ResourceLimitsCPU] + _, memory := hard[corev1.ResourceLimitsMemory] + return cpu || memory +} + // maxResourceList returns the per-resource maximum of a and b. func maxResourceList(a, b corev1.ResourceList) corev1.ResourceList { out := corev1.ResourceList{} diff --git a/internal/controller/tenantquota/reconciler_test.go b/internal/controller/tenantquota/reconciler_test.go index 4624b2e36..5c5778e63 100644 --- a/internal/controller/tenantquota/reconciler_test.go +++ b/internal/controller/tenantquota/reconciler_test.go @@ -363,3 +363,36 @@ func TestReconcile_PoolRootWithoutLimitRange(t *testing.T) { t.Fatalf("no LimitRange can be copied from a root that has none") } } + +// TestReconcile_NoLimitRangeWithoutComputeLimitQuota: the chart renders its +// LimitRange for any non-empty resourceQuotas, compute or not. A member whose +// allocated quota bounds neither limits.cpu nor limits.memory is admitted +// without limits today; copying the root's defaults there would cap or refuse +// pods (one requesting more than the default memory limit, for instance) +// that the quota itself never constrains. +func TestReconcile_NoLimitRangeWithoutComputeLimitQuota(t *testing.T) { + stale := &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: allocatedLimitRangeName, + Namespace: "tenant-foo-bar", + Labels: map[string]string{managedByLabel: managedByValue}, + }, + } + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"services.loadbalancers": "1"}, nil), + chartLimitRange("tenant-foo", "250m"), + stale, + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, ok := allocatedHard(t, c, "tenant-foo-bar", "services.loadbalancers"); !ok { + t.Fatalf("tenant-foo-bar must still get its allocated quota") + } + if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { + t.Fatalf("tenant-foo-bar's quota bounds no compute limit; it must carry no LimitRange copy") + } +} From cc412d44f2f5dbb7f0ed6686feb0e331c6338bcd Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 13 Nov 2026 13:07:08 +0100 Subject: [PATCH 22/28] test(controller): exercise the missing-root-LimitRange path under a compute-limit quota TestReconcile_PoolRootWithoutLimitRange declared its root quota on cpu, which the compute-limit gate does not count, so the sweep never looked for the root's LimitRange and the test passed whatever that path did. Declaring limits.cpu makes it reach the lookup, and it now fails if a missing root LimitRange produces an empty copy. Assisted-by: LLM Signed-off-by: Mattia Eleuteri --- internal/controller/tenantquota/reconciler_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/controller/tenantquota/reconciler_test.go b/internal/controller/tenantquota/reconciler_test.go index 5c5778e63..b869825dc 100644 --- a/internal/controller/tenantquota/reconciler_test.go +++ b/internal/controller/tenantquota/reconciler_test.go @@ -351,12 +351,12 @@ func TestReconcile_PoolRootWithoutLimitRange(t *testing.T) { tenantHR("foo", "tenant-root"), tenantHR("bar", "tenant-foo"), ns("tenant-foo"), ns("tenant-foo-bar"), - chartQuota("tenant-foo", map[string]string{"cpu": "10"}, nil), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), ) if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { t.Fatalf("reconcile: %v", err) } - if _, ok := allocatedHard(t, c, "tenant-foo-bar", "cpu"); !ok { + if _, ok := allocatedHard(t, c, "tenant-foo-bar", "limits.cpu"); !ok { t.Fatalf("tenant-foo-bar must still get its allocated quota") } if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { From 700600ee3c2773f0a96baddef4411600263e6d85 Mon Sep 17 00:00:00 2001 From: Nimesh Kulkarni <167794819+GitNimay@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:31:51 +0530 Subject: [PATCH 23/28] chore(cilium): drop externalIPs.enabled and nodePort.enabled The vendored Cilium chart reads neither key: there is no externalIPs value upstream, and from the nodePort block the templates read only range, addresses, bindProtection, autoProtectPortRange, directRoutingDevice, enableHealthCheck, enableHealthCheckLoadBalancerIP and enableDynamicSourceLookup. Both features follow kubeProxyReplacement, which stays true, so the keys only misled readers into thinking they toggle anything. Fixes #4816 Assisted-by: LLM Signed-off-by: Nimesh Kulkarni <167794819+GitNimay@users.noreply.github.com> --- packages/system/cilium/values.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index 732fd4151..b8bc0b9ee 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -30,10 +30,6 @@ cilium: enabled: true hubble: enabled: false - externalIPs: - enabled: true - nodePort: - enabled: true loadBalancer: algorithm: maglev serviceTopology: true From fd15ad282eb72d8ad6187ff92eb131c99b018fbd Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Sat, 14 Nov 2026 19:51:20 +0700 Subject: [PATCH 24/28] fix(ci): take the main ref for a bare image ref in a drifted file A file whose configuration drifted from the cozystack-packages:main artifact keeps its committed refs. For a digest-pinned ref that runs an older image; for the bare : a new image is first committed with, it names a tag that was never pushed, so the component cannot pull. The merge commit CI tests is fixed when the run starts, so a PR whose merge commit predates a config change to observability-proxy's values.yaml installed observability-proxy:v0.0.0 and failed E2E. In a drifted values file, a bare ref under an image key now takes the artifact line with the same key, the same image name and a digest, when exactly one exists. The rest of the file stays as the PR has it. Assisted-by: LLM Signed-off-by: Maxim Belyy --- hack/overlay-main-images.sh | 89 ++++++++++++++++++++++++++++-- hack/overlay-main-images_test.bats | 85 ++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 4 deletions(-) diff --git a/hack/overlay-main-images.sh b/hack/overlay-main-images.sh index 2c13e34e1..b35be70ba 100755 --- a/hack/overlay-main-images.sh +++ b/hack/overlay-main-images.sh @@ -42,9 +42,13 @@ # a change: `yq -i`, which every `make image` uses to stamp the ref, drops one. # If any OTHER line differs — the PR branched from a # main whose config for that file differs from the artifact's base — the file is -# left on its committed ref (safe degradation, logged, never fatal). CI checks -# out the pull_request merge commit (current main + PR), so for an unbuilt file -# the config already matches current main and only ref lines differ. +# left on its committed ref (safe degradation, logged, never fatal), except +# that a bare digest-less ref in it, which no registry serves, still takes the +# artifact's ref (see replace_bare_refs). CI checks out the pull_request merge +# commit (main + PR), so for an unbuilt file the config matches main as of +# that merge commit. It is fixed when the run starts, so a rerun, or a PR not +# pushed to since, can be older than the artifact and drift on files it never +# touched. # # Usage: hack/overlay-main-images.sh [touched-pkg-dirs] # = extracted cozystack-packages:main tree; its root holds @@ -123,8 +127,76 @@ img_line='(@sha256:|^[[:space:]]*(- )?(image|repository|registry|tag|digest):|^[ # match branch for this same shape and for this same reason. bare_ref_line='^[^[:space:]#][^[:space:]]*/[^[:space:]]+$' +# A drifted file keeps its committed refs. For a ref pinned by digest that is a +# pullable older image, which is the safe answer. A bare `:` with no +# digest is different: it is the first commit of a new image +# (docs/agents/image-refs.md), its tag was never pushed, and keeping it makes the +# component fail to pull instead of run an older image. The merge commit CI +# checks out is fixed when the run starts, so a PR whose merge commit predates a +# config change to such a file drifts on it, and every new image is exposed +# until a release stamps its ref (#5328). +# +# replace_bare_refs rewrites, in only, each line whose value is +# such a bare ref under an image key, taking the line from that has the +# same key and indentation, the same image name and an @sha256: digest. Exactly +# one such line must exist; two are ambiguous and none means build-main did not +# stamp it, so there is nothing more pullable to take. Every other line of +# stays as the PR has it. The current image may expect configuration the PR's +# older chart lacks, but the committed ref cannot run at all. Prints the number +# of lines replaced. +replace_bare_refs() { + tmp="$1.overlay-bare.$$" + n=$(awk -v newf="$2" -v out="$tmp" ' + function key_prefix(s) { + if (match(s, /^[[:space:]]*(- )?([A-Za-z]+[Ii]mage|image):[[:space:]]*/)) return substr(s, 1, RLENGTH) + return "" + } + function ref_value(s, p, v) { + v = substr(s, length(p) + 1) + sub(/[[:space:]]+$/, "", v) + gsub(/"/, "", v) + return v + } + function image_name(v) { + sub(/@.*/, "", v) + sub(/.*\//, "", v) + sub(/:.*/, "", v) + return v + } + BEGIN { + while ((getline l < newf) > 0) { + p = key_prefix(l) + if (p == "" || index(l, "@sha256:") == 0) continue + k = p SUBSEP image_name(ref_value(l, p)) + cnt[k]++ + line[k] = l + } + close(newf) + } + { + p = key_prefix($0) + if (p != "") { + v = ref_value($0, p) + if (v ~ /^[^@[:space:]]+\/[^@[:space:]\/]+:[^@[:space:]\/]+$/) { + k = p SUBSEP image_name(v) + if (cnt[k] == 1) { print line[k] > out; r++; next } + } + } + print > out + } + END { close(out); print r + 0 } + ' "$1") + if [ "$n" -gt 0 ]; then + mv "$tmp" "$1" + else + rm -f "$tmp" + fi + echo "$n" +} + overlaid=0 same=0 +bare=0 skipped=0 drift=0 drift_files="" @@ -164,6 +236,15 @@ for new in $(find "$MAINPKGS" -type d -name charts -prune -o \ | grep -vE '^[[:space:]]*$' | grep -qvE "$line_pat"; then echo "drift (non-ref change) in $cur -> keeping committed ref" drift=$((drift + 1)) + case "$cur" in + *.yaml) + r=$(replace_bare_refs "$cur" "$new") || r=0 + if [ "$r" -gt 0 ]; then + echo "bare ref in $cur -> current-main ($r line(s))" + bare=$((bare + r)) + fi + ;; + esac # Name the PACKAGE, not the directory the file happens to sit in: an # images/*.tag drift would otherwise read as "/images", which is not # something anyone can go and look at. @@ -181,7 +262,7 @@ for new in $(find "$MAINPKGS" -type d -name charts -prune -o \ fi done -echo "Overlay current-main images: overlaid=$overlaid same=$same skipped(rebuilt/owned/edited)=$skipped drift=$drift failed=$failed" +echo "Overlay current-main images: overlaid=$overlaid same=$same skipped(rebuilt/owned/edited)=$skipped drift=$drift bare-refs=$bare failed=$failed" # Keeping the committed ref is the safe answer to a config that does not match # the artifact, but it is not a neutral one: that component then runs its last diff --git a/hack/overlay-main-images_test.bats b/hack/overlay-main-images_test.bats index 7cff2d842..7b22b00ff 100644 --- a/hack/overlay-main-images_test.bats +++ b/hack/overlay-main-images_test.bats @@ -502,3 +502,88 @@ load test_helper cd "$root" rm -rf "$w" } + +# A drifted file keeps its committed refs, and for a digest-pinned ref that is +# a pullable older image. A bare : is the first commit of a new image +# (docs/agents/image-refs.md) and names a tag no registry serves, so keeping it +# turns drift into ImagePullBackOff. That hit every fork PR whose merge commit +# predated a config change to observability-proxy's values.yaml (#5328). Only +# such a bare line takes the artifact's ref; the rest of the file stays as the +# PR has it. +@test "a drifted file still takes the artifact ref for a bare (digest-less) ref line" { + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/proxy" "$w/main/system/proxy" + printf 'proxy:\n image: "ghcr.io/cozystack/cozystack/proxy:v0.0.0"\n lookback: 1h\n' \ + > "$w/packages/system/proxy/values.yaml" + printf 'proxy:\n image: "iad.ocir.io/x/cozystack/proxy:main@sha256:bbbb"\n lookback: 1h\n prefix: cozy-\n' \ + > "$w/main/system/proxy/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q 'drift (non-ref change) in packages/system/proxy/values.yaml' "$w/out.txt" + grep -q ' image: "iad.ocir.io/x/cozystack/proxy:main@sha256:bbbb"' "$w/packages/system/proxy/values.yaml" + # The config stays the PR's: the new key is not introduced. + if grep -q 'prefix:' "$w/packages/system/proxy/values.yaml"; then + echo "FAIL: a config line was taken from the artifact" >&2; exit 1 + fi + grep -q 'bare ref in packages/system/proxy/values.yaml -> current-main' "$w/out.txt" + grep -q 'bare-refs=1' "$w/out.txt" + rm -rf "$w" +} + +@test "a drifted file replaces only its bare ref, not a digest-pinned sibling" { + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/pair" "$w/main/system/pair" + printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/pair:v1.5.0@sha256:aaaa\nclientImage: ghcr.io/cozystack/cozystack/pair-client:latest\n' \ + > "$w/packages/system/pair/values.yaml" + printf 'tuning: new\nimage: iad.ocir.io/x/cozystack/pair:main@sha256:bbbb\nclientImage: iad.ocir.io/x/cozystack/pair-client:main@sha256:cccc\n' \ + > "$w/main/system/pair/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q '^image: ghcr.io/cozystack/cozystack/pair:v1.5.0@sha256:aaaa$' "$w/packages/system/pair/values.yaml" + grep -q '^clientImage: iad.ocir.io/x/cozystack/pair-client:main@sha256:cccc$' "$w/packages/system/pair/values.yaml" + grep -q '^tuning: old$' "$w/packages/system/pair/values.yaml" + rm -rf "$w" +} + +@test "a bare ref is not replaced by an artifact line for a different image" { + # The same key can hold another image in the artifact, e.g. after a rename. + # Matching on the key alone would swap in an image the chart never asked for. + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/ren" "$w/main/system/ren" + printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/old-name:v0.0.0\n' > "$w/packages/system/ren/values.yaml" + printf 'tuning: new\nimage: iad.ocir.io/x/cozystack/new-name:main@sha256:bbbb\n' > "$w/main/system/ren/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q '^image: ghcr.io/cozystack/cozystack/old-name:v0.0.0$' "$w/packages/system/ren/values.yaml" + grep -q 'bare-refs=0' "$w/out.txt" + rm -rf "$w" +} + +@test "a bare ref is kept when the artifact carries no digest for it either" { + # A ref build-main does not stamp (third-party, or not built yet) has no + # digest in the artifact, so there is nothing more pullable to take. + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/tp" "$w/main/system/tp" + printf 'tuning: old\nimage: docker.io/library/busybox:1.37.0\n' > "$w/packages/system/tp/values.yaml" + printf 'tuning: new\nimage: docker.io/library/busybox:1.38.0\n' > "$w/main/system/tp/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q '^image: docker.io/library/busybox:1.37.0$' "$w/packages/system/tp/values.yaml" + grep -q 'bare-refs=0' "$w/out.txt" + rm -rf "$w" +} + +@test "a bare ref is kept when the artifact has two candidate lines for it" { + # Two lines with the same key and image name (e.g. one per component in a + # list) cannot be told apart, so neither is taken. + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/dup" "$w/main/system/dup" + printf 'tuning: old\na:\n image: ghcr.io/cozystack/cozystack/dup:v0.0.0\n' > "$w/packages/system/dup/values.yaml" + printf 'tuning: new\na:\n image: iad.ocir.io/x/cozystack/dup:main@sha256:bbbb\nb:\n image: iad.ocir.io/x/cozystack/dup:main@sha256:cccc\n' \ + > "$w/main/system/dup/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q '^ image: ghcr.io/cozystack/cozystack/dup:v0.0.0$' "$w/packages/system/dup/values.yaml" + grep -q 'bare-refs=0' "$w/out.txt" + rm -rf "$w" +} From 54b76fcdf1ff61402a695eb790381dbf1428e46f Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Sat, 14 Nov 2026 19:51:20 +0700 Subject: [PATCH 25/28] docs(agents): say how PR CI stamps a bare ref in a drifted file Assisted-by: LLM Signed-off-by: Maxim Belyy --- docs/agents/image-refs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/agents/image-refs.md b/docs/agents/image-refs.md index 9c9bb3ca6..764b560c2 100644 --- a/docs/agents/image-refs.md +++ b/docs/agents/image-refs.md @@ -42,7 +42,7 @@ If a package stamps a reference somewhere the two globs do not cover, add that f ## Invariants -- **Every first-party reference is tag+digest.** A bare tag with no digest is not reproducible and must not be committed. The one exception is the first commit of a new image: it has no digest until it is built, so it lands as a bare `:` that the producing `make image` replaces. PR CI and release preparation both stamp it before anything pulls it, but a tree applied without a build pulls a tag that does not exist. +- **Every first-party reference is tag+digest.** A bare tag with no digest is not reproducible and must not be committed. The one exception is the first commit of a new image: it has no digest until it is built, so it lands as a bare `:` that the producing `make image` replaces. PR CI and release preparation both stamp it before anything pulls it, but a tree applied without a build pulls a tag that does not exist. In PR CI the stamp comes from `hack/overlay-main-images.sh` when the PR does not rebuild the package, and it takes a bare ref even from a file whose configuration drifted from the artifact, because keeping a tag that was never pushed cannot install. - **Exactly one producer stamps each reference.** A reference no `Makefile` writes will never be refreshed by anything, including release preparation. - **A package that stamps a reference is in the root `Makefile` `build:` list.** `build-matrix.sh`, the main build and release preparation all iterate that one list, so a package outside it is never rebuilt by any path and its pin freezes permanently. `hack/build-matrix_test.bats` computes the set of packages whose Makefile runs `docker buildx build` and fails on any the list misses. - **Every consumer of references reads all three storage shapes**, via `hack/lib/image-refs.sh`. From 854a3f40e40dbcd8b2bdb8c0c8beec6e590b96b5 Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Sat, 14 Nov 2026 19:54:31 +0700 Subject: [PATCH 26/28] fix(ci): limit the drifted-file bare-ref overlay to first-party images Main can bump a third-party image together with its config, so a PR behind it must keep the committed pair rather than take main's image alone. Only ghcr.io/cozystack/cozystack refs, matched against an artifact line for the same first-party image, are replaced now. State the trade-off for first-party refs that do exist, and name the packages that took main's images in their own annotation. Assisted-by: LLM Signed-off-by: Maxim Belyy --- docs/agents/image-refs.md | 2 +- hack/overlay-main-images.sh | 63 +++++++++++++++++++++--------- hack/overlay-main-images_test.bats | 45 +++++++++++++++++---- 3 files changed, 83 insertions(+), 27 deletions(-) diff --git a/docs/agents/image-refs.md b/docs/agents/image-refs.md index 764b560c2..191bc62c2 100644 --- a/docs/agents/image-refs.md +++ b/docs/agents/image-refs.md @@ -42,7 +42,7 @@ If a package stamps a reference somewhere the two globs do not cover, add that f ## Invariants -- **Every first-party reference is tag+digest.** A bare tag with no digest is not reproducible and must not be committed. The one exception is the first commit of a new image: it has no digest until it is built, so it lands as a bare `:` that the producing `make image` replaces. PR CI and release preparation both stamp it before anything pulls it, but a tree applied without a build pulls a tag that does not exist. In PR CI the stamp comes from `hack/overlay-main-images.sh` when the PR does not rebuild the package, and it takes a bare ref even from a file whose configuration drifted from the artifact, because keeping a tag that was never pushed cannot install. +- **Every first-party reference is tag+digest.** A bare tag with no digest is not reproducible and must not be committed. The one exception is the first commit of a new image: it has no digest until it is built, so it lands as a bare `:` that the producing `make image` replaces. PR CI and release preparation both stamp it before anything pulls it, but a tree applied without a build pulls a tag that does not exist. In PR CI the stamp comes from `hack/overlay-main-images.sh` when the PR does not rebuild the package, and it takes a bare first-party ref even from a file whose configuration drifted from the artifact, because a tag that was never pushed cannot install. The cost is that any other digest-less first-party ref in such a file also runs main's image with the PR's older config; third-party refs in it keep their committed pair. - **Exactly one producer stamps each reference.** A reference no `Makefile` writes will never be refreshed by anything, including release preparation. - **A package that stamps a reference is in the root `Makefile` `build:` list.** `build-matrix.sh`, the main build and release preparation all iterate that one list, so a package outside it is never rebuilt by any path and its pin freezes permanently. `hack/build-matrix_test.bats` computes the set of packages whose Makefile runs `docker buildx build` and fails on any the list misses. - **Every consumer of references reads all three storage shapes**, via `hack/lib/image-refs.sh`. diff --git a/hack/overlay-main-images.sh b/hack/overlay-main-images.sh index b35be70ba..adb6b8638 100755 --- a/hack/overlay-main-images.sh +++ b/hack/overlay-main-images.sh @@ -43,7 +43,7 @@ # If any OTHER line differs — the PR branched from a # main whose config for that file differs from the artifact's base — the file is # left on its committed ref (safe degradation, logged, never fatal), except -# that a bare digest-less ref in it, which no registry serves, still takes the +# that a bare first-party ref in it, one with no digest, still takes the # artifact's ref (see replace_bare_refs). CI checks out the pull_request merge # commit (main + PR), so for an unbuilt file the config matches main as of # that merge commit. It is fixed when the run starts, so a rerun, or a PR not @@ -127,23 +127,33 @@ img_line='(@sha256:|^[[:space:]]*(- )?(image|repository|registry|tag|digest):|^[ # match branch for this same shape and for this same reason. bare_ref_line='^[^[:space:]#][^[:space:]]*/[^[:space:]]+$' -# A drifted file keeps its committed refs. For a ref pinned by digest that is a -# pullable older image, which is the safe answer. A bare `:` with no -# digest is different: it is the first commit of a new image -# (docs/agents/image-refs.md), its tag was never pushed, and keeping it makes the -# component fail to pull instead of run an older image. The merge commit CI -# checks out is fixed when the run starts, so a PR whose merge commit predates a -# config change to such a file drifts on it, and every new image is exposed -# until a release stamps its ref (#5328). +# A drifted file keeps its committed refs. For a ref pinned by digest that is an +# older image which pulls and matches the PR's config, so it is the safe answer. +# A first-party ref with no digest is not: docs/agents/image-refs.md forbids it +# except as the first commit of a new image, whose tag was never pushed, and +# keeping that makes the component fail to pull. The merge commit CI checks out +# is fixed when the run starts, so a PR whose merge commit predates a config +# change to such a file drifts on it, and a new image fails every such PR until +# a release stamps its ref (#5328). # # replace_bare_refs rewrites, in only, each line whose value is -# such a bare ref under an image key, taking the line from that has the -# same key and indentation, the same image name and an @sha256: digest. Exactly -# one such line must exist; two are ambiguous and none means build-main did not -# stamp it, so there is nothing more pullable to take. Every other line of -# stays as the PR has it. The current image may expect configuration the PR's -# older chart lacks, but the committed ref cannot run at all. Prints the number -# of lines replaced. +# a bare first-party ref (ghcr.io/cozystack/cozystack/:, no digest) +# under an image key, taking the line from that has the same key and +# indentation, an @sha256: digest and the same first-party image, i.e. a path +# ending in /cozystack/. Exactly one such line must exist; two are +# ambiguous and none means build-main did not stamp it. Every other line of +# stays as the PR has it. +# +# The trade-off is deliberate. A bare first-party ref that does exist (an older +# release tag or a moving `latest`, #4264) also takes main's image, so it runs +# with the PR's older config rather than as committed. It is the same component +# built from the tree, and a digest-less ref is not reproducible anyway; the +# alternative is a ref that may not pull at all. Third-party refs are never +# touched: main can bump one alongside its config, and the committed pair is +# the one known to work together. The split shape (`repository:` + `tag:` +# without `digest:`) is not handled; no package uses it for a bare ref today. +# +# Prints the number of lines replaced. replace_bare_refs() { tmp="$1.overlay-bare.$$" n=$(awk -v newf="$2" -v out="$tmp" ' @@ -167,7 +177,10 @@ replace_bare_refs() { while ((getline l < newf) > 0) { p = key_prefix(l) if (p == "" || index(l, "@sha256:") == 0) continue - k = p SUBSEP image_name(ref_value(l, p)) + v = ref_value(l, p) + name = image_name(v) + if (index(v, "/cozystack/" name ":") == 0) continue + k = p SUBSEP name cnt[k]++ line[k] = l } @@ -177,7 +190,7 @@ replace_bare_refs() { p = key_prefix($0) if (p != "") { v = ref_value($0, p) - if (v ~ /^[^@[:space:]]+\/[^@[:space:]\/]+:[^@[:space:]\/]+$/) { + if (v ~ /^ghcr\.io\/cozystack\/cozystack\/[^@[:space:]\/]+:[^@[:space:]\/]+$/) { k = p SUBSEP image_name(v) if (cnt[k] == 1) { print line[k] > out; r++; next } } @@ -197,6 +210,7 @@ replace_bare_refs() { overlaid=0 same=0 bare=0 +bare_files="" skipped=0 drift=0 drift_files="" @@ -240,8 +254,10 @@ for new in $(find "$MAINPKGS" -type d -name charts -prune -o \ *.yaml) r=$(replace_bare_refs "$cur" "$new") || r=0 if [ "$r" -gt 0 ]; then - echo "bare ref in $cur -> current-main ($r line(s))" + echo "bare first-party ref in $cur -> current-main ($r line(s)); its other refs stay committed" bare=$((bare + r)) + d=$(dirname "${cur#packages/}") + bare_files="$bare_files $d" fi ;; esac @@ -277,3 +293,12 @@ if [ "$drift" -gt 0 ]; then "$(printf '%s\n' "$drift_pkgs" | wc -l | tr -d ' ')" \ "$(printf '%s' "$drift_pkgs" | tr '\n' ' ' | sed 's/ $//')" fi +# Among those, a package whose bare first-party ref took main's image runs that +# image with the PR's older config, not its committed ref; name it separately +# so the warning above does not read as the whole story for it. +if [ "$bare" -gt 0 ]; then + bare_pkgs=$(echo "$bare_files" | tr ' ' '\n' | grep -v '^$' | sort -u) + printf '::warning title=Image overlay bare refs::%s drifted package(s) took current-main images for bare first-party refs, with the config committed in this PR: %s\n' \ + "$(printf '%s\n' "$bare_pkgs" | wc -l | tr -d ' ')" \ + "$(printf '%s' "$bare_pkgs" | tr '\n' ' ' | sed 's/ $//')" +fi diff --git a/hack/overlay-main-images_test.bats b/hack/overlay-main-images_test.bats index 7b22b00ff..28f04526f 100644 --- a/hack/overlay-main-images_test.bats +++ b/hack/overlay-main-images_test.bats @@ -525,7 +525,8 @@ load test_helper if grep -q 'prefix:' "$w/packages/system/proxy/values.yaml"; then echo "FAIL: a config line was taken from the artifact" >&2; exit 1 fi - grep -q 'bare ref in packages/system/proxy/values.yaml -> current-main' "$w/out.txt" + grep -q 'bare first-party ref in packages/system/proxy/values.yaml -> current-main' "$w/out.txt" + grep -q '::warning title=Image overlay bare refs::1 drifted package(s) .*: system/proxy$' "$w/out.txt" grep -q 'bare-refs=1' "$w/out.txt" rm -rf "$w" } @@ -560,15 +561,45 @@ load test_helper } @test "a bare ref is kept when the artifact carries no digest for it either" { - # A ref build-main does not stamp (third-party, or not built yet) has no - # digest in the artifact, so there is nothing more pullable to take. + # A ref build-main does not stamp has no digest in the artifact, so there is + # nothing more pullable to take. root=$(pwd) w=$(mktemp -d) - mkdir -p "$w/packages/system/tp" "$w/main/system/tp" - printf 'tuning: old\nimage: docker.io/library/busybox:1.37.0\n' > "$w/packages/system/tp/values.yaml" - printf 'tuning: new\nimage: docker.io/library/busybox:1.38.0\n' > "$w/main/system/tp/values.yaml" + mkdir -p "$w/packages/system/nd" "$w/main/system/nd" + printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/nd:v0.0.0\n' > "$w/packages/system/nd/values.yaml" + printf 'tuning: new\nimage: iad.ocir.io/x/cozystack/nd:main\n' > "$w/main/system/nd/values.yaml" ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" - grep -q '^image: docker.io/library/busybox:1.37.0$' "$w/packages/system/tp/values.yaml" + grep -q '^image: ghcr.io/cozystack/cozystack/nd:v0.0.0$' "$w/packages/system/nd/values.yaml" + grep -q 'bare-refs=0' "$w/out.txt" + rm -rf "$w" +} + +@test "a bare third-party ref in a drifted file is kept even when main pins one" { + # Main can bump a third-party image together with its config in one file. A + # PR behind it has the older pair, which is the one known to work together; + # taking main's image alone would run it with the config it replaced. + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/kc" "$w/main/system/kc" + printf 'features: old\nimage: quay.io/keycloak/keycloak:26.6.3\n' > "$w/packages/system/kc/values.yaml" + printf 'features: new\nimage: quay.io/keycloak/keycloak:27.0.0@sha256:abcd\n' > "$w/main/system/kc/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q '^image: quay.io/keycloak/keycloak:26.6.3$' "$w/packages/system/kc/values.yaml" + grep -q 'bare-refs=0' "$w/out.txt" + if grep -q 'Image overlay bare refs' "$w/out.txt"; then + echo "FAIL: a bare-refs annotation with nothing replaced" >&2; exit 1 + fi + rm -rf "$w" +} + +@test "a bare first-party ref is not replaced by a third-party image of the same name" { + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/same" "$w/main/system/same" + printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/redis:v0.0.0\n' > "$w/packages/system/same/values.yaml" + printf 'tuning: new\nimage: docker.io/library/redis:8.0@sha256:abcd\n' > "$w/main/system/same/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q '^image: ghcr.io/cozystack/cozystack/redis:v0.0.0$' "$w/packages/system/same/values.yaml" grep -q 'bare-refs=0' "$w/out.txt" rm -rf "$w" } From 2468b309eea6e6b8e6ec3bbd2ea89b3285fb9b8c Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Sat, 14 Nov 2026 19:58:33 +0700 Subject: [PATCH 27/28] fix(ci): count a failed bare-ref rewrite and word the drift log by result A rewrite that cannot replace the file now counts as failed and leaves no temporary file behind, instead of reporting lines as replaced. A drifted file whose bare refs took main's images no longer logs that it kept its committed ref. Assisted-by: LLM Signed-off-by: Maxim Belyy --- hack/overlay-main-images.sh | 36 +++++++++++++++++++----------- hack/overlay-main-images_test.bats | 20 +++++++++++++++++ 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/hack/overlay-main-images.sh b/hack/overlay-main-images.sh index adb6b8638..2e6c4a1c4 100755 --- a/hack/overlay-main-images.sh +++ b/hack/overlay-main-images.sh @@ -151,7 +151,10 @@ bare_ref_line='^[^[:space:]#][^[:space:]]*/[^[:space:]]+$' # alternative is a ref that may not pull at all. Third-party refs are never # touched: main can bump one alongside its config, and the committed pair is # the one known to work together. The split shape (`repository:` + `tag:` -# without `digest:`) is not handled; no package uses it for a bare ref today. +# without `digest:`) is not handled, nor is a single-quoted value or one +# followed by a `# comment`; such a line stays as committed, and no package +# writes a bare first-party ref that way today. Prints nothing and fails when +# the file could not be rewritten, so the caller counts it as failed. # # Prints the number of lines replaced. replace_bare_refs() { @@ -199,11 +202,11 @@ replace_bare_refs() { } END { close(out); print r + 0 } ' "$1") - if [ "$n" -gt 0 ]; then - mv "$tmp" "$1" - else - rm -f "$tmp" - fi + case "$n" in + ''|*[!0-9]*) rm -f "$tmp"; return 1 ;; + 0) rm -f "$tmp" ;; + *) mv "$tmp" "$1" || { rm -f "$tmp"; return 1; } ;; + esac echo "$n" } @@ -248,19 +251,26 @@ for new in $(find "$MAINPKGS" -type d -name charts -prune -o \ if diff "$cur" "$new" | sed -n 's/^[<>] //p' \ | grep -vE '^[[:space:]]*$' | grep -qvE "$line_pat"; then - echo "drift (non-ref change) in $cur -> keeping committed ref" drift=$((drift + 1)) + r=0 case "$cur" in *.yaml) - r=$(replace_bare_refs "$cur" "$new") || r=0 - if [ "$r" -gt 0 ]; then - echo "bare first-party ref in $cur -> current-main ($r line(s)); its other refs stay committed" - bare=$((bare + r)) - d=$(dirname "${cur#packages/}") - bare_files="$bare_files $d" + if ! r=$(replace_bare_refs "$cur" "$new"); then + echo "WARN: rewriting bare refs failed for $cur (keeping committed refs)" + failed=$((failed + 1)) + r=0 fi ;; esac + if [ "$r" -eq 0 ]; then + echo "drift (non-ref change) in $cur -> keeping committed ref" + else + echo "drift (non-ref change) in $cur -> keeping committed refs except bare first-party ones" + echo "bare first-party ref in $cur -> current-main ($r line(s)); its other refs stay committed" + bare=$((bare + r)) + d=$(dirname "${cur#packages/}") + bare_files="$bare_files $d" + fi # Name the PACKAGE, not the directory the file happens to sit in: an # images/*.tag drift would otherwise read as "/images", which is not # something anyone can go and look at. diff --git a/hack/overlay-main-images_test.bats b/hack/overlay-main-images_test.bats index 28f04526f..4059f9ea1 100644 --- a/hack/overlay-main-images_test.bats +++ b/hack/overlay-main-images_test.bats @@ -618,3 +618,23 @@ load test_helper grep -q 'bare-refs=0' "$w/out.txt" rm -rf "$w" } + +@test "a bare-ref rewrite that cannot write the file is counted as failed, not replaced" { + # Mode bits do not stop root from writing, and the POSIX runner has no + # `skip`, so there is nothing to observe as root. + if [ "$(id -u)" -eq 0 ]; then return 0; fi + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/ro" "$w/main/system/ro" + printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/ro:v0.0.0\n' > "$w/packages/system/ro/values.yaml" + printf 'tuning: new\nimage: iad.ocir.io/x/cozystack/ro:main@sha256:bbbb\n' > "$w/main/system/ro/values.yaml" + chmod a-w "$w/packages/system/ro" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" 2>&1 + chmod u+w "$w/packages/system/ro" + grep -q '^image: ghcr.io/cozystack/cozystack/ro:v0.0.0$' "$w/packages/system/ro/values.yaml" + grep -q 'bare-refs=0 failed=1' "$w/out.txt" + if ls "$w/packages/system/ro" | grep -q 'overlay-bare'; then + echo "FAIL: temp file left behind" >&2; exit 1 + fi + rm -rf "$w" +} From 1597bb609842d961d7d01cf3803dd0ae96f0cb9f Mon Sep 17 00:00:00 2001 From: Maxim Belyy Date: Sat, 14 Nov 2026 20:02:29 +0700 Subject: [PATCH 28/28] fix(ci): keep the drift annotation true when bare refs took main's images Assisted-by: LLM Signed-off-by: Maxim Belyy --- hack/overlay-main-images.sh | 10 +++++++--- hack/overlay-main-images_test.bats | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/hack/overlay-main-images.sh b/hack/overlay-main-images.sh index 2e6c4a1c4..6e8feafd2 100755 --- a/hack/overlay-main-images.sh +++ b/hack/overlay-main-images.sh @@ -141,7 +141,9 @@ bare_ref_line='^[^[:space:]#][^[:space:]]*/[^[:space:]]+$' # under an image key, taking the line from that has the same key and # indentation, an @sha256: digest and the same first-party image, i.e. a path # ending in /cozystack/. Exactly one such line must exist; two are -# ambiguous and none means build-main did not stamp it. Every other line of +# ambiguous, and none means build-main did not stamp it, or the key moved +# (another indentation or parent) or the image was renamed on main since the +# PR's merge commit. Every other line of # stays as the PR has it. # # The trade-off is deliberate. A bare first-party ref that does exist (an older @@ -299,8 +301,10 @@ echo "Overlay current-main images: overlaid=$overlaid same=$same skipped(rebuilt # cost three unrelated PRs, one round each. if [ "$drift" -gt 0 ]; then drift_pkgs=$(echo "$drift_files" | tr ' ' '\n' | grep -v '^$' | sort -u) - printf '::warning title=Image overlay drift::%s package(s) kept committed release refs, so they run release images in this lane rather than current main: %s\n' \ - "$(printf '%s\n' "$drift_pkgs" | wc -l | tr -d ' ')" \ + except="" + [ "$bare" -eq 0 ] || except=" (bare first-party refs excepted, see Image overlay bare refs)" + printf '::warning title=Image overlay drift::%s package(s) kept committed release refs%s, so they run release images in this lane rather than current main: %s\n' \ + "$(printf '%s\n' "$drift_pkgs" | wc -l | tr -d ' ')" "$except" \ "$(printf '%s' "$drift_pkgs" | tr '\n' ' ' | sed 's/ $//')" fi # Among those, a package whose bare first-party ref took main's image runs that diff --git a/hack/overlay-main-images_test.bats b/hack/overlay-main-images_test.bats index 4059f9ea1..68438ffa5 100644 --- a/hack/overlay-main-images_test.bats +++ b/hack/overlay-main-images_test.bats @@ -527,6 +527,7 @@ load test_helper fi grep -q 'bare first-party ref in packages/system/proxy/values.yaml -> current-main' "$w/out.txt" grep -q '::warning title=Image overlay bare refs::1 drifted package(s) .*: system/proxy$' "$w/out.txt" + grep -q '::warning title=Image overlay drift::1 package(s) kept committed release refs (bare first-party refs excepted, see Image overlay bare refs), .*: system/proxy$' "$w/out.txt" grep -q 'bare-refs=1' "$w/out.txt" rm -rf "$w" }