diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 62cbbb1ad..92379083c 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -405,6 +405,11 @@ jobs: - name: Run backport-batch tests if: ${{ !cancelled() && needs.plan.outputs.backport_batch == 'true' }} run: make test-backport-batch + - name: Check new Helm document indices + if: ${{ !cancelled() }} + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: python3 hack/check-helm-document-indices.py "$(git merge-base HEAD "origin/$BASE_REF")" - name: Upload unit test report if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 diff --git a/ADOPTERS.md b/ADOPTERS.md index d6cacdc55..936bfe296 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -33,3 +33,4 @@ This list is sorted in chronological order, based on the submission date. | [Hidora](https://hikube.cloud) | @matthieu-robin | 2025-09-17 | Hidora is a Swiss cloud provider delivering managed services and infrastructure solutions through datacenters located in Switzerland, ensuring data sovereignty and reliability. Its sovereign cloud platform, Hikube, is designed to run workloads with high availability across multiple datacenters, providing enterprises with a secure and scalable foundation for their applications based on Cozystack. | | [QOSI](https://qosi.kz) | @tabu-a | 2025-10-04 | QOSI is a non-profit organization driving open-source adoption and digital sovereignty across Kazakhstan and Central Asia. We use Cozystack as a platform for deploying sovereign, GPU-enabled clouds and educational environments under the National AI Program. Our goal is to accelerate the region’s transition toward open, self-hosted cloud-native technologies | | [Cloupard](https://cloupard.kz/) | @serjiott | 2025-12-18 | Cloupard is a public cloud provider offering IaaS and PaaS services via datacenters in Kazakhstan and Uzbekistan. Uses CozyStack on bare metal to extend its managed PaaS offerings. | +| [CageForge](https://cageforge.com/) | @themoriarti | 2026-11-03 | CageForge is a short-cycle R&D company providing independent expertise in cloud native, Cozystack and security for customers in finance, telecom, hosting and cloud providers. We run Cozystack both as a bare-metal installation and on Proxmox VE through the Proxmox integration, and contribute the Proxmox-related work back upstream. | diff --git a/api/apps/v1alpha1/kubernetesnodes/types.go b/api/apps/v1alpha1/kubernetesnodes/types.go index 79addb215..202807bb1 100644 --- a/api/apps/v1alpha1/kubernetesnodes/types.go +++ b/api/apps/v1alpha1/kubernetesnodes/types.go @@ -51,6 +51,10 @@ type ConfigSpec struct { // List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). // +kubebuilder:default:={} Gpus []GPU `json:"gpus,omitempty"` + // Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=2.147483647e+09 + MaxUnavailable *int `json:"maxUnavailable,omitempty"` // Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. KernelModules []KernelModule `json:"kernelModules,omitempty"` // Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`. diff --git a/api/apps/v1alpha1/kubernetesnodes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetesnodes/zz_generated.deepcopy.go index f858ad79b..dcc3b3e01 100644 --- a/api/apps/v1alpha1/kubernetesnodes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetesnodes/zz_generated.deepcopy.go @@ -67,6 +67,11 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { *out = make([]GPU, len(*in)) copy(*out, *in) } + if in.MaxUnavailable != nil { + in, out := &in.MaxUnavailable, &out.MaxUnavailable + *out = new(int) + **out = **in + } if in.KernelModules != nil { in, out := &in.KernelModules, &out.KernelModules *out = make([]KernelModule, len(*in)) diff --git a/docs/agents/image-refs.md b/docs/agents/image-refs.md index 9c9bb3ca6..191bc62c2 100644 --- a/docs/agents/image-refs.md +++ b/docs/agents/image-refs.md @@ -42,7 +42,7 @@ If a package stamps a reference somewhere the two globs do not cover, add that f ## Invariants -- **Every first-party reference is tag+digest.** A bare tag with no digest is not reproducible and must not be committed. The one exception is the first commit of a new image: it has no digest until it is built, so it lands as a bare `:` that the producing `make image` replaces. PR CI and release preparation both stamp it before anything pulls it, but a tree applied without a build pulls a tag that does not exist. +- **Every first-party reference is tag+digest.** A bare tag with no digest is not reproducible and must not be committed. The one exception is the first commit of a new image: it has no digest until it is built, so it lands as a bare `:` that the producing `make image` replaces. PR CI and release preparation both stamp it before anything pulls it, but a tree applied without a build pulls a tag that does not exist. In PR CI the stamp comes from `hack/overlay-main-images.sh` when the PR does not rebuild the package, and it takes a bare first-party ref even from a file whose configuration drifted from the artifact, because a tag that was never pushed cannot install. The cost is that any other digest-less first-party ref in such a file also runs main's image with the PR's older config; third-party refs in it keep their committed pair. - **Exactly one producer stamps each reference.** A reference no `Makefile` writes will never be refreshed by anything, including release preparation. - **A package that stamps a reference is in the root `Makefile` `build:` list.** `build-matrix.sh`, the main build and release preparation all iterate that one list, so a package outside it is never rebuilt by any path and its pin freezes permanently. `hack/build-matrix_test.bats` computes the set of packages whose Makefile runs `docker buildx build` and fails on any the list misses. - **Every consumer of references reads all three storage shapes**, via `hack/lib/image-refs.sh`. diff --git a/hack/check-helm-document-indices.bats b/hack/check-helm-document-indices.bats new file mode 100644 index 000000000..5188f5b26 --- /dev/null +++ b/hack/check-helm-document-indices.bats @@ -0,0 +1,334 @@ +#!/usr/bin/env bats + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" + +setup() { + strict_setup + tmp=$(mktemp -d) + cd "$tmp" + git init -q + mkdir -p packages/apps/example/templates packages/apps/example/tests + cat > packages/apps/example/templates/multi.yaml <<'EOF' +apiVersion: v1 +kind: ConfigMap +metadata: + name: first +--- +apiVersion: v1 +kind: Secret +metadata: + name: second +EOF + head -n 4 packages/apps/example/templates/multi.yaml > packages/apps/example/templates/single.yaml + git add . + git -c user.name=Test -c user.email=test@example.org commit -qm fixture + base=$(git rev-parse HEAD) +} + +teardown() { + cd "$REPO_ROOT" + rm -rf "$tmp" +} + +write_suite() { + cat > packages/apps/example/tests/resource_test.yaml < output 2>&1; then + echo "accepted a positional selector on a multi-document template" >&2 + false + fi + grep -q 'resource_test.yaml:6: use documentSelector' output +} + +@test "allows an index on a single-document template and ignores comments" { + write_suite single + printf '\n# documentIndex: 1\n' >> packages/apps/example/tests/resource_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "leaves existing indices alone when another assertion changes" { + write_suite multi + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + sed 's/spec.missing/spec.other/' packages/apps/example/tests/resource_test.yaml > changed.yaml + mv changed.yaml packages/apps/example/tests/resource_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "follows assertion template overrides in later YAML suites" { + write_suite single + cat >> packages/apps/example/tests/resource_test.yaml <<'EOF' +--- +suite: overridden template +templates: + - templates/single.yaml +tests: + - it: checks another resource + asserts: + - notExists: + path: spec.missing + template: templates/multi.yaml + documentIndex: 1 +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted an assertion-level positional selector" >&2 + false + fi + grep -q 'use documentSelector.*templates/multi.yaml' output +} + +@test "rejects an invalid base rather than passing an empty diff" { + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" missing-base > output 2>&1; then + echo "accepted an invalid base" >&2 + false + fi +} + +@test "test-level indices follow each assertion's template override" { + write_suite multi + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: assertion override +templates: [templates/single.yaml, templates/multi.yaml] +tests: + - it: checks only the single-document template + template: templates/multi.yaml + documentIndex: 0 + asserts: + - template: templates/single.yaml + notExists: + path: spec.missing +EOF + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "conditional kinds inside one document are not multiple documents" { + write_suite single + cat > packages/apps/example/templates/single.yaml <<'EOF' +apiVersion: v1 +{{ if .Values.secret }} +kind: Secret +{{ else }} +kind: ConfigMap +{{ end }} +metadata: + name: single +EOF + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "rejects a positional selector whose value is on a separate line" { + write_suite multi + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: multiline index +templates: [templates/multi.yaml] +tests: + - it: checks a resource + documentIndex: + 0 + asserts: + - notExists: + path: spec.missing +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a multiline positional selector" >&2 + false + fi + grep -q 'resource_test.yaml:5: use documentSelector' output +} + +@test "rejects a singular test template overriding a single-document suite" { + write_suite single + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/single.yaml] +tests: + - it: checks a resource + template: templates/multi.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a positional selector on the test template override" >&2 + false + fi + grep -q 'use documentSelector.*templates/multi.yaml' output +} + +@test "allows a singular test template selecting one document" { + write_suite multi + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/multi.yaml] +tests: + - it: checks a resource + template: templates/single.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "plural test templates take precedence over the singular test template" { + write_suite multi + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/multi.yaml] +tests: + - it: checks a resource + templates: [templates/single.yaml] + template: templates/multi.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "an empty plural test override falls back to the singular template" { + write_suite single + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: test template override +templates: [templates/single.yaml] +tests: + - it: checks a resource + templates: [] + template: templates/multi.yaml + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a positional selector after an empty plural override" >&2 + false + fi + grep -q 'use documentSelector.*templates/multi.yaml' output +} + +@test "does not check vendored chart suites" { + mkdir -p packages/apps/example/charts/sub/templates packages/apps/example/charts/sub/tests + cp packages/apps/example/templates/multi.yaml packages/apps/example/charts/sub/templates/ + cat > packages/apps/example/charts/sub/tests/resource_test.yaml <<'EOF' +suite: upstream resource selection +templates: [templates/multi.yaml] +tests: + - it: checks an upstream resource + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF + git add . + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "a new test-level index overrides existing assertion indices" { + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: index inheritance +templates: [templates/multi.yaml] +tests: + - it: checks a resource + asserts: + - notExists: + path: spec.missing + documentIndex: 0 + - notExists: + path: spec.other + documentIndex: 0 +EOF + git add . + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: index inheritance +templates: [templates/multi.yaml] +tests: + - it: checks a resource + documentIndex: 1 + asserts: + - notExists: + path: spec.missing + documentIndex: 0 + - notExists: + path: spec.other + documentIndex: 0 +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a new inherited positional selector" >&2 + false + fi + grep -q 'resource_test.yaml:5: use documentSelector' output +} + +@test "allows a pure suite rename with existing indices" { + write_suite multi + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + git mv packages/apps/example/tests/resource_test.yaml packages/apps/example/tests/renamed_test.yaml + python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" +} + +@test "checks added index lines in a renamed suite" { + write_suite multi + git -c user.name=Test -c user.email=test@example.org commit -qm existing + base=$(git rev-parse HEAD) + git mv packages/apps/example/tests/resource_test.yaml packages/apps/example/tests/renamed_test.yaml + cat > packages/apps/example/tests/renamed_test.yaml <<'EOF' +suite: resource selection +templates: + - templates/multi.yaml +tests: + - it: checks a resource + documentIndex: 1 + asserts: + - notExists: + path: spec.missing +EOF + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a new positional selector in a renamed suite" >&2 + false + fi + grep -q 'renamed_test.yaml:6: use documentSelector' output +} + +@test "an empty suite template list includes nested chart templates" { + mkdir packages/apps/example/templates/nested + mv packages/apps/example/templates/multi.yaml packages/apps/example/templates/nested/ + cat > packages/apps/example/tests/resource_test.yaml <<'EOF' +suite: default templates +templates: [] +tests: + - it: checks a resource + documentIndex: 0 + asserts: + - notExists: + path: spec.missing +EOF + git add . + if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then + echo "accepted a positional selector with default templates" >&2 + false + fi + grep -q 'use documentSelector.*templates/nested/multi.yaml' output +} diff --git a/hack/check-helm-document-indices.py b/hack/check-helm-document-indices.py new file mode 100644 index 000000000..dd75e799a --- /dev/null +++ b/hack/check-helm-document-indices.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Reject added positional selectors on multi-resource Helm templates. + +Usage: python3 hack/check-helm-document-indices.py +Requires mikefarah/yq, like the chart test tooling. +""" + +import json +from pathlib import Path +import re +import subprocess +import sys + + +def git(*args): + return subprocess.check_output(["git", *args], text=True) + + +def check(base): + """Check added indices, leaving existing ones alone. + + This static check counts --- separated blocks with a top-level kind; it does + not evaluate conditional documents or resources emitted by helpers/ranges. + """ + root = Path(git("rev-parse", "--show-toplevel").strip()) + additions = {} + path = None + for line in git("diff", "--no-ext-diff", "-M", "--unified=0", base, "--", + ":(glob)packages/*/*/tests/*_test.yaml").splitlines(): + if line.startswith("diff --git "): + path = None + elif line.startswith("+++ b/"): + path = line[6:] + additions[path] = set() + elif line.startswith("@@"): + number = int(re.search(r"\+(\d+)", line).group(1)) + elif path and line.startswith("+"): + if "documentIndex" in line: + additions[path].add(number) + number += 1 + elif line.startswith(" "): + number += 1 + errors = [] + for path, added in additions.items(): + if not added: + continue + + # Preserve the key's source line while yq resolves multi-document YAML, + # flow mappings and anchors; a text-only suite parser misses those. + expression = ('[. | (.. | select(tag == "!!map" and has("documentIndex")) ' + '| .documentIndex) |= {"line": key | line}]') + suites = json.loads(subprocess.check_output( + ["yq", "eval-all", "-o=json", expression, str(root / path)], text=True)) + chart = (root / path).parent.parent + for suite in suites: + if not suite: + continue + for test in suite.get("tests", []): + templates = test.get("templates") or ( + [test["template"]] if test.get("template") + else suite.get("templates") or ["templates/**/*"]) + for assertion in test.get("asserts", []): + index = test.get("documentIndex") or assertion.get("documentIndex", {}) + number = index.get("line") + if number not in added: + continue + selected = [assertion["template"]] if "template" in assertion else templates + for pattern in selected: + matches = list(chart.glob(pattern)) + if not matches: + matches = list((chart / "templates").glob(pattern)) + for template in matches: + if not template.is_file(): + continue + documents = re.split(r"(?m)^---[ \t]*(?:#.*)?$", template.read_text()) + if sum(bool(re.search(r"(?m)^kind\s*:", doc)) for doc in documents) > 1: + errors.append(f"{path}:{number}: use documentSelector, not " + f"documentIndex, for {template.relative_to(root)}") + for error in sorted(set(errors)): + print(error, file=sys.stderr) + return bool(errors) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + sys.exit("usage: python3 hack/check-helm-document-indices.py ") + sys.exit(check(sys.argv[1])) diff --git a/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml b/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml index 4e0c86535..f6e71abcd 100644 --- a/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml @@ -102,7 +102,9 @@ spec: # # nameOverride is outside the module's per-pool allowlist. The ComputePlane # schema accepts it, so it reaches the module and the pool check below can - # see whether the module dropped it. + # see whether the module dropped it. maxUnavailable is set to 0 because a + # zero lost anywhere on the way would let a GPU pool fall back to the + # chart's default of 1; only this path crosses the aggregated API. - script: timeout: 3m content: | @@ -118,6 +120,7 @@ spec: diskSize: 20Gi instanceType: u1.medium logSerialConsole: true + maxUnavailable: 0 minReplicas: 2 maxReplicas: 2 podCpuLimit: 2 @@ -175,8 +178,9 @@ spec: enabled: true exposeMethod: Proxied - # The pool takes the patched shape, and the nameOverride the patch sent - # does not reach it, nor does any other key outside the allowlist. + # The pool takes the patched shape, the explicit maxUnavailable of 0 + # included, and the nameOverride the patch sent does not reach it, nor does + # any other key outside the allowlist. - script: timeout: 1m content: | @@ -186,8 +190,8 @@ spec: sc=$(cozy_e2e_storage_class) values=$(timeout -k 5 30 kubectl -n tenant-cplane get helmrelease kubernetes-nodes-computeplane-cluster-md0 -o json | jq -c '.spec.values') echo "pool values: $values" - echo "$values" | jq -e --arg sc "$sc" '.storageClass == $sc and .minReplicas == 2 and .cluster == "computeplane-cluster" and .clusterReleaseName == "computeplane-cluster"' - extra=$(echo "$values" | jq -r 'keys - ["cluster","clusterReleaseName","minReplicas","maxReplicas","instanceType","diskSize","storageClass","roles","resources","gpus","kubelet","logSerialConsole","maxUnhealthy","nodeStartupTimeout","readyUnknownTimeout","readyFalseTimeout","podCpuLimit","podCpuRequest","maxNodeProvisionTime"] | .[]') + echo "$values" | jq -e --arg sc "$sc" '.storageClass == $sc and .minReplicas == 2 and .maxUnavailable == 0 and .cluster == "computeplane-cluster" and .clusterReleaseName == "computeplane-cluster"' + extra=$(echo "$values" | jq -r 'keys - ["cluster","clusterReleaseName","minReplicas","maxReplicas","instanceType","diskSize","storageClass","roles","resources","gpus","kubelet","logSerialConsole","maxUnhealthy","nodeStartupTimeout","readyUnknownTimeout","readyFalseTimeout","podCpuLimit","podCpuRequest","maxNodeProvisionTime","maxUnavailable"] | .[]') if [ -n "$extra" ]; then echo "pool HelmRelease carries keys outside the module allowlist: $extra" >&2 exit 1 diff --git a/hack/overlay-main-images.sh b/hack/overlay-main-images.sh index 2c13e34e1..6e8feafd2 100755 --- a/hack/overlay-main-images.sh +++ b/hack/overlay-main-images.sh @@ -42,9 +42,13 @@ # a change: `yq -i`, which every `make image` uses to stamp the ref, drops one. # If any OTHER line differs — the PR branched from a # main whose config for that file differs from the artifact's base — the file is -# left on its committed ref (safe degradation, logged, never fatal). CI checks -# out the pull_request merge commit (current main + PR), so for an unbuilt file -# the config already matches current main and only ref lines differ. +# left on its committed ref (safe degradation, logged, never fatal), except +# that a bare first-party ref in it, one with no digest, still takes the +# artifact's ref (see replace_bare_refs). CI checks out the pull_request merge +# commit (main + PR), so for an unbuilt file the config matches main as of +# that merge commit. It is fixed when the run starts, so a rerun, or a PR not +# pushed to since, can be older than the artifact and drift on files it never +# touched. # # Usage: hack/overlay-main-images.sh [touched-pkg-dirs] # = extracted cozystack-packages:main tree; its root holds @@ -123,8 +127,95 @@ img_line='(@sha256:|^[[:space:]]*(- )?(image|repository|registry|tag|digest):|^[ # match branch for this same shape and for this same reason. bare_ref_line='^[^[:space:]#][^[:space:]]*/[^[:space:]]+$' +# A drifted file keeps its committed refs. For a ref pinned by digest that is an +# older image which pulls and matches the PR's config, so it is the safe answer. +# A first-party ref with no digest is not: docs/agents/image-refs.md forbids it +# except as the first commit of a new image, whose tag was never pushed, and +# keeping that makes the component fail to pull. The merge commit CI checks out +# is fixed when the run starts, so a PR whose merge commit predates a config +# change to such a file drifts on it, and a new image fails every such PR until +# a release stamps its ref (#5328). +# +# replace_bare_refs rewrites, in only, each line whose value is +# a bare first-party ref (ghcr.io/cozystack/cozystack/:, no digest) +# under an image key, taking the line from that has the same key and +# indentation, an @sha256: digest and the same first-party image, i.e. a path +# ending in /cozystack/. Exactly one such line must exist; two are +# ambiguous, and none means build-main did not stamp it, or the key moved +# (another indentation or parent) or the image was renamed on main since the +# PR's merge commit. Every other line of +# stays as the PR has it. +# +# The trade-off is deliberate. A bare first-party ref that does exist (an older +# release tag or a moving `latest`, #4264) also takes main's image, so it runs +# with the PR's older config rather than as committed. It is the same component +# built from the tree, and a digest-less ref is not reproducible anyway; the +# alternative is a ref that may not pull at all. Third-party refs are never +# touched: main can bump one alongside its config, and the committed pair is +# the one known to work together. The split shape (`repository:` + `tag:` +# without `digest:`) is not handled, nor is a single-quoted value or one +# followed by a `# comment`; such a line stays as committed, and no package +# writes a bare first-party ref that way today. Prints nothing and fails when +# the file could not be rewritten, so the caller counts it as failed. +# +# Prints the number of lines replaced. +replace_bare_refs() { + tmp="$1.overlay-bare.$$" + n=$(awk -v newf="$2" -v out="$tmp" ' + function key_prefix(s) { + if (match(s, /^[[:space:]]*(- )?([A-Za-z]+[Ii]mage|image):[[:space:]]*/)) return substr(s, 1, RLENGTH) + return "" + } + function ref_value(s, p, v) { + v = substr(s, length(p) + 1) + sub(/[[:space:]]+$/, "", v) + gsub(/"/, "", v) + return v + } + function image_name(v) { + sub(/@.*/, "", v) + sub(/.*\//, "", v) + sub(/:.*/, "", v) + return v + } + BEGIN { + while ((getline l < newf) > 0) { + p = key_prefix(l) + if (p == "" || index(l, "@sha256:") == 0) continue + v = ref_value(l, p) + name = image_name(v) + if (index(v, "/cozystack/" name ":") == 0) continue + k = p SUBSEP name + cnt[k]++ + line[k] = l + } + close(newf) + } + { + p = key_prefix($0) + if (p != "") { + v = ref_value($0, p) + if (v ~ /^ghcr\.io\/cozystack\/cozystack\/[^@[:space:]\/]+:[^@[:space:]\/]+$/) { + k = p SUBSEP image_name(v) + if (cnt[k] == 1) { print line[k] > out; r++; next } + } + } + print > out + } + END { close(out); print r + 0 } + ' "$1") + case "$n" in + ''|*[!0-9]*) rm -f "$tmp"; return 1 ;; + 0) rm -f "$tmp" ;; + *) mv "$tmp" "$1" || { rm -f "$tmp"; return 1; } ;; + esac + echo "$n" +} + overlaid=0 same=0 +bare=0 +bare_files="" skipped=0 drift=0 drift_files="" @@ -162,8 +253,26 @@ for new in $(find "$MAINPKGS" -type d -name charts -prune -o \ if diff "$cur" "$new" | sed -n 's/^[<>] //p' \ | grep -vE '^[[:space:]]*$' | grep -qvE "$line_pat"; then - echo "drift (non-ref change) in $cur -> keeping committed ref" drift=$((drift + 1)) + r=0 + case "$cur" in + *.yaml) + if ! r=$(replace_bare_refs "$cur" "$new"); then + echo "WARN: rewriting bare refs failed for $cur (keeping committed refs)" + failed=$((failed + 1)) + r=0 + fi + ;; + esac + if [ "$r" -eq 0 ]; then + echo "drift (non-ref change) in $cur -> keeping committed ref" + else + echo "drift (non-ref change) in $cur -> keeping committed refs except bare first-party ones" + echo "bare first-party ref in $cur -> current-main ($r line(s)); its other refs stay committed" + bare=$((bare + r)) + d=$(dirname "${cur#packages/}") + bare_files="$bare_files $d" + fi # Name the PACKAGE, not the directory the file happens to sit in: an # images/*.tag drift would otherwise read as "/images", which is not # something anyone can go and look at. @@ -181,7 +290,7 @@ for new in $(find "$MAINPKGS" -type d -name charts -prune -o \ fi done -echo "Overlay current-main images: overlaid=$overlaid same=$same skipped(rebuilt/owned/edited)=$skipped drift=$drift failed=$failed" +echo "Overlay current-main images: overlaid=$overlaid same=$same skipped(rebuilt/owned/edited)=$skipped drift=$drift bare-refs=$bare failed=$failed" # Keeping the committed ref is the safe answer to a config that does not match # the artifact, but it is not a neutral one: that component then runs its last @@ -192,7 +301,18 @@ echo "Overlay current-main images: overlaid=$overlaid same=$same skipped(rebuilt # cost three unrelated PRs, one round each. if [ "$drift" -gt 0 ]; then drift_pkgs=$(echo "$drift_files" | tr ' ' '\n' | grep -v '^$' | sort -u) - printf '::warning title=Image overlay drift::%s package(s) kept committed release refs, so they run release images in this lane rather than current main: %s\n' \ - "$(printf '%s\n' "$drift_pkgs" | wc -l | tr -d ' ')" \ + except="" + [ "$bare" -eq 0 ] || except=" (bare first-party refs excepted, see Image overlay bare refs)" + printf '::warning title=Image overlay drift::%s package(s) kept committed release refs%s, so they run release images in this lane rather than current main: %s\n' \ + "$(printf '%s\n' "$drift_pkgs" | wc -l | tr -d ' ')" "$except" \ "$(printf '%s' "$drift_pkgs" | tr '\n' ' ' | sed 's/ $//')" fi +# Among those, a package whose bare first-party ref took main's image runs that +# image with the PR's older config, not its committed ref; name it separately +# so the warning above does not read as the whole story for it. +if [ "$bare" -gt 0 ]; then + bare_pkgs=$(echo "$bare_files" | tr ' ' '\n' | grep -v '^$' | sort -u) + printf '::warning title=Image overlay bare refs::%s drifted package(s) took current-main images for bare first-party refs, with the config committed in this PR: %s\n' \ + "$(printf '%s\n' "$bare_pkgs" | wc -l | tr -d ' ')" \ + "$(printf '%s' "$bare_pkgs" | tr '\n' ' ' | sed 's/ $//')" +fi diff --git a/hack/overlay-main-images_test.bats b/hack/overlay-main-images_test.bats index 7cff2d842..68438ffa5 100644 --- a/hack/overlay-main-images_test.bats +++ b/hack/overlay-main-images_test.bats @@ -502,3 +502,140 @@ load test_helper cd "$root" rm -rf "$w" } + +# A drifted file keeps its committed refs, and for a digest-pinned ref that is +# a pullable older image. A bare : is the first commit of a new image +# (docs/agents/image-refs.md) and names a tag no registry serves, so keeping it +# turns drift into ImagePullBackOff. That hit every fork PR whose merge commit +# predated a config change to observability-proxy's values.yaml (#5328). Only +# such a bare line takes the artifact's ref; the rest of the file stays as the +# PR has it. +@test "a drifted file still takes the artifact ref for a bare (digest-less) ref line" { + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/proxy" "$w/main/system/proxy" + printf 'proxy:\n image: "ghcr.io/cozystack/cozystack/proxy:v0.0.0"\n lookback: 1h\n' \ + > "$w/packages/system/proxy/values.yaml" + printf 'proxy:\n image: "iad.ocir.io/x/cozystack/proxy:main@sha256:bbbb"\n lookback: 1h\n prefix: cozy-\n' \ + > "$w/main/system/proxy/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q 'drift (non-ref change) in packages/system/proxy/values.yaml' "$w/out.txt" + grep -q ' image: "iad.ocir.io/x/cozystack/proxy:main@sha256:bbbb"' "$w/packages/system/proxy/values.yaml" + # The config stays the PR's: the new key is not introduced. + if grep -q 'prefix:' "$w/packages/system/proxy/values.yaml"; then + echo "FAIL: a config line was taken from the artifact" >&2; exit 1 + fi + grep -q 'bare first-party ref in packages/system/proxy/values.yaml -> current-main' "$w/out.txt" + grep -q '::warning title=Image overlay bare refs::1 drifted package(s) .*: system/proxy$' "$w/out.txt" + grep -q '::warning title=Image overlay drift::1 package(s) kept committed release refs (bare first-party refs excepted, see Image overlay bare refs), .*: system/proxy$' "$w/out.txt" + grep -q 'bare-refs=1' "$w/out.txt" + rm -rf "$w" +} + +@test "a drifted file replaces only its bare ref, not a digest-pinned sibling" { + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/pair" "$w/main/system/pair" + printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/pair:v1.5.0@sha256:aaaa\nclientImage: ghcr.io/cozystack/cozystack/pair-client:latest\n' \ + > "$w/packages/system/pair/values.yaml" + printf 'tuning: new\nimage: iad.ocir.io/x/cozystack/pair:main@sha256:bbbb\nclientImage: iad.ocir.io/x/cozystack/pair-client:main@sha256:cccc\n' \ + > "$w/main/system/pair/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q '^image: ghcr.io/cozystack/cozystack/pair:v1.5.0@sha256:aaaa$' "$w/packages/system/pair/values.yaml" + grep -q '^clientImage: iad.ocir.io/x/cozystack/pair-client:main@sha256:cccc$' "$w/packages/system/pair/values.yaml" + grep -q '^tuning: old$' "$w/packages/system/pair/values.yaml" + rm -rf "$w" +} + +@test "a bare ref is not replaced by an artifact line for a different image" { + # The same key can hold another image in the artifact, e.g. after a rename. + # Matching on the key alone would swap in an image the chart never asked for. + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/ren" "$w/main/system/ren" + printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/old-name:v0.0.0\n' > "$w/packages/system/ren/values.yaml" + printf 'tuning: new\nimage: iad.ocir.io/x/cozystack/new-name:main@sha256:bbbb\n' > "$w/main/system/ren/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q '^image: ghcr.io/cozystack/cozystack/old-name:v0.0.0$' "$w/packages/system/ren/values.yaml" + grep -q 'bare-refs=0' "$w/out.txt" + rm -rf "$w" +} + +@test "a bare ref is kept when the artifact carries no digest for it either" { + # A ref build-main does not stamp has no digest in the artifact, so there is + # nothing more pullable to take. + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/nd" "$w/main/system/nd" + printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/nd:v0.0.0\n' > "$w/packages/system/nd/values.yaml" + printf 'tuning: new\nimage: iad.ocir.io/x/cozystack/nd:main\n' > "$w/main/system/nd/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q '^image: ghcr.io/cozystack/cozystack/nd:v0.0.0$' "$w/packages/system/nd/values.yaml" + grep -q 'bare-refs=0' "$w/out.txt" + rm -rf "$w" +} + +@test "a bare third-party ref in a drifted file is kept even when main pins one" { + # Main can bump a third-party image together with its config in one file. A + # PR behind it has the older pair, which is the one known to work together; + # taking main's image alone would run it with the config it replaced. + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/kc" "$w/main/system/kc" + printf 'features: old\nimage: quay.io/keycloak/keycloak:26.6.3\n' > "$w/packages/system/kc/values.yaml" + printf 'features: new\nimage: quay.io/keycloak/keycloak:27.0.0@sha256:abcd\n' > "$w/main/system/kc/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q '^image: quay.io/keycloak/keycloak:26.6.3$' "$w/packages/system/kc/values.yaml" + grep -q 'bare-refs=0' "$w/out.txt" + if grep -q 'Image overlay bare refs' "$w/out.txt"; then + echo "FAIL: a bare-refs annotation with nothing replaced" >&2; exit 1 + fi + rm -rf "$w" +} + +@test "a bare first-party ref is not replaced by a third-party image of the same name" { + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/same" "$w/main/system/same" + printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/redis:v0.0.0\n' > "$w/packages/system/same/values.yaml" + printf 'tuning: new\nimage: docker.io/library/redis:8.0@sha256:abcd\n' > "$w/main/system/same/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q '^image: ghcr.io/cozystack/cozystack/redis:v0.0.0$' "$w/packages/system/same/values.yaml" + grep -q 'bare-refs=0' "$w/out.txt" + rm -rf "$w" +} + +@test "a bare ref is kept when the artifact has two candidate lines for it" { + # Two lines with the same key and image name (e.g. one per component in a + # list) cannot be told apart, so neither is taken. + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/dup" "$w/main/system/dup" + printf 'tuning: old\na:\n image: ghcr.io/cozystack/cozystack/dup:v0.0.0\n' > "$w/packages/system/dup/values.yaml" + printf 'tuning: new\na:\n image: iad.ocir.io/x/cozystack/dup:main@sha256:bbbb\nb:\n image: iad.ocir.io/x/cozystack/dup:main@sha256:cccc\n' \ + > "$w/main/system/dup/values.yaml" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" + grep -q '^ image: ghcr.io/cozystack/cozystack/dup:v0.0.0$' "$w/packages/system/dup/values.yaml" + grep -q 'bare-refs=0' "$w/out.txt" + rm -rf "$w" +} + +@test "a bare-ref rewrite that cannot write the file is counted as failed, not replaced" { + # Mode bits do not stop root from writing, and the POSIX runner has no + # `skip`, so there is nothing to observe as root. + if [ "$(id -u)" -eq 0 ]; then return 0; fi + root=$(pwd) + w=$(mktemp -d) + mkdir -p "$w/packages/system/ro" "$w/main/system/ro" + printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/ro:v0.0.0\n' > "$w/packages/system/ro/values.yaml" + printf 'tuning: new\nimage: iad.ocir.io/x/cozystack/ro:main@sha256:bbbb\n' > "$w/main/system/ro/values.yaml" + chmod a-w "$w/packages/system/ro" + ( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" 2>&1 + chmod u+w "$w/packages/system/ro" + grep -q '^image: ghcr.io/cozystack/cozystack/ro:v0.0.0$' "$w/packages/system/ro/values.yaml" + grep -q 'bare-refs=0 failed=1' "$w/out.txt" + if ls "$w/packages/system/ro" | grep -q 'overlay-bare'; then + echo "FAIL: temp file left behind" >&2; exit 1 + fi + rm -rf "$w" +} diff --git a/hack/run-kubernetes-block-io_test.bats b/hack/run-kubernetes-block-io_test.bats index b083f9e4f..18b12d253 100644 --- a/hack/run-kubernetes-block-io_test.bats +++ b/hack/run-kubernetes-block-io_test.bats @@ -151,36 +151,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded into an `if`, both land in the passing - # branch and the assertion "this is absent" is satisfied by the matcher giving - # up. Every negative claim in this file rests on the three-way split. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command lands on the very stderr being diff --git a/hack/run-kubernetes-cpu-throttle_test.bats b/hack/run-kubernetes-cpu-throttle_test.bats index d254c7d49..c4ed0dfa4 100644 --- a/hack/run-kubernetes-cpu-throttle_test.bats +++ b/hack/run-kubernetes-cpu-throttle_test.bats @@ -94,45 +94,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". awk exits 2 on an unreadable path, - # which folded into an if reads the same way, so the file is checked first. - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status, not on truthiness. awk exits 1 for "no line - # matched" and 2 for "I could not evaluate this" -- an unparseable regex, most - # of all. Folded into `if ... then fail; fi`, both land in the passing branch, - # so the assertion "this pattern is absent" is satisfied by the matcher giving - # up. That is the wrong direction for a negative assertion: a positive one - # fails loudly when its matcher breaks and gets fixed, this one goes green and - # stays green. Every claim in this file about something NOT appearing rests on - # the three-way split below. - # `|| _rc=$?` rather than a bare call: both runners set -e, and awk exiting 1 - # for "no match" -- the ordinary success case here -- would otherwise abort the - # test before the branch below ever runs. The `if` form this replaces was - # exempt only because conditions are exempt. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # COZY_REPORT_DIR is set inline in each test below rather than through a helper, # which is the opposite of what the node-join suite does. The difference is # deliberate and shrinks to one line: there the assignment carries a ten-line diff --git a/hack/run-kubernetes-kvm-exits_test.bats b/hack/run-kubernetes-kvm-exits_test.bats index 0473e642f..47ff07d9e 100644 --- a/hack/run-kubernetes-kvm-exits_test.bats +++ b/hack/run-kubernetes-kvm-exits_test.bats @@ -81,36 +81,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded together, a negative assertion is - # satisfied by its own matcher giving up, which is the direction that goes - # green and stays green. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command lands on the very stderr being diff --git a/hack/run-kubernetes-network-counters_test.bats b/hack/run-kubernetes-network-counters_test.bats index 73da4c2b2..42f986821 100644 --- a/hack/run-kubernetes-network-counters_test.bats +++ b/hack/run-kubernetes-network-counters_test.bats @@ -148,36 +148,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded into an `if`, both land in the passing - # branch and the assertion "this is absent" is satisfied by the matcher giving - # up. Every negative claim in this file rests on the three-way split. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command lands on the very stderr being diff --git a/hack/run-kubernetes-node-join_test.bats b/hack/run-kubernetes-node-join_test.bats index 42dbe2f11..27bf9bdb9 100644 --- a/hack/run-kubernetes-node-join_test.bats +++ b/hack/run-kubernetes-node-join_test.bats @@ -300,36 +300,6 @@ assert_file_contains() { return 1 } -# The sibling suites' three-way split, and here for the same reason. A bare -# `! grep -q` succeeds on a file that does not exist, which is -# indistinguishable from the file existing without the pattern; and awk exits 1 -# for "no line matched" and 2 for "I could not evaluate this", so folding those -# together lets a broken matcher satisfy a negative assertion. A positive -# assertion fails loudly when its matcher breaks and gets fixed; this one would -# go green and stay green. -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - local _rc=0 - - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The ceilings both budget guards below price their sums from, read out of the # helper rather than restated in each. They are the same numbers held against # two different bounds -- the phase budget from below, the operation window from diff --git a/hack/run-kubernetes-runner-canary_test.bats b/hack/run-kubernetes-runner-canary_test.bats index 7e2f85f90..65c0626f8 100644 --- a/hack/run-kubernetes-runner-canary_test.bats +++ b/hack/run-kubernetes-runner-canary_test.bats @@ -67,37 +67,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded together, a negative assertion is - # satisfied by its own matcher giving up, which is the direction that goes - # green and stays green. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - cat "${file}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The clock the arms are timed by, replaced by a scripted sequence. # # Real durations are the one thing this suite must not depend on: they are set diff --git a/hack/run-kubernetes-runner-cpu_test.bats b/hack/run-kubernetes-runner-cpu_test.bats index 4eca0a3fc..eee4da69a 100644 --- a/hack/run-kubernetes-runner-cpu_test.bats +++ b/hack/run-kubernetes-runner-cpu_test.bats @@ -58,37 +58,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded together, a negative assertion is - # satisfied by its own matcher giving up, which is the direction that goes - # green and stays green. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - cat "${file}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The functions under test redirect a command's stderr into a report file and # decide which artifact to write from it. cozytest.sh runs under `set -x`, so the # tracer's own line for that command lands on the very stderr being redirected, diff --git a/hack/run-kubernetes-sandbox-cpu-time_test.bats b/hack/run-kubernetes-sandbox-cpu-time_test.bats index 0cd1bace4..c6f3028ca 100644 --- a/hack/run-kubernetes-sandbox-cpu-time_test.bats +++ b/hack/run-kubernetes-sandbox-cpu-time_test.bats @@ -83,36 +83,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded together, a negative assertion is - # satisfied by its own matcher giving up, which is the direction that goes - # green and stays green. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command lands on the very stderr being diff --git a/hack/run-kubernetes-serial-console_test.bats b/hack/run-kubernetes-serial-console_test.bats index 3dd91773d..4c77bd269 100644 --- a/hack/run-kubernetes-serial-console_test.bats +++ b/hack/run-kubernetes-serial-console_test.bats @@ -129,23 +129,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: awk exits 2 on an - # unreadable path, which is indistinguishable from "no line matched" once the - # status is folded into an if. - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - if awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}"; then - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi -} - # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command is emitted on the very stderr being diff --git a/hack/run-kubernetes-talos-diagnostics_test.bats b/hack/run-kubernetes-talos-diagnostics_test.bats index c5411837f..5ce8eef66 100644 --- a/hack/run-kubernetes-talos-diagnostics_test.bats +++ b/hack/run-kubernetes-talos-diagnostics_test.bats @@ -114,16 +114,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - if awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}"; then - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi -} - @test "worker VMI rows retain names whose default interface has no IP" { . hack/e2e-chainsaw/_lib/run-kubernetes.sh rows=$(printf '%s\n' '{"items":[{"metadata":{"name":"worker-a"},"status":{"interfaces":[{"name":"default","ipAddress":"10.244.1.92"}]}},{"metadata":{"name":"worker-b"},"status":{"interfaces":[{"name":"other","ipAddress":"192.0.2.1"}]}}]}' | cozy_tenant_worker_vmi_rows) diff --git a/hack/run-kubernetes-thread-cpu_test.bats b/hack/run-kubernetes-thread-cpu_test.bats index 4d9f7c90b..b37c749e5 100644 --- a/hack/run-kubernetes-thread-cpu_test.bats +++ b/hack/run-kubernetes-thread-cpu_test.bats @@ -85,37 +85,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the label". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded into an `if`, both land in the passing - # branch and the assertion "this pattern is absent" is satisfied by the - # matcher giving up. Every claim in this file about something NOT appearing - # rests on the three-way split below. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The function under test redirects a command's stderr into a report file and # decides which artifact to write from it. cozytest.sh runs under `set -x`, so # the tracer's own line for that command lands on the very stderr being diff --git a/hack/runner-identity.bats b/hack/runner-identity.bats index acfa36e32..ea42444f5 100644 --- a/hack/runner-identity.bats +++ b/hack/runner-identity.bats @@ -83,37 +83,6 @@ assert_file_contains() { return 1 } -assert_file_lacks_pattern() { - local pattern="$1" - local file="$2" - - # A missing file must fail rather than vacuously pass: a bare `! grep -q` - # succeeds on an unreadable path, which is indistinguishable from "the file - # exists and does not carry the pattern". - if [ ! -f "${file}" ]; then - printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 - return 1 - fi - # Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for - # "I could not evaluate this"; folded together, a negative assertion is - # satisfied by its own matcher giving up, which is the direction that goes - # green and stays green. - local _rc=0 - awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$? - case "${_rc}" in - 0) - printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 - cat "${file}" >&2 - return 1 - ;; - 1) return 0 ;; - *) - printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 - return 1 - ;; - esac -} - # The shape sizing object, as the `shapeConfig` subpath serves it: a flat object of # sizing numbers with one string in it, pretty-printed because the service is free # to answer either way and the reader buffers its whole input for that reason. diff --git a/hack/test-helper_test.bats b/hack/test-helper_test.bats new file mode 100644 index 000000000..902ed8af9 --- /dev/null +++ b/hack/test-helper_test.bats @@ -0,0 +1,102 @@ +#!/usr/bin/env bats + +load test_helper + +# A separate shell keeps errexit active inside the assertion, unlike calling +# the function directly through Bats' run helper. +run_lacks_pattern() { + run bash -eu -c ' + . "$3" + assert_file_lacks_pattern "$1" "$2" + printf "checked\n" + ' _ "$@" "$BATS_TEST_DIRNAME/test_helper.bash" +} + +@test "a valid non-matching pattern passes under errexit" { + printf 'safe content\n' >"$BATS_TEST_TMPDIR/artifact" + + run_lacks_pattern 'secret:|secretName:' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 0 ] + [ "$output" = checked ] +} + +@test "an empty readable file passes for a valid pattern" { + : >"$BATS_TEST_TMPDIR/artifact" + + run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 0 ] + [ "$output" = checked ] +} + +@test "a matching pattern fails the assertion" { + printf 'safe content\nsecretName: credential\n' >"$BATS_TEST_TMPDIR/artifact" + + run_lacks_pattern 'secret:|secretName:' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 1 ] + [[ "$output" == *'not to match: secret:|secretName:'* ]] +} + +@test "an invalid pattern fails on a populated file" { + printf 'safe content\n' >"$BATS_TEST_TMPDIR/artifact" + + run_lacks_pattern '[' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 1 ] + [[ "$output" == *'could not evaluate pattern ['* ]] +} + +@test "an invalid pattern fails even on an empty file" { + : >"$BATS_TEST_TMPDIR/artifact" + + run_lacks_pattern '[' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 1 ] + [[ "$output" == *'could not evaluate pattern ['* ]] +} + +@test "a missing artifact fails rather than passing as no match" { + run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR/missing" + + [ "$status" -eq 1 ] + [[ "$output" == *'to exist so it could be checked'* ]] +} + +@test "a directory is not an artifact file" { + run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR" + + [ "$status" -eq 1 ] + [[ "$output" == *'to exist so it could be checked'* ]] +} + +@test "an unreadable artifact fails rather than passing as no match" { + printf 'safe content\n' >"$BATS_TEST_TMPDIR/artifact" + chmod 000 "$BATS_TEST_TMPDIR/artifact" + if [ -r "$BATS_TEST_TMPDIR/artifact" ]; then + skip 'this user can read files without read permission' + fi + + run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 1 ] + [[ "$output" == *'could not evaluate pattern secret:'* ]] +} + +@test "a matcher error with status one cannot pass as no match" { + printf 'safe content\n' >"$BATS_TEST_TMPDIR/artifact" + mkdir "$BATS_TEST_TMPDIR/bin" + cat >"$BATS_TEST_TMPDIR/bin/awk" <<'SH' +#!/bin/sh +printf '0\n' +printf 'awk: evaluation failed\n' >&2 +exit 1 +SH + chmod +x "$BATS_TEST_TMPDIR/bin/awk" + + PATH="$BATS_TEST_TMPDIR/bin:$PATH" run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR/artifact" + + [ "$status" -eq 1 ] + [[ "$output" == *'could not evaluate pattern secret:'* ]] +} diff --git a/hack/test_helper.bash b/hack/test_helper.bash index f3944c9c0..bbbef8de8 100644 --- a/hack/test_helper.bash +++ b/hack/test_helper.bash @@ -9,3 +9,31 @@ strict_setup() { setup() { strict_setup } + +# Only a successful matcher explicitly reporting no match passes; missing, +# non-regular or unreadable files and matcher errors fail closed. +assert_file_lacks_pattern() { + local pattern="$1" + local file="$2" + local found + + if [ ! -f "${file}" ]; then + printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2 + return 1 + fi + # Validate even an empty file, and keep the match result separate from awk's + # status: some implementations also use status 1 for evaluation errors. + if ! found=$(awk -v pattern="${pattern}" ' + BEGIN { match("", pattern) } + $0 ~ pattern { found = 1 } + END { print found ? 1 : 0 } + ' "${file}"); then + printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2 + return 1 + fi + if [ "${found}" != 0 ]; then + printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2 + return 1 + fi + return 0 +} diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 3fd620787..76c780676 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -44,6 +44,8 @@ const ( cnpgClusterLabel = "cnpg.io/cluster" cnpgBackupPhaseComplete = "completed" cnpgBackupPhaseFailed = "failed" + cnpgBackupPhaseStarted = "started" + cnpgBackupPhaseRunning = "running" // barmanObjectNameParam / barmanServerNameParam are the barman-cloud plugin // parameter keys on a Cluster's spec.plugins entry. barmanObjectName points @@ -388,9 +390,10 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp // The ObjectStore is the Cluster's WAL archive destination, shared by // every backup of the app, so it is written only before this job's - // Backup exists, and only while no other job's Backup is in flight and - // no other running job without a Backup yet holds the destination it - // last wrote. + // Backup exists, and only while no other job's Backup is in flight, no + // other running job without a Backup yet holds the destination it last + // wrote, and CloudNativePG runs no Backup of the Cluster that no running + // job tracks. writeDestination := existingBackup == nil if writeDestination { // The gate in Reconcile sees only a job that has not started, and @@ -421,6 +424,17 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp return r.waitCNPGBackupJob(ctx, j, "WaitingForOtherBackup", fmt.Sprintf( "waiting for BackupJob %s of the same application to finish its backup, which writes to the shared WAL archive destination", holder)) } + untracked, err := r.untrackedCNPGBackupExecuting(ctx, j, clusterName) + if err != nil { + return ctrl.Result{}, err + } + if untracked != "" { + apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied) + apimeta.SetStatusCondition(&j.Status.Conditions, cnpgDestinationWriteHeld("WaitingForUntrackedBackup")) + return r.waitCNPGBackupJob(ctx, j, "WaitingForUntrackedBackup", fmt.Sprintf( + "waiting for cnpg.io/Backup %s/%s of cnpg.io/Cluster %s/%s, which no running BackupJob tracks, to finish: CloudNativePG is running it against the shared WAL archive destination", + j.Namespace, untracked, j.Namespace, clusterName)) + } if apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld) != nil { apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ Type: backupCondDestinationWriteHeld, @@ -1162,8 +1176,9 @@ func (r *BackupJobReconciler) findCNPGBackupForJob(ctx context.Context, j *backu // otherCNPGBackupInFlight returns the name of another BackupJob that is still // running and whose cnpg.io/Backup of clusterName has not finished, or "" when -// there is none. A Backup left behind by a finished or deleted job does not -// hold the queue. +// there is none. A Backup left behind by a finished or deleted job is not +// counted here; untrackedCNPGBackupExecuting holds for it while +// CloudNativePG executes it. // // The Backups are read from the apiserver, not the cache, which can still // miss a Backup another job's earlier pass created. Only the ones this @@ -1206,6 +1221,51 @@ func (r *BackupJobReconciler) otherCNPGBackupInFlight(ctx context.Context, j *ba return holders[0], nil } +// untrackedCNPGBackupExecuting returns the name of a cnpg.io/Backup of +// clusterName that CloudNativePG is running, whoever created it. It is +// asked after otherCNPGBackupInFlight and destinationClaimHolder found no +// running BackupJob to wait for, so what it finds is a Backup no running +// job tracks: one created by hand or by a ScheduledBackup, or one whose +// BackupJob was deleted or finished first. Its base backup goes to the +// destination it started with, so a rewrite now would send the WAL it +// needs elsewhere. +// +// Only the phases BackupStatus.IsExecuting reports hold, the ones in which +// CloudNativePG's BackupList.CanExecuteBackup lets no other Backup of the +// Cluster start, so the wait never outlasts CloudNativePG's own queue. A pending +// Backup, even one that ran before and was sent back to pending, starts +// its base backup over against the destination current then. +// +// A Backup created by hand need carry no label at all, and CloudNativePG +// labels only the ones a ScheduledBackup creates, so they are matched by +// spec.cluster.name in a list of the whole namespace. That list comes from +// the cache rather than the apiserver, which every waiting job would +// otherwise ask for the namespace's whole Backup history on each poll. The +// cache can miss a Backup that started moments ago, the same race that +// already lies between this check and the write that follows it, which a +// read from the apiserver would not close either. +func (r *BackupJobReconciler) untrackedCNPGBackupExecuting(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string) (string, error) { + list := &cnpgtypes.BackupList{} + if err := r.List(ctx, list, client.InNamespace(j.Namespace)); err != nil { + return "", fmt.Errorf("list cnpg.io/Backups in %s: %w", j.Namespace, err) + } + var executing []string + for i := range list.Items { + b := &list.Items[i] + if b.Spec.Cluster.Name != clusterName { + continue + } + if b.Status.Phase == cnpgBackupPhaseStarted || b.Status.Phase == cnpgBackupPhaseRunning { + executing = append(executing, b.Name) + } + } + if len(executing) == 0 { + return "", nil + } + sort.Strings(executing) + return executing[0], nil +} + func cnpgDestinationWriteHeld(reason string) metav1.Condition { return metav1.Condition{ Type: backupCondDestinationWriteHeld, diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 780f7fb60..9ca6ff987 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -2646,6 +2646,9 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) { owner string cluster string wantWait bool + // wantUntracked expects the hold to name the cnpg.io/Backup rather + // than a running BackupJob: nothing the controller tracks owns it. + wantUntracked bool }{ {name: "another running job with a running Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", wantWait: true}, {name: "another running job with a pending Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "pending", wantWait: true}, @@ -2655,7 +2658,14 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) { {name: "a Backup of a succeeded job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseSucceeded, otherBackup: "pending"}, {name: "a Backup whose job is gone does not", otherBackup: "pending"}, {name: "a Backup of another Cluster does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", cluster: "postgres-other"}, - {name: "a Backup created by hand does not", otherBackup: "running", owner: "-"}, + {name: "a Backup created by hand that has not started does not", otherBackup: "pending", owner: "-"}, + {name: "a Backup created by hand that is running holds it", otherBackup: "running", owner: "-", wantWait: true, wantUntracked: true}, + {name: "a Backup created by hand that has started holds it", otherBackup: "started", owner: "-", wantWait: true, wantUntracked: true}, + {name: "a Backup created by hand of another Cluster does not", otherBackup: "started", owner: "-", cluster: "postgres-other"}, + {name: "a running Backup whose job is gone holds it", otherBackup: "started", wantWait: true, wantUntracked: true}, + {name: "a running Backup of a failed job holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseFailed, otherBackup: "started", wantWait: true, wantUntracked: true}, + {name: "a running Backup of a succeeded job holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseSucceeded, otherBackup: "running", wantWait: true, wantUntracked: true}, + {name: "a Backup created by hand that is finalizing does not", otherBackup: "finalizing", owner: "-"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -2705,8 +2715,15 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) { if ours != 0 { t.Errorf("the waiting job created %d cnpg.io/Backup(s)", ours) } - if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "bj-other") { - t.Errorf("Ready = %+v, want False/WaitingForOtherBackup naming bj-other", ready) + wantReason, wantNamed := "WaitingForOtherBackup", "BackupJob bj-other " + if tc.wantUntracked { + wantReason, wantNamed = "WaitingForUntrackedBackup", "cnpg.io/Backup "+rolloutNS+"/bj-other-cnpg " + } + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != wantReason || !strings.Contains(ready.Message, wantNamed) { + t.Errorf("Ready = %+v, want False/%s naming %q", ready, wantReason, wantNamed) + } + if held := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); held == nil || held.Status != metav1.ConditionTrue || held.Reason != wantReason { + t.Errorf("%s = %+v, want True/%s", backupCondDestinationWriteHeld, held, wantReason) } if res.RequeueAfter == 0 { t.Errorf("a queued job is not requeued, and nothing else wakes it when the holder finishes: %+v", res) @@ -2731,7 +2748,7 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) { if got := f.liveDestination(); got != "s3://bucket/" { t.Errorf("once the holder finished, destination = %q, want the job's own s3://bucket/", got) } - if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" { + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && (ready.Reason == "WaitingForOtherBackup" || ready.Reason == "WaitingForUntrackedBackup") { t.Errorf("Ready still names the finished holder: %+v", ready) } return @@ -2739,7 +2756,7 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) { if got := f.liveDestination(); got != "s3://bucket/" { t.Errorf("destination = %q, want the job's own s3://bucket/", got) } - if ready != nil && ready.Reason == "WaitingForOtherBackup" { + if ready != nil && (ready.Reason == "WaitingForOtherBackup" || ready.Reason == "WaitingForUntrackedBackup") { t.Errorf("Ready = %+v, the job was held", ready) } f.settleDestination() @@ -3881,16 +3898,29 @@ func TestReconcileCNPG_PluginAttachedByAnotherWriterKeepsTheLatch(t *testing.T) // Waiting for another backup resets the retry budget of the destination // write, so the queue does not spend it. func TestReconcileCNPG_QueueResetsDestinationRetryBudget(t *testing.T) { + for name, owner := range map[string]string{"a running BackupJob's Backup": "bj-other", "an untracked Backup": ""} { + t.Run(name, func(t *testing.T) { + testQueueResetsDestinationRetryBudget(t, owner) + }) + } +} + +func testQueueResetsDestinationRetryBudget(t *testing.T, owner string) { expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) f := newAttachedObjectStoreFixture(t, expired, "s3://other/") f.setJobCondition(metav1.Condition{ Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired), }) - f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) - f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + if owner != "" { + f.seedOtherBackupJob(owner, backupsv1alpha1.BackupJobPhaseRunning) + } + f.seedCNPGBackup("bj-other-cnpg", owner, rolloutCluster, "running") j, _ := f.pass() + if held := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); held == nil || held.Status != metav1.ConditionTrue { + t.Fatalf("%s = %+v, want the job held", backupCondDestinationWriteHeld, held) + } if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil { t.Fatalf("%s = %+v kept while waiting for another backup", backupCondDestinationApplied, cond) } @@ -6777,3 +6807,91 @@ func TestReconcileCNPG_LooksUpItsOwnBackupOncePerPass(t *testing.T) { t.Fatalf("got %d cnpg.io/Backups, want 1", len(backups)) } } + +// With two untracked Backups running the wait names the same one on every +// pass, whatever order the cache lists them in, so a waiting job does not +// rewrite its status each poll. +func TestReconcileCNPG_UntrackedHoldNamesOneBackupAcrossPasses(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedCNPGBackup("a-cnpg", "", rolloutCluster, "running") + f.seedCNPGBackup("z-cnpg", "", rolloutCluster, "started") + statusWrites := 0 + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := cl.List(ctx, list, opts...); err != nil { + return err + } + if backups, ok := list.(*cnpgtypes.BackupList); ok { + slices.Reverse(backups.Items) + } + return nil + }, + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + statusWrites++ + return cl.SubResource(sub).Update(ctx, obj, opts...) + }, + }) + + j, _ := f.pass() + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForUntrackedBackup" || !strings.Contains(ready.Message, "/a-cnpg ") { + t.Fatalf("Ready = %+v, want WaitingForUntrackedBackup naming a-cnpg", ready) + } + statusWrites = 0 + f.pass() + if statusWrites != 0 { + t.Errorf("a second pass with the same two untracked Backups wrote the BackupJob status %d time(s)", statusWrites) + } +} + +// An error listing the namespace's Backups is not taken as "nothing runs". +func TestReconcileCNPG_UntrackedBackupListErrorIsReturned(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + lo := &client.ListOptions{} + lo.ApplyOptions(opts) + if _, ok := list.(*cnpgtypes.BackupList); ok && (lo.LabelSelector == nil || lo.LabelSelector.Empty()) { + return apierrors.NewServiceUnavailable("cache not synced") + } + return cl.List(ctx, list, opts...) + }, + }) + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("reconcileCNPG swallowed the error listing untracked Backups") + } + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination written although the running Backups could not be listed: %q", got) + } +} + +// A ScheduledBackup labels its Backups with the Cluster but with no +// BackupJob; such a Backup holds the destination while it runs. +func TestReconcileCNPG_ScheduledBackupHoldsTheDestination(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + b := &cnpgtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "nightly-20261112", Labels: map[string]string{ + cnpgClusterLabel: rolloutCluster, + "cnpg.io/scheduled-backup": "nightly", + "cnpg.io/immediateBackup": "false", + }}, + Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: rolloutCluster}}, + Status: cnpgtypes.BackupStatus{Phase: cnpgBackupPhaseStarted}, + } + if err := f.c.Create(context.Background(), b); err != nil { + t.Fatalf("seed scheduled cnpg.io/Backup: %v", err) + } + j, backups := f.pass() + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination rewritten under a running scheduled backup: %q", got) + } + if len(backups) != 1 { + t.Errorf("got %d cnpg.io/Backups, want only the scheduled one", len(backups)) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForUntrackedBackup" || !strings.Contains(ready.Message, "/nightly-20261112 ") { + t.Errorf("Ready = %+v, want WaitingForUntrackedBackup naming nightly-20261112", ready) + } +} diff --git a/internal/controller/tenantquota/reconciler.go b/internal/controller/tenantquota/reconciler.go index dddd0fbac..ed18499ce 100644 --- a/internal/controller/tenantquota/reconciler.go +++ b/internal/controller/tenantquota/reconciler.go @@ -52,6 +52,17 @@ const ( // whenever it is below the chart quota, without the controller fighting Flux // over the chart-owned object. allocatedQuotaName = "tenant-quota-allocated" + // chartLimitRangeName is the LimitRange the tenant chart renders next to + // chartQuotaName, giving containers default requests and limits. The chart + // renders both only for a tenant that declares resourceQuotas. + chartLimitRangeName = "tenant-range-limits" + // allocatedLimitRangeName is the controller-owned copy of the pool root's + // chart LimitRange, written into every unbounded pool member. Those members + // render no chart LimitRange, yet carry an allocated quota; without + // container defaults, any pod omitting a limit the quota bounds is rejected + // at admission. A distinct name keeps the copy from colliding with the + // chart-owned object if the member later declares its own quota. + allocatedLimitRangeName = "tenant-range-limits-allocated" // managedByLabel marks the controller-owned ResourceQuotas so they can be // listed and garbage-collected. managedByLabel = "quota.cozystack.io/managed-by" @@ -102,6 +113,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result pools := ComputePools(tenants) desired := map[string]corev1.ResourceList{} + // limitRangeSource maps each unbounded pool member whose quota bounds a + // compute limit to the pool root whose chart LimitRange it must inherit. + limitRangeSource := map[string]string{} for _, p := range pools { if over := p.Overcommitted(); len(over) > 0 { logger.Info("tenant quota pool is overcommitted by sub-tenant carve-outs", "pool", p.Root, "overcommit", over) @@ -120,6 +134,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result for _, ns := range p.Members { if existing[ns] { desired[ns] = buffered.EnforcedHard(ns, usedByNS) + if ns != p.Root && boundsComputeLimits(desired[ns]) { + limitRangeSource[ns] = p.Root + } } } } @@ -134,6 +151,22 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result logger.Error(err, "failed to garbage-collect stale allocated quotas") } + written := map[string]bool{} + for ns, root := range limitRangeSource { + ok, err := r.upsertAllocatedLimitRange(ctx, ns, root) + if err != nil { + logger.Error(err, "failed to apply allocated limit range", "namespace", ns, "poolRoot", root) + } + // Keep the existing copy on a transient error rather than dropping the + // defaults the namespace's pods rely on. + if ok || err != nil { + written[ns] = true + } + } + if err := r.gcAllocatedLimitRanges(ctx, written); err != nil { + logger.Error(err, "failed to garbage-collect stale allocated limit ranges") + } + return ctrl.Result{RequeueAfter: resyncInterval}, nil } @@ -224,6 +257,50 @@ func (r *Reconciler) gcAllocatedQuotas(ctx context.Context, desired map[string]c return nil } +// upsertAllocatedLimitRange copies the pool root's chart LimitRange into an +// unbounded member namespace. It reports false, without error, when the root +// has no chart LimitRange to copy. +func (r *Reconciler) upsertAllocatedLimitRange(ctx context.Context, namespace, poolRoot string) (bool, error) { + source := &corev1.LimitRange{} + if err := r.Get(ctx, types.NamespacedName{Namespace: poolRoot, Name: chartLimitRangeName}, source); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + lr := &corev1.LimitRange{ObjectMeta: metav1.ObjectMeta{Name: allocatedLimitRangeName, Namespace: namespace}} + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, lr, func() error { + if lr.Labels == nil { + lr.Labels = map[string]string{} + } + lr.Labels[managedByLabel] = managedByValue + lr.Labels["internal.cozystack.io/managed-by-cozystack"] = "" + lr.Spec = *source.Spec.DeepCopy() + return nil + }) + return err == nil, err +} + +// gcAllocatedLimitRanges deletes controller-owned LimitRanges in namespaces +// that are no longer unbounded pool members (e.g. the tenant gained its own +// quota and now renders the chart LimitRange, or was deleted). +func (r *Reconciler) gcAllocatedLimitRanges(ctx context.Context, keep map[string]bool) error { + managed := &corev1.LimitRangeList{} + if err := r.List(ctx, managed, client.MatchingLabels{managedByLabel: managedByValue}); err != nil { + return err + } + for i := range managed.Items { + m := &managed.Items[i] + if keep[m.Namespace] { + continue + } + if err := r.Delete(ctx, m); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + return nil +} + func (r *Reconciler) recordOvercommit(ctx context.Context, rootNamespace string, over corev1.ResourceList) { if r.Recorder == nil { return @@ -237,8 +314,8 @@ func (r *Reconciler) recordOvercommit(ctx context.Context, rootNamespace string, } // SetupWithManager wires the controller. Every relevant change (a tenant -// HelmRelease, a namespace, or a tenant ResourceQuota whose usage moved) -// coalesces into one full-tree sweep. +// HelmRelease, a namespace, a tenant ResourceQuota whose usage moved, or a +// tenant LimitRange) coalesces into one full-tree sweep. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { toSweep := handler.EnqueueRequestsFromMapFunc(func(context.Context, client.Object) []reconcile.Request { return []reconcile.Request{sweepKey} @@ -249,11 +326,15 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { tenantQuotas := predicate.NewPredicateFuncs(func(o client.Object) bool { return o.GetName() == chartQuotaName || o.GetName() == allocatedQuotaName }) + tenantLimitRanges := predicate.NewPredicateFuncs(func(o client.Object) bool { + return o.GetName() == chartLimitRangeName || o.GetName() == allocatedLimitRangeName + }) return ctrl.NewControllerManagedBy(mgr). Named("tenantquota-controller"). Watches(&helmv2.HelmRelease{}, toSweep, builder.WithPredicates(tenantReleases)). Watches(&corev1.Namespace{}, toSweep). Watches(&corev1.ResourceQuota{}, toSweep, builder.WithPredicates(tenantQuotas)). + Watches(&corev1.LimitRange{}, toSweep, builder.WithPredicates(tenantLimitRanges)). Complete(r) } @@ -271,6 +352,18 @@ func ownedNamespace(hrNamespace, appName string) string { } } +// boundsComputeLimits reports whether a quota bounds limits.cpu or +// limits.memory, the only case in which admission rejects a container that +// omits a limit. The chart renders its LimitRange under any non-empty +// resourceQuotas, so the root may carry one under a quota that bounds no +// compute at all; copying it then would impose default limits the quota +// never asked for. +func boundsComputeLimits(hard corev1.ResourceList) bool { + _, cpu := hard[corev1.ResourceLimitsCPU] + _, memory := hard[corev1.ResourceLimitsMemory] + return cpu || memory +} + // maxResourceList returns the per-resource maximum of a and b. func maxResourceList(a, b corev1.ResourceList) corev1.ResourceList { out := corev1.ResourceList{} diff --git a/internal/controller/tenantquota/reconciler_test.go b/internal/controller/tenantquota/reconciler_test.go index 5f19ed1b1..b869825dc 100644 --- a/internal/controller/tenantquota/reconciler_test.go +++ b/internal/controller/tenantquota/reconciler_test.go @@ -227,3 +227,172 @@ func TestReconcile_GarbageCollect(t *testing.T) { t.Fatalf("stale allocated quota in tenant-gone should have been garbage-collected") } } + +// chartLimitRange builds the chart-rendered "tenant-range-limits" LimitRange, +// which the tenant chart emits alongside its "tenant-quota". +func chartLimitRange(namespace, defaultCPU string) *corev1.LimitRange { + return &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{Name: chartLimitRangeName, Namespace: namespace}, + Spec: corev1.LimitRangeSpec{Limits: []corev1.LimitRangeItem{{ + Type: corev1.LimitTypeContainer, + Default: resourceList(map[string]string{"cpu": defaultCPU, "memory": "128Mi"}), + DefaultRequest: resourceList(map[string]string{"cpu": "25m", "memory": "128Mi"}), + }}}, + } +} + +func getLimitRange(c client.Client, namespace, name string) (*corev1.LimitRange, bool) { + lr := &corev1.LimitRange{} + if err := c.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: name}, lr); err != nil { + return nil, false + } + return lr, true +} + +// TestReconcile_UnboundedMemberGetsPoolLimitRange: an unbounded child renders +// no chart quota and therefore no chart LimitRange, yet the controller clamps +// it with an allocated quota. Without container defaults, every pod that omits +// a limit the quota bounds (e.g. a sidecar without a CPU limit) is rejected at +// admission, so the controller must copy the pool root's LimitRange into it. +func TestReconcile_UnboundedMemberGetsPoolLimitRange(t *testing.T) { + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + chartLimitRange("tenant-foo", "250m"), + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + lr, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName) + if !ok { + t.Fatalf("unbounded member tenant-foo-bar has an allocated quota but no LimitRange") + } + if len(lr.Spec.Limits) != 1 { + t.Fatalf("tenant-foo-bar LimitRange limits = %d, want 1", len(lr.Spec.Limits)) + } + if got := lr.Spec.Limits[0].Default[corev1.ResourceCPU]; got.String() != "250m" { + t.Fatalf("tenant-foo-bar default cpu = %q, want 250m copied from the pool root", got.String()) + } + if lr.Labels[managedByLabel] != managedByValue { + t.Fatalf("tenant-foo-bar LimitRange is not marked as controller-owned") + } + if _, ok := getLimitRange(c, "tenant-foo", allocatedLimitRangeName); ok { + t.Fatalf("pool root tenant-foo already has the chart LimitRange and must not get a second one") + } +} + +// TestReconcile_PoolLimitRangeFollowsRoot: a change to the root's chart +// LimitRange propagates to the members on the next sweep. +func TestReconcile_PoolLimitRangeFollowsRoot(t *testing.T) { + root := chartLimitRange("tenant-foo", "250m") + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + root, + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + root.Spec.Limits[0].Default[corev1.ResourceCPU] = resource.MustParse("500m") + if err := c.Update(context.Background(), root); err != nil { + t.Fatalf("update root LimitRange: %v", err) + } + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + lr, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName) + if !ok { + t.Fatalf("tenant-foo-bar lost its LimitRange") + } + if got := lr.Spec.Limits[0].Default[corev1.ResourceCPU]; got.String() != "500m" { + t.Fatalf("tenant-foo-bar default cpu = %q, want 500m after the root changed", got.String()) + } +} + +// TestReconcile_PoolLimitRangeGarbageCollected: once the child declares its own +// quota, the chart renders its LimitRange and the controller-owned copy goes. +func TestReconcile_PoolLimitRangeGarbageCollected(t *testing.T) { + stale := &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: allocatedLimitRangeName, + Namespace: "tenant-foo-bar", + Labels: map[string]string{managedByLabel: managedByValue}, + }, + } + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + chartLimitRange("tenant-foo", "250m"), + chartQuota("tenant-foo-bar", map[string]string{"limits.cpu": "4"}, nil), + chartLimitRange("tenant-foo-bar", "250m"), + stale, + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { + t.Fatalf("bounded child tenant-foo-bar renders its own LimitRange; the controller copy must be removed") + } + if _, ok := getLimitRange(c, "tenant-foo-bar", chartLimitRangeName); !ok { + t.Fatalf("the chart-owned LimitRange must never be touched") + } +} + +// TestReconcile_PoolRootWithoutLimitRange: if the root has no chart LimitRange +// there is nothing to copy; the quota is still enforced and the sweep succeeds. +func TestReconcile_PoolRootWithoutLimitRange(t *testing.T) { + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil), + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, ok := allocatedHard(t, c, "tenant-foo-bar", "limits.cpu"); !ok { + t.Fatalf("tenant-foo-bar must still get its allocated quota") + } + if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { + t.Fatalf("no LimitRange can be copied from a root that has none") + } +} + +// TestReconcile_NoLimitRangeWithoutComputeLimitQuota: the chart renders its +// LimitRange for any non-empty resourceQuotas, compute or not. A member whose +// allocated quota bounds neither limits.cpu nor limits.memory is admitted +// without limits today; copying the root's defaults there would cap or refuse +// pods (one requesting more than the default memory limit, for instance) +// that the quota itself never constrains. +func TestReconcile_NoLimitRangeWithoutComputeLimitQuota(t *testing.T) { + stale := &corev1.LimitRange{ + ObjectMeta: metav1.ObjectMeta{ + Name: allocatedLimitRangeName, + Namespace: "tenant-foo-bar", + Labels: map[string]string{managedByLabel: managedByValue}, + }, + } + r, c := newReconciler(t, + tenantHR("foo", "tenant-root"), + tenantHR("bar", "tenant-foo"), + ns("tenant-foo"), ns("tenant-foo-bar"), + chartQuota("tenant-foo", map[string]string{"services.loadbalancers": "1"}, nil), + chartLimitRange("tenant-foo", "250m"), + stale, + ) + if _, err := r.Reconcile(context.Background(), sweepKey); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, ok := allocatedHard(t, c, "tenant-foo-bar", "services.loadbalancers"); !ok { + t.Fatalf("tenant-foo-bar must still get its allocated quota") + } + if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok { + t.Fatalf("tenant-foo-bar's quota bounds no compute limit; it must carry no LimitRange copy") + } +} diff --git a/packages/apps/kafka/tests/dashboard-resourcemap_test.yaml b/packages/apps/kafka/tests/dashboard-resourcemap_test.yaml index ace053c58..88da0d69a 100644 --- a/packages/apps/kafka/tests/dashboard-resourcemap_test.yaml +++ b/packages/apps/kafka/tests/dashboard-resourcemap_test.yaml @@ -137,11 +137,13 @@ tests: release: name: test-kafka namespace: tenant-test + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[2].resourceNames content: test-kafka-zookeeper - documentIndex: 0 # External bootstrap service absent when external=false, regardless of tls.enabled - it: does not grant access to external bootstrap service when tls.enabled=true and external=false diff --git a/packages/apps/kafka/tests/jvm_sizing_test.yaml b/packages/apps/kafka/tests/jvm_sizing_test.yaml index 26a065d17..f4569a1e2 100644 --- a/packages/apps/kafka/tests/jvm_sizing_test.yaml +++ b/packages/apps/kafka/tests/jvm_sizing_test.yaml @@ -27,7 +27,9 @@ tests: # kafka-run-class.sh then drops its G1 settings. - it: leaves the node pool JVM options to Strimzi and Kafka template: templates/migration-hook.yaml - documentIndex: 3 + documentSelector: + path: kind + value: Job asserts: - notMatchRegex: path: spec.template.spec.containers[0].args[0] diff --git a/packages/apps/kubernetes-nodes/README.md b/packages/apps/kubernetes-nodes/README.md index 3b8bdd36e..ff1dd2b40 100644 --- a/packages/apps/kubernetes-nodes/README.md +++ b/packages/apps/kubernetes-nodes/README.md @@ -44,6 +44,7 @@ The pool's Kubernetes minor (`version`) may lag the cluster's during an upgrade | `podCpuRequest` | CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. | `quantity` | `""` | | `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | | `gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` | +| `maxUnavailable` | Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. | `*int` | `null` | | `kernelModules` | Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. | `[]object` | `[]` | | `kernelModules[i].name` | Module name as `modprobe` takes it, e.g. `nvidia_uvm`. | `string` | `""` | | `kernelModules[i].parameters` | Module parameters, each as a bare `key=value` string. | `[]string` | `[]` | diff --git a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml index 7ebf42033..440bbfa05 100644 --- a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml +++ b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml @@ -1021,10 +1021,22 @@ spec: # action until the autoscaler notices unschedulable Pods and # scales back up — a real disruption window on every routine # platform bump. + {{- /* maxUnavailable 0 makes a rollout (instance type, image, version) + wait for each replacement Machine to be up before draining an old + one; with 1, a one-node pool lost its only node until the new one + joined. A pool with passed-through GPUs keeps 1: its replacement VM + may need the very GPU the old node holds, and would wait for it + forever. A Proxmox template's GPU is invisible here, hence the + maxUnavailable override. CAPI refuses maxSurge and maxUnavailable both + at 0, and maxReplicas may be 0, hence the floor of 1. */}} + {{- $maxUnavailable := ternary 1 0 (not (empty $group.gpus)) }} + {{- if not (kindIs "invalid" $.Values.maxUnavailable) }} + {{- $maxUnavailable = $.Values.maxUnavailable | int }} + {{- end }} strategy: rollingUpdate: - maxSurge: {{ $group.maxReplicas }} - maxUnavailable: 1 + maxSurge: {{ max 1 ($group.maxReplicas | int) }} + maxUnavailable: {{ $maxUnavailable }} type: RollingUpdate selector: matchLabels: diff --git a/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap b/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap index 281fc1238..c8e990a9c 100644 --- a/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap +++ b/packages/apps/kubernetes-nodes/tests/__snapshot__/render_snapshot_test.yaml.snap @@ -250,7 +250,7 @@ matches the golden render of a kubelet-override pool: strategy: rollingUpdate: maxSurge: 3 - maxUnavailable: 1 + maxUnavailable: 0 type: RollingUpdate template: metadata: @@ -405,7 +405,7 @@ matches the golden render of the pool objects: strategy: rollingUpdate: maxSurge: 3 - maxUnavailable: 1 + maxUnavailable: 0 type: RollingUpdate template: metadata: diff --git a/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml new file mode 100644 index 000000000..17e732d33 --- /dev/null +++ b/packages/apps/kubernetes-nodes/tests/rollout_strategy_test.yaml @@ -0,0 +1,136 @@ +suite: worker pool rollout keeps capacity +templates: + - templates/nodegroup.yaml +release: + name: kubernetes-nodes-myk8s-md0 + namespace: tenant-test +set: + cluster: myk8s + _cluster: + cluster-domain: cozy.local + version: "v1.35" + minReplicas: 1 + maxReplicas: 1 + instanceType: "" + diskSize: 20Gi + storageClass: replicated + resources: + cpu: "2" + memory: 4Gi +tests: + # A rollout replaces every Machine. With maxUnavailable 1 a one-node pool + # drained and deleted its only node as soon as the replacement Machine was + # created, leaving the pool empty until the new node joined. + - it: never drains a node before its replacement is ready + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.type + value: RollingUpdate + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 1 + + - it: surges up to the pool ceiling + set: + maxReplicas: 5 + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 5 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 + + # CAPI rejects a RollingUpdate with maxSurge and maxUnavailable both 0. + - it: keeps a surge of one on a pool capped at zero + set: + minReplicas: 0 + maxReplicas: 0 + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 1 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 + + # A replacement VM in a pool with passed-through GPUs may need the very GPU + # the old node holds; with maxUnavailable 0 it would wait for it forever. + - it: lets a GPU pool remove a node before its replacement is up + set: + gpus: + - name: nvidia.com/AD102GL_L40S + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 1 + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 1 + + # A Proxmox template that passes a GPU through leaves gpus empty, so the + # chart cannot tell; the pool says so itself. + - it: lets a pool set maxUnavailable explicitly + set: + maxUnavailable: 1 + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 1 + + - it: honours an explicit 0 on a GPU pool + set: + maxUnavailable: 0 + gpus: + - name: nvidia.com/AD102GL_L40S + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 0 + + - it: refuses a negative maxUnavailable (schema minimum 0) + set: + maxUnavailable: -1 + asserts: + - failedTemplate: {} + + # CAPI decodes the field as an int32, so a larger value would render and + # then be refused by the MachineDeployment decoder. + - it: refuses a maxUnavailable above the int32 range (schema maximum) + set: + maxUnavailable: 2147483648 + asserts: + - failedTemplate: + errorPattern: "maxUnavailable" + + - it: accepts the int32 maximum + set: + maxUnavailable: 2147483647 + documentSelector: + path: kind + value: MachineDeployment + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 2147483647 diff --git a/packages/apps/kubernetes-nodes/values.schema.json b/packages/apps/kubernetes-nodes/values.schema.json index bbc1ddb8d..0c24c9eb9 100644 --- a/packages/apps/kubernetes-nodes/values.schema.json +++ b/packages/apps/kubernetes-nodes/values.schema.json @@ -146,6 +146,12 @@ } } }, + "maxUnavailable": { + "description": "Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.", + "type": "integer", + "maximum": 2147483647, + "minimum": 0 + }, "kernelModules": { "description": "Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.", "type": "array", diff --git a/packages/apps/kubernetes-nodes/values.yaml b/packages/apps/kubernetes-nodes/values.yaml index 5ca1d8c50..94e1a50f9 100644 --- a/packages/apps/kubernetes-nodes/values.yaml +++ b/packages/apps/kubernetes-nodes/values.yaml @@ -77,6 +77,10 @@ resources: {} ## @param {[]GPU} gpus - List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). gpus: [] +## @param {*int} [maxUnavailable] - Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. +## @minimum 0 +## @maximum 2147483647 + ## @param {[]KernelModule} [kernelModules] - Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. ## @param {Kubelet} [kubelet] - Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`. diff --git a/packages/apps/kubernetes/tests/kamaji_managed_args_test.yaml b/packages/apps/kubernetes/tests/kamaji_managed_args_test.yaml index b09343cf5..85045c9a1 100644 --- a/packages/apps/kubernetes/tests/kamaji_managed_args_test.yaml +++ b/packages/apps/kubernetes/tests/kamaji_managed_args_test.yaml @@ -241,18 +241,22 @@ tests: errorMessage: 'controlPlane.apiServer.extraArgs contains "--secure-port=6444" — the tenant control plane sets --secure-port itself and drops every entry naming it before the kube-apiserver sees the list, so this entry has no effect, and written without =value it leaves its value behind as an argument the apiserver refuses. Remove the entry; the description of controlPlane.apiServer.extraArgs lists every flag the control plane owns.' - it: leaves spec.admissionControllers out by default, so the control plane keeps its own default set + documentSelector: + path: kind + value: KamajiControlPlane asserts: - notExists: path: spec.admissionControllers - documentIndex: 1 - it: leaves spec.admissionControllers out for an explicitly empty list set: controlPlane.apiServer.admissionControllers: [] + documentSelector: + path: kind + value: KamajiControlPlane asserts: - notExists: path: spec.admissionControllers - documentIndex: 1 - it: renders controlPlane.apiServer.admissionControllers into spec.admissionControllers verbatim set: diff --git a/packages/apps/mariadb/tests/tls_test.yaml b/packages/apps/mariadb/tests/tls_test.yaml index 2f5f4548a..ac7b8ae79 100644 --- a/packages/apps/mariadb/tests/tls_test.yaml +++ b/packages/apps/mariadb/tests/tls_test.yaml @@ -530,7 +530,9 @@ tests: - templates/dashboard-resourcemap.yaml set: external: false - documentIndex: 0 + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[1].resourceNames @@ -544,7 +546,9 @@ tests: - templates/dashboard-resourcemap.yaml set: external: true - documentIndex: 0 + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[1].resourceNames @@ -560,7 +564,9 @@ tests: external: true tls: issuer: operator - documentIndex: 0 + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[1].resourceNames @@ -576,7 +582,9 @@ tests: external: false tls: issuer: cert-manager - documentIndex: 0 + documentSelector: + path: kind + value: Role asserts: - notContains: path: rules[1].resourceNames @@ -695,7 +703,9 @@ tests: set: tls: issuer: cert-manager - documentIndex: 3 + documentSelector: + path: metadata.name + value: my-mariadb-tls asserts: - notContains: path: spec.dnsNames @@ -1187,7 +1197,9 @@ tests: - it: "dashboard Role never grants the CA bundle even when TLS is unmanaged" templates: - templates/dashboard-resourcemap.yaml - documentIndex: 0 + documentSelector: + path: kind + value: Role set: tls: issuer: operator @@ -1436,6 +1448,9 @@ tests: enabled: true s3Bucket: s3.example.org/test cleanupStrategy: "--keep-last=3" + documentSelector: + path: kind + value: Secret asserts: - notMatchRegex: path: stringData["backup.sh"] diff --git a/packages/apps/mongodb/tests/tls_test.yaml b/packages/apps/mongodb/tests/tls_test.yaml index 0ddb38e46..8cf64b8cc 100644 --- a/packages/apps/mongodb/tests/tls_test.yaml +++ b/packages/apps/mongodb/tests/tls_test.yaml @@ -53,7 +53,9 @@ tests: value: mg-test-ssl - it: spec.tls has no issuerConf (the operator-managed secret is passed directly) - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: - notExists: path: spec.tls.issuerConf @@ -127,13 +129,17 @@ tests: # test would pass even with the external check removed. _namespace: host: example.com - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: - notExists: path: spec.replsets[0].splitHorizons - it: chart does not set allowInvalidCertificates, leaving the operator default - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # The README states the permissive default is inherited rather than # chosen here. Pin that, so the claim stays falsifiable. @@ -141,7 +147,9 @@ tests: path: spec.tls.allowInvalidCertificates - it: chart does not set certValidityDuration, leaving the operator default - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # Same class as allowInvalidCertificates above: the leaf lifetime is the # operator's to choose, not this chart's. Deliberately left unpinned. @@ -177,7 +185,9 @@ tests: # the CRD and notExists on it can never fail. - it: no TLS-disabling escape hatches with a default replica count - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: - notExists: path: spec.allowUnsafeConfigurations @@ -235,7 +245,9 @@ tests: - name: rs0 replicas: 3 size: 10Gi - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # splitHorizons is replica-set scoped and rendered only for the # non-sharded topology. In sharded mode external access goes through the @@ -255,7 +267,9 @@ tests: replicas: 0 _namespace: host: example.com - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # With replicas: 0 the range body renders nothing. Emitting the key with # a null value would be rejected by the API server: the CRD types @@ -269,7 +283,9 @@ tests: external: true _namespace: host: "" - documentIndex: 0 + documentSelector: + path: kind + value: PerconaServerMongoDB asserts: # The controller may create _namespace before populating host, so the # bootstrap window presents host as an empty string rather than as an diff --git a/packages/apps/redis/tests/tls_test.yaml b/packages/apps/redis/tests/tls_test.yaml index 9b1b3ac0b..c8f55ee91 100644 --- a/packages/apps/redis/tests/tls_test.yaml +++ b/packages/apps/redis/tests/tls_test.yaml @@ -557,7 +557,9 @@ tests: external: false tls: enabled: true - documentIndex: 3 + documentSelector: + path: metadata.name + value: my-redis-tls asserts: - notContains: path: spec.dnsNames @@ -840,7 +842,9 @@ tests: external: false tls: enabled: true - documentIndex: 3 + documentSelector: + path: metadata.name + value: my-redis-tls asserts: - notContains: path: spec.dnsNames diff --git a/packages/extra/computeplane/README.md b/packages/extra/computeplane/README.md index fe6c1af67..15779168f 100644 --- a/packages/extra/computeplane/README.md +++ b/packages/extra/computeplane/README.md @@ -59,4 +59,5 @@ See the full design in | `nodeGroups[name].kubelet.kubeReservedCpu` | CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | | `nodeGroups[name].kubelet.evictionHardMemory` | Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%). | `string` | `7%` | | `nodeGroups[name].kubelet.evictionSoftMemory` | Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%). | `string` | `10%` | +| `nodeGroups[name].maxUnavailable` | Workers a rollout may remove before their replacement is up. Unset, the `kubernetes-nodes` pool chart uses 0, so each replacement joins before an old worker is drained (1 on a pool with `gpus`). Set 1 on a pool without room for one extra VM, whose rollout would otherwise stall with the new Machine Pending. | `*int` | `null` | diff --git a/packages/extra/computeplane/templates/cluster.yaml b/packages/extra/computeplane/templates/cluster.yaml index 82ecdc452..9c598d1af 100644 --- a/packages/extra/computeplane/templates/cluster.yaml +++ b/packages/extra/computeplane/templates/cluster.yaml @@ -211,7 +211,7 @@ spec: clone or repoint itself at another Image Factory, both of which are operator decisions about the platform's own images. Keep this list in sync with the kubernetes-nodes values schema's pool-shape params. */}} - {{- with pick $group "minReplicas" "maxReplicas" "instanceType" "diskSize" "storageClass" "roles" "resources" "gpus" "kubelet" "logSerialConsole" "maxUnhealthy" "nodeStartupTimeout" "readyUnknownTimeout" "readyFalseTimeout" "podCpuLimit" "podCpuRequest" "maxNodeProvisionTime" }} + {{- with pick $group "minReplicas" "maxReplicas" "instanceType" "diskSize" "storageClass" "roles" "resources" "gpus" "kubelet" "logSerialConsole" "maxUnhealthy" "nodeStartupTimeout" "readyUnknownTimeout" "readyFalseTimeout" "podCpuLimit" "podCpuRequest" "maxNodeProvisionTime" "maxUnavailable" }} {{- toYaml . | nindent 4 }} {{- end }} valuesFrom: diff --git a/packages/extra/computeplane/tests/cluster_test.yaml b/packages/extra/computeplane/tests/cluster_test.yaml index 9d1a6768f..ad9af03fd 100644 --- a/packages/extra/computeplane/tests/cluster_test.yaml +++ b/packages/extra/computeplane/tests/cluster_test.yaml @@ -280,6 +280,93 @@ tests: - notExists: path: spec.values.nameOverride + # kubernetes-nodes drains a worker only once its replacement is up; a pool + # without room for one extra VM needs maxUnavailable 1 to finish a rollout, + # so the allowlist must forward it. + - it: forwards maxUnavailable to the pool chart + documentSelector: + path: metadata.name + value: kubernetes-nodes-computeplane-cluster-md0 + set: + nodeGroups: + md0: + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.large + diskSize: 32Gi + roles: ["ingress-nginx"] + maxUnavailable: 1 + asserts: + - equal: + path: spec.values.maxUnavailable + value: 1 + + # An explicit 0 must reach the pool chart: there it overrides the 1 a GPU + # pool gets by default, so dropping it as empty would change the strategy. + - it: forwards an explicit maxUnavailable of 0 to the pool chart + documentSelector: + path: metadata.name + value: kubernetes-nodes-computeplane-cluster-md0 + set: + nodeGroups: + md0: + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.large + diskSize: 32Gi + roles: ["ingress-nginx"] + maxUnavailable: 0 + asserts: + - equal: + path: spec.values.maxUnavailable + value: 0 + + - it: forwards the int32 maximum to the pool chart + documentSelector: + path: metadata.name + value: kubernetes-nodes-computeplane-cluster-md0 + set: + nodeGroups: + md0: + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.large + diskSize: 32Gi + roles: ["ingress-nginx"] + maxUnavailable: 2147483647 + asserts: + - equal: + path: spec.values.maxUnavailable + value: 2147483647 + + - it: refuses a maxUnavailable above the int32 range CAPI decodes + set: + nodeGroups: + md0: + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.large + diskSize: 32Gi + roles: ["ingress-nginx"] + maxUnavailable: 2147483648 + asserts: + - failedTemplate: + errorPattern: "maxUnavailable" + + - it: refuses a negative maxUnavailable + set: + nodeGroups: + md0: + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.large + diskSize: 32Gi + roles: ["ingress-nginx"] + maxUnavailable: -1 + asserts: + - failedTemplate: + errorPattern: "maxUnavailable" + # Helm caps a HelmRelease name at 53 chars and each pool renders a 38-char fixed # prefix, so a pool key over 15 chars would create a release the apiserver # accepts but Flux never reconciles. The render must refuse instead (the same diff --git a/packages/extra/computeplane/values.schema.json b/packages/extra/computeplane/values.schema.json index 41db84758..fa640f3d9 100644 --- a/packages/extra/computeplane/values.schema.json +++ b/packages/extra/computeplane/values.schema.json @@ -93,6 +93,12 @@ "type": "integer", "default": 10 }, + "maxUnavailable": { + "description": "Workers a rollout may remove before their replacement is up. Unset, the `kubernetes-nodes` pool chart uses 0, so each replacement joins before an old worker is drained (1 on a pool with `gpus`). Set 1 on a pool without room for one extra VM, whose rollout would otherwise stall with the new Machine Pending.", + "type": "integer", + "maximum": 2147483647, + "minimum": 0 + }, "minReplicas": { "description": "Minimum number of replicas.", "type": "integer", diff --git a/packages/extra/computeplane/values.yaml b/packages/extra/computeplane/values.yaml index 8ab49ea95..2e568ddf3 100644 --- a/packages/extra/computeplane/values.yaml +++ b/packages/extra/computeplane/values.yaml @@ -30,6 +30,9 @@ ## @field {Resources} [resources] - Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Set both `cpu` and `memory` together or neither. ## @field {[]GPU} gpus - List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). ## @field {Kubelet} [kubelet] - Kubelet resource reservations for this node group. +## @field {*int} [maxUnavailable] - Workers a rollout may remove before their replacement is up. Unset, the `kubernetes-nodes` pool chart uses 0, so each replacement joins before an old worker is drained (1 on a pool with `gpus`). Set 1 on a pool without room for one extra VM, whose rollout would otherwise stall with the new Machine Pending. +## @minimum 0 +## @maximum 2147483647 ## @param {map[string]NodeGroup} nodeGroups - Worker node groups of the ComputePlane cluster (cluster shape only — the security posture is fixed by the module and cannot be overridden through this knob: the module forwards only the documented per-pool fields to the `kubernetes-nodes` pool chart, so any other `NodeGroup` field, e.g. `talos`/`version`/`images`, is silently dropped — accepted but inert). Same structure as the `kubernetes` application's `nodeGroups`. When left empty, the module itself materialises a default scale-from-zero `md0` pool with `roles: [ingress-nginx]` (the wrapped `kubernetes` chart no longer injects one after the Phase 2 split), so the cluster sits idle until a pending Pod (e.g. the ingress-nginx controller) triggers the cluster-autoscaler. Supplied groups are authoritative (not merged with the default), so at least one of them must carry `roles: [ingress-nginx]` — enforced at render time. Each supplied pool key must be at most 15 characters so its `kubernetes-nodes-computeplane-cluster-` HelmRelease name fits Helm's 53-character limit — also enforced at render time. nodeGroups: {} diff --git a/packages/extra/etcd/templates/prometheus-rules.yaml b/packages/extra/etcd/templates/prometheus-rules.yaml index a53706541..6f2be8590 100644 --- a/packages/extra/etcd/templates/prometheus-rules.yaml +++ b/packages/extra/etcd/templates/prometheus-rules.yaml @@ -10,7 +10,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': insufficient members '{{`{{ $value }}`}}'." expr: | - sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2) + sum(up{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == bool 1) by (job) < ((count(up{job="{{ .Release.Namespace }}/etcd-pod-scrape"}) by (job) + 1) / 2) for: 3m labels: severity: critical @@ -19,7 +19,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': member '{{`{{ $labels.instance }}`}}' has no leader." expr: | - etcd_server_has_leader{job=~".*etcd.*"} == 0 + etcd_server_has_leader{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == 0 for: 1m labels: severity: critical @@ -28,7 +28,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': instance '{{`{{ $labels.instance }}`}}' has seen '{{`{{ $value }}`}}' leader changes within the last hour." expr: | - rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3 + rate(etcd_server_leader_changes_seen_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[15m]) > 3 for: 15m labels: severity: warning @@ -37,9 +37,9 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method) + 100 * sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method) / - sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method) + sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (job, instance, grpc_service, grpc_method) > 1 for: 10m labels: @@ -49,9 +49,9 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method) + 100 * sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method) / - sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method) + sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (job, instance, grpc_service, grpc_method) > 5 for: 5m labels: @@ -61,7 +61,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': gRPC requests to '{{`{{ $labels.grpc_method }}`}}' are taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le)) + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le)) > 0.15 for: 10m labels: @@ -71,7 +71,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': member communication with '{{`{{ $labels.To }}`}}' is taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) > 0.15 for: 10m labels: @@ -81,7 +81,7 @@ spec: annotations: summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' proposal failures within the last hour on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + rate(etcd_server_proposals_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[15m]) > 5 for: 15m labels: severity: warning @@ -90,7 +90,7 @@ spec: annotations: summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.01 + sum(rate(etcd_http_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (method) > 0.01 for: 10m labels: severity: warning @@ -99,7 +99,7 @@ spec: annotations: summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'." expr: | - sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.05 + sum(rate(etcd_http_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (method) > 0.05 for: 10m labels: severity: critical @@ -108,7 +108,7 @@ spec: annotations: summary: "etcd instance '{{`{{ $labels.instance }}`}}' HTTP requests to '{{`{{ $labels.method }}`}}' are slow." expr: | - histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) + histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) > 0.15 for: 10m labels: @@ -120,10 +120,10 @@ spec: description: 'etcd cluster "{{`{{ $labels.job }}`}}": members are down {{`{{ $value }}`}}.' expr: | max without (endpoint) ( - sum without (instance, pod) (up{job=~".*etcd.*"} == bool 0) + sum without (instance, pod) (up{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == bool 0) or count without (To) ( - sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01 + sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[120s])) > 0.01 ) ) > 0 diff --git a/packages/extra/etcd/tests/namespace_rules_test.yaml b/packages/extra/etcd/tests/namespace_rules_test.yaml new file mode 100644 index 000000000..e972f3cd0 --- /dev/null +++ b/packages/extra/etcd/tests/namespace_rules_test.yaml @@ -0,0 +1,18 @@ +suite: etcd alerts stay inside their release namespace + +release: + name: etcd + namespace: tenant-scope + +templates: + - templates/prometheus-rules.yaml + +tests: + - it: scopes every alert query to this tenant's scrape job + asserts: + - matchRegex: + path: spec.groups[0].rules[*].expr + pattern: 'job="tenant-scope/etcd-pod-scrape"' + - notMatchRegex: + path: spec.groups[0].rules[*].expr + pattern: 'job=~' diff --git a/packages/extra/monitoring/README.md b/packages/extra/monitoring/README.md index 95e3a5db0..01d189ab2 100644 --- a/packages/extra/monitoring/README.md +++ b/packages/extra/monitoring/README.md @@ -115,6 +115,7 @@ | `alerta.alerts.email.repeatInterval` | Minimum time before a still-firing alert is emailed again, as an Alertmanager duration with units `y`, `w`, `d`, `h`, `m`, `s`, `ms` in descending order, each at most once, such as `5m`, `12h` or `1h30m`. Delivery is evaluated every 5m, so shorter intervals take effect at 5m and other intervals round up to a multiple of 5m. Zero, a unit out of order and a total beyond what Alertmanager accepts fail the render. Empty keeps the 12h default. | `string` | `""` | | `alerta.alerts.email.alertnames` | Alert names emailed whether or not the rule carries the notify_email label, with no severity filter. Earlier platform routes still take precedence. Alerts with `event="Heartbeat"`, including `Watchdog`, go only to Alerta, and alerts with severity `none`, `CDIDefaultStorageClassDegraded` and `CDINoDefaultStorageClass` are blackholed and never reach email. This is the way to opt in alerts the platform ships, since their helm-owned rule definitions cannot be labelled per cluster. Entries must be plain Prometheus alert names such as `KubeAPIDown`, and anything else fails the render. Empty adds no extra route. | `[]string` | `[]` | | `alerta.alerts.email.severities` | Severities the notify_email route accepts (e.g. ["critical","major","warning"]). Empty keeps the default of critical and major. List every severity in use to email opted-in alerts regardless of severity. Entries must be plain severity names, and anything else fails the render. | `[]string` | `[]` | +| `alerta.alerts.email.template` | Text of an Alertmanager template file that changes how the emails look. Define `email.default.subject` and `email.default.html` in it to replace the subject and the HTML body Alertmanager uses for every email, for example to mark the severity or to link to Alerta. The chart stores the text in a ConfigMap named `alertmanager-email-template` and lists it among the Alertmanager templates, so Alertmanager parses it together with its configuration. A template that does not parse fails that load, a running Alertmanager keeps the configuration it already has and a pod that starts with it does not come up. A template that parses but fails on a notification, for example on a missing key, drops that email. The text must contain at least one `define` action. Empty keeps the Alertmanager defaults. | `string` | `""` | ### Grafana configuration diff --git a/packages/extra/monitoring/values.schema.json b/packages/extra/monitoring/values.schema.json index e65feb743..d69046c74 100644 --- a/packages/extra/monitoring/values.schema.json +++ b/packages/extra/monitoring/values.schema.json @@ -557,6 +557,11 @@ "type": "string", "default": "" }, + "template": { + "description": "Text of an Alertmanager template file that changes how the emails look. Define `email.default.subject` and `email.default.html` in it to replace the subject and the HTML body Alertmanager uses for every email, for example to mark the severity or to link to Alerta. The chart stores the text in a ConfigMap named `alertmanager-email-template` and lists it among the Alertmanager templates, so Alertmanager parses it together with its configuration. A template that does not parse fails that load, a running Alertmanager keeps the configuration it already has and a pod that starts with it does not come up. A template that parses but fails on a notification, for example on a missing key, drops that email. The text must contain at least one `define` action. Empty keeps the Alertmanager defaults.", + "type": "string", + "default": "" + }, "to": { "description": "Recipient address, or several separated by commas. A display name is accepted, such as `Oncall \u003concall@example.org\u003e`, as long as it carries no comma. Quote display names containing punctuation such as `:`, `@`, parentheses or `?`. RFC 2047 encoded words are not accepted. IP domain literals accept dotted-quad IPv4 only. IPv6 literals are rejected because Go mail parsers disagree on their syntax across versions. ASCII control characters are rejected. Addresses must be literal, so Alertmanager template actions are rejected. The accepted form is deliberately narrower than RFC 5322. The local part must be a dot-atom, with no leading, trailing or doubled dot, and quoted local parts, comments and a `mailto:` prefix are rejected. An address outside that form fails the render, because Alertmanager parses addresses only when it sends and drops the notification with no record a tenant can read.", "type": "string", diff --git a/packages/extra/monitoring/values.yaml b/packages/extra/monitoring/values.yaml index cd2716acd..f0d16ffac 100644 --- a/packages/extra/monitoring/values.yaml +++ b/packages/extra/monitoring/values.yaml @@ -163,6 +163,7 @@ tracingCollector: ## @field {string} [repeatInterval] - Minimum time before a still-firing alert is emailed again, as an Alertmanager duration with units `y`, `w`, `d`, `h`, `m`, `s`, `ms` in descending order, each at most once, such as `5m`, `12h` or `1h30m`. Delivery is evaluated every 5m, so shorter intervals take effect at 5m and other intervals round up to a multiple of 5m. Zero, a unit out of order and a total beyond what Alertmanager accepts fail the render. Empty keeps the 12h default. ## @field {[]string} [alertnames] - Alert names emailed whether or not the rule carries the notify_email label, with no severity filter. Earlier platform routes still take precedence. Alerts with `event="Heartbeat"`, including `Watchdog`, go only to Alerta, and alerts with severity `none`, `CDIDefaultStorageClassDegraded` and `CDINoDefaultStorageClass` are blackholed and never reach email. This is the way to opt in alerts the platform ships, since their helm-owned rule definitions cannot be labelled per cluster. Entries must be plain Prometheus alert names such as `KubeAPIDown`, and anything else fails the render. Empty adds no extra route. ## @field {[]string} [severities] - Severities the notify_email route accepts (e.g. ["critical","major","warning"]). Empty keeps the default of critical and major. List every severity in use to email opted-in alerts regardless of severity. Entries must be plain severity names, and anything else fails the render. +## @field {string} [template] - Text of an Alertmanager template file that changes how the emails look. Define `email.default.subject` and `email.default.html` in it to replace the subject and the HTML body Alertmanager uses for every email, for example to mark the severity or to link to Alerta. The chart stores the text in a ConfigMap named `alertmanager-email-template` and lists it among the Alertmanager templates, so Alertmanager parses it together with its configuration. A template that does not parse fails that load, a running Alertmanager keeps the configuration it already has and a pod that starts with it does not come up. A template that parses but fails on a notification, for example on a missing key, drops that email. The text must contain at least one `define` action. Empty keeps the Alertmanager defaults. ## @typedef {struct} Alerts - Alert routing configuration. ## @field {TelegramAlerts} [telegram] - Configuration for Telegram alerts. @@ -202,6 +203,7 @@ alerta: repeatInterval: "" alertnames: [] severities: [] + template: "" ## ## @section Grafana configuration ## diff --git a/packages/system/cert-manager-issuers/tests/solver_test.yaml b/packages/system/cert-manager-issuers/tests/solver_test.yaml index 953d978bc..2a9faebbf 100644 --- a/packages/system/cert-manager-issuers/tests/solver_test.yaml +++ b/packages/system/cert-manager-issuers/tests/solver_test.yaml @@ -228,10 +228,14 @@ tests: gateway-class-name: " cloudflare-tunnel " gateway-edge-classes: cloudflare-tunnel asserts: - - documentIndex: 0 + - documentSelector: + path: metadata.name + value: letsencrypt-prod notExists: path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].sectionName - - documentIndex: 1 + - documentSelector: + path: metadata.name + value: letsencrypt-stage notExists: path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].sectionName diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index 732fd4151..b8bc0b9ee 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -30,10 +30,6 @@ cilium: enabled: true hubble: enabled: false - externalIPs: - enabled: true - nodePort: - enabled: true loadBalancer: algorithm: maglev serviceTopology: true diff --git a/packages/system/computeplane-rd/cozyrds/computeplane.yaml b/packages/system/computeplane-rd/cozyrds/computeplane.yaml index c84d39b78..1a2b68a60 100644 --- a/packages/system/computeplane-rd/cozyrds/computeplane.yaml +++ b/packages/system/computeplane-rd/cozyrds/computeplane.yaml @@ -8,7 +8,7 @@ spec: singular: computeplane plural: computeplanes openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"nodeGroups":{"description":"Worker node groups of the ComputePlane cluster (cluster shape only — the security posture is fixed by the module and cannot be overridden through this knob: the module forwards only the documented per-pool fields to the `kubernetes-nodes` pool chart, so any other `NodeGroup` field, e.g. `talos`/`version`/`images`, is silently dropped — accepted but inert). Same structure as the `kubernetes` application's `nodeGroups`. When left empty, the module itself materialises a default scale-from-zero `md0` pool with `roles: [ingress-nginx]` (the wrapped `kubernetes` chart no longer injects one after the Phase 2 split), so the cluster sits idle until a pending Pod (e.g. the ingress-nginx controller) triggers the cluster-autoscaler. Supplied groups are authoritative (not merged with the default), so at least one of them must carry `roles: [ingress-nginx]` — enforced at render time. Each supplied pool key must be at most 15 characters so its `kubernetes-nodes-computeplane-cluster-` HelmRelease name fits Helm's 53-character limit — also enforced at render time.","type":"object","default":{},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas"],"properties":{"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Set both `cpu` and `memory` together or neither.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles. At least one group must carry `ingress-nginx` — the module pins the ingress-nginx addon on, and its controller only schedules onto nodes with that role (enforced at render time).","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, the `kubernetes-nodes` pool chart applies its own default (`replicated`).","type":"string","x-cozystack-options":{"source":"storageclass"}}}}}}} + {"title":"Chart Values","type":"object","properties":{"nodeGroups":{"description":"Worker node groups of the ComputePlane cluster (cluster shape only — the security posture is fixed by the module and cannot be overridden through this knob: the module forwards only the documented per-pool fields to the `kubernetes-nodes` pool chart, so any other `NodeGroup` field, e.g. `talos`/`version`/`images`, is silently dropped — accepted but inert). Same structure as the `kubernetes` application's `nodeGroups`. When left empty, the module itself materialises a default scale-from-zero `md0` pool with `roles: [ingress-nginx]` (the wrapped `kubernetes` chart no longer injects one after the Phase 2 split), so the cluster sits idle until a pending Pod (e.g. the ingress-nginx controller) triggers the cluster-autoscaler. Supplied groups are authoritative (not merged with the default), so at least one of them must carry `roles: [ingress-nginx]` — enforced at render time. Each supplied pool key must be at most 15 characters so its `kubernetes-nodes-computeplane-cluster-` HelmRelease name fits Helm's 53-character limit — also enforced at render time.","type":"object","default":{},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas"],"properties":{"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"maxUnavailable":{"description":"Workers a rollout may remove before their replacement is up. Unset, the `kubernetes-nodes` pool chart uses 0, so each replacement joins before an old worker is drained (1 on a pool with `gpus`). Set 1 on a pool without room for one extra VM, whose rollout would otherwise stall with the new Machine Pending.","type":"integer","maximum":2147483647,"minimum":0},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Set both `cpu` and `memory` together or neither.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles. At least one group must carry `ingress-nginx` — the module pins the ingress-nginx addon on, and its controller only schedules onto nodes with that role (enforced at render time).","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, the `kubernetes-nodes` pool chart applies its own default (`replicated`).","type":"string","x-cozystack-options":{"source":"storageclass"}}}}}}} release: prefix: "" labels: diff --git a/packages/system/cozystack-controller/templates/rbac.yaml b/packages/system/cozystack-controller/templates/rbac.yaml index e40ab9080..da8f71f49 100644 --- a/packages/system/cozystack-controller/templates/rbac.yaml +++ b/packages/system/cozystack-controller/templates/rbac.yaml @@ -49,10 +49,12 @@ rules: resources: ["namespaces"] verbs: ["get", "list", "watch", "patch", "update"] # TenantQuotaReconciler manages the per-namespace "tenant-quota-allocated" -# ResourceQuota that enforces each tenant's share of its parent pool, and emits -# events on the tenant namespace when sub-tenant quotas overcommit the budget. +# ResourceQuota that enforces each tenant's share of its parent pool, copies the +# pool root's "tenant-range-limits" LimitRange into unbounded members as +# "tenant-range-limits-allocated", and emits events on the tenant namespace when +# sub-tenant quotas overcommit the budget. - apiGroups: [""] - resources: ["resourcequotas"] + resources: ["resourcequotas", "limitranges"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] - apiGroups: [""] resources: ["events"] diff --git a/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml b/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml index 71de2dd63..ba1d7e01e 100644 --- a/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml +++ b/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml @@ -21,7 +21,7 @@ spec: singular: kubernetesnodes plural: kubernetesnodeses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool's objects are named `kubernetes--`; when a MachineDeployment of that name already exists and belongs to another Helm release, the render fails instead of taking it over.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"KubeVirt instance type that sizes each worker VM when `resources` is not set. Ignored when `resources.cpu` and `resources.memory` are both set, and therefore on `substrate: proxmox`, which requires them.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. Has no effect on `substrate: proxmox`, where the worker disk is a clone of the VM template's disk and keeps its size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `\"010\"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `\"010\"` reserves what `\"10\"` does.","type":"object","default":{},"properties":{"cpu":{"description":"Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"podCpuLimit":{"description":"CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"podCpuRequest":{"description":"CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"kernelModules":{"description":"Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Module name as `modprobe` takes it, e.g. `nvidia_uvm`.","type":"string"},"parameters":{"description":"Module parameters, each as a bare `key=value` string.","type":"array","items":{"type":"string"}}}}},"kubelet":{"description":"Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`.","type":"object","default":{},"properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. When empty, 5% of the worker's CPU, clamped to 50m..500m.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. When empty, 5% of the worker's CPU, clamped to 50m..500m.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.","type":"string"}}},"logSerialConsole":{"description":"Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it.","type":"boolean","default":false},"maxUnhealthy":{"description":"Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is a percentage. Drop to \"0%\" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At \"50%\" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two.","type":"string","default":"50%"},"nodeStartupTimeout":{"description":"Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (the Talos image import from the Image Factory, or a storage class slow to provision or clone the worker disk). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.","type":"string","default":"20m"},"readyUnknownTimeout":{"description":"How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.","type":"string","default":"5m"},"readyFalseTimeout":{"description":"How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.","type":"string","default":"10m"},"maxNodeProvisionTime":{"description":"How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted.","type":"string","default":"30m"},"version":{"description":"Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool. The render fails when it is ahead. Its pairing with the Talos release the pool resolves (`talos.version`, or the `osImage` override) is checked against the chart's Talos<->Kubernetes support matrix: for a Talos minor the matrix lists, a Kubernetes minor outside that minor's list is refused; a Talos minor the matrix does not list is not checked.","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"osImage":{"description":"Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool.","type":"object","default":{},"properties":{"builtin":{"description":"Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled.","type":"object","properties":{"schematicID":{"description":"Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version of the golden to clone. Defaults to `talos.version`.","type":"string"}}},"factory":{"description":"Import from an Image Factory / mirror (default).","type":"object","properties":{"imageFactoryURL":{"description":"Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. This redirects only the worker boot-disk import; the installer image in the worker machine config is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.","type":"string"},"schematicID":{"description":"Talos schematic ID. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version. Defaults to `talos.version`.","type":"string"}}}}},"talos":{"description":"Talos worker image configuration for this pool.","type":"object","default":{},"required":["imageFactoryURL","installerRepository","registryMirrors","schematicID","version"],"properties":{"imageFactoryCA":{"description":"Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there. On `substrate: proxmox` it has no effect on the workers either: the ConfigMap is still rendered, but nothing references it.","type":"string","default":""},"imageFactoryURL":{"description":"Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal. On `substrate: proxmox` the workers clone a VM template that already carries the OS, so this field has no effect there beyond that format check.","type":"string","default":"https://factory.talos.dev"},"installerRepository":{"description":"OCI repository prefix for the Talos installer image, written into the worker machine config as `machine.install.image`. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.","type":"string","default":"factory.talos.dev/installer"},"registryMirrors":{"description":"Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true},"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"sysctls":{"description":"Sysctl overrides rendered into Talos `machine.sysctls` on this pool's guest workers only. Values must be strings (quote numeric values); for example, `user.max_user_namespaces: \"11255\"` enables the user namespaces required by the Talos gVisor extension. An empty map preserves Talos defaults and the existing bootstrap hash. Adding, changing or removing an override changes the content-hashed TalosConfigTemplate and rolls the pool's workers. Supported names and values depend on the worker's Talos release and kernel; unsafe settings can weaken guest isolation or prevent workers from booting. This does not install gVisor or configure a RuntimeClass, and does not change the management hosts, control plane or other pools.","type":"object","default":{},"additionalProperties":{"type":"string"}},"version":{"description":"Default Talos release for this pool's worker OS image and for the installer image written into the worker machine config. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix lists, for each Talos minor the chart knows, the Kubernetes minors it supports and refuses a pool `version` outside that list; a Talos minor it does not list passes unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.","type":"string","default":"v1.13.6"}}},"substrate":{"description":"Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["network"],"properties":{"dnsServers":{"description":"Nameservers written into the worker machineconfig. On kubevirt the workers use the management cluster's CoreDNS, which they reach over the pod network; an off-cluster Proxmox worker cannot, so it needs reachable resolvers of its own. Required when `substrate` is `proxmox`. Keep in sync with the parent kubernetes chart's `proxmox.dnsServers`.","type":"array","default":[],"items":{"type":"string"}},"full":{"description":"Full clone rather than linked. Default is a linked clone: it costs kilobytes at creation instead of the whole disk, because a ZFS-backed linked clone writes only its own increment (measured: 8K against 10.2G for the same worker full-cloned onto a `sparse 0` pool). The cost is that the clone holds the template's base snapshot, so the template cannot be rotated while any linked clone still references it — which is what the consolidation CronJob resolves 24h after creation. Set true for a pool that must be independent of the template from the first second.","type":"boolean","default":false},"network":{"description":"NIC configuration.","type":"object","default":{},"required":["bridge"],"properties":{"bridge":{"description":"Proxmox bridge the VM NIC attaches to (e.g. `vmbr0`). Required when `substrate` is `proxmox`.","type":"string","default":""},"mtu":{"description":"NIC MTU. Omitted when unset.","type":"integer"},"vlan":{"description":"L2 VLAN tag. Omitted when unset.","type":"integer"}}},"pool":{"description":"Proxmox resource pool the worker VMs are added to. Set the same pool on every node pool of a tenant cluster and grant the tenant's tokens their privileges on `/pool/`: that is the one ACL path that follows every VM capmox creates, replacements included. The pool must exist on the hypervisor; capmox does not create it. Empty adds the workers to no pool.","type":"string","default":""},"storage":{"description":"Proxmox storage for the cloned disk; needs `full: true`. A target storage is a full-clone parameter — a linked clone always lives on the template's storage, and Proxmox refuses the pair — so the render refuses it too. Empty keeps the template's storage.","type":"string","default":""},"templateTags":{"description":"Tags identifying the Talos VM template to clone. capmox matches a template when its tag set is EQUAL to this list, not when it contains it, so every tag on the template must be listed here — a missing one yields `found 0 VM templates with tags ...` while the tags are plainly present. The template has to carry this pool's `talos.schematicID` at `talos.version` in the nocloud flavour, with the Proxmox guest agent enabled for the `qemu-guest-agent` extension of the default schematic; `hack/proxmox-talos-template.sh` builds it from these values and prints the tags to list here. Do not build it from the platform release's `nocloud-amd64.raw.xz`: that is the bare-metal node image, whose `zfs` extension keeps a VM from finishing its boot sequence and reboots it at the Talos boot timeout. Required when `substrate` is `proxmox`.","type":"array","default":[],"items":{"type":"string"}}}}}} + {"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool's objects are named `kubernetes--`; when a MachineDeployment of that name already exists and belongs to another Helm release, the render fails instead of taking it over.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"KubeVirt instance type that sizes each worker VM when `resources` is not set. Ignored when `resources.cpu` and `resources.memory` are both set, and therefore on `substrate: proxmox`, which requires them.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. Has no effect on `substrate: proxmox`, where the worker disk is a clone of the VM template's disk and keeps its size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `\"010\"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `\"010\"` reserves what `\"10\"` does.","type":"object","default":{},"properties":{"cpu":{"description":"Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"podCpuLimit":{"description":"CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"podCpuRequest":{"description":"CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"maxUnavailable":{"description":"Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.","type":"integer","maximum":2147483647,"minimum":0},"kernelModules":{"description":"Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Module name as `modprobe` takes it, e.g. `nvidia_uvm`.","type":"string"},"parameters":{"description":"Module parameters, each as a bare `key=value` string.","type":"array","items":{"type":"string"}}}}},"kubelet":{"description":"Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`.","type":"object","default":{},"properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. When empty, 5% of the worker's CPU, clamped to 50m..500m.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. When empty, 5% of the worker's CPU, clamped to 50m..500m.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.","type":"string"}}},"logSerialConsole":{"description":"Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it.","type":"boolean","default":false},"maxUnhealthy":{"description":"Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is a percentage. Drop to \"0%\" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At \"50%\" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two.","type":"string","default":"50%"},"nodeStartupTimeout":{"description":"Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (the Talos image import from the Image Factory, or a storage class slow to provision or clone the worker disk). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.","type":"string","default":"20m"},"readyUnknownTimeout":{"description":"How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.","type":"string","default":"5m"},"readyFalseTimeout":{"description":"How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.","type":"string","default":"10m"},"maxNodeProvisionTime":{"description":"How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted.","type":"string","default":"30m"},"version":{"description":"Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool. The render fails when it is ahead. Its pairing with the Talos release the pool resolves (`talos.version`, or the `osImage` override) is checked against the chart's Talos<->Kubernetes support matrix: for a Talos minor the matrix lists, a Kubernetes minor outside that minor's list is refused; a Talos minor the matrix does not list is not checked.","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"osImage":{"description":"Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool.","type":"object","default":{},"properties":{"builtin":{"description":"Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled.","type":"object","properties":{"schematicID":{"description":"Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version of the golden to clone. Defaults to `talos.version`.","type":"string"}}},"factory":{"description":"Import from an Image Factory / mirror (default).","type":"object","properties":{"imageFactoryURL":{"description":"Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. This redirects only the worker boot-disk import; the installer image in the worker machine config is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.","type":"string"},"schematicID":{"description":"Talos schematic ID. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version. Defaults to `talos.version`.","type":"string"}}}}},"talos":{"description":"Talos worker image configuration for this pool.","type":"object","default":{},"required":["imageFactoryURL","installerRepository","registryMirrors","schematicID","version"],"properties":{"imageFactoryCA":{"description":"Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there. On `substrate: proxmox` it has no effect on the workers either: the ConfigMap is still rendered, but nothing references it.","type":"string","default":""},"imageFactoryURL":{"description":"Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal. On `substrate: proxmox` the workers clone a VM template that already carries the OS, so this field has no effect there beyond that format check.","type":"string","default":"https://factory.talos.dev"},"installerRepository":{"description":"OCI repository prefix for the Talos installer image, written into the worker machine config as `machine.install.image`. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.","type":"string","default":"factory.talos.dev/installer"},"registryMirrors":{"description":"Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true},"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"sysctls":{"description":"Sysctl overrides rendered into Talos `machine.sysctls` on this pool's guest workers only. Values must be strings (quote numeric values); for example, `user.max_user_namespaces: \"11255\"` enables the user namespaces required by the Talos gVisor extension. An empty map preserves Talos defaults and the existing bootstrap hash. Adding, changing or removing an override changes the content-hashed TalosConfigTemplate and rolls the pool's workers. Supported names and values depend on the worker's Talos release and kernel; unsafe settings can weaken guest isolation or prevent workers from booting. This does not install gVisor or configure a RuntimeClass, and does not change the management hosts, control plane or other pools.","type":"object","default":{},"additionalProperties":{"type":"string"}},"version":{"description":"Default Talos release for this pool's worker OS image and for the installer image written into the worker machine config. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix lists, for each Talos minor the chart knows, the Kubernetes minors it supports and refuses a pool `version` outside that list; a Talos minor it does not list passes unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.","type":"string","default":"v1.13.6"}}},"substrate":{"description":"Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["network"],"properties":{"dnsServers":{"description":"Nameservers written into the worker machineconfig. On kubevirt the workers use the management cluster's CoreDNS, which they reach over the pod network; an off-cluster Proxmox worker cannot, so it needs reachable resolvers of its own. Required when `substrate` is `proxmox`. Keep in sync with the parent kubernetes chart's `proxmox.dnsServers`.","type":"array","default":[],"items":{"type":"string"}},"full":{"description":"Full clone rather than linked. Default is a linked clone: it costs kilobytes at creation instead of the whole disk, because a ZFS-backed linked clone writes only its own increment (measured: 8K against 10.2G for the same worker full-cloned onto a `sparse 0` pool). The cost is that the clone holds the template's base snapshot, so the template cannot be rotated while any linked clone still references it — which is what the consolidation CronJob resolves 24h after creation. Set true for a pool that must be independent of the template from the first second.","type":"boolean","default":false},"network":{"description":"NIC configuration.","type":"object","default":{},"required":["bridge"],"properties":{"bridge":{"description":"Proxmox bridge the VM NIC attaches to (e.g. `vmbr0`). Required when `substrate` is `proxmox`.","type":"string","default":""},"mtu":{"description":"NIC MTU. Omitted when unset.","type":"integer"},"vlan":{"description":"L2 VLAN tag. Omitted when unset.","type":"integer"}}},"pool":{"description":"Proxmox resource pool the worker VMs are added to. Set the same pool on every node pool of a tenant cluster and grant the tenant's tokens their privileges on `/pool/`: that is the one ACL path that follows every VM capmox creates, replacements included. The pool must exist on the hypervisor; capmox does not create it. Empty adds the workers to no pool.","type":"string","default":""},"storage":{"description":"Proxmox storage for the cloned disk; needs `full: true`. A target storage is a full-clone parameter — a linked clone always lives on the template's storage, and Proxmox refuses the pair — so the render refuses it too. Empty keeps the template's storage.","type":"string","default":""},"templateTags":{"description":"Tags identifying the Talos VM template to clone. capmox matches a template when its tag set is EQUAL to this list, not when it contains it, so every tag on the template must be listed here — a missing one yields `found 0 VM templates with tags ...` while the tags are plainly present. The template has to carry this pool's `talos.schematicID` at `talos.version` in the nocloud flavour, with the Proxmox guest agent enabled for the `qemu-guest-agent` extension of the default schematic; `hack/proxmox-talos-template.sh` builds it from these values and prints the tags to list here. Do not build it from the platform release's `nocloud-amd64.raw.xz`: that is the bare-metal node image, whose `zfs` extension keeps a VM from finishing its boot sequence and reboots it at the Talos boot timeout. Required when `substrate` is `proxmox`.","type":"array","default":[],"items":{"type":"string"}}}}}} release: labels: sharding.fluxcd.io/key: tenants diff --git a/packages/system/monitoring-agents/alerts/etcd.yaml b/packages/system/monitoring-agents/alerts/etcd.yaml index 72c5ef7b2..48cdc84d9 100644 --- a/packages/system/monitoring-agents/alerts/etcd.yaml +++ b/packages/system/monitoring-agents/alerts/etcd.yaml @@ -116,8 +116,9 @@ spec: {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.' summary: etcd cluster member communication is slow. + # Tenant peer RTT is covered by the tenant etcd rules. expr: |- - histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*", job!~"tenant-.*/etcd-pod-scrape"}[5m])) > 0.15 for: 10m labels: diff --git a/packages/system/monitoring-agents/tests/etcd_rules_test.yaml b/packages/system/monitoring-agents/tests/etcd_rules_test.yaml new file mode 100644 index 000000000..78651297c --- /dev/null +++ b/packages/system/monitoring-agents/tests/etcd_rules_test.yaml @@ -0,0 +1,20 @@ +suite: tenant peer RTT is evaluated once + +templates: + - templates/alerts.yaml + +tests: + - it: leaves tenant peer RTT to the tenant rules + documentSelector: + path: metadata.name + value: alerts-etcd + asserts: + - matchRegex: + path: spec.groups[0].rules[7].expr + pattern: 'job!~"tenant-\.\*/etcd-pod-scrape"' + - notMatchRegex: + path: spec.groups[0].rules[9].expr + pattern: 'job!~' + - notMatchRegex: + path: spec.groups[0].rules[10].expr + pattern: 'job!~' diff --git a/packages/system/monitoring-rd/cozyrds/monitoring.yaml b/packages/system/monitoring-rd/cozyrds/monitoring.yaml index 4e02ea731..51b1a2dd7 100644 --- a/packages/system/monitoring-rd/cozyrds/monitoring.yaml +++ b/packages/system/monitoring-rd/cozyrds/monitoring.yaml @@ -20,7 +20,7 @@ spec: singular: monitoring plural: monitorings openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access Grafana externally (defaults to 'grafana' subdomain for the tenant host).","type":"string","default":""},"metricsStorages":{"description":"Configuration of metrics storage instances.","type":"array","default":[{"deduplicationInterval":"15s","name":"shortterm","retentionPeriod":"3d","storage":"10Gi","storageClassName":""},{"deduplicationInterval":"5m","name":"longterm","retentionPeriod":"14d","storage":"10Gi","storageClassName":""}],"items":{"type":"object","required":["deduplicationInterval","name","retentionPeriod","storage"],"properties":{"deduplicationInterval":{"description":"Deduplication interval for metrics.","type":"string"},"name":{"description":"Name of the storage instance.","type":"string"},"retentionPeriod":{"description":"Retention period for metrics.","type":"string"},"storage":{"description":"Persistent volume size.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used for the data.","type":"string"},"vminsert":{"description":"Configuration for vminsert.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"vmselect":{"description":"Configuration for vmselect.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"vmstorage":{"description":"Configuration for vmstorage.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}}}}},"logsStorages":{"description":"Configuration of logs storage instances.","type":"array","default":[{"name":"generic","retentionPeriod":"1","storage":"10Gi","storageClassName":"replicated"}],"items":{"type":"object","required":["name","retentionPeriod","storage","storageClassName"],"properties":{"name":{"description":"Name of the storage instance.","type":"string"},"retentionPeriod":{"description":"Retention period for logs.","type":"string","default":"1"},"storage":{"description":"Persistent volume size.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"}}}},"tracingStorages":{"description":"Configuration of traces storage instances. Empty by default (tracing off): no backend and no trace collector are rendered. Adding an entry provisions a VictoriaTraces backend and, while tracingCollector.enabled is true, deploys the OTLP collector in front of it. A tenant with shared-central tracing (the Tenant's tracingCentral) gets the collector, while tracingCollector.enabled is true, without an entry, which then exports to the shared store; entries still listed keep their store and datasource, so traces stored before the switch stay readable until the entry is removed.","type":"array","default":[],"items":{"type":"object","required":["mode","name","retentionPeriod","storage","storageClassName"],"properties":{"mode":{"description":"Deployment mode: \"cluster\" renders a VTCluster; \"single\" renders a single-binary VTSingle for edge/dev/small clusters. Treat as immutable once a backend exists: mode selects which Kind is rendered under the same name, so changing it on a running backend re-identifies the object rather than reconfiguring it. Switching from \"single\" to \"cluster\" prunes the VTSingle and the operator garbage-collects its PVC (stored traces are lost); switching from \"cluster\" to \"single\" (or removing the entry) leaves the vtstorage PVCs Bound and billed until the whole Monitoring release is deleted. The API cannot guard this (transition rules are not available on an array-item field), so this note is the only guard: migrate deliberately, accepting the loss.","type":"string","default":"cluster","enum":["cluster","single"]},"name":{"description":"Name of the storage instance: a lowercase RFC 1123 label of at most 42 characters. Treat as immutable once a backend exists: the VTCluster/VTSingle, its Services, PVCs and Grafana datasource are all named after it, so renaming an entry deploys a new, empty backend and removes the old one, the same way changing mode does (single mode loses the old PVC, cluster mode leaves the old vtstorage PVCs Bound until the whole Monitoring release is deleted).","type":"string"},"retentionDiskUsageBytes":{"description":"Optional disk-usage cap as a VictoriaMetrics BytesString (e.g. \"8GB\" or \"8GiB\"), bounding disk use on top of the age-based retentionPeriod. Uses byte-suffix grammar (kb/mb/gb/tb, KiB/MiB/GiB/TiB), NOT the Kubernetes-quantity grammar of the sibling storage field, so \"8Gi\" is rejected by the operator. In cluster mode the cap is per vtstorage replica (replicaCount 2), so the effective total is twice the value. Unset, the store drops its oldest day once its filesystem is 80% full instead, so every store is disk-bounded. Either way VictoriaTraces keeps the last two days, so spans arriving faster than the cap allows for two days can still fill the disk.","type":"string"},"retentionPeriod":{"description":"Retention period for traces.","type":"string","default":"14d"},"storage":{"description":"Persistent volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClassName":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"}}}},"tracingCentralHost":{"description":"tenant-root only: accept spans from tenants that turned on shared-central tracing (the Tenant's tracingCentral) into this Monitoring's tracingStorages entry named `generic`, which must be in `cluster` mode. Every central tenant then shares that store's disk and retention with tenant-root, and nothing reserves space per tenant: once the store's disk cap is reached (retentionDiskUsageBytes, or 80% of its filesystem) the oldest spans of all of them are evicted together. Each tenant is held to the platform's monitoring.tracingCentralTenantBytesPerSecond, which by default lets one tenant write at most the cap of a default 10Gi store, counted uncompressed and before the tenant's sampling, over the two days VictoriaTraces always keeps, so raise it with the store; several tenants writing at their limit together still can fill it. That bound holds for a tenant whose collector keeps running: each collector pod starts with a full bucket, up to 5 MiB or one second of its share of the rate when that is larger, and any change to the tenant's Monitoring rolls every collector pod, so a tenant that keeps changing it writes that much more on each rollout, for up to ten replicas; closing that needs a limiter with shared state, which the collector does not run. Ignored outside tenant-root. The tenant-root namespace is inside the trust boundary of the shared store, as it is for the metrics of tenants that rely on a parent's monitoring: vmauth pins each central tenant to its own account, but workloads in tenant-root reach the store directly, and one that learns a tenant's account ID (a hash of its namespace name and UID, which its VMUsers carry) can read and write that account. Whoever tenant-root admits to its Grafana is inside it too, since tenant-root's own datasource for the store reaches vtselect without vmauth; who that is follows its Grafana's oidc mode (with System, tenant-root's own groups, which already hold the same access level in every tenant under tenant-root). The render refuses to stop hosting while central tenants remain, but removing the tenant-root Monitoring altogether is not a render and is not refused: turn tracingCentral off on the central tenants first, since until then every Monitoring render of theirs fails, their metrics, logs and alerting changes included. The refusals read each other's objects when they render, so a tenant turning tracingCentral on while tenant-root stops hosting can pass both: that tenant's collector then exports to a vmauth that is gone while every release reports Ready, until its Monitoring renders again. Recover by hosting again; turning that tenant's tracingCentral off instead ends the silent drop, and turning it on again is then refused until tenant-root hosts.","type":"boolean","default":false},"tracingCollector":{"description":"OpenTelemetry Collector for trace ingestion. With shared-central tracing it refuses spans above the platform's per-tenant write rate: OTLP/HTTP senders retry them, OTLP/gRPC senders drop them, so export over HTTP. It also refuses a request larger than 3 MiB (decompressed) with a 400, which senders do not retry, so keep export batches below that.","type":"object","default":{},"required":["enabled","replicas","samplingPercentage"],"properties":{"enabled":{"description":"Deploy the OTLP collector. When disabled, applications must send OTLP straight to the traces backend (no central sampling/redaction); with shared-central tracing they have no path to it, so no new spans arrive, while the traces already stored stay readable. The collector fronts a single backend, and the limit holds with shared-central tracing too, since turning that off makes it export to the local entry again: while it is enabled, tracingStorages must hold at most one entry, and adding a second one fails the whole Monitoring render until the extra entry is removed or the collector is disabled.","type":"boolean","default":true},"redactAttributes":{"description":"Span attribute keys to drop before export (the PII/secret control point, e.g. db.statement).","type":"array","default":[],"items":{"type":"string"}},"replicas":{"description":"Number of collector replicas. Head sampling is trace-id-deterministic, so replicas>1 are safe and remove the single-ingest-path SPOF. With shared-central tracing each replica sends up to two requests at a time against this tenant's write limit of four per vmauth replica, so more replicas do not raise the tenant's throughput; the tenant's write rate (the platform's monitoring.tracingCentralTenantBytesPerSecond) is split across them as well, so a sender whose connection stays on one replica gets that replica's share, and each replica that starts lets through above the rate up to 5 MiB, or one second of its share when that is larger, which is why shared-central tracing allows at most 10 replicas and fails the render above that.","type":"integer","default":1,"minimum":1},"resources":{"description":"Resource configuration for the collector.","type":"object","default":{},"properties":{"limits":{"description":"Resource limits.","type":"object","default":{},"properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"requests":{"description":"Resource requests.","type":"object","default":{},"properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"samplingPercentage":{"description":"Head-sampling percentage applied at the collector before export.","type":"integer","default":10,"maximum":100,"minimum":0}}},"alerta":{"description":"Configuration for the Alerta service.","type":"object","default":{},"properties":{"alerts":{"description":"Alert routing configuration.","type":"object","default":{},"properties":{"email":{"description":"Configuration for email alerts. Alerts carrying the label notify_email with the string value true are emailed when their severity is in the severities list (default critical and major). A bare YAML boolean does not match. Alerts listed in alertnames are emailed unless matched by an earlier platform route.","type":"object","default":{},"required":["from","smarthost","to"],"properties":{"alertnames":{"description":"Alert names emailed whether or not the rule carries the notify_email label, with no severity filter. Earlier platform routes still take precedence. Alerts with `event=\"Heartbeat\"`, including `Watchdog`, go only to Alerta, and alerts with severity `none`, `CDIDefaultStorageClassDegraded` and `CDINoDefaultStorageClass` are blackholed and never reach email. This is the way to opt in alerts the platform ships, since their helm-owned rule definitions cannot be labelled per cluster. Entries must be plain Prometheus alert names such as `KubeAPIDown`, and anything else fails the render. Empty adds no extra route.","type":"array","default":[],"items":{"type":"string"}},"authUsername":{"description":"Username for SMTP LOGIN/PLAIN authentication. The password is read from the `password` key of a Secret named `alertmanager-email-password` in the release namespace, which the platform operator creates. The name is fixed and the Secret is mounted into Alertmanager, so a missing Secret blocks the whole Alertmanager rollout, not only email delivery. The credential in it must belong to this tenant alone. `smarthost` is chosen by whoever writes the Monitoring, so one relay credential provisioned under the same name in every namespace can be collected by pointing `smarthost` at a host the tenant controls.","type":"string","default":""},"from":{"description":"Sender address, one only. A display name is accepted. It takes the same narrower form as `to`, and an address outside it fails the render.","type":"string","default":""},"repeatInterval":{"description":"Minimum time before a still-firing alert is emailed again, as an Alertmanager duration with units `y`, `w`, `d`, `h`, `m`, `s`, `ms` in descending order, each at most once, such as `5m`, `12h` or `1h30m`. Delivery is evaluated every 5m, so shorter intervals take effect at 5m and other intervals round up to a multiple of 5m. Zero, a unit out of order and a total beyond what Alertmanager accepts fail the render. Empty keeps the 12h default.","type":"string","default":""},"severities":{"description":"Severities the notify_email route accepts (e.g. [\"critical\",\"major\",\"warning\"]). Empty keeps the default of critical and major. List every severity in use to email opted-in alerts regardless of severity. Entries must be plain severity names, and anything else fails the render.","type":"array","default":[],"items":{"type":"string"}},"smarthost":{"description":"SMTP server as `host:port` (e.g. \"smtp.example.com:465\"). A value without a port, an invalid bracketed IP address or a port with leading zeros fails the render. Port 465 uses implicit TLS, other ports use STARTTLS.","type":"string","default":""},"to":{"description":"Recipient address, or several separated by commas. A display name is accepted, such as `Oncall `, as long as it carries no comma. Quote display names containing punctuation such as `:`, `@`, parentheses or `?`. RFC 2047 encoded words are not accepted. IP domain literals accept dotted-quad IPv4 only. IPv6 literals are rejected because Go mail parsers disagree on their syntax across versions. ASCII control characters are rejected. Addresses must be literal, so Alertmanager template actions are rejected. The accepted form is deliberately narrower than RFC 5322. The local part must be a dot-atom, with no leading, trailing or doubled dot, and quoted local parts, comments and a `mailto:` prefix are rejected. An address outside that form fails the render, because Alertmanager parses addresses only when it sends and drops the notification with no record a tenant can read.","type":"string","default":""}}},"slack":{"description":"Configuration for Slack alerts.","type":"object","default":{},"required":["url"],"properties":{"disabledSeverity":{"description":"List of severities without alerts (e.g. [\"informational\",\"warning\"]).","type":"array","default":[],"items":{"type":"string"}},"url":{"description":"Configuration uri for Slack alerts.","type":"string","default":""}}},"telegram":{"description":"Configuration for Telegram alerts.","type":"object","default":{},"required":["chatID","token"],"properties":{"chatID":{"description":"Telegram chat ID(s), separated by commas.","type":"string","default":""},"disabledSeverity":{"description":"List of severities without alerts (e.g. [\"informational\",\"warning\"]).","type":"array","default":[],"items":{"type":"string"}},"token":{"description":"Telegram bot token.","type":"string","default":""}}}}},"resources":{"description":"Resource configuration.","type":"object","default":{},"properties":{"limits":{"description":"Resource limits.","type":"object","default":{},"properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"requests":{"description":"Resource requests.","type":"object","default":{},"properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"storage":{"description":"Persistent volume size for the database.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used for the database.","type":"string","default":""}}},"grafana":{"description":"Configuration for Grafana.","type":"object","default":{},"properties":{"db":{"description":"Database configuration.","type":"object","default":{},"properties":{"size":{"description":"Persistent volume size for the database.","type":"string","default":"10Gi"}}},"resources":{"description":"Resource configuration.","type":"object","default":{},"properties":{"limits":{"description":"Resource limits.","type":"object","default":{},"properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"requests":{"description":"Resource requests.","type":"object","default":{},"properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}}}},"vmagent":{"description":"Configuration for VictoriaMetrics Agent.","type":"object","default":{},"properties":{"externalLabels":{"description":"External labels applied to all metrics.","type":"object","default":{"cluster":"cozystack"}},"remoteWrite":{"description":"Remote write configuration.","type":"object","default":{"urls":["http://vminsert-shortterm:8480/insert/0/prometheus","http://vminsert-longterm:8480/insert/0/prometheus"]}}}},"oidc":{"description":"OIDC authentication for the Grafana instance. See https://cozystack.io/docs/v1.6/operations/services/monitoring/oidc-authentication/ for the operator guide.","type":"object","default":{},"required":["mode"],"properties":{"customConfig":{"description":"Tenant-supplied `[auth.generic_oauth]` payload; consumed only when `mode: CustomConfig`.","type":"object","default":{},"properties":{"config":{"description":"Inline map of `[auth.generic_oauth]` keys. Written verbatim into the Grafana CR `spec.config`. Mutually exclusive with `secretRef.name`.","type":"object","default":{}},"secretRef":{"description":"Reference to a Secret holding an `auth.ini` fragment; mounted under `/etc/grafana/oidc` and loaded via `GF_PATHS_CUSTOM_INI`. With shared-central tracing only its `auth.ini` key is mounted, so the fragment cannot read another key of the Secret with `$__file{}`.","type":"object","default":{},"properties":{"name":{"description":"Name of an existing Secret in the release namespace whose `auth.ini` key holds a ready-made `[auth.generic_oauth]` block. Mutually exclusive with `customConfig.config`.","type":"string","default":""}}}}},"mode":{"description":"Identity mode. `None`: no OIDC, only the `grafana-admin-password` Secret works. `System`: trust the platform `cozy` realm via a per-instance confidential client and audience binding; zero-config default. `CustomConfig`: trust a tenant-supplied issuer directly (BYO); `cozy` is not in the path.","type":"string","default":"None","enum":["None","System","CustomConfig"]},"users":{"description":"Users granted access to this Grafana instance's org. Each entry is reconciled into Grafana org membership + role by a chart-owned post-install/post-upgrade Job. Works for both `System` and `CustomConfig` modes; ignored when `mode: None`. Users not listed here get nothing (no default org role from group mapping).","type":"array","default":[],"items":{"type":"object","required":["email","role"],"properties":{"email":{"description":"Email address matched against the `email` claim from the issuer. The chart pre-provisions a Grafana account with this email so the users-Job can set the org role before the operator's first login (`oauth_allow_insecure_email_lookup = true` glues the two together). Must match a conservative pattern: local part is `[A-Za-z0-9._%+-]+`, domain is `[A-Za-z0-9.-]+\\.[A-Za-z]{2,}`. Whitespace, quoted literals, and unusual URL-reserved characters are rejected on schema level so the reconcile-Job's shell handling never has to defend against malformed input.","type":"string"},"role":{"description":"Grafana org role to bind: `Admin`, `Editor`, or `Viewer`. Applied to the release's Grafana org (`Main Org.`); server-level `GrafanaAdmin` promotion is out of scope for Phase 1. Required — the users-Job POSTs `{loginOrEmail, role}` to `/api/orgs/1/users` and PATCHes `{role}` to converge across re-runs; omitting the field would render `role: \"null\"` into the body and fail with HTTP 400.","type":"string","enum":["Admin","Editor","Viewer"]}}}}}}}} + {"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access Grafana externally (defaults to 'grafana' subdomain for the tenant host).","type":"string","default":""},"metricsStorages":{"description":"Configuration of metrics storage instances.","type":"array","default":[{"deduplicationInterval":"15s","name":"shortterm","retentionPeriod":"3d","storage":"10Gi","storageClassName":""},{"deduplicationInterval":"5m","name":"longterm","retentionPeriod":"14d","storage":"10Gi","storageClassName":""}],"items":{"type":"object","required":["deduplicationInterval","name","retentionPeriod","storage"],"properties":{"deduplicationInterval":{"description":"Deduplication interval for metrics.","type":"string"},"name":{"description":"Name of the storage instance.","type":"string"},"retentionPeriod":{"description":"Retention period for metrics.","type":"string"},"storage":{"description":"Persistent volume size.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used for the data.","type":"string"},"vminsert":{"description":"Configuration for vminsert.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"vmselect":{"description":"Configuration for vmselect.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"vmstorage":{"description":"Configuration for vmstorage.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}}}}},"logsStorages":{"description":"Configuration of logs storage instances.","type":"array","default":[{"name":"generic","retentionPeriod":"1","storage":"10Gi","storageClassName":"replicated"}],"items":{"type":"object","required":["name","retentionPeriod","storage","storageClassName"],"properties":{"name":{"description":"Name of the storage instance.","type":"string"},"retentionPeriod":{"description":"Retention period for logs.","type":"string","default":"1"},"storage":{"description":"Persistent volume size.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"}}}},"tracingStorages":{"description":"Configuration of traces storage instances. Empty by default (tracing off): no backend and no trace collector are rendered. Adding an entry provisions a VictoriaTraces backend and, while tracingCollector.enabled is true, deploys the OTLP collector in front of it. A tenant with shared-central tracing (the Tenant's tracingCentral) gets the collector, while tracingCollector.enabled is true, without an entry, which then exports to the shared store; entries still listed keep their store and datasource, so traces stored before the switch stay readable until the entry is removed.","type":"array","default":[],"items":{"type":"object","required":["mode","name","retentionPeriod","storage","storageClassName"],"properties":{"mode":{"description":"Deployment mode: \"cluster\" renders a VTCluster; \"single\" renders a single-binary VTSingle for edge/dev/small clusters. Treat as immutable once a backend exists: mode selects which Kind is rendered under the same name, so changing it on a running backend re-identifies the object rather than reconfiguring it. Switching from \"single\" to \"cluster\" prunes the VTSingle and the operator garbage-collects its PVC (stored traces are lost); switching from \"cluster\" to \"single\" (or removing the entry) leaves the vtstorage PVCs Bound and billed until the whole Monitoring release is deleted. The API cannot guard this (transition rules are not available on an array-item field), so this note is the only guard: migrate deliberately, accepting the loss.","type":"string","default":"cluster","enum":["cluster","single"]},"name":{"description":"Name of the storage instance: a lowercase RFC 1123 label of at most 42 characters. Treat as immutable once a backend exists: the VTCluster/VTSingle, its Services, PVCs and Grafana datasource are all named after it, so renaming an entry deploys a new, empty backend and removes the old one, the same way changing mode does (single mode loses the old PVC, cluster mode leaves the old vtstorage PVCs Bound until the whole Monitoring release is deleted).","type":"string"},"retentionDiskUsageBytes":{"description":"Optional disk-usage cap as a VictoriaMetrics BytesString (e.g. \"8GB\" or \"8GiB\"), bounding disk use on top of the age-based retentionPeriod. Uses byte-suffix grammar (kb/mb/gb/tb, KiB/MiB/GiB/TiB), NOT the Kubernetes-quantity grammar of the sibling storage field, so \"8Gi\" is rejected by the operator. In cluster mode the cap is per vtstorage replica (replicaCount 2), so the effective total is twice the value. Unset, the store drops its oldest day once its filesystem is 80% full instead, so every store is disk-bounded. Either way VictoriaTraces keeps the last two days, so spans arriving faster than the cap allows for two days can still fill the disk.","type":"string"},"retentionPeriod":{"description":"Retention period for traces.","type":"string","default":"14d"},"storage":{"description":"Persistent volume size.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClassName":{"description":"StorageClass used to store the data.","type":"string","default":"replicated"}}}},"tracingCentralHost":{"description":"tenant-root only: accept spans from tenants that turned on shared-central tracing (the Tenant's tracingCentral) into this Monitoring's tracingStorages entry named `generic`, which must be in `cluster` mode. Every central tenant then shares that store's disk and retention with tenant-root, and nothing reserves space per tenant: once the store's disk cap is reached (retentionDiskUsageBytes, or 80% of its filesystem) the oldest spans of all of them are evicted together. Each tenant is held to the platform's monitoring.tracingCentralTenantBytesPerSecond, which by default lets one tenant write at most the cap of a default 10Gi store, counted uncompressed and before the tenant's sampling, over the two days VictoriaTraces always keeps, so raise it with the store; several tenants writing at their limit together still can fill it. That bound holds for a tenant whose collector keeps running: each collector pod starts with a full bucket, up to 5 MiB or one second of its share of the rate when that is larger, and any change to the tenant's Monitoring rolls every collector pod, so a tenant that keeps changing it writes that much more on each rollout, for up to ten replicas; closing that needs a limiter with shared state, which the collector does not run. Ignored outside tenant-root. The tenant-root namespace is inside the trust boundary of the shared store, as it is for the metrics of tenants that rely on a parent's monitoring: vmauth pins each central tenant to its own account, but workloads in tenant-root reach the store directly, and one that learns a tenant's account ID (a hash of its namespace name and UID, which its VMUsers carry) can read and write that account. Whoever tenant-root admits to its Grafana is inside it too, since tenant-root's own datasource for the store reaches vtselect without vmauth; who that is follows its Grafana's oidc mode (with System, tenant-root's own groups, which already hold the same access level in every tenant under tenant-root). The render refuses to stop hosting while central tenants remain, but removing the tenant-root Monitoring altogether is not a render and is not refused: turn tracingCentral off on the central tenants first, since until then every Monitoring render of theirs fails, their metrics, logs and alerting changes included. The refusals read each other's objects when they render, so a tenant turning tracingCentral on while tenant-root stops hosting can pass both: that tenant's collector then exports to a vmauth that is gone while every release reports Ready, until its Monitoring renders again. Recover by hosting again; turning that tenant's tracingCentral off instead ends the silent drop, and turning it on again is then refused until tenant-root hosts.","type":"boolean","default":false},"tracingCollector":{"description":"OpenTelemetry Collector for trace ingestion. With shared-central tracing it refuses spans above the platform's per-tenant write rate: OTLP/HTTP senders retry them, OTLP/gRPC senders drop them, so export over HTTP. It also refuses a request larger than 3 MiB (decompressed) with a 400, which senders do not retry, so keep export batches below that.","type":"object","default":{},"required":["enabled","replicas","samplingPercentage"],"properties":{"enabled":{"description":"Deploy the OTLP collector. When disabled, applications must send OTLP straight to the traces backend (no central sampling/redaction); with shared-central tracing they have no path to it, so no new spans arrive, while the traces already stored stay readable. The collector fronts a single backend, and the limit holds with shared-central tracing too, since turning that off makes it export to the local entry again: while it is enabled, tracingStorages must hold at most one entry, and adding a second one fails the whole Monitoring render until the extra entry is removed or the collector is disabled.","type":"boolean","default":true},"redactAttributes":{"description":"Span attribute keys to drop before export (the PII/secret control point, e.g. db.statement).","type":"array","default":[],"items":{"type":"string"}},"replicas":{"description":"Number of collector replicas. Head sampling is trace-id-deterministic, so replicas>1 are safe and remove the single-ingest-path SPOF. With shared-central tracing each replica sends up to two requests at a time against this tenant's write limit of four per vmauth replica, so more replicas do not raise the tenant's throughput; the tenant's write rate (the platform's monitoring.tracingCentralTenantBytesPerSecond) is split across them as well, so a sender whose connection stays on one replica gets that replica's share, and each replica that starts lets through above the rate up to 5 MiB, or one second of its share when that is larger, which is why shared-central tracing allows at most 10 replicas and fails the render above that.","type":"integer","default":1,"minimum":1},"resources":{"description":"Resource configuration for the collector.","type":"object","default":{},"properties":{"limits":{"description":"Resource limits.","type":"object","default":{},"properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"requests":{"description":"Resource requests.","type":"object","default":{},"properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"samplingPercentage":{"description":"Head-sampling percentage applied at the collector before export.","type":"integer","default":10,"maximum":100,"minimum":0}}},"alerta":{"description":"Configuration for the Alerta service.","type":"object","default":{},"properties":{"alerts":{"description":"Alert routing configuration.","type":"object","default":{},"properties":{"email":{"description":"Configuration for email alerts. Alerts carrying the label notify_email with the string value true are emailed when their severity is in the severities list (default critical and major). A bare YAML boolean does not match. Alerts listed in alertnames are emailed unless matched by an earlier platform route.","type":"object","default":{},"required":["from","smarthost","to"],"properties":{"alertnames":{"description":"Alert names emailed whether or not the rule carries the notify_email label, with no severity filter. Earlier platform routes still take precedence. Alerts with `event=\"Heartbeat\"`, including `Watchdog`, go only to Alerta, and alerts with severity `none`, `CDIDefaultStorageClassDegraded` and `CDINoDefaultStorageClass` are blackholed and never reach email. This is the way to opt in alerts the platform ships, since their helm-owned rule definitions cannot be labelled per cluster. Entries must be plain Prometheus alert names such as `KubeAPIDown`, and anything else fails the render. Empty adds no extra route.","type":"array","default":[],"items":{"type":"string"}},"authUsername":{"description":"Username for SMTP LOGIN/PLAIN authentication. The password is read from the `password` key of a Secret named `alertmanager-email-password` in the release namespace, which the platform operator creates. The name is fixed and the Secret is mounted into Alertmanager, so a missing Secret blocks the whole Alertmanager rollout, not only email delivery. The credential in it must belong to this tenant alone. `smarthost` is chosen by whoever writes the Monitoring, so one relay credential provisioned under the same name in every namespace can be collected by pointing `smarthost` at a host the tenant controls.","type":"string","default":""},"from":{"description":"Sender address, one only. A display name is accepted. It takes the same narrower form as `to`, and an address outside it fails the render.","type":"string","default":""},"repeatInterval":{"description":"Minimum time before a still-firing alert is emailed again, as an Alertmanager duration with units `y`, `w`, `d`, `h`, `m`, `s`, `ms` in descending order, each at most once, such as `5m`, `12h` or `1h30m`. Delivery is evaluated every 5m, so shorter intervals take effect at 5m and other intervals round up to a multiple of 5m. Zero, a unit out of order and a total beyond what Alertmanager accepts fail the render. Empty keeps the 12h default.","type":"string","default":""},"severities":{"description":"Severities the notify_email route accepts (e.g. [\"critical\",\"major\",\"warning\"]). Empty keeps the default of critical and major. List every severity in use to email opted-in alerts regardless of severity. Entries must be plain severity names, and anything else fails the render.","type":"array","default":[],"items":{"type":"string"}},"smarthost":{"description":"SMTP server as `host:port` (e.g. \"smtp.example.com:465\"). A value without a port, an invalid bracketed IP address or a port with leading zeros fails the render. Port 465 uses implicit TLS, other ports use STARTTLS.","type":"string","default":""},"template":{"description":"Text of an Alertmanager template file that changes how the emails look. Define `email.default.subject` and `email.default.html` in it to replace the subject and the HTML body Alertmanager uses for every email, for example to mark the severity or to link to Alerta. The chart stores the text in a ConfigMap named `alertmanager-email-template` and lists it among the Alertmanager templates, so Alertmanager parses it together with its configuration. A template that does not parse fails that load, a running Alertmanager keeps the configuration it already has and a pod that starts with it does not come up. A template that parses but fails on a notification, for example on a missing key, drops that email. The text must contain at least one `define` action. Empty keeps the Alertmanager defaults.","type":"string","default":""},"to":{"description":"Recipient address, or several separated by commas. A display name is accepted, such as `Oncall `, as long as it carries no comma. Quote display names containing punctuation such as `:`, `@`, parentheses or `?`. RFC 2047 encoded words are not accepted. IP domain literals accept dotted-quad IPv4 only. IPv6 literals are rejected because Go mail parsers disagree on their syntax across versions. ASCII control characters are rejected. Addresses must be literal, so Alertmanager template actions are rejected. The accepted form is deliberately narrower than RFC 5322. The local part must be a dot-atom, with no leading, trailing or doubled dot, and quoted local parts, comments and a `mailto:` prefix are rejected. An address outside that form fails the render, because Alertmanager parses addresses only when it sends and drops the notification with no record a tenant can read.","type":"string","default":""}}},"slack":{"description":"Configuration for Slack alerts.","type":"object","default":{},"required":["url"],"properties":{"disabledSeverity":{"description":"List of severities without alerts (e.g. [\"informational\",\"warning\"]).","type":"array","default":[],"items":{"type":"string"}},"url":{"description":"Configuration uri for Slack alerts.","type":"string","default":""}}},"telegram":{"description":"Configuration for Telegram alerts.","type":"object","default":{},"required":["chatID","token"],"properties":{"chatID":{"description":"Telegram chat ID(s), separated by commas.","type":"string","default":""},"disabledSeverity":{"description":"List of severities without alerts (e.g. [\"informational\",\"warning\"]).","type":"array","default":[],"items":{"type":"string"}},"token":{"description":"Telegram bot token.","type":"string","default":""}}}}},"resources":{"description":"Resource configuration.","type":"object","default":{},"properties":{"limits":{"description":"Resource limits.","type":"object","default":{},"properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"requests":{"description":"Resource requests.","type":"object","default":{},"properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"storage":{"description":"Persistent volume size for the database.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used for the database.","type":"string","default":""}}},"grafana":{"description":"Configuration for Grafana.","type":"object","default":{},"properties":{"db":{"description":"Database configuration.","type":"object","default":{},"properties":{"size":{"description":"Persistent volume size for the database.","type":"string","default":"10Gi"}}},"resources":{"description":"Resource configuration.","type":"object","default":{},"properties":{"limits":{"description":"Resource limits.","type":"object","default":{},"properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"requests":{"description":"Resource requests.","type":"object","default":{},"properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}}}},"vmagent":{"description":"Configuration for VictoriaMetrics Agent.","type":"object","default":{},"properties":{"externalLabels":{"description":"External labels applied to all metrics.","type":"object","default":{"cluster":"cozystack"}},"remoteWrite":{"description":"Remote write configuration.","type":"object","default":{"urls":["http://vminsert-shortterm:8480/insert/0/prometheus","http://vminsert-longterm:8480/insert/0/prometheus"]}}}},"oidc":{"description":"OIDC authentication for the Grafana instance. See https://cozystack.io/docs/v1.6/operations/services/monitoring/oidc-authentication/ for the operator guide.","type":"object","default":{},"required":["mode"],"properties":{"customConfig":{"description":"Tenant-supplied `[auth.generic_oauth]` payload; consumed only when `mode: CustomConfig`.","type":"object","default":{},"properties":{"config":{"description":"Inline map of `[auth.generic_oauth]` keys. Written verbatim into the Grafana CR `spec.config`. Mutually exclusive with `secretRef.name`.","type":"object","default":{}},"secretRef":{"description":"Reference to a Secret holding an `auth.ini` fragment; mounted under `/etc/grafana/oidc` and loaded via `GF_PATHS_CUSTOM_INI`. With shared-central tracing only its `auth.ini` key is mounted, so the fragment cannot read another key of the Secret with `$__file{}`.","type":"object","default":{},"properties":{"name":{"description":"Name of an existing Secret in the release namespace whose `auth.ini` key holds a ready-made `[auth.generic_oauth]` block. Mutually exclusive with `customConfig.config`.","type":"string","default":""}}}}},"mode":{"description":"Identity mode. `None`: no OIDC, only the `grafana-admin-password` Secret works. `System`: trust the platform `cozy` realm via a per-instance confidential client and audience binding; zero-config default. `CustomConfig`: trust a tenant-supplied issuer directly (BYO); `cozy` is not in the path.","type":"string","default":"None","enum":["None","System","CustomConfig"]},"users":{"description":"Users granted access to this Grafana instance's org. Each entry is reconciled into Grafana org membership + role by a chart-owned post-install/post-upgrade Job. Works for both `System` and `CustomConfig` modes; ignored when `mode: None`. Users not listed here get nothing (no default org role from group mapping).","type":"array","default":[],"items":{"type":"object","required":["email","role"],"properties":{"email":{"description":"Email address matched against the `email` claim from the issuer. The chart pre-provisions a Grafana account with this email so the users-Job can set the org role before the operator's first login (`oauth_allow_insecure_email_lookup = true` glues the two together). Must match a conservative pattern: local part is `[A-Za-z0-9._%+-]+`, domain is `[A-Za-z0-9.-]+\\.[A-Za-z]{2,}`. Whitespace, quoted literals, and unusual URL-reserved characters are rejected on schema level so the reconcile-Job's shell handling never has to defend against malformed input.","type":"string"},"role":{"description":"Grafana org role to bind: `Admin`, `Editor`, or `Viewer`. Applied to the release's Grafana org (`Main Org.`); server-level `GrafanaAdmin` promotion is out of scope for Phase 1. Required — the users-Job POSTs `{loginOrEmail, role}` to `/api/orgs/1/users` and PATCHes `{role}` to converge across re-runs; omitting the field would render `role: \"null\"` into the body and fail with HTTP 400.","type":"string","enum":["Admin","Editor","Viewer"]}}}}}}}} release: prefix: "" labels: @@ -38,7 +38,7 @@ spec: description: Monitoring and observability stack module: true icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODdfMzI2OCkiLz4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzY4N18zMjY4KSI+CjxwYXRoIGQ9Ik04OS41MDM5IDExMS43MDdINTQuNDk3QzU0LjE3MjcgMTExLjcwNyA1NC4wMTA4IDExMS4yMjEgNTQuMzM0OSAxMTEuMDU5TDU3LjI1MjIgMTA4Ljk1MkM2MC4zMzE0IDEwNi42ODMgNjEuOTUyMiAxMDIuNjMxIDYwLjk3OTcgOTguNzQxMkg4My4wMjFDODIuMDQ4NSAxMDIuNjMxIDgzLjY2OTMgMTA2LjY4MyA4Ni43NDg1IDEwOC45NTJMODkuNjY1OCAxMTEuMDU5Qzg5Ljk5IDExMS4yMjEgODkuODI3OSAxMTEuNzA3IDg5LjUwMzkgMTExLjcwN1oiIGZpbGw9IiNCMEI2QkIiLz4KPHBhdGggZD0iTTExMy4zMjggOTguNzQxSDMwLjY3MjVDMjcuNTkzMSA5OC43NDEgMjUgOTYuMTQ4IDI1IDkzLjA2ODdWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJWOTMuMDY4N0MxMTkgOTYuMTQ4IDExNi40MDcgOTguNzQxIDExMy4zMjggOTguNzQxWiIgZmlsbD0iI0U4RURFRSIvPgo8cGF0aCBkPSJNMTE5IDg0LjE1NDlIMjVWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJMMTE5IDg0LjE1NDlaIiBmaWxsPSIjMzg0NTRGIi8+CjxwYXRoIGQ9Ik05MC42Mzc0IDExNi41NjlINTMuMzYxNkM1Mi4wNjUxIDExNi41NjkgNTAuOTMwNyAxMTUuNDM1IDUwLjkzMDcgMTE0LjEzOEM1MC45MzA3IDExMi44NDEgNTIuMDY1MSAxMTEuNzA3IDUzLjM2MTYgMTExLjcwN0g5MC42Mzc0QzkxLjkzMzkgMTExLjcwNyA5My4wNjg0IDExMi44NDEgOTMuMDY4NCAxMTQuMTM4QzkzLjA2ODQgMTE1LjQzNSA5MS45MzM5IDExNi41NjkgOTAuNjM3NCAxMTYuNTY5WiIgZmlsbD0iI0U4RURFRSIvPgo8cGF0aCBkPSJNNTQuMTcyMiAzOC43NzU3SDMzLjEwMzJDMzIuMTMwNyAzOC43NzU3IDMxLjQ4MjQgMzguMTI3NCAzMS40ODI0IDM3LjE1NDlDMzEuNDgyNCAzNi4xODI0IDMyLjEzMDcgMzUuNTM0MiAzMy4xMDMyIDM1LjUzNDJINTQuMTcyMkM1NS4xNDQ3IDM1LjUzNDIgNTUuNzkzIDM2LjE4MjQgNTUuNzkzIDM3LjE1NDlDNTUuNzkyOCAzOC4xMjc0IDU1LjE0NDUgMzguNzc1NyA1NC4xNzIyIDM4Ljc3NTdaIiBmaWxsPSIjREQzNDJFIi8+CjxwYXRoIGQ9Ik02My44OTYzIDQ1LjI1OTFINDEuMjA2N0M0MC4yMzQyIDQ1LjI1OTEgMzkuNTg1OSA0NC42MTA4IDM5LjU4NTkgNDMuNjM4M0MzOS41ODU5IDQyLjY2NTggNDAuMjM0MiA0Mi4wMTc2IDQxLjIwNjcgNDIuMDE3Nkg2My44OTYzQzY0Ljg2ODggNDIuMDE3NiA2NS41MTcxIDQyLjY2NTggNjUuNTE3MSA0My42MzgzQzY1LjUxNzEgNDQuNjEwOCA2NC44Njg4IDQ1LjI1OTEgNjMuODk2MyA0NS4yNTkxWiIgZmlsbD0iIzczODNCRiIvPgo8cGF0aCBkPSJNMzQuNzI0IDQ1LjI1OTFIMzMuMTAzMkMzMi4xMzA3IDQ1LjI1OTEgMzEuNDgyNCA0NC42MTA4IDMxLjQ4MjQgNDMuNjM4M0MzMS40ODI0IDQyLjY2NTggMzIuMTMwNyA0Mi4wMTc2IDMzLjEwMzIgNDIuMDE3NkgzNC43MjRDMzUuNjk2NCA0Mi4wMTc2IDM2LjM0NDcgNDIuNjY1OCAzNi4zNDQ3IDQzLjYzODNDMzYuMzQ0NyA0NC42MTA4IDM1LjY5NjMgNDUuMjU5MSAzNC43MjQgNDUuMjU5MVoiIGZpbGw9IiM0MkIwNUMiLz4KPHBhdGggZD0iTTYzLjg5NjMgMzguNzc1N0g2MC42NTQ5QzU5LjY4MjQgMzguNzc1NyA1OS4wMzQyIDM4LjEyNzQgNTkuMDM0MiAzNy4xNTQ5QzU5LjAzNDIgMzYuMTgyNCA1OS42ODI0IDM1LjUzNDIgNjAuNjU0OSAzNS41MzQySDYzLjg5NjNDNjQuODY4OCAzNS41MzQyIDY1LjUxNzEgMzYuMTgyNCA2NS41MTcxIDM3LjE1NDlDNjUuNTE3MSAzOC4xMjc0IDY0Ljg2ODggMzguNzc1NyA2My44OTYzIDM4Ljc3NTdaIiBmaWxsPSIjRUNCQTE2Ii8+CjxwYXRoIGQ9Ik00Ny42ODkzIDUxLjc0MTNIMzMuMTAzMkMzMi4xMzA3IDUxLjc0MTMgMzEuNDgyNCA1MS4wOTMxIDMxLjQ4MjQgNTAuMTIwNkMzMS40ODI0IDQ5LjE0ODEgMzIuMTMwNyA0OC41IDMzLjEwMzIgNDguNUg0Ny42ODkzQzQ4LjY2MTggNDguNSA0OS4zMTAxIDQ5LjE0ODMgNDkuMzEwMSA1MC4xMjA4QzQ5LjMxMDEgNTEuMDkzMyA0OC42NjE4IDUxLjc0MTMgNDcuNjg5MyA1MS43NDEzWiIgZmlsbD0iI0REMzQyRSIvPgo8cGF0aCBkPSJNNjMuODk2OCA1MS43NDEzSDU0LjE3MjVDNTMuMiA1MS43NDEzIDUyLjU1MTggNTEuMDkzMSA1Mi41NTE4IDUwLjEyMDZDNTIuNTUxOCA0OS4xNDgxIDUzLjIwMDIgNDguNSA1NC4xNzI3IDQ4LjVINjMuODk2OUM2NC44Njk0IDQ4LjUgNjUuNTE3NyA0OS4xNDgzIDY1LjUxNzcgNTAuMTIwOEM2NS41MTc3IDUxLjA5MzMgNjQuODY5MiA1MS43NDEzIDYzLjg5NjggNTEuNzQxM1oiIGZpbGw9IiNFQ0JBMTYiLz4KPHBhdGggZD0iTTU0LjE3MjIgNTguMjI0SDMzLjEwMzJDMzIuMTMwNyA1OC4yMjQgMzEuNDgyNCA1Ny41NzU3IDMxLjQ4MjQgNTYuNjAzMkMzMS40ODI0IDU1LjYzMDcgMzIuMTMwNyA1NC45ODI0IDMzLjEwMzIgNTQuOTgyNEg1NC4xNzIyQzU1LjE0NDcgNTQuOTgyNCA1NS43OTMgNTUuNjMwNyA1NS43OTMgNTYuNjAzMkM1NS43OTMgNTcuNTc1NyA1NS4xNDQ1IDU4LjIyNCA1NC4xNzIyIDU4LjIyNFoiIGZpbGw9IiM0MkIwNUMiLz4KPHBhdGggZD0iTTYzLjg5NjMgNjQuNzA3NEg0MS4yMDY3QzQwLjIzNDIgNjQuNzA3NCAzOS41ODU5IDY0LjA1OTEgMzkuNTg1OSA2My4wODY2QzM5LjU4NTkgNjIuMTE0MSA0MC4yMzQyIDYxLjQ2NTggNDEuMjA2NyA2MS40NjU4SDYzLjg5NjNDNjQuODY4OCA2MS40NjU4IDY1LjUxNzEgNjIuMTE0MSA2NS41MTcxIDYzLjA4NjZDNjUuNTE3MSA2NC4wNTkxIDY0Ljg2ODggNjQuNzA3NCA2My44OTYzIDY0LjcwNzRaIiBmaWxsPSIjRUNCQTE2Ii8+CjxwYXRoIGQ9Ik0zNC43MjQgNjQuNzA3NEgzMy4xMDMyQzMyLjEzMDcgNjQuNzA3NCAzMS40ODI0IDY0LjA1OTEgMzEuNDgyNCA2My4wODY2QzMxLjQ4MjQgNjIuMTE0MSAzMi4xMzA3IDYxLjQ2NTggMzMuMTAzMiA2MS40NjU4SDM0LjcyNEMzNS42OTY0IDYxLjQ2NTggMzYuMzQ0NyA2Mi4xMTQxIDM2LjM0NDcgNjMuMDg2NkMzNi4zNDQ3IDY0LjA1OTEgMzUuNjk2MyA2NC43MDc0IDM0LjcyNCA2NC43MDc0WiIgZmlsbD0iI0REMzQyRSIvPgo8cGF0aCBkPSJNNDcuNjg5MyA3MS4xODk4SDMzLjEwMzJDMzIuMTMwNyA3MS4xODk4IDMxLjQ4MjQgNzAuNTQxNSAzMS40ODI0IDY5LjU2OUMzMS40ODI0IDY4LjU5NjUgMzIuMTMwNyA2Ny45NDgyIDMzLjEwMzIgNjcuOTQ4Mkg0Ny42ODkzQzQ4LjY2MTggNjcuOTQ4MiA0OS4zMTAxIDY4LjU5NjUgNDkuMzEwMSA2OS41NjlDNDkuMzEwMSA3MC41NDE1IDQ4LjY2MTggNzEuMTg5OCA0Ny42ODkzIDcxLjE4OThaIiBmaWxsPSIjNDJCMDVDIi8+CjxwYXRoIGQ9Ik02My44OTY4IDcxLjE4OThINTQuMTcyNUM1My4yIDcxLjE4OTggNTIuNTUxOCA3MC41NDE1IDUyLjU1MTggNjkuNTY5QzUyLjU1MTggNjguNTk2NSA1My4yIDY3Ljk0ODIgNTQuMTcyNSA2Ny45NDgySDYzLjg5NjhDNjQuODY5MiA2Ny45NDgyIDY1LjUxNzUgNjguNTk2NSA2NS41MTc1IDY5LjU2OUM2NS41MTc1IDcwLjU0MTUgNjQuODY5MiA3MS4xODk4IDYzLjg5NjggNzEuMTg5OFoiIGZpbGw9IiM3MzgzQkYiLz4KPHBhdGggZD0iTTU0LjE3MjIgNzcuNjcyMkgzMy4xMDMyQzMyLjEzMDcgNzcuNjcyMiAzMS40ODI0IDc3LjAyMzkgMzEuNDgyNCA3Ni4wNTE0QzMxLjQ4MjQgNzUuMDc4OSAzMi4xMzA3IDc0LjQzMDcgMzMuMTAzMiA3NC40MzA3SDU0LjE3MjJDNTUuMTQ0NyA3NC40MzA3IDU1Ljc5MyA3NS4wNzg5IDU1Ljc5MyA3Ni4wNTE0QzU1Ljc5MjggNzcuMDIzOSA1NS4xNDQ1IDc3LjY3MjIgNTQuMTcyMiA3Ny42NzIyWiIgZmlsbD0iI0VDQkExNiIvPgo8cGF0aCBkPSJNNjMuODk2MyA3Ny42NzIySDYwLjY1NDlDNTkuNjgyNCA3Ny42NzIyIDU5LjAzNDIgNzcuMDIzOSA1OS4wMzQyIDc2LjA1MTRDNTkuMDM0MiA3NS4wNzg5IDU5LjY4MjQgNzQuNDMwNyA2MC42NTQ5IDc0LjQzMDdINjMuODk2M0M2NC44Njg4IDc0LjQzMDcgNjUuNTE3MSA3NS4wNzg5IDY1LjUxNzEgNzYuMDUxNEM2NS41MTcxIDc3LjAyMzkgNjQuODY4OCA3Ny42NzIyIDYzLjg5NjMgNzcuNjcyMloiIGZpbGw9IiM0MkIwNUMiLz4KPHBhdGggZD0iTTEwMS4xNzIgNzcuNjcyMkg4MC4xMDMyQzc5LjEzMDcgNzcuNjcyMiA3OC40ODI0IDc3LjAyMzkgNzguNDgyNCA3Ni4wNTE0Qzc4LjQ4MjQgNzUuMDc4OSA3OS4xMzA3IDc0LjQzMDcgODAuMTAzMiA3NC40MzA3SDEwMS4xNzJDMTAyLjE0NSA3NC40MzA3IDEwMi43OTMgNzUuMDc4OSAxMDIuNzkzIDc2LjA1MTRDMTAyLjc5MyA3Ny4wMjM5IDEwMi4xNDUgNzcuNjcyMiAxMDEuMTcyIDc3LjY3MjJaIiBmaWxsPSIjREQzNDJFIi8+CjxwYXRoIGQ9Ik0xMTAuODk2IDc3LjY3MjJIMTA3LjY1NUMxMDYuNjgyIDc3LjY3MjIgMTA2LjAzNCA3Ny4wMjM5IDEwNi4wMzQgNzYuMDUxNEMxMDYuMDM0IDc1LjA3ODkgMTA2LjY4MiA3NC40MzA3IDEwNy42NTUgNzQuNDMwN0gxMTAuODk2QzExMS44NjkgNzQuNDMwNyAxMTIuNTE3IDc1LjA3ODkgMTEyLjUxNyA3Ni4wNTE0QzExMi41MTcgNzcuMDIzOSAxMTEuODY5IDc3LjY3MjIgMTEwLjg5NiA3Ny42NzIyWiIgZmlsbD0iIzQyQjA1QyIvPgo8cGF0aCBkPSJNNjMuODk2MyA1OC4yMjRINjAuNjU0OUM1OS42ODI0IDU4LjIyNCA1OS4wMzQyIDU3LjU3NTcgNTkuMDM0MiA1Ni42MDMyQzU5LjAzNDIgNTUuNjMwNyA1OS42ODI0IDU0Ljk4MjQgNjAuNjU0OSA1NC45ODI0SDYzLjg5NjNDNjQuODY4OCA1NC45ODI0IDY1LjUxNzEgNTUuNjMwNyA2NS41MTcxIDU2LjYwMzJDNjUuNTE3MSA1Ny41NzU3IDY0Ljg2ODggNTguMjI0IDYzLjg5NjMgNTguMjI0WiIgZmlsbD0iIzczODNCRiIvPgo8cGF0aCBkPSJNMTEyLjUxNyA1MS43NDExQzExMi41MTcgNjAuNjU0OSAxMDUuMjI0IDY3Ljk0OCA5Ni4zMTA0IDY3Ljk0OEM4Ny4zOTY2IDY3Ljk0OCA4MC4xMDM1IDYwLjY1NDkgODAuMTAzNSA1MS43NDExQzgwLjEwMzUgNDIuODI3MyA4Ny4zOTY2IDM1LjUzNDIgOTYuMzEwNCAzNS41MzQyQzEwNS4yMjQgMzUuNTM0MiAxMTIuNTE3IDQyLjgyNzMgMTEyLjUxNyA1MS43NDExWiIgZmlsbD0iI0VDQkExNiIvPgo8cGF0aCBkPSJNODAuMTAzNSA1MS43NDExQzgwLjEwMzUgNTIuMjI3MyA4MC4xMDM1IDUyLjg3NTUgODAuMTAzNSA1My4zNjE5SDk2LjMxMDRWMzUuNTM0MkM4Ny4zOTY2IDM1LjUzNDIgODAuMTAzNSA0Mi44MjczIDgwLjEwMzUgNTEuNzQxMVoiIGZpbGw9IiM0MkIwNUMiLz4KPC9nPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGlkPSJwYWludDBfbGluZWFyXzY4N18zMjY4IiB4MT0iMS4yMzIzOWUtMDYiIHkxPSItOS41MDAwMSIgeDI9IjE2OCIgeTI9IjE2MiIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjOEZEREZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzAwNzVGRiIvPgo8L2xpbmVhckdyYWRpZW50Pgo8Y2xpcFBhdGggaWQ9ImNsaXAwXzY4N18zMjY4Ij4KPHJlY3Qgd2lkdGg9Ijk0IiBoZWlnaHQ9Ijk0IiBmaWxsPSJ3aGl0ZSIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoMjUgMjUpIi8+CjwvY2xpcFBhdGg+CjwvZGVmcz4KPC9zdmc+Cg== - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "metricsStorages"], ["spec", "logsStorages"], ["spec", "tracingStorages"], ["spec", "tracingCentralHost"], ["spec", "tracingCollector"], ["spec", "tracingCollector", "enabled"], ["spec", "tracingCollector", "replicas"], ["spec", "tracingCollector", "samplingPercentage"], ["spec", "tracingCollector", "redactAttributes"], ["spec", "tracingCollector", "resources"], ["spec", "tracingCollector", "resources", "limits"], ["spec", "tracingCollector", "resources", "limits", "cpu"], ["spec", "tracingCollector", "resources", "limits", "memory"], ["spec", "tracingCollector", "resources", "requests"], ["spec", "tracingCollector", "resources", "requests", "cpu"], ["spec", "tracingCollector", "resources", "requests", "memory"], ["spec", "alerta"], ["spec", "alerta", "storage"], ["spec", "alerta", "storageClassName"], ["spec", "alerta", "resources"], ["spec", "alerta", "resources", "limits"], ["spec", "alerta", "resources", "limits", "cpu"], ["spec", "alerta", "resources", "limits", "memory"], ["spec", "alerta", "resources", "requests"], ["spec", "alerta", "resources", "requests", "cpu"], ["spec", "alerta", "resources", "requests", "memory"], ["spec", "alerta", "alerts"], ["spec", "alerta", "alerts", "telegram"], ["spec", "alerta", "alerts", "telegram", "token"], ["spec", "alerta", "alerts", "telegram", "chatID"], ["spec", "alerta", "alerts", "telegram", "disabledSeverity"], ["spec", "alerta", "alerts", "slack"], ["spec", "alerta", "alerts", "slack", "url"], ["spec", "alerta", "alerts", "slack", "disabledSeverity"], ["spec", "alerta", "alerts", "email"], ["spec", "alerta", "alerts", "email", "to"], ["spec", "alerta", "alerts", "email", "from"], ["spec", "alerta", "alerts", "email", "smarthost"], ["spec", "alerta", "alerts", "email", "authUsername"], ["spec", "alerta", "alerts", "email", "repeatInterval"], ["spec", "alerta", "alerts", "email", "alertnames"], ["spec", "alerta", "alerts", "email", "severities"], ["spec", "grafana"], ["spec", "grafana", "db"], ["spec", "grafana", "db", "size"], ["spec", "grafana", "resources"], ["spec", "grafana", "resources", "limits"], ["spec", "grafana", "resources", "limits", "cpu"], ["spec", "grafana", "resources", "limits", "memory"], ["spec", "grafana", "resources", "requests"], ["spec", "grafana", "resources", "requests", "cpu"], ["spec", "grafana", "resources", "requests", "memory"], ["spec", "vmagent"], ["spec", "vmagent", "externalLabels"], ["spec", "vmagent", "externalLabels", "cluster"], ["spec", "vmagent", "remoteWrite"], ["spec", "vmagent", "remoteWrite", "urls"], ["spec", "oidc"], ["spec", "oidc", "mode"], ["spec", "oidc", "customConfig"], ["spec", "oidc", "customConfig", "config"], ["spec", "oidc", "customConfig", "secretRef"], ["spec", "oidc", "customConfig", "secretRef", "name"], ["spec", "oidc", "users"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "metricsStorages"], ["spec", "logsStorages"], ["spec", "tracingStorages"], ["spec", "tracingCentralHost"], ["spec", "tracingCollector"], ["spec", "tracingCollector", "enabled"], ["spec", "tracingCollector", "replicas"], ["spec", "tracingCollector", "samplingPercentage"], ["spec", "tracingCollector", "redactAttributes"], ["spec", "tracingCollector", "resources"], ["spec", "tracingCollector", "resources", "limits"], ["spec", "tracingCollector", "resources", "limits", "cpu"], ["spec", "tracingCollector", "resources", "limits", "memory"], ["spec", "tracingCollector", "resources", "requests"], ["spec", "tracingCollector", "resources", "requests", "cpu"], ["spec", "tracingCollector", "resources", "requests", "memory"], ["spec", "alerta"], ["spec", "alerta", "storage"], ["spec", "alerta", "storageClassName"], ["spec", "alerta", "resources"], ["spec", "alerta", "resources", "limits"], ["spec", "alerta", "resources", "limits", "cpu"], ["spec", "alerta", "resources", "limits", "memory"], ["spec", "alerta", "resources", "requests"], ["spec", "alerta", "resources", "requests", "cpu"], ["spec", "alerta", "resources", "requests", "memory"], ["spec", "alerta", "alerts"], ["spec", "alerta", "alerts", "telegram"], ["spec", "alerta", "alerts", "telegram", "token"], ["spec", "alerta", "alerts", "telegram", "chatID"], ["spec", "alerta", "alerts", "telegram", "disabledSeverity"], ["spec", "alerta", "alerts", "slack"], ["spec", "alerta", "alerts", "slack", "url"], ["spec", "alerta", "alerts", "slack", "disabledSeverity"], ["spec", "alerta", "alerts", "email"], ["spec", "alerta", "alerts", "email", "to"], ["spec", "alerta", "alerts", "email", "from"], ["spec", "alerta", "alerts", "email", "smarthost"], ["spec", "alerta", "alerts", "email", "authUsername"], ["spec", "alerta", "alerts", "email", "repeatInterval"], ["spec", "alerta", "alerts", "email", "alertnames"], ["spec", "alerta", "alerts", "email", "severities"], ["spec", "alerta", "alerts", "email", "template"], ["spec", "grafana"], ["spec", "grafana", "db"], ["spec", "grafana", "db", "size"], ["spec", "grafana", "resources"], ["spec", "grafana", "resources", "limits"], ["spec", "grafana", "resources", "limits", "cpu"], ["spec", "grafana", "resources", "limits", "memory"], ["spec", "grafana", "resources", "requests"], ["spec", "grafana", "resources", "requests", "cpu"], ["spec", "grafana", "resources", "requests", "memory"], ["spec", "vmagent"], ["spec", "vmagent", "externalLabels"], ["spec", "vmagent", "externalLabels", "cluster"], ["spec", "vmagent", "remoteWrite"], ["spec", "vmagent", "remoteWrite", "urls"], ["spec", "oidc"], ["spec", "oidc", "mode"], ["spec", "oidc", "customConfig"], ["spec", "oidc", "customConfig", "config"], ["spec", "oidc", "customConfig", "secretRef"], ["spec", "oidc", "customConfig", "secretRef", "name"], ["spec", "oidc", "users"]] secrets: exclude: [] include: diff --git a/packages/system/monitoring/templates/alerta/alerta.yaml b/packages/system/monitoring/templates/alerta/alerta.yaml index 6376eaa9e..0938f7e0b 100644 --- a/packages/system/monitoring/templates/alerta/alerta.yaml +++ b/packages/system/monitoring/templates/alerta/alerta.yaml @@ -10,8 +10,14 @@ {{- $emailAlertnames := $email.alertnames | default list }} {{- $emailSeverities := $email.severities | default (list "critical" "major") }} {{- $emailPasswordSecret := "alertmanager-email-password" }} +{{- $emailTemplateConfigMap := "alertmanager-email-template" }} +{{- $emailTemplateRaw := index $email "template" }} +{{- if not (or (kindIs "string" $emailTemplateRaw) (kindIs "invalid" $emailTemplateRaw)) }} +{{- fail (printf "alerta.alerts.email.template must be the text of an Alertmanager template file, got a %s" (kindOf $emailTemplateRaw)) }} +{{- end }} +{{- $emailTemplate := $emailTemplateRaw | default "" }} {{- $emailSet := list }} -{{- range $field := list "to" "from" "smarthost" "authUsername" "repeatInterval" "alertnames" "severities" }} +{{- range $field := list "to" "from" "smarthost" "authUsername" "repeatInterval" "alertnames" "severities" "template" }} {{- if index $email $field }}{{ $emailSet = append $emailSet $field }}{{ end }} {{- end }} {{- $emailEnabled := gt (len $emailSet) 0 }} @@ -101,6 +107,11 @@ {{- fail (printf "alerta.alerts.email.severities entries must be plain severity names, got %q" (toString $severity)) }} {{- end }} {{- end }} +{{- if $emailTemplate }} +{{- if not (regexMatch "\\{\\{-?\\s*define\\s" $emailTemplate) }} +{{- fail "alerta.alerts.email.template must define named templates, such as email.default.subject and email.default.html, because Alertmanager only uses the file through the names it defines" }} +{{- end }} +{{- end }} {{- end }} {{- $apiKey := randAlphaNum 32 }} @@ -408,6 +419,15 @@ stringData: {{- end }} send_resolved: true {{- end }} +{{- if and $emailEnabled $emailTemplate }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $emailTemplateConfigMap }} +data: + email.tmpl: {{ $emailTemplate | quote }} +{{- end }} --- apiVersion: operator.victoriametrics.com/v1beta1 kind: VMAlertmanager @@ -425,6 +445,11 @@ spec: secrets: - {{ $emailPasswordSecret | quote }} {{- end }} + {{- if and $emailEnabled $emailTemplate }} + templates: + - name: {{ $emailTemplateConfigMap }} + key: email.tmpl + {{- end }} podMetadata: labels: policy.cozystack.io/allow-to-apiserver: "true" diff --git a/packages/system/monitoring/tests/alertmanager_email_test.yaml b/packages/system/monitoring/tests/alertmanager_email_test.yaml index 0d92cd2ed..c84d5479c 100644 --- a/packages/system/monitoring/tests/alertmanager_email_test.yaml +++ b/packages/system/monitoring/tests/alertmanager_email_test.yaml @@ -1149,3 +1149,70 @@ tests: asserts: - failedTemplate: errorMessage: "alerta.alerts.email.from must be one address, such as alerts@example.org or Alerts , got \"alerts@[IPv6:2001:db8::25]\"" + + # alerta.alerts.email.template carries an Alertmanager template file. It is + # stored in a ConfigMap and listed in VMAlertmanager spec.templates, so + # Alertmanager parses it while loading its configuration, and the definitions + # it holds can replace email.default.subject and email.default.html. + - it: mounts no email template when template is empty + set: + alerta: + alerts: + email: + to: "oncall@example.org" + from: "alerts@example.org" + smarthost: "smtp.example.org:465" + asserts: + - containsDocument: + apiVersion: v1 + kind: ConfigMap + name: alertmanager-email-template + any: true + not: true + - documentSelector: + path: kind + value: VMAlertmanager + notExists: + path: spec.templates + + - it: stores the template text verbatim and lists it among the Alertmanager templates + set: + alerta: + alerts: + email: + to: "oncall@example.org" + from: "alerts@example.org" + smarthost: "smtp.example.org:465" + authUsername: "mailer@example.org" + template: | + {{ define "email.default.subject" }}[prod] {{ if eq .Status "resolved" }}✅ resolved{{ else }}🔴 {{ .CommonLabels.alertname }}{{ end }}{{ end }} + {{ define "email.default.html" }}

{{ .CommonAnnotations.summary }}, проверка "кавычек" и \ слеша

{{ end }} + asserts: + - containsDocument: + apiVersion: v1 + kind: ConfigMap + name: alertmanager-email-template + any: true + - documentSelector: + path: metadata.name + value: alertmanager-email-template + equal: + path: data["email.tmpl"] + value: | + {{ define "email.default.subject" }}[prod] {{ if eq .Status "resolved" }}✅ resolved{{ else }}🔴 {{ .CommonLabels.alertname }}{{ end }}{{ end }} + {{ define "email.default.html" }}

{{ .CommonAnnotations.summary }}, проверка "кавычек" и \ слеша

{{ end }} + - documentSelector: + path: kind + value: VMAlertmanager + equal: + path: spec.templates + value: + - name: alertmanager-email-template + key: email.tmpl + - documentSelector: + path: kind + value: VMAlertmanager + equal: + path: spec.secrets + value: + - alertmanager-email-password diff --git a/packages/system/monitoring/tests/alertmanager_email_validation_test.yaml b/packages/system/monitoring/tests/alertmanager_email_validation_test.yaml index 9691a5746..5edc54242 100644 --- a/packages/system/monitoring/tests/alertmanager_email_validation_test.yaml +++ b/packages/system/monitoring/tests/alertmanager_email_validation_test.yaml @@ -226,3 +226,35 @@ tests: asserts: - failedTemplate: errorMessage: "alerta.alerts.email.to has an empty entry, got \" \"" + + - it: rejects a template without a define action + set: + alerta.alerts.email.template: "plain text {{ .Status }}" + asserts: + - failedTemplate: + errorMessage: "alerta.alerts.email.template must define named templates, such as email.default.subject and email.default.html, because Alertmanager only uses the file through the names it defines" + + - it: rejects a template that is not text + set: + alerta.alerts.email.template: + subject: "[prod]" + asserts: + - failedTemplate: + errorMessage: "alerta.alerts.email.template must be the text of an Alertmanager template file, got a map" + + - it: names template among the fields still set when the addresses are cleared + set: + alerta.alerts.email.to: "" + alerta.alerts.email.from: "" + alerta.alerts.email.smarthost: "" + alerta.alerts.email.template: '{{ define "email.default.subject" }}[prod]{{ end }}' + asserts: + - failedTemplate: + errorMessage: "alerta.alerts.email to, from and smarthost must all be set to enable email alerts. To turn email off, clear every field under alerta.alerts.email. Fields still set are template" + + - it: rejects a template given as a boolean instead of being dropped by the default + set: + alerta.alerts.email.template: false + asserts: + - failedTemplate: + errorMessage: "alerta.alerts.email.template must be the text of an Alertmanager template file, got a bool" diff --git a/packages/system/monitoring/values.yaml b/packages/system/monitoring/values.yaml index 2abaed368..aba63aab8 100644 --- a/packages/system/monitoring/values.yaml +++ b/packages/system/monitoring/values.yaml @@ -95,6 +95,7 @@ alerta: repeatInterval: "" alertnames: [] severities: [] + template: "" grafana: # Upstream Grafana image, pinned by digest and routed through # cozy-lib.image (so mirrored-registry installs work). Plugins (VictoriaLogs diff --git a/packages/system/seaweedfs/tests/volume_resize_hook_test.yaml b/packages/system/seaweedfs/tests/volume_resize_hook_test.yaml index 402748312..011a3de94 100644 --- a/packages/system/seaweedfs/tests/volume_resize_hook_test.yaml +++ b/packages/system/seaweedfs/tests/volume_resize_hook_test.yaml @@ -156,9 +156,11 @@ tests: resources: requests: storage: 10Gi + documentSelector: + path: kind + value: Job asserts: - - documentIndex: 0 - notMatchRegex: + - notMatchRegex: path: spec.template.spec.containers[0].args[0] pattern: kubectl patch pvc