Merge branch 'main' into feat/kubernetes-nodes-talos-sysctls

The generated cozyrds file is regenerated from the merged values.yaml.

Assisted-by: LLM
Signed-off-by: Aleksei Sviridkin <f@lex.la>
This commit is contained in:
Aleksei Sviridkin committed 2026-11-16 17:17:33 +03:00
commit 0d239fab81
63 files changed
+1826 -424

No files matched your search

+5
View File
@@ -405,6 +405,11 @@ jobs:
- name: Run backport-batch tests
if: ${{ !cancelled() && needs.plan.outputs.backport_batch == 'true' }}
run: make test-backport-batch
- name: Check new Helm document indices
if: ${{ !cancelled() }}
env:
BASE_REF: ${{ github.event.pull_request.base.ref }}
run: python3 hack/check-helm-document-indices.py "$(git merge-base HEAD "origin/$BASE_REF")"
- name: Upload unit test report
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+1
View File
@@ -33,3 +33,4 @@ This list is sorted in chronological order, based on the submission date.
| [Hidora](https://hikube.cloud) | @matthieu-robin | 2025-09-17 | Hidora is a Swiss cloud provider delivering managed services and infrastructure solutions through datacenters located in Switzerland, ensuring data sovereignty and reliability. Its sovereign cloud platform, Hikube, is designed to run workloads with high availability across multiple datacenters, providing enterprises with a secure and scalable foundation for their applications based on Cozystack. |
| [QOSI](https://qosi.kz) | @tabu-a | 2025-10-04 | QOSI is a non-profit organization driving open-source adoption and digital sovereignty across Kazakhstan and Central Asia. We use Cozystack as a platform for deploying sovereign, GPU-enabled clouds and educational environments under the National AI Program. Our goal is to accelerate the region’s transition toward open, self-hosted cloud-native technologies |
| [Cloupard](https://cloupard.kz/) | @serjiott | 2025-12-18 | Cloupard is a public cloud provider offering IaaS and PaaS services via datacenters in Kazakhstan and Uzbekistan. Uses CozyStack on bare metal to extend its managed PaaS offerings. |
| [CageForge](https://cageforge.com/) | @themoriarti | 2026-11-03 | CageForge is a short-cycle R&D company providing independent expertise in cloud native, Cozystack and security for customers in finance, telecom, hosting and cloud providers. We run Cozystack both as a bare-metal installation and on Proxmox VE through the Proxmox integration, and contribute the Proxmox-related work back upstream. |
+4
View File
@@ -51,6 +51,10 @@ type ConfigSpec struct {
// List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).
// +kubebuilder:default:={}
Gpus []GPU `json:"gpus,omitempty"`
// Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.
// +kubebuilder:validation:Minimum=0
// +kubebuilder:validation:Maximum=2.147483647e+09
MaxUnavailable *int `json:"maxUnavailable,omitempty"`
// Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.
KernelModules []KernelModule `json:"kernelModules,omitempty"`
// Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`.
@@ -67,6 +67,11 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) {
*out = make([]GPU, len(*in))
copy(*out, *in)
}
if in.MaxUnavailable != nil {
in, out := &in.MaxUnavailable, &out.MaxUnavailable
*out = new(int)
**out = **in
}
if in.KernelModules != nil {
in, out := &in.KernelModules, &out.KernelModules
*out = make([]KernelModule, len(*in))
+1 -1
View File
@@ -42,7 +42,7 @@ If a package stamps a reference somewhere the two globs do not cover, add that f
## Invariants
- **Every first-party reference is tag+digest.** A bare tag with no digest is not reproducible and must not be committed. The one exception is the first commit of a new image: it has no digest until it is built, so it lands as a bare `<repo>:<tag>` that the producing `make image` replaces. PR CI and release preparation both stamp it before anything pulls it, but a tree applied without a build pulls a tag that does not exist.
- **Every first-party reference is tag+digest.** A bare tag with no digest is not reproducible and must not be committed. The one exception is the first commit of a new image: it has no digest until it is built, so it lands as a bare `<repo>:<tag>` that the producing `make image` replaces. PR CI and release preparation both stamp it before anything pulls it, but a tree applied without a build pulls a tag that does not exist. In PR CI the stamp comes from `hack/overlay-main-images.sh` when the PR does not rebuild the package, and it takes a bare first-party ref even from a file whose configuration drifted from the artifact, because a tag that was never pushed cannot install. The cost is that any other digest-less first-party ref in such a file also runs main's image with the PR's older config; third-party refs in it keep their committed pair.
- **Exactly one producer stamps each reference.** A reference no `Makefile` writes will never be refreshed by anything, including release preparation.
- **A package that stamps a reference is in the root `Makefile` `build:` list.** `build-matrix.sh`, the main build and release preparation all iterate that one list, so a package outside it is never rebuilt by any path and its pin freezes permanently. `hack/build-matrix_test.bats` computes the set of packages whose Makefile runs `docker buildx build` and fails on any the list misses.
- **Every consumer of references reads all three storage shapes**, via `hack/lib/image-refs.sh`.
+334
View File
@@ -0,0 +1,334 @@
#!/usr/bin/env bats
load test_helper
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
setup() {
strict_setup
tmp=$(mktemp -d)
cd "$tmp"
git init -q
mkdir -p packages/apps/example/templates packages/apps/example/tests
cat > packages/apps/example/templates/multi.yaml <<'EOF'
apiVersion: v1
kind: ConfigMap
metadata:
name: first
---
apiVersion: v1
kind: Secret
metadata:
name: second
EOF
head -n 4 packages/apps/example/templates/multi.yaml > packages/apps/example/templates/single.yaml
git add .
git -c user.name=Test -c user.email=test@example.org commit -qm fixture
base=$(git rev-parse HEAD)
}
teardown() {
cd "$REPO_ROOT"
rm -rf "$tmp"
}
write_suite() {
cat > packages/apps/example/tests/resource_test.yaml <<EOF
suite: resource selection
templates:
- templates/$1.yaml
tests:
- it: checks a resource
documentIndex: 0
asserts:
- notExists:
path: spec.missing
EOF
git add .
}
@test "rejects a new test-level index on a multi-document template" {
write_suite multi
if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then
echo "accepted a positional selector on a multi-document template" >&2
false
fi
grep -q 'resource_test.yaml:6: use documentSelector' output
}
@test "allows an index on a single-document template and ignores comments" {
write_suite single
printf '\n# documentIndex: 1\n' >> packages/apps/example/tests/resource_test.yaml
python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base"
}
@test "leaves existing indices alone when another assertion changes" {
write_suite multi
git -c user.name=Test -c user.email=test@example.org commit -qm existing
base=$(git rev-parse HEAD)
sed 's/spec.missing/spec.other/' packages/apps/example/tests/resource_test.yaml > changed.yaml
mv changed.yaml packages/apps/example/tests/resource_test.yaml
python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base"
}
@test "follows assertion template overrides in later YAML suites" {
write_suite single
cat >> packages/apps/example/tests/resource_test.yaml <<'EOF'
---
suite: overridden template
templates:
- templates/single.yaml
tests:
- it: checks another resource
asserts:
- notExists:
path: spec.missing
template: templates/multi.yaml
documentIndex: 1
EOF
if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then
echo "accepted an assertion-level positional selector" >&2
false
fi
grep -q 'use documentSelector.*templates/multi.yaml' output
}
@test "rejects an invalid base rather than passing an empty diff" {
if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" missing-base > output 2>&1; then
echo "accepted an invalid base" >&2
false
fi
}
@test "test-level indices follow each assertion's template override" {
write_suite multi
cat > packages/apps/example/tests/resource_test.yaml <<'EOF'
suite: assertion override
templates: [templates/single.yaml, templates/multi.yaml]
tests:
- it: checks only the single-document template
template: templates/multi.yaml
documentIndex: 0
asserts:
- template: templates/single.yaml
notExists:
path: spec.missing
EOF
python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base"
}
@test "conditional kinds inside one document are not multiple documents" {
write_suite single
cat > packages/apps/example/templates/single.yaml <<'EOF'
apiVersion: v1
{{ if .Values.secret }}
kind: Secret
{{ else }}
kind: ConfigMap
{{ end }}
metadata:
name: single
EOF
python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base"
}
@test "rejects a positional selector whose value is on a separate line" {
write_suite multi
cat > packages/apps/example/tests/resource_test.yaml <<'EOF'
suite: multiline index
templates: [templates/multi.yaml]
tests:
- it: checks a resource
documentIndex:
0
asserts:
- notExists:
path: spec.missing
EOF
if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then
echo "accepted a multiline positional selector" >&2
false
fi
grep -q 'resource_test.yaml:5: use documentSelector' output
}
@test "rejects a singular test template overriding a single-document suite" {
write_suite single
cat > packages/apps/example/tests/resource_test.yaml <<'EOF'
suite: test template override
templates: [templates/single.yaml]
tests:
- it: checks a resource
template: templates/multi.yaml
documentIndex: 0
asserts:
- notExists:
path: spec.missing
EOF
if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then
echo "accepted a positional selector on the test template override" >&2
false
fi
grep -q 'use documentSelector.*templates/multi.yaml' output
}
@test "allows a singular test template selecting one document" {
write_suite multi
cat > packages/apps/example/tests/resource_test.yaml <<'EOF'
suite: test template override
templates: [templates/multi.yaml]
tests:
- it: checks a resource
template: templates/single.yaml
documentIndex: 0
asserts:
- notExists:
path: spec.missing
EOF
python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base"
}
@test "plural test templates take precedence over the singular test template" {
write_suite multi
cat > packages/apps/example/tests/resource_test.yaml <<'EOF'
suite: test template override
templates: [templates/multi.yaml]
tests:
- it: checks a resource
templates: [templates/single.yaml]
template: templates/multi.yaml
documentIndex: 0
asserts:
- notExists:
path: spec.missing
EOF
python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base"
}
@test "an empty plural test override falls back to the singular template" {
write_suite single
cat > packages/apps/example/tests/resource_test.yaml <<'EOF'
suite: test template override
templates: [templates/single.yaml]
tests:
- it: checks a resource
templates: []
template: templates/multi.yaml
documentIndex: 0
asserts:
- notExists:
path: spec.missing
EOF
if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then
echo "accepted a positional selector after an empty plural override" >&2
false
fi
grep -q 'use documentSelector.*templates/multi.yaml' output
}
@test "does not check vendored chart suites" {
mkdir -p packages/apps/example/charts/sub/templates packages/apps/example/charts/sub/tests
cp packages/apps/example/templates/multi.yaml packages/apps/example/charts/sub/templates/
cat > packages/apps/example/charts/sub/tests/resource_test.yaml <<'EOF'
suite: upstream resource selection
templates: [templates/multi.yaml]
tests:
- it: checks an upstream resource
documentIndex: 0
asserts:
- notExists:
path: spec.missing
EOF
git add .
python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base"
}
@test "a new test-level index overrides existing assertion indices" {
cat > packages/apps/example/tests/resource_test.yaml <<'EOF'
suite: index inheritance
templates: [templates/multi.yaml]
tests:
- it: checks a resource
asserts:
- notExists:
path: spec.missing
documentIndex: 0
- notExists:
path: spec.other
documentIndex: 0
EOF
git add .
git -c user.name=Test -c user.email=test@example.org commit -qm existing
base=$(git rev-parse HEAD)
cat > packages/apps/example/tests/resource_test.yaml <<'EOF'
suite: index inheritance
templates: [templates/multi.yaml]
tests:
- it: checks a resource
documentIndex: 1
asserts:
- notExists:
path: spec.missing
documentIndex: 0
- notExists:
path: spec.other
documentIndex: 0
EOF
if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then
echo "accepted a new inherited positional selector" >&2
false
fi
grep -q 'resource_test.yaml:5: use documentSelector' output
}
@test "allows a pure suite rename with existing indices" {
write_suite multi
git -c user.name=Test -c user.email=test@example.org commit -qm existing
base=$(git rev-parse HEAD)
git mv packages/apps/example/tests/resource_test.yaml packages/apps/example/tests/renamed_test.yaml
python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base"
}
@test "checks added index lines in a renamed suite" {
write_suite multi
git -c user.name=Test -c user.email=test@example.org commit -qm existing
base=$(git rev-parse HEAD)
git mv packages/apps/example/tests/resource_test.yaml packages/apps/example/tests/renamed_test.yaml
cat > packages/apps/example/tests/renamed_test.yaml <<'EOF'
suite: resource selection
templates:
- templates/multi.yaml
tests:
- it: checks a resource
documentIndex: 1
asserts:
- notExists:
path: spec.missing
EOF
if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then
echo "accepted a new positional selector in a renamed suite" >&2
false
fi
grep -q 'renamed_test.yaml:6: use documentSelector' output
}
@test "an empty suite template list includes nested chart templates" {
mkdir packages/apps/example/templates/nested
mv packages/apps/example/templates/multi.yaml packages/apps/example/templates/nested/
cat > packages/apps/example/tests/resource_test.yaml <<'EOF'
suite: default templates
templates: []
tests:
- it: checks a resource
documentIndex: 0
asserts:
- notExists:
path: spec.missing
EOF
git add .
if python3 "$REPO_ROOT/hack/check-helm-document-indices.py" "$base" > output 2>&1; then
echo "accepted a positional selector with default templates" >&2
false
fi
grep -q 'use documentSelector.*templates/nested/multi.yaml' output
}
+87
View File
@@ -0,0 +1,87 @@
#!/usr/bin/env python3
"""Reject added positional selectors on multi-resource Helm templates.
Usage: python3 hack/check-helm-document-indices.py <base-ref>
Requires mikefarah/yq, like the chart test tooling.
"""
import json
from pathlib import Path
import re
import subprocess
import sys
def git(*args):
return subprocess.check_output(["git", *args], text=True)
def check(base):
"""Check added indices, leaving existing ones alone.
This static check counts --- separated blocks with a top-level kind; it does
not evaluate conditional documents or resources emitted by helpers/ranges.
"""
root = Path(git("rev-parse", "--show-toplevel").strip())
additions = {}
path = None
for line in git("diff", "--no-ext-diff", "-M", "--unified=0", base, "--",
":(glob)packages/*/*/tests/*_test.yaml").splitlines():
if line.startswith("diff --git "):
path = None
elif line.startswith("+++ b/"):
path = line[6:]
additions[path] = set()
elif line.startswith("@@"):
number = int(re.search(r"\+(\d+)", line).group(1))
elif path and line.startswith("+"):
if "documentIndex" in line:
additions[path].add(number)
number += 1
elif line.startswith(" "):
number += 1
errors = []
for path, added in additions.items():
if not added:
continue
# Preserve the key's source line while yq resolves multi-document YAML,
# flow mappings and anchors; a text-only suite parser misses those.
expression = ('[. | (.. | select(tag == "!!map" and has("documentIndex")) '
'| .documentIndex) |= {"line": key | line}]')
suites = json.loads(subprocess.check_output(
["yq", "eval-all", "-o=json", expression, str(root / path)], text=True))
chart = (root / path).parent.parent
for suite in suites:
if not suite:
continue
for test in suite.get("tests", []):
templates = test.get("templates") or (
[test["template"]] if test.get("template")
else suite.get("templates") or ["templates/**/*"])
for assertion in test.get("asserts", []):
index = test.get("documentIndex") or assertion.get("documentIndex", {})
number = index.get("line")
if number not in added:
continue
selected = [assertion["template"]] if "template" in assertion else templates
for pattern in selected:
matches = list(chart.glob(pattern))
if not matches:
matches = list((chart / "templates").glob(pattern))
for template in matches:
if not template.is_file():
continue
documents = re.split(r"(?m)^---[ \t]*(?:#.*)?$", template.read_text())
if sum(bool(re.search(r"(?m)^kind\s*:", doc)) for doc in documents) > 1:
errors.append(f"{path}:{number}: use documentSelector, not "
f"documentIndex, for {template.relative_to(root)}")
for error in sorted(set(errors)):
print(error, file=sys.stderr)
return bool(errors)
if __name__ == "__main__":
if len(sys.argv) != 2:
sys.exit("usage: python3 hack/check-helm-document-indices.py <base-ref>")
sys.exit(check(sys.argv[1]))
@@ -102,7 +102,9 @@ spec:
#
# nameOverride is outside the module's per-pool allowlist. The ComputePlane
# schema accepts it, so it reaches the module and the pool check below can
# see whether the module dropped it.
# see whether the module dropped it. maxUnavailable is set to 0 because a
# zero lost anywhere on the way would let a GPU pool fall back to the
# chart's default of 1; only this path crosses the aggregated API.
- script:
timeout: 3m
content: |
@@ -118,6 +120,7 @@ spec:
diskSize: 20Gi
instanceType: u1.medium
logSerialConsole: true
maxUnavailable: 0
minReplicas: 2
maxReplicas: 2
podCpuLimit: 2
@@ -175,8 +178,9 @@ spec:
enabled: true
exposeMethod: Proxied
# The pool takes the patched shape, and the nameOverride the patch sent
# does not reach it, nor does any other key outside the allowlist.
# The pool takes the patched shape, the explicit maxUnavailable of 0
# included, and the nameOverride the patch sent does not reach it, nor does
# any other key outside the allowlist.
- script:
timeout: 1m
content: |
@@ -186,8 +190,8 @@ spec:
sc=$(cozy_e2e_storage_class)
values=$(timeout -k 5 30 kubectl -n tenant-cplane get helmrelease kubernetes-nodes-computeplane-cluster-md0 -o json | jq -c '.spec.values')
echo "pool values: $values"
echo "$values" | jq -e --arg sc "$sc" '.storageClass == $sc and .minReplicas == 2 and .cluster == "computeplane-cluster" and .clusterReleaseName == "computeplane-cluster"'
extra=$(echo "$values" | jq -r 'keys - ["cluster","clusterReleaseName","minReplicas","maxReplicas","instanceType","diskSize","storageClass","roles","resources","gpus","kubelet","logSerialConsole","maxUnhealthy","nodeStartupTimeout","readyUnknownTimeout","readyFalseTimeout","podCpuLimit","podCpuRequest","maxNodeProvisionTime"] | .[]')
echo "$values" | jq -e --arg sc "$sc" '.storageClass == $sc and .minReplicas == 2 and .maxUnavailable == 0 and .cluster == "computeplane-cluster" and .clusterReleaseName == "computeplane-cluster"'
extra=$(echo "$values" | jq -r 'keys - ["cluster","clusterReleaseName","minReplicas","maxReplicas","instanceType","diskSize","storageClass","roles","resources","gpus","kubelet","logSerialConsole","maxUnhealthy","nodeStartupTimeout","readyUnknownTimeout","readyFalseTimeout","podCpuLimit","podCpuRequest","maxNodeProvisionTime","maxUnavailable"] | .[]')
if [ -n "$extra" ]; then
echo "pool HelmRelease carries keys outside the module allowlist: $extra" >&2
exit 1
+127 -7
View File
@@ -42,9 +42,13 @@
# a change: `yq -i`, which every `make image` uses to stamp the ref, drops one.
# If any OTHER line differs — the PR branched from a
# main whose config for that file differs from the artifact's base — the file is
# left on its committed ref (safe degradation, logged, never fatal). CI checks
# out the pull_request merge commit (current main + PR), so for an unbuilt file
# the config already matches current main and only ref lines differ.
# left on its committed ref (safe degradation, logged, never fatal), except
# that a bare first-party ref in it, one with no digest, still takes the
# artifact's ref (see replace_bare_refs). CI checks out the pull_request merge
# commit (main + PR), so for an unbuilt file the config matches main as of
# that merge commit. It is fixed when the run starts, so a rerun, or a PR not
# pushed to since, can be older than the artifact and drift on files it never
# touched.
#
# Usage: hack/overlay-main-images.sh <mainpkgs-dir> <built-matrix-json> [touched-pkg-dirs]
# <mainpkgs-dir> = extracted cozystack-packages:main tree; its root holds
@@ -123,8 +127,95 @@ img_line='(@sha256:|^[[:space:]]*(- )?(image|repository|registry|tag|digest):|^[
# match branch for this same shape and for this same reason.
bare_ref_line='^[^[:space:]#][^[:space:]]*/[^[:space:]]+$'
# A drifted file keeps its committed refs. For a ref pinned by digest that is an
# older image which pulls and matches the PR's config, so it is the safe answer.
# A first-party ref with no digest is not: docs/agents/image-refs.md forbids it
# except as the first commit of a new image, whose tag was never pushed, and
# keeping that makes the component fail to pull. The merge commit CI checks out
# is fixed when the run starts, so a PR whose merge commit predates a config
# change to such a file drifts on it, and a new image fails every such PR until
# a release stamps its ref (#5328).
#
# replace_bare_refs <cur> <new> rewrites, in <cur> only, each line whose value is
# a bare first-party ref (ghcr.io/cozystack/cozystack/<name>:<tag>, no digest)
# under an image key, taking the line from <new> that has the same key and
# indentation, an @sha256: digest and the same first-party image, i.e. a path
# ending in /cozystack/<name>. Exactly one such line must exist; two are
# ambiguous, and none means build-main did not stamp it, or the key moved
# (another indentation or parent) or the image was renamed on main since the
# PR's merge commit. Every other line of
# <cur> stays as the PR has it.
#
# The trade-off is deliberate. A bare first-party ref that does exist (an older
# release tag or a moving `latest`, #4264) also takes main's image, so it runs
# with the PR's older config rather than as committed. It is the same component
# built from the tree, and a digest-less ref is not reproducible anyway; the
# alternative is a ref that may not pull at all. Third-party refs are never
# touched: main can bump one alongside its config, and the committed pair is
# the one known to work together. The split shape (`repository:` + `tag:`
# without `digest:`) is not handled, nor is a single-quoted value or one
# followed by a `# comment`; such a line stays as committed, and no package
# writes a bare first-party ref that way today. Prints nothing and fails when
# the file could not be rewritten, so the caller counts it as failed.
#
# Prints the number of lines replaced.
replace_bare_refs() {
tmp="$1.overlay-bare.$$"
n=$(awk -v newf="$2" -v out="$tmp" '
function key_prefix(s) {
if (match(s, /^[[:space:]]*(- )?([A-Za-z]+[Ii]mage|image):[[:space:]]*/)) return substr(s, 1, RLENGTH)
return ""
}
function ref_value(s, p, v) {
v = substr(s, length(p) + 1)
sub(/[[:space:]]+$/, "", v)
gsub(/"/, "", v)
return v
}
function image_name(v) {
sub(/@.*/, "", v)
sub(/.*\//, "", v)
sub(/:.*/, "", v)
return v
}
BEGIN {
while ((getline l < newf) > 0) {
p = key_prefix(l)
if (p == "" || index(l, "@sha256:") == 0) continue
v = ref_value(l, p)
name = image_name(v)
if (index(v, "/cozystack/" name ":") == 0) continue
k = p SUBSEP name
cnt[k]++
line[k] = l
}
close(newf)
}
{
p = key_prefix($0)
if (p != "") {
v = ref_value($0, p)
if (v ~ /^ghcr\.io\/cozystack\/cozystack\/[^@[:space:]\/]+:[^@[:space:]\/]+$/) {
k = p SUBSEP image_name(v)
if (cnt[k] == 1) { print line[k] > out; r++; next }
}
}
print > out
}
END { close(out); print r + 0 }
' "$1")
case "$n" in
''|*[!0-9]*) rm -f "$tmp"; return 1 ;;
0) rm -f "$tmp" ;;
*) mv "$tmp" "$1" || { rm -f "$tmp"; return 1; } ;;
esac
echo "$n"
}
overlaid=0
same=0
bare=0
bare_files=""
skipped=0
drift=0
drift_files=""
@@ -162,8 +253,26 @@ for new in $(find "$MAINPKGS" -type d -name charts -prune -o \
if diff "$cur" "$new" | sed -n 's/^[<>] //p' \
| grep -vE '^[[:space:]]*$' | grep -qvE "$line_pat"; then
echo "drift (non-ref change) in $cur -> keeping committed ref"
drift=$((drift + 1))
r=0
case "$cur" in
*.yaml)
if ! r=$(replace_bare_refs "$cur" "$new"); then
echo "WARN: rewriting bare refs failed for $cur (keeping committed refs)"
failed=$((failed + 1))
r=0
fi
;;
esac
if [ "$r" -eq 0 ]; then
echo "drift (non-ref change) in $cur -> keeping committed ref"
else
echo "drift (non-ref change) in $cur -> keeping committed refs except bare first-party ones"
echo "bare first-party ref in $cur -> current-main ($r line(s)); its other refs stay committed"
bare=$((bare + r))
d=$(dirname "${cur#packages/}")
bare_files="$bare_files $d"
fi
# Name the PACKAGE, not the directory the file happens to sit in: an
# images/*.tag drift would otherwise read as "<pkg>/images", which is not
# something anyone can go and look at.
@@ -181,7 +290,7 @@ for new in $(find "$MAINPKGS" -type d -name charts -prune -o \
fi
done
echo "Overlay current-main images: overlaid=$overlaid same=$same skipped(rebuilt/owned/edited)=$skipped drift=$drift failed=$failed"
echo "Overlay current-main images: overlaid=$overlaid same=$same skipped(rebuilt/owned/edited)=$skipped drift=$drift bare-refs=$bare failed=$failed"
# Keeping the committed ref is the safe answer to a config that does not match
# the artifact, but it is not a neutral one: that component then runs its last
@@ -192,7 +301,18 @@ echo "Overlay current-main images: overlaid=$overlaid same=$same skipped(rebuilt
# cost three unrelated PRs, one round each.
if [ "$drift" -gt 0 ]; then
drift_pkgs=$(echo "$drift_files" | tr ' ' '\n' | grep -v '^$' | sort -u)
printf '::warning title=Image overlay drift::%s package(s) kept committed release refs, so they run release images in this lane rather than current main: %s\n' \
"$(printf '%s\n' "$drift_pkgs" | wc -l | tr -d ' ')" \
except=""
[ "$bare" -eq 0 ] || except=" (bare first-party refs excepted, see Image overlay bare refs)"
printf '::warning title=Image overlay drift::%s package(s) kept committed release refs%s, so they run release images in this lane rather than current main: %s\n' \
"$(printf '%s\n' "$drift_pkgs" | wc -l | tr -d ' ')" "$except" \
"$(printf '%s' "$drift_pkgs" | tr '\n' ' ' | sed 's/ $//')"
fi
# Among those, a package whose bare first-party ref took main's image runs that
# image with the PR's older config, not its committed ref; name it separately
# so the warning above does not read as the whole story for it.
if [ "$bare" -gt 0 ]; then
bare_pkgs=$(echo "$bare_files" | tr ' ' '\n' | grep -v '^$' | sort -u)
printf '::warning title=Image overlay bare refs::%s drifted package(s) took current-main images for bare first-party refs, with the config committed in this PR: %s\n' \
"$(printf '%s\n' "$bare_pkgs" | wc -l | tr -d ' ')" \
"$(printf '%s' "$bare_pkgs" | tr '\n' ' ' | sed 's/ $//')"
fi
+137
View File
@@ -502,3 +502,140 @@ load test_helper
cd "$root"
rm -rf "$w"
}
# A drifted file keeps its committed refs, and for a digest-pinned ref that is
# a pullable older image. A bare <repo>:<tag> is the first commit of a new image
# (docs/agents/image-refs.md) and names a tag no registry serves, so keeping it
# turns drift into ImagePullBackOff. That hit every fork PR whose merge commit
# predated a config change to observability-proxy's values.yaml (#5328). Only
# such a bare line takes the artifact's ref; the rest of the file stays as the
# PR has it.
@test "a drifted file still takes the artifact ref for a bare (digest-less) ref line" {
root=$(pwd)
w=$(mktemp -d)
mkdir -p "$w/packages/system/proxy" "$w/main/system/proxy"
printf 'proxy:\n image: "ghcr.io/cozystack/cozystack/proxy:v0.0.0"\n lookback: 1h\n' \
> "$w/packages/system/proxy/values.yaml"
printf 'proxy:\n image: "iad.ocir.io/x/cozystack/proxy:main@sha256:bbbb"\n lookback: 1h\n prefix: cozy-\n' \
> "$w/main/system/proxy/values.yaml"
( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt"
grep -q 'drift (non-ref change) in packages/system/proxy/values.yaml' "$w/out.txt"
grep -q ' image: "iad.ocir.io/x/cozystack/proxy:main@sha256:bbbb"' "$w/packages/system/proxy/values.yaml"
# The config stays the PR's: the new key is not introduced.
if grep -q 'prefix:' "$w/packages/system/proxy/values.yaml"; then
echo "FAIL: a config line was taken from the artifact" >&2; exit 1
fi
grep -q 'bare first-party ref in packages/system/proxy/values.yaml -> current-main' "$w/out.txt"
grep -q '::warning title=Image overlay bare refs::1 drifted package(s) .*: system/proxy$' "$w/out.txt"
grep -q '::warning title=Image overlay drift::1 package(s) kept committed release refs (bare first-party refs excepted, see Image overlay bare refs), .*: system/proxy$' "$w/out.txt"
grep -q 'bare-refs=1' "$w/out.txt"
rm -rf "$w"
}
@test "a drifted file replaces only its bare ref, not a digest-pinned sibling" {
root=$(pwd)
w=$(mktemp -d)
mkdir -p "$w/packages/system/pair" "$w/main/system/pair"
printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/pair:v1.5.0@sha256:aaaa\nclientImage: ghcr.io/cozystack/cozystack/pair-client:latest\n' \
> "$w/packages/system/pair/values.yaml"
printf 'tuning: new\nimage: iad.ocir.io/x/cozystack/pair:main@sha256:bbbb\nclientImage: iad.ocir.io/x/cozystack/pair-client:main@sha256:cccc\n' \
> "$w/main/system/pair/values.yaml"
( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt"
grep -q '^image: ghcr.io/cozystack/cozystack/pair:v1.5.0@sha256:aaaa$' "$w/packages/system/pair/values.yaml"
grep -q '^clientImage: iad.ocir.io/x/cozystack/pair-client:main@sha256:cccc$' "$w/packages/system/pair/values.yaml"
grep -q '^tuning: old$' "$w/packages/system/pair/values.yaml"
rm -rf "$w"
}
@test "a bare ref is not replaced by an artifact line for a different image" {
# The same key can hold another image in the artifact, e.g. after a rename.
# Matching on the key alone would swap in an image the chart never asked for.
root=$(pwd)
w=$(mktemp -d)
mkdir -p "$w/packages/system/ren" "$w/main/system/ren"
printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/old-name:v0.0.0\n' > "$w/packages/system/ren/values.yaml"
printf 'tuning: new\nimage: iad.ocir.io/x/cozystack/new-name:main@sha256:bbbb\n' > "$w/main/system/ren/values.yaml"
( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt"
grep -q '^image: ghcr.io/cozystack/cozystack/old-name:v0.0.0$' "$w/packages/system/ren/values.yaml"
grep -q 'bare-refs=0' "$w/out.txt"
rm -rf "$w"
}
@test "a bare ref is kept when the artifact carries no digest for it either" {
# A ref build-main does not stamp has no digest in the artifact, so there is
# nothing more pullable to take.
root=$(pwd)
w=$(mktemp -d)
mkdir -p "$w/packages/system/nd" "$w/main/system/nd"
printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/nd:v0.0.0\n' > "$w/packages/system/nd/values.yaml"
printf 'tuning: new\nimage: iad.ocir.io/x/cozystack/nd:main\n' > "$w/main/system/nd/values.yaml"
( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt"
grep -q '^image: ghcr.io/cozystack/cozystack/nd:v0.0.0$' "$w/packages/system/nd/values.yaml"
grep -q 'bare-refs=0' "$w/out.txt"
rm -rf "$w"
}
@test "a bare third-party ref in a drifted file is kept even when main pins one" {
# Main can bump a third-party image together with its config in one file. A
# PR behind it has the older pair, which is the one known to work together;
# taking main's image alone would run it with the config it replaced.
root=$(pwd)
w=$(mktemp -d)
mkdir -p "$w/packages/system/kc" "$w/main/system/kc"
printf 'features: old\nimage: quay.io/keycloak/keycloak:26.6.3\n' > "$w/packages/system/kc/values.yaml"
printf 'features: new\nimage: quay.io/keycloak/keycloak:27.0.0@sha256:abcd\n' > "$w/main/system/kc/values.yaml"
( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt"
grep -q '^image: quay.io/keycloak/keycloak:26.6.3$' "$w/packages/system/kc/values.yaml"
grep -q 'bare-refs=0' "$w/out.txt"
if grep -q 'Image overlay bare refs' "$w/out.txt"; then
echo "FAIL: a bare-refs annotation with nothing replaced" >&2; exit 1
fi
rm -rf "$w"
}
@test "a bare first-party ref is not replaced by a third-party image of the same name" {
root=$(pwd)
w=$(mktemp -d)
mkdir -p "$w/packages/system/same" "$w/main/system/same"
printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/redis:v0.0.0\n' > "$w/packages/system/same/values.yaml"
printf 'tuning: new\nimage: docker.io/library/redis:8.0@sha256:abcd\n' > "$w/main/system/same/values.yaml"
( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt"
grep -q '^image: ghcr.io/cozystack/cozystack/redis:v0.0.0$' "$w/packages/system/same/values.yaml"
grep -q 'bare-refs=0' "$w/out.txt"
rm -rf "$w"
}
@test "a bare ref is kept when the artifact has two candidate lines for it" {
# Two lines with the same key and image name (e.g. one per component in a
# list) cannot be told apart, so neither is taken.
root=$(pwd)
w=$(mktemp -d)
mkdir -p "$w/packages/system/dup" "$w/main/system/dup"
printf 'tuning: old\na:\n image: ghcr.io/cozystack/cozystack/dup:v0.0.0\n' > "$w/packages/system/dup/values.yaml"
printf 'tuning: new\na:\n image: iad.ocir.io/x/cozystack/dup:main@sha256:bbbb\nb:\n image: iad.ocir.io/x/cozystack/dup:main@sha256:cccc\n' \
> "$w/main/system/dup/values.yaml"
( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt"
grep -q '^ image: ghcr.io/cozystack/cozystack/dup:v0.0.0$' "$w/packages/system/dup/values.yaml"
grep -q 'bare-refs=0' "$w/out.txt"
rm -rf "$w"
}
@test "a bare-ref rewrite that cannot write the file is counted as failed, not replaced" {
# Mode bits do not stop root from writing, and the POSIX runner has no
# `skip`, so there is nothing to observe as root.
if [ "$(id -u)" -eq 0 ]; then return 0; fi
root=$(pwd)
w=$(mktemp -d)
mkdir -p "$w/packages/system/ro" "$w/main/system/ro"
printf 'tuning: old\nimage: ghcr.io/cozystack/cozystack/ro:v0.0.0\n' > "$w/packages/system/ro/values.yaml"
printf 'tuning: new\nimage: iad.ocir.io/x/cozystack/ro:main@sha256:bbbb\n' > "$w/main/system/ro/values.yaml"
chmod a-w "$w/packages/system/ro"
( cd "$w" && "$root/hack/overlay-main-images.sh" main '[]' ) > "$w/out.txt" 2>&1
chmod u+w "$w/packages/system/ro"
grep -q '^image: ghcr.io/cozystack/cozystack/ro:v0.0.0$' "$w/packages/system/ro/values.yaml"
grep -q 'bare-refs=0 failed=1' "$w/out.txt"
if ls "$w/packages/system/ro" | grep -q 'overlay-bare'; then
echo "FAIL: temp file left behind" >&2; exit 1
fi
rm -rf "$w"
}
-30
View File
@@ -151,36 +151,6 @@ assert_file_contains() {
return 1
}
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
# A missing file must fail rather than vacuously pass: a bare `! grep -q`
# succeeds on an unreadable path, which is indistinguishable from "the file
# exists and does not carry the label".
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
# Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for
# "I could not evaluate this"; folded into an `if`, both land in the passing
# branch and the assertion "this is absent" is satisfied by the matcher giving
# up. Every negative claim in this file rests on the three-way split.
local _rc=0
awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$?
case "${_rc}" in
0)
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
return 1
;;
1) return 0 ;;
*)
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
return 1
;;
esac
}
# The function under test redirects a command's stderr into a report file and
# decides which artifact to write from it. cozytest.sh runs under `set -x`, so
# the tracer's own line for that command lands on the very stderr being
@@ -94,45 +94,6 @@ assert_file_contains() {
return 1
}
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
# A missing file must fail rather than vacuously pass: a bare `! grep -q`
# succeeds on an unreadable path, which is indistinguishable from "the file
# exists and does not carry the label". awk exits 2 on an unreadable path,
# which folded into an if reads the same way, so the file is checked first.
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
# Branched on awk's exact status, not on truthiness. awk exits 1 for "no line
# matched" and 2 for "I could not evaluate this" -- an unparseable regex, most
# of all. Folded into `if ... then fail; fi`, both land in the passing branch,
# so the assertion "this pattern is absent" is satisfied by the matcher giving
# up. That is the wrong direction for a negative assertion: a positive one
# fails loudly when its matcher breaks and gets fixed, this one goes green and
# stays green. Every claim in this file about something NOT appearing rests on
# the three-way split below.
# `|| _rc=$?` rather than a bare call: both runners set -e, and awk exiting 1
# for "no match" -- the ordinary success case here -- would otherwise abort the
# test before the branch below ever runs. The `if` form this replaces was
# exempt only because conditions are exempt.
local _rc=0
awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$?
case "${_rc}" in
0)
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
return 1
;;
1) return 0 ;;
*)
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
return 1
;;
esac
}
# COZY_REPORT_DIR is set inline in each test below rather than through a helper,
# which is the opposite of what the node-join suite does. The difference is
# deliberate and shrinks to one line: there the assignment carries a ten-line
-30
View File
@@ -81,36 +81,6 @@ assert_file_contains() {
return 1
}
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
# A missing file must fail rather than vacuously pass: a bare `! grep -q`
# succeeds on an unreadable path, which is indistinguishable from "the file
# exists and does not carry the label".
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
# Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for
# "I could not evaluate this"; folded together, a negative assertion is
# satisfied by its own matcher giving up, which is the direction that goes
# green and stays green.
local _rc=0
awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$?
case "${_rc}" in
0)
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
return 1
;;
1) return 0 ;;
*)
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
return 1
;;
esac
}
# The function under test redirects a command's stderr into a report file and
# decides which artifact to write from it. cozytest.sh runs under `set -x`, so
# the tracer's own line for that command lands on the very stderr being
@@ -148,36 +148,6 @@ assert_file_contains() {
return 1
}
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
# A missing file must fail rather than vacuously pass: a bare `! grep -q`
# succeeds on an unreadable path, which is indistinguishable from "the file
# exists and does not carry the label".
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
# Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for
# "I could not evaluate this"; folded into an `if`, both land in the passing
# branch and the assertion "this is absent" is satisfied by the matcher giving
# up. Every negative claim in this file rests on the three-way split.
local _rc=0
awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$?
case "${_rc}" in
0)
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
return 1
;;
1) return 0 ;;
*)
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
return 1
;;
esac
}
# The function under test redirects a command's stderr into a report file and
# decides which artifact to write from it. cozytest.sh runs under `set -x`, so
# the tracer's own line for that command lands on the very stderr being
-30
View File
@@ -300,36 +300,6 @@ assert_file_contains() {
return 1
}
# The sibling suites' three-way split, and here for the same reason. A bare
# `! grep -q` succeeds on a file that does not exist, which is
# indistinguishable from the file existing without the pattern; and awk exits 1
# for "no line matched" and 2 for "I could not evaluate this", so folding those
# together lets a broken matcher satisfy a negative assertion. A positive
# assertion fails loudly when its matcher breaks and gets fixed; this one would
# go green and stay green.
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
local _rc=0
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$?
case "${_rc}" in
0)
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
return 1
;;
1) return 0 ;;
*)
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
return 1
;;
esac
}
# The ceilings both budget guards below price their sums from, read out of the
# helper rather than restated in each. They are the same numbers held against
# two different bounds -- the phase budget from below, the operation window from
@@ -67,37 +67,6 @@ assert_file_contains() {
return 1
}
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
# A missing file must fail rather than vacuously pass: a bare `! grep -q`
# succeeds on an unreadable path, which is indistinguishable from "the file
# exists and does not carry the label".
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
# Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for
# "I could not evaluate this"; folded together, a negative assertion is
# satisfied by its own matcher giving up, which is the direction that goes
# green and stays green.
local _rc=0
awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$?
case "${_rc}" in
0)
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
cat "${file}" >&2
return 1
;;
1) return 0 ;;
*)
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
return 1
;;
esac
}
# The clock the arms are timed by, replaced by a scripted sequence.
#
# Real durations are the one thing this suite must not depend on: they are set
-31
View File
@@ -58,37 +58,6 @@ assert_file_contains() {
return 1
}
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
# A missing file must fail rather than vacuously pass: a bare `! grep -q`
# succeeds on an unreadable path, which is indistinguishable from "the file
# exists and does not carry the label".
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
# Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for
# "I could not evaluate this"; folded together, a negative assertion is
# satisfied by its own matcher giving up, which is the direction that goes
# green and stays green.
local _rc=0
awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$?
case "${_rc}" in
0)
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
cat "${file}" >&2
return 1
;;
1) return 0 ;;
*)
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
return 1
;;
esac
}
# The functions under test redirect a command's stderr into a report file and
# decide which artifact to write from it. cozytest.sh runs under `set -x`, so the
# tracer's own line for that command lands on the very stderr being redirected,
@@ -83,36 +83,6 @@ assert_file_contains() {
return 1
}
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
# A missing file must fail rather than vacuously pass: a bare `! grep -q`
# succeeds on an unreadable path, which is indistinguishable from "the file
# exists and does not carry the label".
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
# Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for
# "I could not evaluate this"; folded together, a negative assertion is
# satisfied by its own matcher giving up, which is the direction that goes
# green and stays green.
local _rc=0
awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$?
case "${_rc}" in
0)
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
return 1
;;
1) return 0 ;;
*)
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
return 1
;;
esac
}
# The function under test redirects a command's stderr into a report file and
# decides which artifact to write from it. cozytest.sh runs under `set -x`, so
# the tracer's own line for that command lands on the very stderr being
@@ -129,23 +129,6 @@ assert_file_contains() {
return 1
}
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
# A missing file must fail rather than vacuously pass: awk exits 2 on an
# unreadable path, which is indistinguishable from "no line matched" once the
# status is folded into an if.
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
if awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}"; then
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
return 1
fi
}
# The function under test redirects a command's stderr into a report file and
# decides which artifact to write from it. cozytest.sh runs under `set -x`, so
# the tracer's own line for that command is emitted on the very stderr being
@@ -114,16 +114,6 @@ assert_file_contains() {
return 1
}
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
if awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}"; then
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
return 1
fi
}
@test "worker VMI rows retain names whose default interface has no IP" {
. hack/e2e-chainsaw/_lib/run-kubernetes.sh
rows=$(printf '%s\n' '{"items":[{"metadata":{"name":"worker-a"},"status":{"interfaces":[{"name":"default","ipAddress":"10.244.1.92"}]}},{"metadata":{"name":"worker-b"},"status":{"interfaces":[{"name":"other","ipAddress":"192.0.2.1"}]}}]}' | cozy_tenant_worker_vmi_rows)
-31
View File
@@ -85,37 +85,6 @@ assert_file_contains() {
return 1
}
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
# A missing file must fail rather than vacuously pass: a bare `! grep -q`
# succeeds on an unreadable path, which is indistinguishable from "the file
# exists and does not carry the label".
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
# Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for
# "I could not evaluate this"; folded into an `if`, both land in the passing
# branch and the assertion "this pattern is absent" is satisfied by the
# matcher giving up. Every claim in this file about something NOT appearing
# rests on the three-way split below.
local _rc=0
awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$?
case "${_rc}" in
0)
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
return 1
;;
1) return 0 ;;
*)
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
return 1
;;
esac
}
# The function under test redirects a command's stderr into a report file and
# decides which artifact to write from it. cozytest.sh runs under `set -x`, so
# the tracer's own line for that command lands on the very stderr being
-31
View File
@@ -83,37 +83,6 @@ assert_file_contains() {
return 1
}
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
# A missing file must fail rather than vacuously pass: a bare `! grep -q`
# succeeds on an unreadable path, which is indistinguishable from "the file
# exists and does not carry the pattern".
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
# Branched on awk's exact status. awk exits 1 for "no line matched" and 2 for
# "I could not evaluate this"; folded together, a negative assertion is
# satisfied by its own matcher giving up, which is the direction that goes
# green and stays green.
local _rc=0
awk -v pattern="${pattern}" '$0 ~ pattern { found = 1 } END { exit found ? 0 : 1 }' "${file}" || _rc=$?
case "${_rc}" in
0)
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
cat "${file}" >&2
return 1
;;
1) return 0 ;;
*)
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
return 1
;;
esac
}
# The shape sizing object, as the `shapeConfig` subpath serves it: a flat object of
# sizing numbers with one string in it, pretty-printed because the service is free
# to answer either way and the reader buffers its whole input for that reason.
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env bats
load test_helper
# A separate shell keeps errexit active inside the assertion, unlike calling
# the function directly through Bats' run helper.
run_lacks_pattern() {
run bash -eu -c '
. "$3"
assert_file_lacks_pattern "$1" "$2"
printf "checked\n"
' _ "$@" "$BATS_TEST_DIRNAME/test_helper.bash"
}
@test "a valid non-matching pattern passes under errexit" {
printf 'safe content\n' >"$BATS_TEST_TMPDIR/artifact"
run_lacks_pattern 'secret:|secretName:' "$BATS_TEST_TMPDIR/artifact"
[ "$status" -eq 0 ]
[ "$output" = checked ]
}
@test "an empty readable file passes for a valid pattern" {
: >"$BATS_TEST_TMPDIR/artifact"
run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR/artifact"
[ "$status" -eq 0 ]
[ "$output" = checked ]
}
@test "a matching pattern fails the assertion" {
printf 'safe content\nsecretName: credential\n' >"$BATS_TEST_TMPDIR/artifact"
run_lacks_pattern 'secret:|secretName:' "$BATS_TEST_TMPDIR/artifact"
[ "$status" -eq 1 ]
[[ "$output" == *'not to match: secret:|secretName:'* ]]
}
@test "an invalid pattern fails on a populated file" {
printf 'safe content\n' >"$BATS_TEST_TMPDIR/artifact"
run_lacks_pattern '[' "$BATS_TEST_TMPDIR/artifact"
[ "$status" -eq 1 ]
[[ "$output" == *'could not evaluate pattern ['* ]]
}
@test "an invalid pattern fails even on an empty file" {
: >"$BATS_TEST_TMPDIR/artifact"
run_lacks_pattern '[' "$BATS_TEST_TMPDIR/artifact"
[ "$status" -eq 1 ]
[[ "$output" == *'could not evaluate pattern ['* ]]
}
@test "a missing artifact fails rather than passing as no match" {
run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR/missing"
[ "$status" -eq 1 ]
[[ "$output" == *'to exist so it could be checked'* ]]
}
@test "a directory is not an artifact file" {
run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR"
[ "$status" -eq 1 ]
[[ "$output" == *'to exist so it could be checked'* ]]
}
@test "an unreadable artifact fails rather than passing as no match" {
printf 'safe content\n' >"$BATS_TEST_TMPDIR/artifact"
chmod 000 "$BATS_TEST_TMPDIR/artifact"
if [ -r "$BATS_TEST_TMPDIR/artifact" ]; then
skip 'this user can read files without read permission'
fi
run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR/artifact"
[ "$status" -eq 1 ]
[[ "$output" == *'could not evaluate pattern secret:'* ]]
}
@test "a matcher error with status one cannot pass as no match" {
printf 'safe content\n' >"$BATS_TEST_TMPDIR/artifact"
mkdir "$BATS_TEST_TMPDIR/bin"
cat >"$BATS_TEST_TMPDIR/bin/awk" <<'SH'
#!/bin/sh
printf '0\n'
printf 'awk: evaluation failed\n' >&2
exit 1
SH
chmod +x "$BATS_TEST_TMPDIR/bin/awk"
PATH="$BATS_TEST_TMPDIR/bin:$PATH" run_lacks_pattern 'secret:' "$BATS_TEST_TMPDIR/artifact"
[ "$status" -eq 1 ]
[[ "$output" == *'could not evaluate pattern secret:'* ]]
}
+28
View File
@@ -9,3 +9,31 @@ strict_setup() {
setup() {
strict_setup
}
# Only a successful matcher explicitly reporting no match passes; missing,
# non-regular or unreadable files and matcher errors fail closed.
assert_file_lacks_pattern() {
local pattern="$1"
local file="$2"
local found
if [ ! -f "${file}" ]; then
printf 'expected %s to exist so it could be checked for: %s\n' "${file}" "${pattern}" >&2
return 1
fi
# Validate even an empty file, and keep the match result separate from awk's
# status: some implementations also use status 1 for evaluation errors.
if ! found=$(awk -v pattern="${pattern}" '
BEGIN { match("", pattern) }
$0 ~ pattern { found = 1 }
END { print found ? 1 : 0 }
' "${file}"); then
printf 'awk could not evaluate pattern %s against %s\n' "${pattern}" "${file}" >&2
return 1
fi
if [ "${found}" != 0 ]; then
printf 'expected %s not to match: %s\n' "${file}" "${pattern}" >&2
return 1
fi
return 0
}
@@ -44,6 +44,8 @@ const (
cnpgClusterLabel = "cnpg.io/cluster"
cnpgBackupPhaseComplete = "completed"
cnpgBackupPhaseFailed = "failed"
cnpgBackupPhaseStarted = "started"
cnpgBackupPhaseRunning = "running"
// barmanObjectNameParam / barmanServerNameParam are the barman-cloud plugin
// parameter keys on a Cluster's spec.plugins entry. barmanObjectName points
@@ -388,9 +390,10 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
// The ObjectStore is the Cluster's WAL archive destination, shared by
// every backup of the app, so it is written only before this job's
// Backup exists, and only while no other job's Backup is in flight and
// no other running job without a Backup yet holds the destination it
// last wrote.
// Backup exists, and only while no other job's Backup is in flight, no
// other running job without a Backup yet holds the destination it last
// wrote, and CloudNativePG runs no Backup of the Cluster that no running
// job tracks.
writeDestination := existingBackup == nil
if writeDestination {
// The gate in Reconcile sees only a job that has not started, and
@@ -421,6 +424,17 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp
return r.waitCNPGBackupJob(ctx, j, "WaitingForOtherBackup", fmt.Sprintf(
"waiting for BackupJob %s of the same application to finish its backup, which writes to the shared WAL archive destination", holder))
}
untracked, err := r.untrackedCNPGBackupExecuting(ctx, j, clusterName)
if err != nil {
return ctrl.Result{}, err
}
if untracked != "" {
apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied)
apimeta.SetStatusCondition(&j.Status.Conditions, cnpgDestinationWriteHeld("WaitingForUntrackedBackup"))
return r.waitCNPGBackupJob(ctx, j, "WaitingForUntrackedBackup", fmt.Sprintf(
"waiting for cnpg.io/Backup %s/%s of cnpg.io/Cluster %s/%s, which no running BackupJob tracks, to finish: CloudNativePG is running it against the shared WAL archive destination",
j.Namespace, untracked, j.Namespace, clusterName))
}
if apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld) != nil {
apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{
Type: backupCondDestinationWriteHeld,
@@ -1162,8 +1176,9 @@ func (r *BackupJobReconciler) findCNPGBackupForJob(ctx context.Context, j *backu
// otherCNPGBackupInFlight returns the name of another BackupJob that is still
// running and whose cnpg.io/Backup of clusterName has not finished, or "" when
// there is none. A Backup left behind by a finished or deleted job does not
// hold the queue.
// there is none. A Backup left behind by a finished or deleted job is not
// counted here; untrackedCNPGBackupExecuting holds for it while
// CloudNativePG executes it.
//
// The Backups are read from the apiserver, not the cache, which can still
// miss a Backup another job's earlier pass created. Only the ones this
@@ -1206,6 +1221,51 @@ func (r *BackupJobReconciler) otherCNPGBackupInFlight(ctx context.Context, j *ba
return holders[0], nil
}
// untrackedCNPGBackupExecuting returns the name of a cnpg.io/Backup of
// clusterName that CloudNativePG is running, whoever created it. It is
// asked after otherCNPGBackupInFlight and destinationClaimHolder found no
// running BackupJob to wait for, so what it finds is a Backup no running
// job tracks: one created by hand or by a ScheduledBackup, or one whose
// BackupJob was deleted or finished first. Its base backup goes to the
// destination it started with, so a rewrite now would send the WAL it
// needs elsewhere.
//
// Only the phases BackupStatus.IsExecuting reports hold, the ones in which
// CloudNativePG's BackupList.CanExecuteBackup lets no other Backup of the
// Cluster start, so the wait never outlasts CloudNativePG's own queue. A pending
// Backup, even one that ran before and was sent back to pending, starts
// its base backup over against the destination current then.
//
// A Backup created by hand need carry no label at all, and CloudNativePG
// labels only the ones a ScheduledBackup creates, so they are matched by
// spec.cluster.name in a list of the whole namespace. That list comes from
// the cache rather than the apiserver, which every waiting job would
// otherwise ask for the namespace's whole Backup history on each poll. The
// cache can miss a Backup that started moments ago, the same race that
// already lies between this check and the write that follows it, which a
// read from the apiserver would not close either.
func (r *BackupJobReconciler) untrackedCNPGBackupExecuting(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string) (string, error) {
list := &cnpgtypes.BackupList{}
if err := r.List(ctx, list, client.InNamespace(j.Namespace)); err != nil {
return "", fmt.Errorf("list cnpg.io/Backups in %s: %w", j.Namespace, err)
}
var executing []string
for i := range list.Items {
b := &list.Items[i]
if b.Spec.Cluster.Name != clusterName {
continue
}
if b.Status.Phase == cnpgBackupPhaseStarted || b.Status.Phase == cnpgBackupPhaseRunning {
executing = append(executing, b.Name)
}
}
if len(executing) == 0 {
return "", nil
}
sort.Strings(executing)
return executing[0], nil
}
func cnpgDestinationWriteHeld(reason string) metav1.Condition {
return metav1.Condition{
Type: backupCondDestinationWriteHeld,
@@ -2646,6 +2646,9 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) {
owner string
cluster string
wantWait bool
// wantUntracked expects the hold to name the cnpg.io/Backup rather
// than a running BackupJob: nothing the controller tracks owns it.
wantUntracked bool
}{
{name: "another running job with a running Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", wantWait: true},
{name: "another running job with a pending Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "pending", wantWait: true},
@@ -2655,7 +2658,14 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) {
{name: "a Backup of a succeeded job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseSucceeded, otherBackup: "pending"},
{name: "a Backup whose job is gone does not", otherBackup: "pending"},
{name: "a Backup of another Cluster does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", cluster: "postgres-other"},
{name: "a Backup created by hand does not", otherBackup: "running", owner: "-"},
{name: "a Backup created by hand that has not started does not", otherBackup: "pending", owner: "-"},
{name: "a Backup created by hand that is running holds it", otherBackup: "running", owner: "-", wantWait: true, wantUntracked: true},
{name: "a Backup created by hand that has started holds it", otherBackup: "started", owner: "-", wantWait: true, wantUntracked: true},
{name: "a Backup created by hand of another Cluster does not", otherBackup: "started", owner: "-", cluster: "postgres-other"},
{name: "a running Backup whose job is gone holds it", otherBackup: "started", wantWait: true, wantUntracked: true},
{name: "a running Backup of a failed job holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseFailed, otherBackup: "started", wantWait: true, wantUntracked: true},
{name: "a running Backup of a succeeded job holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseSucceeded, otherBackup: "running", wantWait: true, wantUntracked: true},
{name: "a Backup created by hand that is finalizing does not", otherBackup: "finalizing", owner: "-"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
@@ -2705,8 +2715,15 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) {
if ours != 0 {
t.Errorf("the waiting job created %d cnpg.io/Backup(s)", ours)
}
if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "bj-other") {
t.Errorf("Ready = %+v, want False/WaitingForOtherBackup naming bj-other", ready)
wantReason, wantNamed := "WaitingForOtherBackup", "BackupJob bj-other "
if tc.wantUntracked {
wantReason, wantNamed = "WaitingForUntrackedBackup", "cnpg.io/Backup "+rolloutNS+"/bj-other-cnpg "
}
if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != wantReason || !strings.Contains(ready.Message, wantNamed) {
t.Errorf("Ready = %+v, want False/%s naming %q", ready, wantReason, wantNamed)
}
if held := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); held == nil || held.Status != metav1.ConditionTrue || held.Reason != wantReason {
t.Errorf("%s = %+v, want True/%s", backupCondDestinationWriteHeld, held, wantReason)
}
if res.RequeueAfter == 0 {
t.Errorf("a queued job is not requeued, and nothing else wakes it when the holder finishes: %+v", res)
@@ -2731,7 +2748,7 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) {
if got := f.liveDestination(); got != "s3://bucket/" {
t.Errorf("once the holder finished, destination = %q, want the job's own s3://bucket/", got)
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" {
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && (ready.Reason == "WaitingForOtherBackup" || ready.Reason == "WaitingForUntrackedBackup") {
t.Errorf("Ready still names the finished holder: %+v", ready)
}
return
@@ -2739,7 +2756,7 @@ func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) {
if got := f.liveDestination(); got != "s3://bucket/" {
t.Errorf("destination = %q, want the job's own s3://bucket/", got)
}
if ready != nil && ready.Reason == "WaitingForOtherBackup" {
if ready != nil && (ready.Reason == "WaitingForOtherBackup" || ready.Reason == "WaitingForUntrackedBackup") {
t.Errorf("Ready = %+v, the job was held", ready)
}
f.settleDestination()
@@ -3881,16 +3898,29 @@ func TestReconcileCNPG_PluginAttachedByAnotherWriterKeepsTheLatch(t *testing.T)
// Waiting for another backup resets the retry budget of the destination
// write, so the queue does not spend it.
func TestReconcileCNPG_QueueResetsDestinationRetryBudget(t *testing.T) {
for name, owner := range map[string]string{"a running BackupJob's Backup": "bj-other", "an untracked Backup": ""} {
t.Run(name, func(t *testing.T) {
testQueueResetsDestinationRetryBudget(t, owner)
})
}
}
func testQueueResetsDestinationRetryBudget(t *testing.T, owner string) {
expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute))
f := newAttachedObjectStoreFixture(t, expired, "s3://other/")
f.setJobCondition(metav1.Condition{
Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying",
Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired),
})
f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning)
f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running")
if owner != "" {
f.seedOtherBackupJob(owner, backupsv1alpha1.BackupJobPhaseRunning)
}
f.seedCNPGBackup("bj-other-cnpg", owner, rolloutCluster, "running")
j, _ := f.pass()
if held := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); held == nil || held.Status != metav1.ConditionTrue {
t.Fatalf("%s = %+v, want the job held", backupCondDestinationWriteHeld, held)
}
if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil {
t.Fatalf("%s = %+v kept while waiting for another backup", backupCondDestinationApplied, cond)
}
@@ -6777,3 +6807,91 @@ func TestReconcileCNPG_LooksUpItsOwnBackupOncePerPass(t *testing.T) {
t.Fatalf("got %d cnpg.io/Backups, want 1", len(backups))
}
}
// With two untracked Backups running the wait names the same one on every
// pass, whatever order the cache lists them in, so a waiting job does not
// rewrite its status each poll.
func TestReconcileCNPG_UntrackedHoldNamesOneBackupAcrossPasses(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
f.seedCNPGBackup("a-cnpg", "", rolloutCluster, "running")
f.seedCNPGBackup("z-cnpg", "", rolloutCluster, "started")
statusWrites := 0
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
if err := cl.List(ctx, list, opts...); err != nil {
return err
}
if backups, ok := list.(*cnpgtypes.BackupList); ok {
slices.Reverse(backups.Items)
}
return nil
},
SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error {
statusWrites++
return cl.SubResource(sub).Update(ctx, obj, opts...)
},
})
j, _ := f.pass()
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForUntrackedBackup" || !strings.Contains(ready.Message, "/a-cnpg ") {
t.Fatalf("Ready = %+v, want WaitingForUntrackedBackup naming a-cnpg", ready)
}
statusWrites = 0
f.pass()
if statusWrites != 0 {
t.Errorf("a second pass with the same two untracked Backups wrote the BackupJob status %d time(s)", statusWrites)
}
}
// An error listing the namespace's Backups is not taken as "nothing runs".
func TestReconcileCNPG_UntrackedBackupListErrorIsReturned(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
lo := &client.ListOptions{}
lo.ApplyOptions(opts)
if _, ok := list.(*cnpgtypes.BackupList); ok && (lo.LabelSelector == nil || lo.LabelSelector.Empty()) {
return apierrors.NewServiceUnavailable("cache not synced")
}
return cl.List(ctx, list, opts...)
},
})
j := &backupsv1alpha1.BackupJob{}
if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil {
t.Fatalf("get BackupJob: %v", err)
}
if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil {
t.Fatalf("reconcileCNPG swallowed the error listing untracked Backups")
}
if got := f.liveDestination(); got != "s3://other/" {
t.Errorf("destination written although the running Backups could not be listed: %q", got)
}
}
// A ScheduledBackup labels its Backups with the Cluster but with no
// BackupJob; such a Backup holds the destination while it runs.
func TestReconcileCNPG_ScheduledBackupHoldsTheDestination(t *testing.T) {
f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/")
b := &cnpgtypes.Backup{
ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "nightly-20261112", Labels: map[string]string{
cnpgClusterLabel: rolloutCluster,
"cnpg.io/scheduled-backup": "nightly",
"cnpg.io/immediateBackup": "false",
}},
Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: rolloutCluster}},
Status: cnpgtypes.BackupStatus{Phase: cnpgBackupPhaseStarted},
}
if err := f.c.Create(context.Background(), b); err != nil {
t.Fatalf("seed scheduled cnpg.io/Backup: %v", err)
}
j, backups := f.pass()
if got := f.liveDestination(); got != "s3://other/" {
t.Errorf("destination rewritten under a running scheduled backup: %q", got)
}
if len(backups) != 1 {
t.Errorf("got %d cnpg.io/Backups, want only the scheduled one", len(backups))
}
if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForUntrackedBackup" || !strings.Contains(ready.Message, "/nightly-20261112 ") {
t.Errorf("Ready = %+v, want WaitingForUntrackedBackup naming nightly-20261112", ready)
}
}
+95 -2
View File
@@ -52,6 +52,17 @@ const (
// whenever it is below the chart quota, without the controller fighting Flux
// over the chart-owned object.
allocatedQuotaName = "tenant-quota-allocated"
// chartLimitRangeName is the LimitRange the tenant chart renders next to
// chartQuotaName, giving containers default requests and limits. The chart
// renders both only for a tenant that declares resourceQuotas.
chartLimitRangeName = "tenant-range-limits"
// allocatedLimitRangeName is the controller-owned copy of the pool root's
// chart LimitRange, written into every unbounded pool member. Those members
// render no chart LimitRange, yet carry an allocated quota; without
// container defaults, any pod omitting a limit the quota bounds is rejected
// at admission. A distinct name keeps the copy from colliding with the
// chart-owned object if the member later declares its own quota.
allocatedLimitRangeName = "tenant-range-limits-allocated"
// managedByLabel marks the controller-owned ResourceQuotas so they can be
// listed and garbage-collected.
managedByLabel = "quota.cozystack.io/managed-by"
@@ -102,6 +113,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result
pools := ComputePools(tenants)
desired := map[string]corev1.ResourceList{}
// limitRangeSource maps each unbounded pool member whose quota bounds a
// compute limit to the pool root whose chart LimitRange it must inherit.
limitRangeSource := map[string]string{}
for _, p := range pools {
if over := p.Overcommitted(); len(over) > 0 {
logger.Info("tenant quota pool is overcommitted by sub-tenant carve-outs", "pool", p.Root, "overcommit", over)
@@ -120,6 +134,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result
for _, ns := range p.Members {
if existing[ns] {
desired[ns] = buffered.EnforcedHard(ns, usedByNS)
if ns != p.Root && boundsComputeLimits(desired[ns]) {
limitRangeSource[ns] = p.Root
}
}
}
}
@@ -134,6 +151,22 @@ func (r *Reconciler) Reconcile(ctx context.Context, _ ctrl.Request) (ctrl.Result
logger.Error(err, "failed to garbage-collect stale allocated quotas")
}
written := map[string]bool{}
for ns, root := range limitRangeSource {
ok, err := r.upsertAllocatedLimitRange(ctx, ns, root)
if err != nil {
logger.Error(err, "failed to apply allocated limit range", "namespace", ns, "poolRoot", root)
}
// Keep the existing copy on a transient error rather than dropping the
// defaults the namespace's pods rely on.
if ok || err != nil {
written[ns] = true
}
}
if err := r.gcAllocatedLimitRanges(ctx, written); err != nil {
logger.Error(err, "failed to garbage-collect stale allocated limit ranges")
}
return ctrl.Result{RequeueAfter: resyncInterval}, nil
}
@@ -224,6 +257,50 @@ func (r *Reconciler) gcAllocatedQuotas(ctx context.Context, desired map[string]c
return nil
}
// upsertAllocatedLimitRange copies the pool root's chart LimitRange into an
// unbounded member namespace. It reports false, without error, when the root
// has no chart LimitRange to copy.
func (r *Reconciler) upsertAllocatedLimitRange(ctx context.Context, namespace, poolRoot string) (bool, error) {
source := &corev1.LimitRange{}
if err := r.Get(ctx, types.NamespacedName{Namespace: poolRoot, Name: chartLimitRangeName}, source); err != nil {
if apierrors.IsNotFound(err) {
return false, nil
}
return false, err
}
lr := &corev1.LimitRange{ObjectMeta: metav1.ObjectMeta{Name: allocatedLimitRangeName, Namespace: namespace}}
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, lr, func() error {
if lr.Labels == nil {
lr.Labels = map[string]string{}
}
lr.Labels[managedByLabel] = managedByValue
lr.Labels["internal.cozystack.io/managed-by-cozystack"] = ""
lr.Spec = *source.Spec.DeepCopy()
return nil
})
return err == nil, err
}
// gcAllocatedLimitRanges deletes controller-owned LimitRanges in namespaces
// that are no longer unbounded pool members (e.g. the tenant gained its own
// quota and now renders the chart LimitRange, or was deleted).
func (r *Reconciler) gcAllocatedLimitRanges(ctx context.Context, keep map[string]bool) error {
managed := &corev1.LimitRangeList{}
if err := r.List(ctx, managed, client.MatchingLabels{managedByLabel: managedByValue}); err != nil {
return err
}
for i := range managed.Items {
m := &managed.Items[i]
if keep[m.Namespace] {
continue
}
if err := r.Delete(ctx, m); err != nil && !apierrors.IsNotFound(err) {
return err
}
}
return nil
}
func (r *Reconciler) recordOvercommit(ctx context.Context, rootNamespace string, over corev1.ResourceList) {
if r.Recorder == nil {
return
@@ -237,8 +314,8 @@ func (r *Reconciler) recordOvercommit(ctx context.Context, rootNamespace string,
}
// SetupWithManager wires the controller. Every relevant change (a tenant
// HelmRelease, a namespace, or a tenant ResourceQuota whose usage moved)
// coalesces into one full-tree sweep.
// HelmRelease, a namespace, a tenant ResourceQuota whose usage moved, or a
// tenant LimitRange) coalesces into one full-tree sweep.
func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
toSweep := handler.EnqueueRequestsFromMapFunc(func(context.Context, client.Object) []reconcile.Request {
return []reconcile.Request{sweepKey}
@@ -249,11 +326,15 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
tenantQuotas := predicate.NewPredicateFuncs(func(o client.Object) bool {
return o.GetName() == chartQuotaName || o.GetName() == allocatedQuotaName
})
tenantLimitRanges := predicate.NewPredicateFuncs(func(o client.Object) bool {
return o.GetName() == chartLimitRangeName || o.GetName() == allocatedLimitRangeName
})
return ctrl.NewControllerManagedBy(mgr).
Named("tenantquota-controller").
Watches(&helmv2.HelmRelease{}, toSweep, builder.WithPredicates(tenantReleases)).
Watches(&corev1.Namespace{}, toSweep).
Watches(&corev1.ResourceQuota{}, toSweep, builder.WithPredicates(tenantQuotas)).
Watches(&corev1.LimitRange{}, toSweep, builder.WithPredicates(tenantLimitRanges)).
Complete(r)
}
@@ -271,6 +352,18 @@ func ownedNamespace(hrNamespace, appName string) string {
}
}
// boundsComputeLimits reports whether a quota bounds limits.cpu or
// limits.memory, the only case in which admission rejects a container that
// omits a limit. The chart renders its LimitRange under any non-empty
// resourceQuotas, so the root may carry one under a quota that bounds no
// compute at all; copying it then would impose default limits the quota
// never asked for.
func boundsComputeLimits(hard corev1.ResourceList) bool {
_, cpu := hard[corev1.ResourceLimitsCPU]
_, memory := hard[corev1.ResourceLimitsMemory]
return cpu || memory
}
// maxResourceList returns the per-resource maximum of a and b.
func maxResourceList(a, b corev1.ResourceList) corev1.ResourceList {
out := corev1.ResourceList{}
@@ -227,3 +227,172 @@ func TestReconcile_GarbageCollect(t *testing.T) {
t.Fatalf("stale allocated quota in tenant-gone should have been garbage-collected")
}
}
// chartLimitRange builds the chart-rendered "tenant-range-limits" LimitRange,
// which the tenant chart emits alongside its "tenant-quota".
func chartLimitRange(namespace, defaultCPU string) *corev1.LimitRange {
return &corev1.LimitRange{
ObjectMeta: metav1.ObjectMeta{Name: chartLimitRangeName, Namespace: namespace},
Spec: corev1.LimitRangeSpec{Limits: []corev1.LimitRangeItem{{
Type: corev1.LimitTypeContainer,
Default: resourceList(map[string]string{"cpu": defaultCPU, "memory": "128Mi"}),
DefaultRequest: resourceList(map[string]string{"cpu": "25m", "memory": "128Mi"}),
}}},
}
}
func getLimitRange(c client.Client, namespace, name string) (*corev1.LimitRange, bool) {
lr := &corev1.LimitRange{}
if err := c.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: name}, lr); err != nil {
return nil, false
}
return lr, true
}
// TestReconcile_UnboundedMemberGetsPoolLimitRange: an unbounded child renders
// no chart quota and therefore no chart LimitRange, yet the controller clamps
// it with an allocated quota. Without container defaults, every pod that omits
// a limit the quota bounds (e.g. a sidecar without a CPU limit) is rejected at
// admission, so the controller must copy the pool root's LimitRange into it.
func TestReconcile_UnboundedMemberGetsPoolLimitRange(t *testing.T) {
r, c := newReconciler(t,
tenantHR("foo", "tenant-root"),
tenantHR("bar", "tenant-foo"),
ns("tenant-foo"), ns("tenant-foo-bar"),
chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil),
chartLimitRange("tenant-foo", "250m"),
)
if _, err := r.Reconcile(context.Background(), sweepKey); err != nil {
t.Fatalf("reconcile: %v", err)
}
lr, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName)
if !ok {
t.Fatalf("unbounded member tenant-foo-bar has an allocated quota but no LimitRange")
}
if len(lr.Spec.Limits) != 1 {
t.Fatalf("tenant-foo-bar LimitRange limits = %d, want 1", len(lr.Spec.Limits))
}
if got := lr.Spec.Limits[0].Default[corev1.ResourceCPU]; got.String() != "250m" {
t.Fatalf("tenant-foo-bar default cpu = %q, want 250m copied from the pool root", got.String())
}
if lr.Labels[managedByLabel] != managedByValue {
t.Fatalf("tenant-foo-bar LimitRange is not marked as controller-owned")
}
if _, ok := getLimitRange(c, "tenant-foo", allocatedLimitRangeName); ok {
t.Fatalf("pool root tenant-foo already has the chart LimitRange and must not get a second one")
}
}
// TestReconcile_PoolLimitRangeFollowsRoot: a change to the root's chart
// LimitRange propagates to the members on the next sweep.
func TestReconcile_PoolLimitRangeFollowsRoot(t *testing.T) {
root := chartLimitRange("tenant-foo", "250m")
r, c := newReconciler(t,
tenantHR("foo", "tenant-root"),
tenantHR("bar", "tenant-foo"),
ns("tenant-foo"), ns("tenant-foo-bar"),
chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil),
root,
)
if _, err := r.Reconcile(context.Background(), sweepKey); err != nil {
t.Fatalf("reconcile: %v", err)
}
root.Spec.Limits[0].Default[corev1.ResourceCPU] = resource.MustParse("500m")
if err := c.Update(context.Background(), root); err != nil {
t.Fatalf("update root LimitRange: %v", err)
}
if _, err := r.Reconcile(context.Background(), sweepKey); err != nil {
t.Fatalf("reconcile: %v", err)
}
lr, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName)
if !ok {
t.Fatalf("tenant-foo-bar lost its LimitRange")
}
if got := lr.Spec.Limits[0].Default[corev1.ResourceCPU]; got.String() != "500m" {
t.Fatalf("tenant-foo-bar default cpu = %q, want 500m after the root changed", got.String())
}
}
// TestReconcile_PoolLimitRangeGarbageCollected: once the child declares its own
// quota, the chart renders its LimitRange and the controller-owned copy goes.
func TestReconcile_PoolLimitRangeGarbageCollected(t *testing.T) {
stale := &corev1.LimitRange{
ObjectMeta: metav1.ObjectMeta{
Name: allocatedLimitRangeName,
Namespace: "tenant-foo-bar",
Labels: map[string]string{managedByLabel: managedByValue},
},
}
r, c := newReconciler(t,
tenantHR("foo", "tenant-root"),
tenantHR("bar", "tenant-foo"),
ns("tenant-foo"), ns("tenant-foo-bar"),
chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil),
chartLimitRange("tenant-foo", "250m"),
chartQuota("tenant-foo-bar", map[string]string{"limits.cpu": "4"}, nil),
chartLimitRange("tenant-foo-bar", "250m"),
stale,
)
if _, err := r.Reconcile(context.Background(), sweepKey); err != nil {
t.Fatalf("reconcile: %v", err)
}
if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok {
t.Fatalf("bounded child tenant-foo-bar renders its own LimitRange; the controller copy must be removed")
}
if _, ok := getLimitRange(c, "tenant-foo-bar", chartLimitRangeName); !ok {
t.Fatalf("the chart-owned LimitRange must never be touched")
}
}
// TestReconcile_PoolRootWithoutLimitRange: if the root has no chart LimitRange
// there is nothing to copy; the quota is still enforced and the sweep succeeds.
func TestReconcile_PoolRootWithoutLimitRange(t *testing.T) {
r, c := newReconciler(t,
tenantHR("foo", "tenant-root"),
tenantHR("bar", "tenant-foo"),
ns("tenant-foo"), ns("tenant-foo-bar"),
chartQuota("tenant-foo", map[string]string{"limits.cpu": "10"}, nil),
)
if _, err := r.Reconcile(context.Background(), sweepKey); err != nil {
t.Fatalf("reconcile: %v", err)
}
if _, ok := allocatedHard(t, c, "tenant-foo-bar", "limits.cpu"); !ok {
t.Fatalf("tenant-foo-bar must still get its allocated quota")
}
if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok {
t.Fatalf("no LimitRange can be copied from a root that has none")
}
}
// TestReconcile_NoLimitRangeWithoutComputeLimitQuota: the chart renders its
// LimitRange for any non-empty resourceQuotas, compute or not. A member whose
// allocated quota bounds neither limits.cpu nor limits.memory is admitted
// without limits today; copying the root's defaults there would cap or refuse
// pods (one requesting more than the default memory limit, for instance)
// that the quota itself never constrains.
func TestReconcile_NoLimitRangeWithoutComputeLimitQuota(t *testing.T) {
stale := &corev1.LimitRange{
ObjectMeta: metav1.ObjectMeta{
Name: allocatedLimitRangeName,
Namespace: "tenant-foo-bar",
Labels: map[string]string{managedByLabel: managedByValue},
},
}
r, c := newReconciler(t,
tenantHR("foo", "tenant-root"),
tenantHR("bar", "tenant-foo"),
ns("tenant-foo"), ns("tenant-foo-bar"),
chartQuota("tenant-foo", map[string]string{"services.loadbalancers": "1"}, nil),
chartLimitRange("tenant-foo", "250m"),
stale,
)
if _, err := r.Reconcile(context.Background(), sweepKey); err != nil {
t.Fatalf("reconcile: %v", err)
}
if _, ok := allocatedHard(t, c, "tenant-foo-bar", "services.loadbalancers"); !ok {
t.Fatalf("tenant-foo-bar must still get its allocated quota")
}
if _, ok := getLimitRange(c, "tenant-foo-bar", allocatedLimitRangeName); ok {
t.Fatalf("tenant-foo-bar's quota bounds no compute limit; it must carry no LimitRange copy")
}
}
@@ -137,11 +137,13 @@ tests:
release:
name: test-kafka
namespace: tenant-test
documentSelector:
path: kind
value: Role
asserts:
- notContains:
path: rules[2].resourceNames
content: test-kafka-zookeeper
documentIndex: 0
# External bootstrap service absent when external=false, regardless of tls.enabled
- it: does not grant access to external bootstrap service when tls.enabled=true and external=false
@@ -27,7 +27,9 @@ tests:
# kafka-run-class.sh then drops its G1 settings.
- it: leaves the node pool JVM options to Strimzi and Kafka
template: templates/migration-hook.yaml
documentIndex: 3
documentSelector:
path: kind
value: Job
asserts:
- notMatchRegex:
path: spec.template.spec.containers[0].args[0]
+1
View File
@@ -44,6 +44,7 @@ The pool's Kubernetes minor (`version`) may lag the cluster's during an upgrade
| `podCpuRequest` | CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. | `quantity` | `""` |
| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` |
| `gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` |
| `maxUnavailable` | Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever. | `*int` | `null` |
| `kernelModules` | Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. | `[]object` | `[]` |
| `kernelModules[i].name` | Module name as `modprobe` takes it, e.g. `nvidia_uvm`. | `string` | `""` |
| `kernelModules[i].parameters` | Module parameters, each as a bare `key=value` string. | `[]string` | `[]` |
@@ -1021,10 +1021,22 @@ spec:
# action until the autoscaler notices unschedulable Pods and
# scales back up — a real disruption window on every routine
# platform bump.
{{- /* maxUnavailable 0 makes a rollout (instance type, image, version)
wait for each replacement Machine to be up before draining an old
one; with 1, a one-node pool lost its only node until the new one
joined. A pool with passed-through GPUs keeps 1: its replacement VM
may need the very GPU the old node holds, and would wait for it
forever. A Proxmox template's GPU is invisible here, hence the
maxUnavailable override. CAPI refuses maxSurge and maxUnavailable both
at 0, and maxReplicas may be 0, hence the floor of 1. */}}
{{- $maxUnavailable := ternary 1 0 (not (empty $group.gpus)) }}
{{- if not (kindIs "invalid" $.Values.maxUnavailable) }}
{{- $maxUnavailable = $.Values.maxUnavailable | int }}
{{- end }}
strategy:
rollingUpdate:
maxSurge: {{ $group.maxReplicas }}
maxUnavailable: 1
maxSurge: {{ max 1 ($group.maxReplicas | int) }}
maxUnavailable: {{ $maxUnavailable }}
type: RollingUpdate
selector:
matchLabels:
@@ -250,7 +250,7 @@ matches the golden render of a kubelet-override pool:
strategy:
rollingUpdate:
maxSurge: 3
maxUnavailable: 1
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
@@ -405,7 +405,7 @@ matches the golden render of the pool objects:
strategy:
rollingUpdate:
maxSurge: 3
maxUnavailable: 1
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
@@ -0,0 +1,136 @@
suite: worker pool rollout keeps capacity
templates:
- templates/nodegroup.yaml
release:
name: kubernetes-nodes-myk8s-md0
namespace: tenant-test
set:
cluster: myk8s
_cluster:
cluster-domain: cozy.local
version: "v1.35"
minReplicas: 1
maxReplicas: 1
instanceType: ""
diskSize: 20Gi
storageClass: replicated
resources:
cpu: "2"
memory: 4Gi
tests:
# A rollout replaces every Machine. With maxUnavailable 1 a one-node pool
# drained and deleted its only node as soon as the replacement Machine was
# created, leaving the pool empty until the new node joined.
- it: never drains a node before its replacement is ready
documentSelector:
path: kind
value: MachineDeployment
asserts:
- equal:
path: spec.strategy.type
value: RollingUpdate
- equal:
path: spec.strategy.rollingUpdate.maxUnavailable
value: 0
- equal:
path: spec.strategy.rollingUpdate.maxSurge
value: 1
- it: surges up to the pool ceiling
set:
maxReplicas: 5
documentSelector:
path: kind
value: MachineDeployment
asserts:
- equal:
path: spec.strategy.rollingUpdate.maxSurge
value: 5
- equal:
path: spec.strategy.rollingUpdate.maxUnavailable
value: 0
# CAPI rejects a RollingUpdate with maxSurge and maxUnavailable both 0.
- it: keeps a surge of one on a pool capped at zero
set:
minReplicas: 0
maxReplicas: 0
documentSelector:
path: kind
value: MachineDeployment
asserts:
- equal:
path: spec.strategy.rollingUpdate.maxSurge
value: 1
- equal:
path: spec.strategy.rollingUpdate.maxUnavailable
value: 0
# A replacement VM in a pool with passed-through GPUs may need the very GPU
# the old node holds; with maxUnavailable 0 it would wait for it forever.
- it: lets a GPU pool remove a node before its replacement is up
set:
gpus:
- name: nvidia.com/AD102GL_L40S
documentSelector:
path: kind
value: MachineDeployment
asserts:
- equal:
path: spec.strategy.rollingUpdate.maxUnavailable
value: 1
- equal:
path: spec.strategy.rollingUpdate.maxSurge
value: 1
# A Proxmox template that passes a GPU through leaves gpus empty, so the
# chart cannot tell; the pool says so itself.
- it: lets a pool set maxUnavailable explicitly
set:
maxUnavailable: 1
documentSelector:
path: kind
value: MachineDeployment
asserts:
- equal:
path: spec.strategy.rollingUpdate.maxUnavailable
value: 1
- it: honours an explicit 0 on a GPU pool
set:
maxUnavailable: 0
gpus:
- name: nvidia.com/AD102GL_L40S
documentSelector:
path: kind
value: MachineDeployment
asserts:
- equal:
path: spec.strategy.rollingUpdate.maxUnavailable
value: 0
- it: refuses a negative maxUnavailable (schema minimum 0)
set:
maxUnavailable: -1
asserts:
- failedTemplate: {}
# CAPI decodes the field as an int32, so a larger value would render and
# then be refused by the MachineDeployment decoder.
- it: refuses a maxUnavailable above the int32 range (schema maximum)
set:
maxUnavailable: 2147483648
asserts:
- failedTemplate:
errorPattern: "maxUnavailable"
- it: accepts the int32 maximum
set:
maxUnavailable: 2147483647
documentSelector:
path: kind
value: MachineDeployment
asserts:
- equal:
path: spec.strategy.rollingUpdate.maxUnavailable
value: 2147483647
@@ -146,6 +146,12 @@
}
}
},
"maxUnavailable": {
"description": "Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.",
"type": "integer",
"maximum": 2147483647,
"minimum": 0
},
"kernelModules": {
"description": "Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.",
"type": "array",
@@ -77,6 +77,10 @@ resources: {}
## @param {[]GPU} gpus - List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).
gpus: []
## @param {*int} [maxUnavailable] - Workers a rollout (instance type, image or version change) may remove before their replacement is up. Unset, it is 0, so each replacement joins before an old worker is drained, and 1 on a pool with `gpus`, whose replacement may need the very GPU the old worker holds. With 0, a rollout needs room for one extra VM: a pool short of quota or capacity for it stalls, with the new Machine Pending, until room is made or `maxUnavailable` is set to 1. Set 1 on a Proxmox pool whose template passes a GPU through: the chart cannot see that GPU, and with 0 the rollout would wait for it forever.
## @minimum 0
## @maximum 2147483647
## @param {[]KernelModule} [kernelModules] - Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.
## @param {Kubelet} [kubelet] - Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`.
@@ -241,18 +241,22 @@ tests:
errorMessage: 'controlPlane.apiServer.extraArgs contains "--secure-port=6444" — the tenant control plane sets --secure-port itself and drops every entry naming it before the kube-apiserver sees the list, so this entry has no effect, and written without =value it leaves its value behind as an argument the apiserver refuses. Remove the entry; the description of controlPlane.apiServer.extraArgs lists every flag the control plane owns.'
- it: leaves spec.admissionControllers out by default, so the control plane keeps its own default set
documentSelector:
path: kind
value: KamajiControlPlane
asserts:
- notExists:
path: spec.admissionControllers
documentIndex: 1
- it: leaves spec.admissionControllers out for an explicitly empty list
set:
controlPlane.apiServer.admissionControllers: []
documentSelector:
path: kind
value: KamajiControlPlane
asserts:
- notExists:
path: spec.admissionControllers
documentIndex: 1
- it: renders controlPlane.apiServer.admissionControllers into spec.admissionControllers verbatim
set:
+21 -6
View File
@@ -530,7 +530,9 @@ tests:
- templates/dashboard-resourcemap.yaml
set:
external: false
documentIndex: 0
documentSelector:
path: kind
value: Role
asserts:
- notContains:
path: rules[1].resourceNames
@@ -544,7 +546,9 @@ tests:
- templates/dashboard-resourcemap.yaml
set:
external: true
documentIndex: 0
documentSelector:
path: kind
value: Role
asserts:
- notContains:
path: rules[1].resourceNames
@@ -560,7 +564,9 @@ tests:
external: true
tls:
issuer: operator
documentIndex: 0
documentSelector:
path: kind
value: Role
asserts:
- notContains:
path: rules[1].resourceNames
@@ -576,7 +582,9 @@ tests:
external: false
tls:
issuer: cert-manager
documentIndex: 0
documentSelector:
path: kind
value: Role
asserts:
- notContains:
path: rules[1].resourceNames
@@ -695,7 +703,9 @@ tests:
set:
tls:
issuer: cert-manager
documentIndex: 3
documentSelector:
path: metadata.name
value: my-mariadb-tls
asserts:
- notContains:
path: spec.dnsNames
@@ -1187,7 +1197,9 @@ tests:
- it: "dashboard Role never grants the CA bundle even when TLS is unmanaged"
templates:
- templates/dashboard-resourcemap.yaml
documentIndex: 0
documentSelector:
path: kind
value: Role
set:
tls:
issuer: operator
@@ -1436,6 +1448,9 @@ tests:
enabled: true
s3Bucket: s3.example.org/test
cleanupStrategy: "--keep-last=3"
documentSelector:
path: kind
value: Secret
asserts:
- notMatchRegex:
path: stringData["backup.sh"]
+24 -8
View File
@@ -53,7 +53,9 @@ tests:
value: mg-test-ssl
- it: spec.tls has no issuerConf (the operator-managed secret is passed directly)
documentIndex: 0
documentSelector:
path: kind
value: PerconaServerMongoDB
asserts:
- notExists:
path: spec.tls.issuerConf
@@ -127,13 +129,17 @@ tests:
# test would pass even with the external check removed.
_namespace:
host: example.com
documentIndex: 0
documentSelector:
path: kind
value: PerconaServerMongoDB
asserts:
- notExists:
path: spec.replsets[0].splitHorizons
- it: chart does not set allowInvalidCertificates, leaving the operator default
documentIndex: 0
documentSelector:
path: kind
value: PerconaServerMongoDB
asserts:
# The README states the permissive default is inherited rather than
# chosen here. Pin that, so the claim stays falsifiable.
@@ -141,7 +147,9 @@ tests:
path: spec.tls.allowInvalidCertificates
- it: chart does not set certValidityDuration, leaving the operator default
documentIndex: 0
documentSelector:
path: kind
value: PerconaServerMongoDB
asserts:
# Same class as allowInvalidCertificates above: the leaf lifetime is the
# operator's to choose, not this chart's. Deliberately left unpinned.
@@ -177,7 +185,9 @@ tests:
# the CRD and notExists on it can never fail.
- it: no TLS-disabling escape hatches with a default replica count
documentIndex: 0
documentSelector:
path: kind
value: PerconaServerMongoDB
asserts:
- notExists:
path: spec.allowUnsafeConfigurations
@@ -235,7 +245,9 @@ tests:
- name: rs0
replicas: 3
size: 10Gi
documentIndex: 0
documentSelector:
path: kind
value: PerconaServerMongoDB
asserts:
# splitHorizons is replica-set scoped and rendered only for the
# non-sharded topology. In sharded mode external access goes through the
@@ -255,7 +267,9 @@ tests:
replicas: 0
_namespace:
host: example.com
documentIndex: 0
documentSelector:
path: kind
value: PerconaServerMongoDB
asserts:
# With replicas: 0 the range body renders nothing. Emitting the key with
# a null value would be rejected by the API server: the CRD types
@@ -269,7 +283,9 @@ tests:
external: true
_namespace:
host: ""
documentIndex: 0
documentSelector:
path: kind
value: PerconaServerMongoDB
asserts:
# The controller may create _namespace before populating host, so the
# bootstrap window presents host as an empty string rather than as an
+6 -2
View File
@@ -557,7 +557,9 @@ tests:
external: false
tls:
enabled: true
documentIndex: 3
documentSelector:
path: metadata.name
value: my-redis-tls
asserts:
- notContains:
path: spec.dnsNames
@@ -840,7 +842,9 @@ tests:
external: false
tls:
enabled: true
documentIndex: 3
documentSelector:
path: metadata.name
value: my-redis-tls
asserts:
- notContains:
path: spec.dnsNames
+1
View File
@@ -59,4 +59,5 @@ See the full design in
| `nodeGroups[name].kubelet.kubeReservedCpu` | CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` |
| `nodeGroups[name].kubelet.evictionHardMemory` | Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%). | `string` | `7%` |
| `nodeGroups[name].kubelet.evictionSoftMemory` | Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%). | `string` | `10%` |
| `nodeGroups[name].maxUnavailable` | Workers a rollout may remove before their replacement is up. Unset, the `kubernetes-nodes` pool chart uses 0, so each replacement joins before an old worker is drained (1 on a pool with `gpus`). Set 1 on a pool without room for one extra VM, whose rollout would otherwise stall with the new Machine Pending. | `*int` | `null` |
@@ -211,7 +211,7 @@ spec:
clone or repoint itself at another Image Factory, both of which are
operator decisions about the platform's own images. Keep this list in
sync with the kubernetes-nodes values schema's pool-shape params. */}}
{{- with pick $group "minReplicas" "maxReplicas" "instanceType" "diskSize" "storageClass" "roles" "resources" "gpus" "kubelet" "logSerialConsole" "maxUnhealthy" "nodeStartupTimeout" "readyUnknownTimeout" "readyFalseTimeout" "podCpuLimit" "podCpuRequest" "maxNodeProvisionTime" }}
{{- with pick $group "minReplicas" "maxReplicas" "instanceType" "diskSize" "storageClass" "roles" "resources" "gpus" "kubelet" "logSerialConsole" "maxUnhealthy" "nodeStartupTimeout" "readyUnknownTimeout" "readyFalseTimeout" "podCpuLimit" "podCpuRequest" "maxNodeProvisionTime" "maxUnavailable" }}
{{- toYaml . | nindent 4 }}
{{- end }}
valuesFrom:
@@ -280,6 +280,93 @@ tests:
- notExists:
path: spec.values.nameOverride
# kubernetes-nodes drains a worker only once its replacement is up; a pool
# without room for one extra VM needs maxUnavailable 1 to finish a rollout,
# so the allowlist must forward it.
- it: forwards maxUnavailable to the pool chart
documentSelector:
path: metadata.name
value: kubernetes-nodes-computeplane-cluster-md0
set:
nodeGroups:
md0:
minReplicas: 1
maxReplicas: 3
instanceType: u1.large
diskSize: 32Gi
roles: ["ingress-nginx"]
maxUnavailable: 1
asserts:
- equal:
path: spec.values.maxUnavailable
value: 1
# An explicit 0 must reach the pool chart: there it overrides the 1 a GPU
# pool gets by default, so dropping it as empty would change the strategy.
- it: forwards an explicit maxUnavailable of 0 to the pool chart
documentSelector:
path: metadata.name
value: kubernetes-nodes-computeplane-cluster-md0
set:
nodeGroups:
md0:
minReplicas: 1
maxReplicas: 3
instanceType: u1.large
diskSize: 32Gi
roles: ["ingress-nginx"]
maxUnavailable: 0
asserts:
- equal:
path: spec.values.maxUnavailable
value: 0
- it: forwards the int32 maximum to the pool chart
documentSelector:
path: metadata.name
value: kubernetes-nodes-computeplane-cluster-md0
set:
nodeGroups:
md0:
minReplicas: 1
maxReplicas: 3
instanceType: u1.large
diskSize: 32Gi
roles: ["ingress-nginx"]
maxUnavailable: 2147483647
asserts:
- equal:
path: spec.values.maxUnavailable
value: 2147483647
- it: refuses a maxUnavailable above the int32 range CAPI decodes
set:
nodeGroups:
md0:
minReplicas: 1
maxReplicas: 3
instanceType: u1.large
diskSize: 32Gi
roles: ["ingress-nginx"]
maxUnavailable: 2147483648
asserts:
- failedTemplate:
errorPattern: "maxUnavailable"
- it: refuses a negative maxUnavailable
set:
nodeGroups:
md0:
minReplicas: 1
maxReplicas: 3
instanceType: u1.large
diskSize: 32Gi
roles: ["ingress-nginx"]
maxUnavailable: -1
asserts:
- failedTemplate:
errorPattern: "maxUnavailable"
# Helm caps a HelmRelease name at 53 chars and each pool renders a 38-char fixed
# prefix, so a pool key over 15 chars would create a release the apiserver
# accepts but Flux never reconciles. The render must refuse instead (the same
@@ -93,6 +93,12 @@
"type": "integer",
"default": 10
},
"maxUnavailable": {
"description": "Workers a rollout may remove before their replacement is up. Unset, the `kubernetes-nodes` pool chart uses 0, so each replacement joins before an old worker is drained (1 on a pool with `gpus`). Set 1 on a pool without room for one extra VM, whose rollout would otherwise stall with the new Machine Pending.",
"type": "integer",
"maximum": 2147483647,
"minimum": 0
},
"minReplicas": {
"description": "Minimum number of replicas.",
"type": "integer",
+3
View File
@@ -30,6 +30,9 @@
## @field {Resources} [resources] - Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Set both `cpu` and `memory` together or neither.
## @field {[]GPU} gpus - List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).
## @field {Kubelet} [kubelet] - Kubelet resource reservations for this node group.
## @field {*int} [maxUnavailable] - Workers a rollout may remove before their replacement is up. Unset, the `kubernetes-nodes` pool chart uses 0, so each replacement joins before an old worker is drained (1 on a pool with `gpus`). Set 1 on a pool without room for one extra VM, whose rollout would otherwise stall with the new Machine Pending.
## @minimum 0
## @maximum 2147483647
## @param {map[string]NodeGroup} nodeGroups - Worker node groups of the ComputePlane cluster (cluster shape only — the security posture is fixed by the module and cannot be overridden through this knob: the module forwards only the documented per-pool fields to the `kubernetes-nodes` pool chart, so any other `NodeGroup` field, e.g. `talos`/`version`/`images`, is silently dropped — accepted but inert). Same structure as the `kubernetes` application's `nodeGroups`. When left empty, the module itself materialises a default scale-from-zero `md0` pool with `roles: [ingress-nginx]` (the wrapped `kubernetes` chart no longer injects one after the Phase 2 split), so the cluster sits idle until a pending Pod (e.g. the ingress-nginx controller) triggers the cluster-autoscaler. Supplied groups are authoritative (not merged with the default), so at least one of them must carry `roles: [ingress-nginx]` — enforced at render time. Each supplied pool key must be at most 15 characters so its `kubernetes-nodes-computeplane-cluster-<pool>` HelmRelease name fits Helm's 53-character limit — also enforced at render time.
nodeGroups: {}
@@ -10,7 +10,7 @@ spec:
annotations:
summary: "etcd cluster '{{`{{ $labels.job }}`}}': insufficient members '{{`{{ $value }}`}}'."
expr: |
sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2)
sum(up{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == bool 1) by (job) < ((count(up{job="{{ .Release.Namespace }}/etcd-pod-scrape"}) by (job) + 1) / 2)
for: 3m
labels:
severity: critical
@@ -19,7 +19,7 @@ spec:
annotations:
summary: "etcd cluster '{{`{{ $labels.job }}`}}': member '{{`{{ $labels.instance }}`}}' has no leader."
expr: |
etcd_server_has_leader{job=~".*etcd.*"} == 0
etcd_server_has_leader{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == 0
for: 1m
labels:
severity: critical
@@ -28,7 +28,7 @@ spec:
annotations:
summary: "etcd cluster '{{`{{ $labels.job }}`}}': instance '{{`{{ $labels.instance }}`}}' has seen '{{`{{ $value }}`}}' leader changes within the last hour."
expr: |
rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3
rate(etcd_server_leader_changes_seen_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[15m]) > 3
for: 15m
labels:
severity: warning
@@ -37,9 +37,9 @@ spec:
annotations:
summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
expr: |
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method)
100 * sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method)
/
sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (job, instance, grpc_service, grpc_method)
> 1
for: 10m
labels:
@@ -49,9 +49,9 @@ spec:
annotations:
summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' of requests for '{{`{{ $labels.grpc_method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
expr: |
100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method)
100 * sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) BY (job, instance, grpc_service, grpc_method)
/
sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method)
sum(rate(grpc_server_handled_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (job, instance, grpc_service, grpc_method)
> 5
for: 5m
labels:
@@ -61,7 +61,7 @@ spec:
annotations:
summary: "etcd cluster '{{`{{ $labels.job }}`}}': gRPC requests to '{{`{{ $labels.grpc_method }}`}}' are taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'."
expr: |
histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le))
histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le))
> 0.15
for: 10m
labels:
@@ -71,7 +71,7 @@ spec:
annotations:
summary: "etcd cluster '{{`{{ $labels.job }}`}}': member communication with '{{`{{ $labels.To }}`}}' is taking '{{`{{ $value }}`}}' on etcd instance '{{`{{ $labels.instance }}`}}'."
expr: |
histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m]))
> 0.15
for: 10m
labels:
@@ -81,7 +81,7 @@ spec:
annotations:
summary: "etcd cluster '{{`{{ $labels.job }}`}}': '{{`{{ $value }}`}}' proposal failures within the last hour on etcd instance '{{`{{ $labels.instance }}`}}'."
expr: |
rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
rate(etcd_server_proposals_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[15m]) > 5
for: 15m
labels:
severity: warning
@@ -90,7 +90,7 @@ spec:
annotations:
summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
expr: |
sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.01
sum(rate(etcd_http_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (method) > 0.01
for: 10m
labels:
severity: warning
@@ -99,7 +99,7 @@ spec:
annotations:
summary: "'{{`{{ $value }}`}}' of requests for '{{`{{ $labels.method }}`}}' failed on etcd instance '{{`{{ $labels.instance }}`}}'."
expr: |
sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) BY (method) > 0.05
sum(rate(etcd_http_failed_total{job="{{ .Release.Namespace }}/etcd-pod-scrape", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m])) BY (method) > 0.05
for: 10m
labels:
severity: critical
@@ -108,7 +108,7 @@ spec:
annotations:
summary: "etcd instance '{{`{{ $labels.instance }}`}}' HTTP requests to '{{`{{ $labels.method }}`}}' are slow."
expr: |
histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))
histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[5m]))
> 0.15
for: 10m
labels:
@@ -120,10 +120,10 @@ spec:
description: 'etcd cluster "{{`{{ $labels.job }}`}}": members are down {{`{{ $value }}`}}.'
expr: |
max without (endpoint) (
sum without (instance, pod) (up{job=~".*etcd.*"} == bool 0)
sum without (instance, pod) (up{job="{{ .Release.Namespace }}/etcd-pod-scrape"} == bool 0)
or
count without (To) (
sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01
sum without (instance, pod) (rate(etcd_network_peer_sent_failures_total{job="{{ .Release.Namespace }}/etcd-pod-scrape"}[120s])) > 0.01
)
)
> 0
@@ -0,0 +1,18 @@
suite: etcd alerts stay inside their release namespace
release:
name: etcd
namespace: tenant-scope
templates:
- templates/prometheus-rules.yaml
tests:
- it: scopes every alert query to this tenant's scrape job
asserts:
- matchRegex:
path: spec.groups[0].rules[*].expr
pattern: 'job="tenant-scope/etcd-pod-scrape"'
- notMatchRegex:
path: spec.groups[0].rules[*].expr
pattern: 'job=~'
+1
View File
@@ -115,6 +115,7 @@
| `alerta.alerts.email.repeatInterval` | Minimum time before a still-firing alert is emailed again, as an Alertmanager duration with units `y`, `w`, `d`, `h`, `m`, `s`, `ms` in descending order, each at most once, such as `5m`, `12h` or `1h30m`. Delivery is evaluated every 5m, so shorter intervals take effect at 5m and other intervals round up to a multiple of 5m. Zero, a unit out of order and a total beyond what Alertmanager accepts fail the render. Empty keeps the 12h default. | `string` | `""` |
| `alerta.alerts.email.alertnames` | Alert names emailed whether or not the rule carries the notify_email label, with no severity filter. Earlier platform routes still take precedence. Alerts with `event="Heartbeat"`, including `Watchdog`, go only to Alerta, and alerts with severity `none`, `CDIDefaultStorageClassDegraded` and `CDINoDefaultStorageClass` are blackholed and never reach email. This is the way to opt in alerts the platform ships, since their helm-owned rule definitions cannot be labelled per cluster. Entries must be plain Prometheus alert names such as `KubeAPIDown`, and anything else fails the render. Empty adds no extra route. | `[]string` | `[]` |
| `alerta.alerts.email.severities` | Severities the notify_email route accepts (e.g. ["critical","major","warning"]). Empty keeps the default of critical and major. List every severity in use to email opted-in alerts regardless of severity. Entries must be plain severity names, and anything else fails the render. | `[]string` | `[]` |
| `alerta.alerts.email.template` | Text of an Alertmanager template file that changes how the emails look. Define `email.default.subject` and `email.default.html` in it to replace the subject and the HTML body Alertmanager uses for every email, for example to mark the severity or to link to Alerta. The chart stores the text in a ConfigMap named `alertmanager-email-template` and lists it among the Alertmanager templates, so Alertmanager parses it together with its configuration. A template that does not parse fails that load, a running Alertmanager keeps the configuration it already has and a pod that starts with it does not come up. A template that parses but fails on a notification, for example on a missing key, drops that email. The text must contain at least one `define` action. Empty keeps the Alertmanager defaults. | `string` | `""` |
### Grafana configuration
@@ -557,6 +557,11 @@
"type": "string",
"default": ""
},
"template": {
"description": "Text of an Alertmanager template file that changes how the emails look. Define `email.default.subject` and `email.default.html` in it to replace the subject and the HTML body Alertmanager uses for every email, for example to mark the severity or to link to Alerta. The chart stores the text in a ConfigMap named `alertmanager-email-template` and lists it among the Alertmanager templates, so Alertmanager parses it together with its configuration. A template that does not parse fails that load, a running Alertmanager keeps the configuration it already has and a pod that starts with it does not come up. A template that parses but fails on a notification, for example on a missing key, drops that email. The text must contain at least one `define` action. Empty keeps the Alertmanager defaults.",
"type": "string",
"default": ""
},
"to": {
"description": "Recipient address, or several separated by commas. A display name is accepted, such as `Oncall \u003concall@example.org\u003e`, as long as it carries no comma. Quote display names containing punctuation such as `:`, `@`, parentheses or `?`. RFC 2047 encoded words are not accepted. IP domain literals accept dotted-quad IPv4 only. IPv6 literals are rejected because Go mail parsers disagree on their syntax across versions. ASCII control characters are rejected. Addresses must be literal, so Alertmanager template actions are rejected. The accepted form is deliberately narrower than RFC 5322. The local part must be a dot-atom, with no leading, trailing or doubled dot, and quoted local parts, comments and a `mailto:` prefix are rejected. An address outside that form fails the render, because Alertmanager parses addresses only when it sends and drops the notification with no record a tenant can read.",
"type": "string",
+2
View File
@@ -163,6 +163,7 @@ tracingCollector:
## @field {string} [repeatInterval] - Minimum time before a still-firing alert is emailed again, as an Alertmanager duration with units `y`, `w`, `d`, `h`, `m`, `s`, `ms` in descending order, each at most once, such as `5m`, `12h` or `1h30m`. Delivery is evaluated every 5m, so shorter intervals take effect at 5m and other intervals round up to a multiple of 5m. Zero, a unit out of order and a total beyond what Alertmanager accepts fail the render. Empty keeps the 12h default.
## @field {[]string} [alertnames] - Alert names emailed whether or not the rule carries the notify_email label, with no severity filter. Earlier platform routes still take precedence. Alerts with `event="Heartbeat"`, including `Watchdog`, go only to Alerta, and alerts with severity `none`, `CDIDefaultStorageClassDegraded` and `CDINoDefaultStorageClass` are blackholed and never reach email. This is the way to opt in alerts the platform ships, since their helm-owned rule definitions cannot be labelled per cluster. Entries must be plain Prometheus alert names such as `KubeAPIDown`, and anything else fails the render. Empty adds no extra route.
## @field {[]string} [severities] - Severities the notify_email route accepts (e.g. ["critical","major","warning"]). Empty keeps the default of critical and major. List every severity in use to email opted-in alerts regardless of severity. Entries must be plain severity names, and anything else fails the render.
## @field {string} [template] - Text of an Alertmanager template file that changes how the emails look. Define `email.default.subject` and `email.default.html` in it to replace the subject and the HTML body Alertmanager uses for every email, for example to mark the severity or to link to Alerta. The chart stores the text in a ConfigMap named `alertmanager-email-template` and lists it among the Alertmanager templates, so Alertmanager parses it together with its configuration. A template that does not parse fails that load, a running Alertmanager keeps the configuration it already has and a pod that starts with it does not come up. A template that parses but fails on a notification, for example on a missing key, drops that email. The text must contain at least one `define` action. Empty keeps the Alertmanager defaults.
## @typedef {struct} Alerts - Alert routing configuration.
## @field {TelegramAlerts} [telegram] - Configuration for Telegram alerts.
@@ -202,6 +203,7 @@ alerta:
repeatInterval: ""
alertnames: []
severities: []
template: ""
##
## @section Grafana configuration
##
@@ -228,10 +228,14 @@ tests:
gateway-class-name: " cloudflare-tunnel "
gateway-edge-classes: cloudflare-tunnel
asserts:
- documentIndex: 0
- documentSelector:
path: metadata.name
value: letsencrypt-prod
notExists:
path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].sectionName
- documentIndex: 1
- documentSelector:
path: metadata.name
value: letsencrypt-stage
notExists:
path: spec.acme.solvers[0].http01.gatewayHTTPRoute.parentRefs[0].sectionName
-4
View File
@@ -30,10 +30,6 @@ cilium:
enabled: true
hubble:
enabled: false
externalIPs:
enabled: true
nodePort:
enabled: true
loadBalancer:
algorithm: maglev
serviceTopology: true
+1 -1
View File
@@ -8,7 +8,7 @@ spec:
singular: computeplane
plural: computeplanes
openAPISchema: |-
{"title":"Chart Values","type":"object","properties":{"nodeGroups":{"description":"Worker node groups of the ComputePlane cluster (cluster shape only — the security posture is fixed by the module and cannot be overridden through this knob: the module forwards only the documented per-pool fields to the `kubernetes-nodes` pool chart, so any other `NodeGroup` field, e.g. `talos`/`version`/`images`, is silently dropped — accepted but inert). Same structure as the `kubernetes` application's `nodeGroups`. When left empty, the module itself materialises a default scale-from-zero `md0` pool with `roles: [ingress-nginx]` (the wrapped `kubernetes` chart no longer injects one after the Phase 2 split), so the cluster sits idle until a pending Pod (e.g. the ingress-nginx controller) triggers the cluster-autoscaler. Supplied groups are authoritative (not merged with the default), so at least one of them must carry `roles: [ingress-nginx]` — enforced at render time. Each supplied pool key must be at most 15 characters so its `kubernetes-nodes-computeplane-cluster-<pool>` HelmRelease name fits Helm's 53-character limit — also enforced at render time.","type":"object","default":{},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas"],"properties":{"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Set both `cpu` and `memory` together or neither.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles. At least one group must carry `ingress-nginx` — the module pins the ingress-nginx addon on, and its controller only schedules onto nodes with that role (enforced at render time).","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, the `kubernetes-nodes` pool chart applies its own default (`replicated`).","type":"string","x-cozystack-options":{"source":"storageclass"}}}}}}}
{"title":"Chart Values","type":"object","properties":{"nodeGroups":{"description":"Worker node groups of the ComputePlane cluster (cluster shape only — the security posture is fixed by the module and cannot be overridden through this knob: the module forwards only the documented per-pool fields to the `kubernetes-nodes` pool chart, so any other `NodeGroup` field, e.g. `talos`/`version`/`images`, is silently dropped — accepted but inert). Same structure as the `kubernetes` application's `nodeGroups`. When left empty, the module itself materialises a default scale-from-zero `md0` pool with `roles: [ingress-nginx]` (the wrapped `kubernetes` chart no longer injects one after the Phase 2 split), so the cluster sits idle until a pending Pod (e.g. the ingress-nginx controller) triggers the cluster-autoscaler. Supplied groups are authoritative (not merged with the default), so at least one of them must carry `roles: [ingress-nginx]` — enforced at render time. Each supplied pool key must be at most 15 characters so its `kubernetes-nodes-computeplane-cluster-<pool>` HelmRelease name fits Helm's 53-character limit — also enforced at render time.","type":"object","default":{},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas"],"properties":{"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"maxUnavailable":{"description":"Workers a rollout may remove before their replacement is up. Unset, the `kubernetes-nodes` pool chart uses 0, so each replacement joins before an old worker is drained (1 on a pool with `gpus`). Set 1 on a pool without room for one extra VM, whose rollout would otherwise stall with the new Machine Pending.","type":"integer","maximum":2147483647,"minimum":0},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Set both `cpu` and `memory` together or neither.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles. At least one group must carry `ingress-nginx` — the module pins the ingress-nginx addon on, and its controller only schedules onto nodes with that role (enforced at render time).","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, the `kubernetes-nodes` pool chart applies its own default (`replicated`).","type":"string","x-cozystack-options":{"source":"storageclass"}}}}}}}
release:
prefix: ""
labels:
@@ -49,10 +49,12 @@ rules:
resources: ["namespaces"]
verbs: ["get", "list", "watch", "patch", "update"]
# TenantQuotaReconciler manages the per-namespace "tenant-quota-allocated"
# ResourceQuota that enforces each tenant's share of its parent pool, and emits
# events on the tenant namespace when sub-tenant quotas overcommit the budget.
# ResourceQuota that enforces each tenant's share of its parent pool, copies the
# pool root's "tenant-range-limits" LimitRange into unbounded members as
# "tenant-range-limits-allocated", and emits events on the tenant namespace when
# sub-tenant quotas overcommit the budget.
- apiGroups: [""]
resources: ["resourcequotas"]
resources: ["resourcequotas", "limitranges"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: [""]
resources: ["events"]
@@ -21,7 +21,7 @@ spec:
singular: kubernetesnodes
plural: kubernetesnodeses
openAPISchema: |-
{"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-<cluster>`; the KubernetesNodes CR must be named `<cluster>-<pool>`. The pool's objects are named `kubernetes-<cluster>-<pool>`; when a MachineDeployment of that name already exists and belongs to another Helm release, the render fails instead of taking it over.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"KubeVirt instance type that sizes each worker VM when `resources` is not set. Ignored when `resources.cpu` and `resources.memory` are both set, and therefore on `substrate: proxmox`, which requires them.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. Has no effect on `substrate: proxmox`, where the worker disk is a clone of the VM template's disk and keeps its size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/<r>`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `iLine truncated
{"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-<cluster>`; the KubernetesNodes CR must be named `<cluster>-<pool>`. The pool's objects are named `kubernetes-<cluster>-<pool>`; when a MachineDeployment of that name already exists and belongs to another Helm release, the render fails instead of taking it over.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"KubeVirt instance type that sizes each worker VM when `resources` is not set. Ignored when `resources.cpu` and `resources.memory` are both set, and therefore on `substrate: proxmox`, which requires them.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. Has no effect on `substrate: proxmox`, where the worker disk is a clone of the VM template's disk and keeps its size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/<r>`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `iLine truncated
release:
labels:
sharding.fluxcd.io/key: tenants
@@ -116,8 +116,9 @@ spec:
{{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance
}}.'
summary: etcd cluster member communication is slow.
# Tenant peer RTT is covered by the tenant etcd rules.
expr: |-
histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*", job!~"tenant-.*/etcd-pod-scrape"}[5m]))
> 0.15
for: 10m
labels:
@@ -0,0 +1,20 @@
suite: tenant peer RTT is evaluated once
templates:
- templates/alerts.yaml
tests:
- it: leaves tenant peer RTT to the tenant rules
documentSelector:
path: metadata.name
value: alerts-etcd
asserts:
- matchRegex:
path: spec.groups[0].rules[7].expr
pattern: 'job!~"tenant-\.\*/etcd-pod-scrape"'
- notMatchRegex:
path: spec.groups[0].rules[9].expr
pattern: 'job!~'
- notMatchRegex:
path: spec.groups[0].rules[10].expr
pattern: 'job!~'
+2 -2
View File
@@ -20,7 +20,7 @@ spec:
singular: monitoring
plural: monitorings
openAPISchema: |-
{"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access Grafana externally (defaults to 'grafana' subdomain for the tenant host).","type":"string","default":""},"metricsStorages":{"description":"Configuration of metrics storage instances.","type":"array","default":[{"deduplicationInterval":"15s","name":"shortterm","retentionPeriod":"3d","storage":"10Gi","storageClassName":""},{"deduplicationInterval":"5m","name":"longterm","retentionPeriod":"14d","storage":"10Gi","storageClassName":""}],"items":{"type":"object","required":["deduplicationInterval","name","retentionPeriod","storage"],"properties":{"deduplicationInterval":{"description":"Deduplication interval for metrics.","type":"string"},"name":{"description":"Name of the storage instance.","type":"string"},"retentionPeriod":{"description":"Retention period for metrics.","type":"string"},"storage":{"description":"Persistent volume size.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used for the data.","type":"string"},"vminsert":{"description":"Configuration for vminsert.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"vmselect":{"description":"Configuration for vmselect.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"vmstorage":{"description":"Configuration for vmstorage.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0Line truncated
{"title":"Chart Values","type":"object","properties":{"host":{"description":"The hostname used to access Grafana externally (defaults to 'grafana' subdomain for the tenant host).","type":"string","default":""},"metricsStorages":{"description":"Configuration of metrics storage instances.","type":"array","default":[{"deduplicationInterval":"15s","name":"shortterm","retentionPeriod":"3d","storage":"10Gi","storageClassName":""},{"deduplicationInterval":"5m","name":"longterm","retentionPeriod":"14d","storage":"10Gi","storageClassName":""}],"items":{"type":"object","required":["deduplicationInterval","name","retentionPeriod","storage"],"properties":{"deduplicationInterval":{"description":"Deduplication interval for metrics.","type":"string"},"name":{"description":"Name of the storage instance.","type":"string"},"retentionPeriod":{"description":"Retention period for metrics.","type":"string"},"storage":{"description":"Persistent volume size.","type":"string","default":"10Gi"},"storageClassName":{"description":"StorageClass used for the data.","type":"string"},"vminsert":{"description":"Configuration for vminsert.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"vmselect":{"description":"Configuration for vmselect.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}}}},"vmstorage":{"description":"Configuration for vmstorage.","type":"object","properties":{"maxAllowed":{"description":"Maximum allowed resources.","type":"object","properties":{"cpu":{"description":"CPU limit.","default":1,"pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory limit.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"minAllowed":{"description":"Minimum guaranteed resources.","type":"object","properties":{"cpu":{"description":"CPU request.","default":"100m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory request.","default":"256Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0Line truncated
release:
prefix: ""
labels:
@@ -38,7 +38,7 @@ spec:
description: Monitoring and observability stack
module: true
icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODdfMzI2OCkiLz4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzY4N18zMjY4KSI+CjxwYXRoIGQ9Ik04OS41MDM5IDExMS43MDdINTQuNDk3QzU0LjE3MjcgMTExLjcwNyA1NC4wMTA4IDExMS4yMjEgNTQuMzM0OSAxMTEuMDU5TDU3LjI1MjIgMTA4Ljk1MkM2MC4zMzE0IDEwNi42ODMgNjEuOTUyMiAxMDIuNjMxIDYwLjk3OTcgOTguNzQxMkg4My4wMjFDODIuMDQ4NSAxMDIuNjMxIDgzLjY2OTMgMTA2LjY4MyA4Ni43NDg1IDEwOC45NTJMODkuNjY1OCAxMTEuMDU5Qzg5Ljk5IDExMS4yMjEgODkuODI3OSAxMTEuNzA3IDg5LjUwMzkgMTExLjcwN1oiIGZpbGw9IiNCMEI2QkIiLz4KPHBhdGggZD0iTTExMy4zMjggOTguNzQxSDMwLjY3MjVDMjcuNTkzMSA5OC43NDEgMjUgOTYuMTQ4IDI1IDkzLjA2ODdWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJWOTMuMDY4N0MxMTkgOTYuMTQ4IDExNi40MDcgOTguNzQxIDExMy4zMjggOTguNzQxWiIgZmlsbD0iI0U4RURFRSIvPgo8cGF0aCBkPSJNMTE5IDg0LjE1NDlIMjVWMzMuMTAzMkMyNSAzMC4wMjM5IDI3LjU5MzEgMjcuNDMwNyAzMC42NzI1IDI3LjQzMDdIMTEzLjMyOEMxMTYuNDA3IDI3LjQzMDcgMTE5IDMwLjAyMzcgMTE5IDMzLjEwMzJMMTE5IDg0LjE1NDlaIiBmaWxsPSIjMzg0NTRGIi8+CjxwYXRoIGQ9Ik05MC42Mzc0IDExNi41NjlINTMuMzYxNkM1Mi4wNjUxIDExNi41NjkgNTAuOTMwNyAxMTUuNDM1IDUwLjkzMDcgMTE0LjEzOEM1MC45MzA3IDExMi44NDEgNTIuMDY1MSAxMTEuNzA3IDUzLjM2MTYgMTExLjcwN0g5MC42Mzc0QzkxLjkzMzkgMTExLjcwNyA5My4wNjg0IDExMi44NDEgOTMuMDY4NCAxMTQuMTM4QzkzLjA2ODQgMTE1LjQzNSA5MS45MzM5IDExNi41NjkgOTAuNjM3NCAxMTYuNTY5WiIgZmlsbD0iI0U4RURFRSIvPgo8cGF0aCBkPSJNNTQuMTcyMiAzOC43NzU3SDMzLjEwMzJDMzIuMTMwNyAzOC43NzU3IDMxLjQ4MjQgMzguMTI3NCAzMS40ODI0IDM3LjE1NDlDMzEuNDgyNCAzNi4xODI0IDMyLjEzMDcgMzUuNTM0MiAzMy4xMDMyIDM1LjUzNDJINTQuMTcyMkM1NS4xNDQ3IDM1LjUzNDIgNTUuNzkzIDM2LjE4MjQgNTUuNzkzIDM3LjE1NDlDNTUuNzkyOCAzOC4xMjc0IDU1LjE0NDUgMzguNzc1NyA1NC4xNzIyIDM4Ljc3NTdaIiBmaWxsPSIjREQzNDJFIi8+CjxwYXRoIGQ9Ik02My44OTYzIDQ1LjI1OTFINDEuMjA2N0M0MC4yMzQyIDQ1LjI1OTEgMzkuNTg1OSA0NC42MTA4IDM5LjU4NTkgNDMuNjM4M0MzOS41ODU5IDQyLjY2NTggNDAuMjM0MiA0Mi4wMTc2IDQxLjIwNjcgNDIuMDE3Nkg2My44OTYzQzY0Ljg2ODggNDIuMDE3NiA2NS41MTcxIDQyLjY2NTggNjUuNTE3MSA0My42MzgzQzY1LjUxNzEgNDQuNjEwOCA2NC44Njg4IDQ1LjI1OTEgNjMuODk2MyA0NS4yNTkxWiIgZmlsbD0iIzczODNCRiIvPgo8cGF0aCBkPSJNMzQuNzI0IDQ1LjI1OTFIMzMuMTAzMkMzMi4xMzA3IDQ1LjI1OTEgMzEuNDgyNCA0NC42MTA4IDMxLjQ4MjQgNDMuNjM4M0MzMS40ODI0IDQyLjY2NTggMzIuMTMwNyA0Mi4wMTc2IDMzLjEwMzIgNDIuMDE3NkgzNC43MjRDMzUuNjk2NCA0Mi4wMTc2IDM2LjM0NDcgNDIuNjY1OCAzNi4zNDQ3IDQzLjYzODNDMzYuMzQ0NyA0NC42MTA4IDM1LjY5NjMgNDUuMjU5MSAzNC43MjQgNDUuMjU5MVoiIGZpbGw9IiM0MkIwNUMiLz4KPHBhdGggZD0iTTYzLjg5NjMgMzguNzc1N0g2MC42NTQ5QzU5LjY4MjQgMzguNzc1NyA1OS4wMzQyIDM4LjEyNzQgNTkuMDM0MiAzNy4xNTQ5QzU5LjAzNDIgMzYuMTgyNCA1OS42ODI0IDM1LjUzNDIgNjAuNjU0OSAzNS41MzQySDYzLjg5NjNDNjQuODY4OCAzNS41MzQyIDY1LjUxNzEgMzYuMTgyNCA2NS41MTcxIDM3LjE1NDlDNjUuNTE3MSAzOC4xMjc0IDY0Ljg2ODggMzguNzc1NyA2My44OTYzIDM4Ljc3NTdaIiBmaWxsPSIjRUNCQTE2Ii8+CjxwYXRoIGQ9Ik00Ny42ODkzIDUxLjc0MTNIMzMuMTAzMkMzMi4xMzA3IDUxLjc0MTMgMzEuNDgyNCA1MS4wOTMxIDMxLjQ4MjQgNTAuMTIwNkMzMS40ODI0IDQ5LjE0ODEgMzIuMTMwNyA0OC41IDMzLjEwMzIgNDguNUg0Ny42ODkzQzQ4LjY2MTggNDguNSA0OS4zMTAxIDQ5LjE0ODMgNDkuMzEwMSA1MC4xMjA4QzQ5LjMxMDEgNTEuMDkzMyA0OC42NjE4IDUxLjc0MTMgNDcuNjg5MyA1MS43NDEzWiIgZmlsbD0iI0REMzQyRSIvPgo8cGF0aCBkPSJNNjMuODk2OCA1MS43NDEzSDU0LjE3MjVDNTMuMiA1MS43NDEzIDUyLjU1MTggNTEuMDkzMSA1Mi41NTE4IDUwLjEyMDZDNTIuNTUxOCA0OS4xNDgxIDUzLjIwMDIgNDguNSA1NC4xNzI3IDQ4LjVINjMuODk2OUM2NC44Njk0IDQ4LjUgNjUuNTE3NyA0OS4xNDgzIDY1LjUxNzcgNTAuMTIwOEM2NS41MTc3IDUxLjA5MzMgNjQuODY5MiA1MS43NDEzIDYzLjg5NjggNTEuNzQxM1oiIGZpbGw9IiNFQ0JBMTYiLz4KPHBhdGggZD0iTTU0LjE3MjIgNTguMjI0SDMzLjEwMzJDMzIuMTMwNyA1OC4yMjQgMzEuNDgyNCA1Ny41NzU3IDMxLjQ4MjQgNTYuNjAzMkMzMS40ODI0IDU1LjYzMDcgMzIuMTMwNyA1NC45ODI0IDMzLjEwMzIgNTQuOTgyNEg1NC4xNzIyQzU1LjE0NDcgNTQuOTgyNCA1NS43OTMgNTUuNjMwNyA1NS43OTMgNTYuNjAzMkM1NS43OTMgNTcuNTc1NyA1NS4xNDQ1IDU4LjIyNCA1NC4xNzIyIDU4LjIyNFoiIGZpbGw9IiM0MkIwNUMiLz4KPHBhdGggZD0iTTYzLjg5NjMgNjQuNzA3NEg0MS4yMDY3QzQwLjIzNDIgNjQuNzA3NCAzOS41ODU5IDY0LjA1OTEgMzkuNTg1OSA2My4wODY2QzM5LjU4NTkgNjIuMTE0MSA0MC4yMzQyIDYxLjQ2NTggNDEuMjA2NyA2MS40NjU4SDYzLjg5NjNDNjQuODY4OCA2MS40NjU4IDY1LjUxNzEgNjIuMTE0MSA2NS41MTcxIDYzLjA4NjZDNjUuNTE3MSA2NC4wNTkxIDY0Ljg2ODggNjQuNzA3NCA2My44OTYzIDY0LjcwNzRaIiBmaWxsPSIjRUNCQTE2Ii8+CjxwYXRoIGQ9Ik0zNC43MjQgNjQuNzA3NEgzMy4xMDMyQzMyLjEzMDcgNjQuNzA3NCAzMS40ODI0IDY0LjA1OTEgMzEuNDgyNCA2My4wODY2QzMxLjQ4MjQgNjIuMTE0MSAzMi4xMzA3IDYxLjQ2NTggMzMuMTAzMiA2MS40NjU4SDM0LjcyNEMzNS42OTY0IDYxLjQ2NTggMzYuMzQ0NyA2Mi4xMTQxIDM2LjM0NDcgNjMuMDg2NkMzNi4zNDQ3IDY0LjA1OTEgMzUuNjk2MyA2NC43MDc0IDM0LjcyNCA2NC43MDc0WiIgZmlsbD0iI0REMzQyRSIvPgo8cGF0aCBkPSJNNDcuNjg5MyA3MS4xODk4SDMzLjEwMzJDMzIuMTMwNyA3MS4xODk4IDMxLjQ4MjQgNzAuNTQxNSAzMS40ODI0IDY5LjU2OUMzMS40ODI0IDY4LjU5NjUgMzIuMTMwNyA2Ny45NDgyIDMzLjEwMzIgNjcuOTQ4Mkg0Ny42ODkzQzQ4LjY2MTggNjcuOTQ4MiA0OS4zMTAxIDY4LjU5NjUgNDkuMzEwMSA2OS41NjlDNDkuMzEwMSA3MC41NDE1IDQ4LjY2MTggNzEuMTg5OCA0Ny42ODkzIDcxLjE4OThaIiBmaWxsPSIjNDJCMDVDIi8+CjxwYXRoIGQ9Ik02My44OTY4IDcxLjE4OThINTQuMTcyNUM1My4yIDcxLjE4OTggNTIuNTUxOCA3MC41NDE1IDUyLjU1MTggNjkuNTY5QzUyLjU1MTggNjguNTk2NSA1My4yIDY3Ljk0ODIgNTQuMLine truncated
keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "metricsStorages"], ["spec", "logsStorages"], ["spec", "tracingStorages"], ["spec", "tracingCentralHost"], ["spec", "tracingCollector"], ["spec", "tracingCollector", "enabled"], ["spec", "tracingCollector", "replicas"], ["spec", "tracingCollector", "samplingPercentage"], ["spec", "tracingCollector", "redactAttributes"], ["spec", "tracingCollector", "resources"], ["spec", "tracingCollector", "resources", "limits"], ["spec", "tracingCollector", "resources", "limits", "cpu"], ["spec", "tracingCollector", "resources", "limits", "memory"], ["spec", "tracingCollector", "resources", "requests"], ["spec", "tracingCollector", "resources", "requests", "cpu"], ["spec", "tracingCollector", "resources", "requests", "memory"], ["spec", "alerta"], ["spec", "alerta", "storage"], ["spec", "alerta", "storageClassName"], ["spec", "alerta", "resources"], ["spec", "alerta", "resources", "limits"], ["spec", "alerta", "resources", "limits", "cpu"], ["spec", "alerta", "resources", "limits", "memory"], ["spec", "alerta", "resources", "requests"], ["spec", "alerta", "resources", "requests", "cpu"], ["spec", "alerta", "resources", "requests", "memory"], ["spec", "alerta", "alerts"], ["spec", "alerta", "alerts", "telegram"], ["spec", "alerta", "alerts", "telegram", "token"], ["spec", "alerta", "alerts", "telegram", "chatID"], ["spec", "alerta", "alerts", "telegram", "disabledSeverity"], ["spec", "alerta", "alerts", "slack"], ["spec", "alerta", "alerts", "slack", "url"], ["spec", "alerta", "alerts", "slack", "disabledSeverity"], ["spec", "alerta", "alerts", "email"], ["spec", "alerta", "alerts", "email", "to"], ["spec", "alerta", "alerts", "email", "from"], ["spec", "alerta", "alerts", "email", "smarthost"], ["spec", "alerta", "alerts", "email", "authUsername"], ["spec", "alerta", "alerts", "email", "repeatInterval"], ["spec", "alerta", "alerts", "email", "alertnames"], ["spec", "alerta", "alerts", "email", "severities"], ["spec", "grafana"], ["spec", "grafana", "db"], ["spec", "grafana", "db", "size"], ["spec", "grafana", "resources"], ["spec", "grafana", "resources", "limits"], ["spec", "grafana", "resources", "limits", "cpu"], ["spec", "grafana", "resources", "limits", "memory"], ["spec", "grafana", "resources", "requests"], ["spec", "grafana", "resources", "requests", "cpu"], ["spec", "grafana", "resources", "requests", "memory"], ["spec", "vmagent"], ["spec", "vmagent", "externalLabels"], ["spec", "vmagent", "externalLabels", "cluster"], ["spec", "vmagent", "remoteWrite"], ["spec", "vmagent", "remoteWrite", "urls"], ["spec", "oidc"], ["spec", "oidc", "mode"], ["spec", "oidc", "customConfig"], ["spec", "oidc", "customConfig", "config"], ["spec", "oidc", "customConfig", "secretRef"], ["spec", "oidc", "customConfig", "secretRef", "name"], ["spec", "oidc", "users"]]
keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "host"], ["spec", "metricsStorages"], ["spec", "logsStorages"], ["spec", "tracingStorages"], ["spec", "tracingCentralHost"], ["spec", "tracingCollector"], ["spec", "tracingCollector", "enabled"], ["spec", "tracingCollector", "replicas"], ["spec", "tracingCollector", "samplingPercentage"], ["spec", "tracingCollector", "redactAttributes"], ["spec", "tracingCollector", "resources"], ["spec", "tracingCollector", "resources", "limits"], ["spec", "tracingCollector", "resources", "limits", "cpu"], ["spec", "tracingCollector", "resources", "limits", "memory"], ["spec", "tracingCollector", "resources", "requests"], ["spec", "tracingCollector", "resources", "requests", "cpu"], ["spec", "tracingCollector", "resources", "requests", "memory"], ["spec", "alerta"], ["spec", "alerta", "storage"], ["spec", "alerta", "storageClassName"], ["spec", "alerta", "resources"], ["spec", "alerta", "resources", "limits"], ["spec", "alerta", "resources", "limits", "cpu"], ["spec", "alerta", "resources", "limits", "memory"], ["spec", "alerta", "resources", "requests"], ["spec", "alerta", "resources", "requests", "cpu"], ["spec", "alerta", "resources", "requests", "memory"], ["spec", "alerta", "alerts"], ["spec", "alerta", "alerts", "telegram"], ["spec", "alerta", "alerts", "telegram", "token"], ["spec", "alerta", "alerts", "telegram", "chatID"], ["spec", "alerta", "alerts", "telegram", "disabledSeverity"], ["spec", "alerta", "alerts", "slack"], ["spec", "alerta", "alerts", "slack", "url"], ["spec", "alerta", "alerts", "slack", "disabledSeverity"], ["spec", "alerta", "alerts", "email"], ["spec", "alerta", "alerts", "email", "to"], ["spec", "alerta", "alerts", "email", "from"], ["spec", "alerta", "alerts", "email", "smarthost"], ["spec", "alerta", "alerts", "email", "authUsername"], ["spec", "alerta", "alerts", "email", "repeatInterval"], ["spec", "alerta", "alerts", "email", "alertnames"], ["spec", "alerta", "alerts", "email", "severities"], ["spec", "alerta", "alerts", "email", "template"], ["spec", "grafana"], ["spec", "grafana", "db"], ["spec", "grafana", "db", "size"], ["spec", "grafana", "resources"], ["spec", "grafana", "resources", "limits"], ["spec", "grafana", "resources", "limits", "cpu"], ["spec", "grafana", "resources", "limits", "memory"], ["spec", "grafana", "resources", "requests"], ["spec", "grafana", "resources", "requests", "cpu"], ["spec", "grafana", "resources", "requests", "memory"], ["spec", "vmagent"], ["spec", "vmagent", "externalLabels"], ["spec", "vmagent", "externalLabels", "cluster"], ["spec", "vmagent", "remoteWrite"], ["spec", "vmagent", "remoteWrite", "urls"], ["spec", "oidc"], ["spec", "oidc", "mode"], ["spec", "oidc", "customConfig"], ["spec", "oidc", "customConfig", "config"], ["spec", "oidc", "customConfig", "secretRef"], ["spec", "oidc", "customConfig", "secretRef", "name"], ["spec", "oidc", "users"]]
secrets:
exclude: []
include:
@@ -10,8 +10,14 @@
{{- $emailAlertnames := $email.alertnames | default list }}
{{- $emailSeverities := $email.severities | default (list "critical" "major") }}
{{- $emailPasswordSecret := "alertmanager-email-password" }}
{{- $emailTemplateConfigMap := "alertmanager-email-template" }}
{{- $emailTemplateRaw := index $email "template" }}
{{- if not (or (kindIs "string" $emailTemplateRaw) (kindIs "invalid" $emailTemplateRaw)) }}
{{- fail (printf "alerta.alerts.email.template must be the text of an Alertmanager template file, got a %s" (kindOf $emailTemplateRaw)) }}
{{- end }}
{{- $emailTemplate := $emailTemplateRaw | default "" }}
{{- $emailSet := list }}
{{- range $field := list "to" "from" "smarthost" "authUsername" "repeatInterval" "alertnames" "severities" }}
{{- range $field := list "to" "from" "smarthost" "authUsername" "repeatInterval" "alertnames" "severities" "template" }}
{{- if index $email $field }}{{ $emailSet = append $emailSet $field }}{{ end }}
{{- end }}
{{- $emailEnabled := gt (len $emailSet) 0 }}
@@ -101,6 +107,11 @@
{{- fail (printf "alerta.alerts.email.severities entries must be plain severity names, got %q" (toString $severity)) }}
{{- end }}
{{- end }}
{{- if $emailTemplate }}
{{- if not (regexMatch "\\{\\{-?\\s*define\\s" $emailTemplate) }}
{{- fail "alerta.alerts.email.template must define named templates, such as email.default.subject and email.default.html, because Alertmanager only uses the file through the names it defines" }}
{{- end }}
{{- end }}
{{- end }}
{{- $apiKey := randAlphaNum 32 }}
@@ -408,6 +419,15 @@ stringData:
{{- end }}
send_resolved: true
{{- end }}
{{- if and $emailEnabled $emailTemplate }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ $emailTemplateConfigMap }}
data:
email.tmpl: {{ $emailTemplate | quote }}
{{- end }}
---
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMAlertmanager
@@ -425,6 +445,11 @@ spec:
secrets:
- {{ $emailPasswordSecret | quote }}
{{- end }}
{{- if and $emailEnabled $emailTemplate }}
templates:
- name: {{ $emailTemplateConfigMap }}
key: email.tmpl
{{- end }}
podMetadata:
labels:
policy.cozystack.io/allow-to-apiserver: "true"
@@ -1149,3 +1149,70 @@ tests:
asserts:
- failedTemplate:
errorMessage: "alerta.alerts.email.from must be one address, such as alerts@example.org or Alerts <alerts@example.org>, got \"alerts@[IPv6:2001:db8::25]\""
# alerta.alerts.email.template carries an Alertmanager template file. It is
# stored in a ConfigMap and listed in VMAlertmanager spec.templates, so
# Alertmanager parses it while loading its configuration, and the definitions
# it holds can replace email.default.subject and email.default.html.
- it: mounts no email template when template is empty
set:
alerta:
alerts:
email:
to: "oncall@example.org"
from: "alerts@example.org"
smarthost: "smtp.example.org:465"
asserts:
- containsDocument:
apiVersion: v1
kind: ConfigMap
name: alertmanager-email-template
any: true
not: true
- documentSelector:
path: kind
value: VMAlertmanager
notExists:
path: spec.templates
- it: stores the template text verbatim and lists it among the Alertmanager templates
set:
alerta:
alerts:
email:
to: "oncall@example.org"
from: "alerts@example.org"
smarthost: "smtp.example.org:465"
authUsername: "mailer@example.org"
template: |
{{ define "email.default.subject" }}[prod] {{ if eq .Status "resolved" }}✅ resolved{{ else }}🔴 {{ .CommonLabels.alertname }}{{ end }}{{ end }}
{{ define "email.default.html" }}<p style="color:#c0392b">{{ .CommonAnnotations.summary }}, проверка "кавычек" и \ слеша</p>{{ end }}
asserts:
- containsDocument:
apiVersion: v1
kind: ConfigMap
name: alertmanager-email-template
any: true
- documentSelector:
path: metadata.name
value: alertmanager-email-template
equal:
path: data["email.tmpl"]
value: |
{{ define "email.default.subject" }}[prod] {{ if eq .Status "resolved" }}✅ resolved{{ else }}🔴 {{ .CommonLabels.alertname }}{{ end }}{{ end }}
{{ define "email.default.html" }}<p style="color:#c0392b">{{ .CommonAnnotations.summary }}, проверка "кавычек" и \ слеша</p>{{ end }}
- documentSelector:
path: kind
value: VMAlertmanager
equal:
path: spec.templates
value:
- name: alertmanager-email-template
key: email.tmpl
- documentSelector:
path: kind
value: VMAlertmanager
equal:
path: spec.secrets
value:
- alertmanager-email-password
@@ -226,3 +226,35 @@ tests:
asserts:
- failedTemplate:
errorMessage: "alerta.alerts.email.to has an empty entry, got \" \""
- it: rejects a template without a define action
set:
alerta.alerts.email.template: "plain text {{ .Status }}"
asserts:
- failedTemplate:
errorMessage: "alerta.alerts.email.template must define named templates, such as email.default.subject and email.default.html, because Alertmanager only uses the file through the names it defines"
- it: rejects a template that is not text
set:
alerta.alerts.email.template:
subject: "[prod]"
asserts:
- failedTemplate:
errorMessage: "alerta.alerts.email.template must be the text of an Alertmanager template file, got a map"
- it: names template among the fields still set when the addresses are cleared
set:
alerta.alerts.email.to: ""
alerta.alerts.email.from: ""
alerta.alerts.email.smarthost: ""
alerta.alerts.email.template: '{{ define "email.default.subject" }}[prod]{{ end }}'
asserts:
- failedTemplate:
errorMessage: "alerta.alerts.email to, from and smarthost must all be set to enable email alerts. To turn email off, clear every field under alerta.alerts.email. Fields still set are template"
- it: rejects a template given as a boolean instead of being dropped by the default
set:
alerta.alerts.email.template: false
asserts:
- failedTemplate:
errorMessage: "alerta.alerts.email.template must be the text of an Alertmanager template file, got a bool"
+1
View File
@@ -95,6 +95,7 @@ alerta:
repeatInterval: ""
alertnames: []
severities: []
template: ""
grafana:
# Upstream Grafana image, pinned by digest and routed through
# cozy-lib.image (so mirrored-registry installs work). Plugins (VictoriaLogs
@@ -156,9 +156,11 @@ tests:
resources:
requests:
storage: 10Gi
documentSelector:
path: kind
value: Job
asserts:
- documentIndex: 0
notMatchRegex:
- notMatchRegex:
path: spec.template.spec.containers[0].args[0]
pattern: kubectl patch pvc