diff --git a/.github/workflows/e2e-fork.yaml b/.github/workflows/e2e-fork.yaml index f988632e9..03c568395 100644 --- a/.github/workflows/e2e-fork.yaml +++ b/.github/workflows/e2e-fork.yaml @@ -61,7 +61,8 @@ jobs: # base-repo PR file list — NOT from the presence/absence of an artifact the # fork's own build controls. Fail-closed if the fork build failed or the # pr-patch artifact is missing; green-and-stop only for a PR whose file list is - # entirely under docs/ (verified against the base-repo API). + # entirely under docs/ or the CI-only tools cmd/backport-batch/ and + # cmd/api-gate/ (verified against the base-repo API). resolve: name: Resolve fork PR # Only fork PRs: head_repository differs from the base repository. Same-repo @@ -266,15 +267,21 @@ jobs: await decide('failure', 'PR file list hit the 3000-file API cap — cannot verify docs-only'); return; } + // Paths with nothing to build and nothing for e2e to install: docs, + // and the CI-only Go tools no image builds. pull-requests.yaml's + // plan job answers the same question with the same prefixes, and + // hack/gate-concurrency-contract.bats holds the two lists equal. + // // A rename reports the NEW path in `filename` and the old one in // `previous_filename`. Checking only `filename` would read a // packages/… -> docs/… rename as docs-only, when it in fact removes - // a code path. BOTH sides must be under docs/ to count as docs-only. - const isDocs = p => typeof p === 'string' && p.startsWith('docs/'); - const hasNonDocs = files.some(f => - !isDocs(f.filename) || (f.previous_filename && !isDocs(f.previous_filename))); - if (!hasNonDocs) { - await decide('success', 'docs-only PR (verified via PR file list) — e2e not required'); + // a code path. BOTH sides must be under these prefixes to count. + const noRuntimePrefixes = ['docs/', 'cmd/backport-batch/', 'cmd/api-gate/']; + const isNoRuntime = p => typeof p === 'string' && noRuntimePrefixes.some(x => p.startsWith(x)); + const hasRuntime = files.some(f => + !isNoRuntime(f.filename) || (f.previous_filename && !isNoRuntime(f.previous_filename))); + if (!hasRuntime) { + await decide('success', 'docs or CI-only tools (verified via PR file list) — e2e not required'); return; } @@ -543,7 +550,7 @@ jobs: # Covers a green full suite. It does not cover a tenant-Kubernetes suite # running into its node-join deadline, so a red there can end as a # cancelled job. Held to one figure across every lane that runs the suite. - timeout-minutes: 215 + timeout-minutes: 265 permissions: contents: read packages: read diff --git a/.github/workflows/e2e-tag.yaml b/.github/workflows/e2e-tag.yaml index f390077d0..d9bba8559 100644 --- a/.github/workflows/e2e-tag.yaml +++ b/.github/workflows/e2e-tag.yaml @@ -137,7 +137,7 @@ jobs: # Covers a green full suite. It does not cover a tenant-Kubernetes suite # running into its node-join deadline, so a red there can end as a # cancelled job. Held to one figure across every lane that runs the suite. - timeout-minutes: 215 + timeout-minutes: 265 permissions: contents: read checks: write # Breakpoint action updates its dedicated "Breakpoint Open" Check Run. @@ -292,7 +292,7 @@ jobs: # collect-report packages cozyreport plus per-failed-test crust-gather # snapshots captured before Chainsaw cleanup. # - # A ceiling of its own, not the job's 215 minutes. `Upload cozyreport.tgz` + # A ceiling of its own, not the job's 265 minutes. `Upload cozyreport.tgz` # is the next step and the collector writes its tarball at the very end of # its run, so a read that never returns spends what is left of the job # budget, the job is cancelled, and the upload never runs -- the artifact diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 495ccec38..57cce6790 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -250,7 +250,7 @@ jobs: # Covers a green full suite. It does not cover a tenant-Kubernetes suite # running into its node-join deadline, so a red there can end as a # cancelled job. Held to one figure across every lane that runs the suite. - timeout-minutes: 215 + timeout-minutes: 265 permissions: contents: read packages: read @@ -385,7 +385,7 @@ jobs: path: _out/chainsaw-report.xml if-no-files-found: ignore - # A ceiling of its own, not the job's 215 minutes. `Upload cozyreport.tgz` + # A ceiling of its own, not the job's 265 minutes. `Upload cozyreport.tgz` # is the next step and the collector writes its tarball at the very end of # its run, so a read that never returns spends what is left of the job # budget, the job is cancelled, and the upload never runs -- the artifact diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 7d3787ff9..62cbbb1ad 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -75,6 +75,8 @@ jobs: statuses: write outputs: code: ${{ steps.p.outputs.code }} + docs: ${{ steps.p.outputs.docs }} + backport_batch: ${{ steps.p.outputs.backport_batch }} matrix: ${{ steps.p.outputs.matrix }} any: ${{ steps.p.outputs.any }} touched: ${{ steps.p.outputs.touched }} @@ -167,11 +169,27 @@ jobs: # alone here rather than widened by a side effect. git diff --name-only --no-renames "origin/${BASE_REF}...HEAD" > /tmp/changed_norenames.txt echo "Changed files:"; sed 's/^/ /' /tmp/changed_norenames.txt - if grep -qvE '^docs/' /tmp/changed_norenames.txt; then + # `code` is false when every path is under docs/ or in a CI-only Go + # tool that no image builds (cmd/backport-batch, cmd/api-gate): there + # is nothing to build and nothing for e2e to install. e2e-fork.yaml + # answers the same question from the PR file list with the same + # prefixes, and hack/gate-concurrency-contract.bats holds the two lists + # equal. + if grep -qvE '^(docs/|cmd/backport-batch/|cmd/api-gate/)' /tmp/changed_norenames.txt; then echo "code=true" >> "$GITHUB_OUTPUT" else echo "code=false" >> "$GITHUB_OUTPUT" fi + # The Bats contracts read docs, so a docs change runs them even when + # `code` is false; a change confined to the CI-only tools does not. + if grep -qE '^docs/' /tmp/changed_norenames.txt; then + echo "docs=true" >> "$GITHUB_OUTPUT" + else + echo "docs=false" >> "$GITHUB_OUTPUT" + fi + backport_batch="$(hack/backport-batch-tests-needed.sh /tmp/changed_norenames.txt)" + echo "backport-batch tests needed: $backport_batch" + echo "backport_batch=$backport_batch" >> "$GITHUB_OUTPUT" matrix="$(hack/build-matrix.sh /tmp/changed.txt)" echo "Build matrix: $matrix" echo "matrix=$matrix" >> "$GITHUB_OUTPUT" @@ -293,9 +311,11 @@ jobs: contents: read needs: ["plan"] # Unit contracts read both code and documentation. Run this job for every - # non-release PR: code changes take the complete unit path, while docs-only - # changes run the Bats lane that owns those cross-tree contracts. Builds and - # E2E remain gated on `code` below. + # non-release PR: code changes take the complete unit path, docs changes run + # the Bats lane that owns those cross-tree contracts, and the backport-batch + # tests run whenever one of their inputs changed and at no other time. A + # change confined to the CI-only tools runs no other lane. Builds and E2E + # remain gated on `code` below. if: | !contains(github.event.pull_request.labels.*.name, 'release') steps: @@ -306,20 +326,21 @@ jobs: # The module cache is restored from the key build-main.yaml writes on # main, which every PR run can read, so a PR's first run starts warm. The # download fetches whatever the cache lacks, the whole set on a miss. The - # Go steps are for code changes only: a docs-only run is the Bats lane alone. + # Go steps are for code changes and the backport-batch tests only: a + # docs-only run is the Bats lane alone. - name: Locate Go module cache - if: needs.plan.outputs.code == 'true' + if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true' id: gomodcache run: echo "dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" - name: Cache Go modules - if: needs.plan.outputs.code == 'true' + if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true' uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ steps.gomodcache.outputs.dir }} key: go-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }} restore-keys: go-mod-${{ runner.os }}- - name: Download Go modules - if: needs.plan.outputs.code == 'true' + if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true' run: | for mod in $(git ls-files go.mod '*/go.mod' | xargs -n1 dirname); do (cd "$mod" && "$GITHUB_WORKSPACE/hack/go-mod-download.sh") @@ -327,8 +348,9 @@ jobs: # Ephemeral runners don't carry the helm-unittest plugin the persistent # self-hosted runner accreted. The plugin embeds its own Helm, so its # version is pinned: a plugin release must not change test results. Bats is - # installed on every run, docs-only ones included. + # installed whenever a Bats lane runs, docs-only ones included. - name: Set up test toolchain + if: needs.plan.outputs.code == 'true' || needs.plan.outputs.docs == 'true' env: CODE_CHANGED: ${{ needs.plan.outputs.code }} HELM_UNITTEST_VERSION: "1.1.2" @@ -374,8 +396,15 @@ jobs: if: needs.plan.outputs.code == 'true' run: make unit-tests test-controllers test-controllers-envtest -j4 -k --output-sync=target - name: Run Bats unit tests for docs-only changes - if: needs.plan.outputs.code == 'false' + if: needs.plan.outputs.code == 'false' && needs.plan.outputs.docs == 'true' run: make bats-unit-tests + # Not part of `make unit-tests`: hack/backport-batch-tests-needed.sh + # decides from cmd/backport-batch/test-inputs.txt, whatever `code` is. + # `!cancelled()` for the reason the unit lane runs with -k: a red step + # above must not hide this suite's own result. + - name: Run backport-batch tests + if: ${{ !cancelled() && needs.plan.outputs.backport_batch == 'true' }} + run: make test-backport-batch - name: Upload unit test report if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -953,7 +982,7 @@ jobs: # Covers a green full suite. It does not cover a tenant-Kubernetes suite # running into its node-join deadline, so a red there can end as a # cancelled job. Held to one figure across every lane that runs the suite. - timeout-minutes: 215 + timeout-minutes: 265 permissions: contents: read packages: read @@ -1224,7 +1253,7 @@ jobs: # ▸ Collect debug information (always runs) # - # A ceiling of its own, not the job's 215 minutes. `Upload + # A ceiling of its own, not the job's 265 minutes. `Upload # cozyreport.tgz` is the next step and the collector writes its tarball # at the very end of its run, so a read that never returns spends what # is left of the job budget, the job is cancelled, and the upload never @@ -1235,7 +1264,7 @@ jobs: # 30 minutes is measured against the step: it takes 3-42s across sampled # runs, on failed e2e jobs as well as green ones. It RESERVES nothing -- # `timeout-minutes` caps a step, it does not carve budget out of the job, - # so a suite that has already burned 210 of the 215 leaves this step five + # so a suite that has already burned 260 of the 265 leaves this step five # minutes and the upload is cancelled with everything else. What the cap # buys is that a hung read cannot be the reason. # `continue-on-error` because the ceiling must not change what the job @@ -1427,7 +1456,7 @@ jobs: description = `release packages candidate did not verify (result: ${CANDIDATE_RESULT})`; } else if (CODE !== 'true') { state = 'success'; - description = 'docs-only PR — e2e not required'; + description = 'docs or CI-only tools — e2e not required'; } else if (IS_RELEASE === 'true' && IS_FULL_E2E !== 'true') { // Release promotion PRs deliberately do not repeat e2e: the rc was // already verified against the immutable candidate whose digests diff --git a/Makefile b/Makefile index 2feea00e9..aac1d6617 100644 --- a/Makefile +++ b/Makefile @@ -120,7 +120,7 @@ test: make -C packages/core/testing apply make -C packages/core/testing e2e -unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness test-backport-batch migrations-target-check +unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness migrations-target-check helm-unit-tests: hack/helm-unit-tests.sh @@ -139,13 +139,14 @@ rd-presets-check: migrations-target-check: hack/check-migrations-target.sh -# Scoped go test over the cozystack-api surface that this repo owns. Kept -# narrow intentionally - running `go test ./...` pulls in generated code -# round-trip suites whose behavior depends on tool versions outside this -# repo's control (kubebuilder, openapi-gen, etc.) and is better exercised -# from their generator workflows. +# Go tests of the cozystack-api surface and of the standalone packages and +# commands listed here, whose tests need no cluster. Kept narrow +# intentionally - running `go test ./...` pulls in generated code round-trip +# suites whose behavior depends on tool versions outside this repo's control +# (kubebuilder, openapi-gen, etc.) and is better exercised from their +# generator workflows. go-unit-tests: - go test ./pkg/registry/... ./pkg/config/... ./pkg/cmd/server/... + go test ./pkg/registry/... ./pkg/config/... ./pkg/cmd/server/... ./pkg/lineage/... ./pkg/ovnstatus/... ./pkg/reservation/... ./cmd/cozypkg/... ./cmd/check-readiness/... # The nested Go modules (image sources and the published API types) are # outside ./... of the root module, so no other target reaches their tests. @@ -158,9 +159,7 @@ go-module-tests: done # Go tests for the controllers and supporting packages under ./internal. -# Excludes ./pkg/... and ./cmd/... — those are run separately by -# go-unit-tests above (pkg subset) and skipped (cmd) until their tests -# stabilise. CI schedules this target in the same four-slot make invocation as +# CI schedules this target in the same four-slot make invocation as # unit-tests; locally invoke it directly or chain the two targets. test-controllers: go test ./internal/... -count=1 @@ -195,7 +194,10 @@ test-controllers-envtest: $(SETUP_ENVTEST) test-check-readiness: go test ./test/check-readiness/ -count=1 -# ./cmd/... is excluded from go-unit-tests, so the batch tool needs its own target. +# The batch tool's suite drives real git repositories for minutes, so it is +# kept out of go-unit-tests in a target of its own. It is not a prerequisite +# of unit-tests: the checks job in pull-requests.yaml runs it only when the +# diff touches a path listed in cmd/backport-batch/test-inputs.txt. test-backport-batch: go test ./cmd/backport-batch/... -count=1 diff --git a/api/apps/v1alpha1/bucket/types.go b/api/apps/v1alpha1/bucket/types.go index 6632ca530..8a4f331d5 100644 --- a/api/apps/v1alpha1/bucket/types.go +++ b/api/apps/v1alpha1/bucket/types.go @@ -25,6 +25,9 @@ type ConfigSpec struct { // Deploys the web UI (s3manager) with its Service and its Ingress or HTTPRoute at `.`. Set to `false` to run the bucket without them. // +kubebuilder:default:=true Ui bool `json:"ui"` + // Serves the bucket's objects to anyone without credentials at `//`. Anonymous callers can read objects but cannot list the bucket or write to it. + // +kubebuilder:default:=false + PublicRead bool `json:"publicRead"` // Users configuration map. // +kubebuilder:default:={} Users map[string]User `json:"users,omitempty"` diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index df989c5c5..8500d70ee 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -25,10 +25,10 @@ type ConfigSpec struct { // Kubernetes major.minor version to deploy // +kubebuilder:default:="v1.35" Version Version `json:"version"` - // External hostname for Kubernetes cluster. Defaults to `.` if empty. + // External hostname for Kubernetes cluster. Defaults to `kubernetes-.` if empty. // +kubebuilder:default:="" Host string `json:"host"` - // Which infrastructure provider backs this cluster's worker VMs. `kubevirt` runs them inside this cluster; `proxmox` runs them on an external Proxmox VE cluster through capmox. The control plane is Kamaji either way — this selects the infrastructure half only. Must match the `substrate` of every KubernetesNodes pool attached to this cluster: the pools reference this cluster's infrastructure object by kind, and a mismatch leaves their Machines unreconciled. Switching an existing cluster is not supported; create a new one. + // Which infrastructure provider backs this cluster's worker VMs. `kubevirt` runs them inside this cluster; `proxmox` runs them on an external Proxmox VE cluster through capmox. The control plane is Kamaji either way — this selects the infrastructure half only. Every KubernetesNodes pool of this cluster must set the same `substrate`: a pool set to the other one refuses to render once this cluster exists. Switching an existing cluster is not supported; create a new one. // +kubebuilder:default:="kubevirt" // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="substrate is immutable" Substrate string `json:"substrate"` @@ -41,9 +41,6 @@ type ConfigSpec struct { // Kubernetes control-plane configuration. // +kubebuilder:default:={} ControlPlane ControlPlane `json:"controlPlane"` - // Talos worker image configuration. - // +kubebuilder:default:={} - Talos Talos `json:"talos"` // OIDC authentication and per-user RBAC for the tenant kube-apiserver. See https://cozystack.io/docs/v1.6/kubernetes/oidc-authentication/ for the operator guide. // +kubebuilder:default:={} Oidc OIDC `json:"oidc"` @@ -53,7 +50,7 @@ type APIServer struct { // Admission plugins the tenant kube-apiserver enables, passed through to KamajiControlPlane `spec.admissionControllers`, which the control plane renders as `--enable-admission-plugins`. A non-empty list replaces the control plane's default set instead of adding to it. The kube-apiserver enables these plugins in addition to its own default ones, so leaving a plugin out does not turn it off; to turn one off, name it in `--disable-admission-plugins` in `controlPlane.apiServer.extraArgs` and keep it out of this list, since the apiserver refuses to start with a plugin in both, and the control plane's default set names several. Empty by default, which keeps that default set, taken from the TenantControlPlane CRD: CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, LimitRanger, MutatingAdmissionWebhook, NamespaceLifecycle, PersistentVolumeClaimResize, Priority, ResourceQuota, RuntimeClass, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. `NodeRestriction` is not in it. // +kubebuilder:default:={} AdmissionControllers []AdmissionController `json:"admissionControllers,omitempty"` - // Extra command-line flags appended to the tenant kube-apiserver, passed through to KamajiControlPlane `spec.apiServer.extraArgs`. For OIDC use `spec.oidc.mode` — this passthrough is the escape hatch for unrelated apiserver flags. The tenant control plane sets each of the following flags itself and drops an entry naming one before the apiserver sees it. The chart moves an `--enable-admission-plugins=` entry into `controlPlane.apiServer.admissionControllers` and a `--kubelet-preferred-address-types=` entry into `controlPlane.kubelet.preferredAddressTypes` while that field is empty, and refuses any other entry naming one of them: `--advertise-address`, `--client-ca-file`, `--enable-admission-plugins`, `--service-cluster-ip-range`, `--kubelet-client-certificate`, `--kubelet-client-key`, `--kubelet-preferred-address-types`, `--proxy-client-cert-file`, `--proxy-client-key-file`, `--requestheader-allowed-names`, `--requestheader-client-ca-file`, `--secure-port`, `--service-account-key-file`, `--service-account-signing-key-file`, `--tls-cert-file`, `--tls-private-key-file`, `--etcd-servers`, `--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, `--etcd-prefix`, `--etcd-compaction-interval`, `--egress-selector-config-file`. When `spec.oidc.mode` is not `None` the chart also renders OIDC flags here, reads what you have written on them, and refuses some combinations outright; the Aggregated API servers section of the Kubernetes OIDC guide on cozystack.io carries which flags, which combinations, and why. Do NOT add legacy `--oidc-*` flags here when `spec.oidc.mode` is not `None`; the chart injects `--authentication-config` and the apiserver refuses to boot with both. Empty by default (no change to current behavior). + // Extra command-line flags appended to the tenant kube-apiserver, passed through to KamajiControlPlane `spec.apiServer.extraArgs`. For OIDC use `spec.oidc.mode` — this passthrough is the escape hatch for unrelated apiserver flags. The tenant control plane sets each of the following flags itself and drops an entry naming one before the apiserver sees it. The chart moves an `--enable-admission-plugins=` entry into `controlPlane.apiServer.admissionControllers` and a `--kubelet-preferred-address-types=` entry into `controlPlane.kubelet.preferredAddressTypes` while that field is empty, and refuses any other entry naming one of them: `--advertise-address`, `--client-ca-file`, `--enable-admission-plugins`, `--service-cluster-ip-range`, `--kubelet-client-certificate`, `--kubelet-client-key`, `--kubelet-preferred-address-types`, `--proxy-client-cert-file`, `--proxy-client-key-file`, `--requestheader-allowed-names`, `--requestheader-client-ca-file`, `--secure-port`, `--service-account-key-file`, `--service-account-signing-key-file`, `--tls-cert-file`, `--tls-private-key-file`, `--etcd-servers`, `--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, `--etcd-prefix`, `--etcd-compaction-interval`, `--egress-selector-config-file`. When `spec.oidc.mode` is not `None` the chart also renders OIDC flags here, reads what you have written on them, and refuses some combinations outright; the Aggregated API servers section of the Kubernetes OIDC guide on cozystack.io carries which flags, which combinations, and why. Do NOT add legacy `--oidc-*` flags here when `spec.oidc.mode` is not `None`; the chart injects `--authentication-config` and the apiserver refuses to boot with both. Empty by default. // +kubebuilder:default:={} ExtraArgs []string `json:"extraArgs,omitempty"` // Extra volume mounts added to the tenant kube-apiserver container, passed through to KamajiControlPlane `spec.apiServer.extraVolumeMounts`. Each `name` must reference a volume declared in `extraVolumes`; the chart-managed talos secret volumes cannot be mounted. Each item is a core/v1 VolumeMount. Empty by default. @@ -349,24 +346,6 @@ type Scheduler struct { ResourcesPreset ResourcesPreset `json:"resourcesPreset"` } -type Talos struct { - // Base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. - // +kubebuilder:default:="https://factory.talos.dev" - ImageFactoryURL string `json:"imageFactoryURL"` - // OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:`. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash. - // +kubebuilder:default:="factory.talos.dev/installer" - InstallerRepository string `json:"installerRepository"` - // Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. - // +kubebuilder:default:={} - RegistryMirrors k8sRuntime.RawExtension `json:"registryMirrors"` - // Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. - // +kubebuilder:default:="ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515" - SchematicID string `json:"schematicID"` - // Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`. - // +kubebuilder:default:="v1.13.6" - Version string `json:"version"` -} - type VeleroAddon struct { // Enable Velero. // +kubebuilder:default:=false diff --git a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go index 6b033290f..69129b193 100644 --- a/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/kubernetes/zz_generated.deepcopy.go @@ -154,7 +154,6 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { in.Proxmox.DeepCopyInto(&out.Proxmox) in.Addons.DeepCopyInto(&out.Addons) in.ControlPlane.DeepCopyInto(&out.ControlPlane) - in.Talos.DeepCopyInto(&out.Talos) in.Oidc.DeepCopyInto(&out.Oidc) } @@ -570,22 +569,6 @@ func (in *Scheduler) DeepCopy() *Scheduler { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Talos) DeepCopyInto(out *Talos) { - *out = *in - in.RegistryMirrors.DeepCopyInto(&out.RegistryMirrors) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Talos. -func (in *Talos) DeepCopy() *Talos { - if in == nil { - return nil - } - out := new(Talos) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VeleroAddon) DeepCopyInto(out *VeleroAddon) { *out = *in diff --git a/api/apps/v1alpha1/kubernetesnodes/types.go b/api/apps/v1alpha1/kubernetesnodes/types.go index 7966e3565..79addb215 100644 --- a/api/apps/v1alpha1/kubernetesnodes/types.go +++ b/api/apps/v1alpha1/kubernetesnodes/types.go @@ -18,11 +18,11 @@ type Config struct { } type ConfigSpec struct { - // Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool name (the part after `-`) must not collide with a nodeGroup still managed by the parent kubernetes chart (e.g. the default `md0`), or the render fails on ownership conflict. + // Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool's objects are named `kubernetes--`; when a MachineDeployment of that name already exists and belongs to another Helm release, the render fails instead of taking it over. // +kubebuilder:default:="" // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="cluster is immutable" Cluster string `json:"cluster"` - // StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. + // StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`. // +kubebuilder:default:="replicated" // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="storageClass is immutable" StorageClass string `json:"storageClass"` @@ -32,10 +32,10 @@ type ConfigSpec struct { // Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling. // +kubebuilder:default:=10 MaxReplicas int `json:"maxReplicas"` - // Virtual machine instance type. + // KubeVirt instance type that sizes each worker VM when `resources` is not set. Ignored when `resources.cpu` and `resources.memory` are both set, and therefore on `substrate: proxmox`, which requires them. // +kubebuilder:default:="u1.medium" InstanceType string `json:"instanceType"` - // System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. + // System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. Has no effect on `substrate: proxmox`, where the worker disk is a clone of the VM template's disk and keeps its size. // +kubebuilder:default:="20Gi" DiskSize resource.Quantity `json:"diskSize"` // List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller. @@ -44,16 +44,16 @@ type ConfigSpec struct { // Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `"010"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `"010"` reserves what `"10"` does. // +kubebuilder:default:={} Resources Resources `json:"resources,omitempty"` - // CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart. + // CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. PodCpuLimit resource.Quantity `json:"podCpuLimit,omitempty"` - // CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart. + // CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. PodCpuRequest resource.Quantity `json:"podCpuRequest,omitempty"` // List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). // +kubebuilder:default:={} Gpus []GPU `json:"gpus,omitempty"` - // Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls. + // Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. KernelModules []KernelModule `json:"kernelModules,omitempty"` - // Kubelet resource reservations for this pool. + // Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`. // +kubebuilder:default:={} Kubelet Kubelet `json:"kubelet,omitempty"` // Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it. @@ -62,25 +62,25 @@ type ConfigSpec struct { // Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is a percentage. Drop to "0%" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At "50%" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two. // +kubebuilder:default:="50%" MaxUnhealthy string `json:"maxUnhealthy"` - // Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning. + // Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (the Talos image import from the Image Factory, or a storage class slow to provision or clone the worker disk). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning. // +kubebuilder:default:="20m" NodeStartupTimeout string `json:"nodeStartupTimeout"` - // How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart. + // How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. // +kubebuilder:default:="5m" ReadyUnknownTimeout string `json:"readyUnknownTimeout"` - // How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart. + // How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. // +kubebuilder:default:="10m" ReadyFalseTimeout string `json:"readyFalseTimeout"` - // How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours ("30m", "1h30m"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to ("20m0s", "1h0m0s") is accepted. Keep in sync with the parent kubernetes chart. + // How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours ("30m", "1h30m"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to ("20m0s", "1h0m0s") is accepted. // +kubebuilder:default:="30m" MaxNodeProvisionTime string `json:"maxNodeProvisionTime"` - // Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool -- and must satisfy the Talos<->Kubernetes support matrix against `talos.version`. + // Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool. The render fails when it is ahead. Its pairing with the Talos release the pool resolves (`talos.version`, or the `osImage` override) is checked against the chart's Talos<->Kubernetes support matrix: for a Talos minor the matrix lists, a Kubernetes minor outside that minor's list is refused; a Talos minor the matrix does not list is not checked. // +kubebuilder:default:="v1.35" Version Version `json:"version"` // Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool. // +kubebuilder:default:={} OsImage WorkerImage `json:"osImage,omitempty"` - // Talos worker image configuration. Keep in sync with the parent cluster's `talos`. + // Talos worker image configuration for this pool. // +kubebuilder:default:={} Talos Talos `json:"talos"` // Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down. @@ -111,13 +111,13 @@ type Kubelet struct { // Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%). // +kubebuilder:default:="10%" EvictionSoftMemory string `json:"evictionSoftMemory,omitempty"` - // CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty. + // CPU reserved for kubelet and container runtime. When empty, 5% of the worker's CPU, clamped to 50m..500m. KubeReservedCpu string `json:"kubeReservedCpu,omitempty"` - // Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty. + // Memory reserved for kubelet and container runtime. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi. KubeReservedMemory string `json:"kubeReservedMemory,omitempty"` - // CPU reserved for host OS. Auto-computed from instanceType if empty. + // CPU reserved for host OS. When empty, 5% of the worker's CPU, clamped to 50m..500m. SystemReservedCpu string `json:"systemReservedCpu,omitempty"` - // Memory reserved for host OS. Auto-computed from instanceType if empty. + // Memory reserved for host OS. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi. SystemReservedMemory string `json:"systemReservedMemory,omitempty"` } @@ -160,13 +160,13 @@ type Resources struct { } type Talos struct { - // Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there. + // Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there. On `substrate: proxmox` it has no effect on the workers either: the ConfigMap is still rendered, but nothing references it. // +kubebuilder:default:="" ImageFactoryCA string `json:"imageFactoryCA,omitempty"` - // Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal. + // Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal. On `substrate: proxmox` the workers clone a VM template that already carries the OS, so this field has no effect there beyond that format check. // +kubebuilder:default:="https://factory.talos.dev" ImageFactoryURL string `json:"imageFactoryURL"` - // OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash. + // OCI repository prefix for the Talos installer image, written into the worker machine config as `machine.install.image`. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash. // +kubebuilder:default:="factory.talos.dev/installer" InstallerRepository string `json:"installerRepository"` // Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. @@ -178,7 +178,7 @@ type Talos struct { // Sysctl overrides rendered into Talos `machine.sysctls` on this pool's guest workers only. Values must be strings (quote numeric values); for example, `user.max_user_namespaces: "11255"` enables the user namespaces required by the Talos gVisor extension. An empty map preserves Talos defaults and the existing bootstrap hash. Adding, changing or removing an override changes the content-hashed TalosConfigTemplate and rolls the pool's workers. Supported names and values depend on the worker's Talos release and kernel; unsafe settings can weaken guest isolation or prevent workers from booting. This does not install gVisor or configure a RuntimeClass, and does not change the management hosts, control plane or other pools. // +kubebuilder:default:={} Sysctls map[string]string `json:"sysctls,omitempty"` - // Default Talos release for this pool's worker OS image and in-guest installer. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix rejects a pairing it lists as unsupported; it lists the Talos minors this chart ships and passes any other minor unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead. + // Default Talos release for this pool's worker OS image and for the installer image written into the worker machine config. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix lists, for each Talos minor the chart knows, the Kubernetes minors it supports and refuses a pool `version` outside that list; a Talos minor it does not list passes unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead. // +kubebuilder:default:="v1.13.6" Version string `json:"version"` } @@ -198,7 +198,7 @@ type WorkerImageBuiltin struct { } type WorkerImageFactory struct { - // Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. NOTE: this redirects only the worker boot-disk import; the in-guest upgrade installer image is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry. + // Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. This redirects only the worker boot-disk import; the installer image in the worker machine config is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry. ImageFactoryURL string `json:"imageFactoryURL,omitempty"` // Talos schematic ID. Defaults to `talos.schematicID`. SchematicID string `json:"schematicID,omitempty"` diff --git a/api/apps/v1alpha1/postgresql/types.go b/api/apps/v1alpha1/postgresql/types.go index 4c0d80ab8..70beb678e 100644 --- a/api/apps/v1alpha1/postgresql/types.go +++ b/api/apps/v1alpha1/postgresql/types.go @@ -124,7 +124,7 @@ type Backup struct { // DEPRECATED. See `s3AccessKey`. // +kubebuilder:default:="" S3SecretKey string `json:"s3SecretKey,omitempty"` - // Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. + // Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. // +kubebuilder:default:="" Schedule string `json:"schedule,omitempty"` // Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`. diff --git a/api/apps/v1alpha1/redis/types.go b/api/apps/v1alpha1/redis/types.go index c2363749c..41cf03c14 100644 --- a/api/apps/v1alpha1/redis/types.go +++ b/api/apps/v1alpha1/redis/types.go @@ -51,6 +51,14 @@ type ConfigSpec struct { // Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it. // +kubebuilder:validation:Minimum=0 MinReplicasMaxLag *int `json:"minReplicasMaxLag,omitempty"` + // Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers. + Maxmemory resource.Quantity `json:"maxmemory,omitempty"` + // Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset. + // +kubebuilder:default:="noeviction" + MaxmemoryPolicy MaxmemoryPolicy `json:"maxmemoryPolicy"` + // Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty. + // +kubebuilder:default:=true + Snapshots bool `json:"snapshots"` // Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does. // +kubebuilder:default:={} Sentinel Sentinel `json:"sentinel,omitempty"` @@ -80,6 +88,9 @@ type TLS struct { // +kubebuilder:validation:Enum="no";"optional";"yes" type AuthClients string +// +kubebuilder:validation:Enum="noeviction";"allkeys-lru";"allkeys-lfu";"allkeys-random";"volatile-lru";"volatile-lfu";"volatile-random";"volatile-ttl" +type MaxmemoryPolicy string + // +kubebuilder:validation:Enum="t1.nano";"t1.micro";"t1.small";"t1.medium";"t1.large";"t1.xlarge";"t1.2xlarge";"t1.4xlarge";"c1.nano";"c1.micro";"c1.small";"c1.medium";"c1.large";"c1.xlarge";"c1.2xlarge";"c1.4xlarge";"s1.nano";"s1.micro";"s1.small";"s1.medium";"s1.large";"s1.xlarge";"s1.2xlarge";"s1.4xlarge";"u1.nano";"u1.micro";"u1.small";"u1.medium";"u1.large";"u1.xlarge";"u1.2xlarge";"u1.4xlarge";"m1.nano";"m1.micro";"m1.small";"m1.medium";"m1.large";"m1.xlarge";"m1.2xlarge";"m1.4xlarge";"nano";"micro";"small";"medium";"large";"xlarge";"2xlarge" type ResourcesPreset string diff --git a/api/apps/v1alpha1/redis/zz_generated.deepcopy.go b/api/apps/v1alpha1/redis/zz_generated.deepcopy.go index 489c28e85..86437ea58 100644 --- a/api/apps/v1alpha1/redis/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/redis/zz_generated.deepcopy.go @@ -66,6 +66,7 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { *out = new(int) **out = **in } + out.Maxmemory = in.Maxmemory.DeepCopy() in.Sentinel.DeepCopyInto(&out.Sentinel) } diff --git a/api/apps/v1alpha1/vminstance/types.go b/api/apps/v1alpha1/vminstance/types.go index 95c800aae..e457f4cac 100644 --- a/api/apps/v1alpha1/vminstance/types.go +++ b/api/apps/v1alpha1/vminstance/types.go @@ -74,6 +74,9 @@ type ConfigSpec struct { } type Disk struct { + // Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both. + // +kubebuilder:validation:Minimum=1 + BootOrder *int `json:"bootOrder,omitempty"` // Disk bus type (e.g. "sata"). Bus string `json:"bus,omitempty"` // Disk name. diff --git a/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go b/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go index 060281042..84f7315ac 100644 --- a/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go +++ b/api/apps/v1alpha1/vminstance/zz_generated.deepcopy.go @@ -61,7 +61,9 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { if in.Disks != nil { in, out := &in.Disks, &out.Disks *out = make([]Disk, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.Networks != nil { in, out := &in.Networks, &out.Networks @@ -100,6 +102,11 @@ func (in *ConfigSpec) DeepCopy() *ConfigSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Disk) DeepCopyInto(out *Disk) { *out = *in + if in.BootOrder != nil { + in, out := &in.BootOrder, &out.BootOrder + *out = new(int) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Disk. diff --git a/api/backups/strategy/v1alpha1/kubernetes_types.go b/api/backups/strategy/v1alpha1/kubernetes_types.go new file mode 100644 index 000000000..704fc5d9f --- /dev/null +++ b/api/backups/strategy/v1alpha1/kubernetes_types.go @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: Apache-2.0 +// Package v1alpha1 defines strategy.backups.cozystack.io API types. +// +// Group: strategy.backups.cozystack.io +// Version: v1alpha1 +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func init() { + SchemeBuilder.Register(func(s *runtime.Scheme) error { + s.AddKnownTypes(GroupVersion, + &Kubernetes{}, + &KubernetesList{}, + ) + return nil + }) +} + +const ( + KubernetesStrategyKind = "Kubernetes" +) + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster + +// Kubernetes defines a backup strategy for a tenant Kubernetes application. +// A tenant cluster keeps its state in two places the platform can reach: the +// cluster's key prefix in the tenant's shared etcd DataStore, and host-side +// objects (control-plane PKI Secrets, the application HelmReleases, and the +// DataVolumes that back the tenant's PersistentVolumes). The strategy captures +// the first with a one-shot Job and the second with Velero. The two are taken +// one after another, so a backup is crash-consistent at best. +type Kubernetes struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec KubernetesSpec `json:"spec,omitempty"` + Status KubernetesStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// KubernetesList contains a list of Kubernetes backup strategies. +type KubernetesList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Kubernetes `json:"items"` +} + +// KubernetesSpec specifies the desired behavior of a Kubernetes backup strategy. +type KubernetesSpec struct { + // Template holds a PodTemplateSpec for the Job that dumps the cluster's + // key prefix from its etcd DataStore to object storage, writes it back on + // restore, and deletes the object on cleanup. The Job runs in the + // application namespace. Helm-like Go templates are supported in every + // string field and are rendered against: + // - `.Release.Name` and `.Release.Namespace` — the application's name and + // namespace; `.Release.Prefix` is `kubernetes-`. + // - `.Mode` — `"backup"`, `"restore"` or `"cleanup"`. + // - `.ObjectKey` — the object-storage key of this backup's dump. + // - `.Datastore.Prefix` — the cluster's key prefix, without slashes. + // - `.Datastore.Endpoints` — comma-separated etcd client URLs. + // - `.Datastore.CertificateSecret` — the Secret holding the cluster's + // etcd client certificate (`ca.crt`, `server.crt`, `server.key`). + // - `.Parameters` — the resolved BackupClass strategy parameters. + // + // Stored schemaless for the same CRD-size reason as redis_types.go. + // + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + Template corev1.PodTemplateSpec `json:"template"` + + // Velero holds the Velero Backup and Restore specs for the host-side + // objects and volumes. Rendered against `.Application` and `.Parameters`, + // like the Velero strategy. + // + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + Velero VeleroTemplate `json:"velero"` + + // CertificateWarningThreshold is how close to expiry a control-plane + // certificate may be before a backup records a warning. Defaults to 720h. + // +optional + CertificateWarningThreshold *metav1.Duration `json:"certificateWarningThreshold,omitempty"` +} + +type KubernetesStatus struct { + Conditions []metav1.Condition `json:"conditions,omitempty"` +} diff --git a/api/backups/strategy/v1alpha1/zz_generated.deepcopy.go b/api/backups/strategy/v1alpha1/zz_generated.deepcopy.go index 3ad54f0fe..9efd1fb9b 100644 --- a/api/backups/strategy/v1alpha1/zz_generated.deepcopy.go +++ b/api/backups/strategy/v1alpha1/zz_generated.deepcopy.go @@ -839,6 +839,109 @@ func (in *KafkaStatus) DeepCopy() *KafkaStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Kubernetes) DeepCopyInto(out *Kubernetes) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Kubernetes. +func (in *Kubernetes) DeepCopy() *Kubernetes { + if in == nil { + return nil + } + out := new(Kubernetes) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Kubernetes) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesList) DeepCopyInto(out *KubernetesList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Kubernetes, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesList. +func (in *KubernetesList) DeepCopy() *KubernetesList { + if in == nil { + return nil + } + out := new(KubernetesList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KubernetesList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesSpec) DeepCopyInto(out *KubernetesSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) + in.Velero.DeepCopyInto(&out.Velero) + if in.CertificateWarningThreshold != nil { + in, out := &in.CertificateWarningThreshold, &out.CertificateWarningThreshold + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesSpec. +func (in *KubernetesSpec) DeepCopy() *KubernetesSpec { + if in == nil { + return nil + } + out := new(KubernetesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesStatus) DeepCopyInto(out *KubernetesStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesStatus. +func (in *KubernetesStatus) DeepCopy() *KubernetesStatus { + if in == nil { + return nil + } + out := new(KubernetesStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MariaDB) DeepCopyInto(out *MariaDB) { *out = *in diff --git a/api/v1alpha1/workloadmonitor_types.go b/api/v1alpha1/workloadmonitor_types.go index 0a4aa04cc..b5089652a 100644 --- a/api/v1alpha1/workloadmonitor_types.go +++ b/api/v1alpha1/workloadmonitor_types.go @@ -39,6 +39,10 @@ type WorkloadMonitorSpec struct { // that is not operational because a DataVolume it selects is not ready. const WorkloadMonitorReasonDataVolumeNotReady = "DataVolumeNotReady" +// WorkloadMonitorReasonBucketClaimNotReady is the status Reason of a monitor +// that is not operational because a BucketClaim it selects is not provisioned. +const WorkloadMonitorReasonBucketClaimNotReady = "BucketClaimNotReady" + // WorkloadMonitorStatus defines the observed state of WorkloadMonitor type WorkloadMonitorStatus struct { // Operational indicates if the workload meets all operational requirements diff --git a/cmd/backport-batch/README.md b/cmd/backport-batch/README.md index b1837f489..7137bdaf0 100644 --- a/cmd/backport-batch/README.md +++ b/cmd/backport-batch/README.md @@ -27,16 +27,22 @@ Requires `git` and an authenticated `gh` on PATH, and a remote called `origin`. Build the binary wherever an exit code matters: `go run` turns any non-zero exit into its own `1`, so under it a check that could not be completed looks the same as one that found something missing. +Output is written before a push or a publish. Once a write to stdout or stderr fails, a closed pipe included, the run changes nothing more: no push, no GitHub write, no local branch move, and no new pick; a pick already in flight still finishes its rollback. It exits 2 where it would have exited 0, saying on stderr, while it still can, that the output was lost. A confirmation lost after a push or a publish fails the run the same way. + ### run Switches the current working tree to `backport-batch/-`, created at `origin/`, and picks onto it one original at a time, in the order they merged into `main`, as one unit each on the branch's first-parent line. A pull request merged into `main` with a merge commit `M` goes on as its own commits, `git rev-list --reverse --topo-order --no-merges M^1..M^2`, each picked with `git cherry-pick -x --signoff`, so each keeps its author, message and sign-offs and gains the `(cherry picked from commit …)` line; the merges that brought `main` into its branch are left out, since `main` already has what they brought. A merge commit then joins the picks to the batch: its first parent is the batch before the original, its second the last pick, and its message is `M`'s with a paragraph of its own, `Backported-from: M`. A pull request squash-merged into `main` goes on as one pick. When any commit of an original conflicts, the pick is aborted, the branch goes back to where it was before that original, and the original is recorded with the commit that failed and its files; the run goes on with the next one. A commit the branch already carries is skipped as empty, and an original none of whose commits changes anything is reported as carried. The tree has to be clean. A local branch of that name is reset to where the run starts; when that drops commits only it has, the run first prints its tip and the `git branch` command that restores them, and so does `--defer`. Signing follows your git configuration; hooks are off for the picks and the merge, so each commit carries its message as git writes it. -The pull request body goes to stdout, built from the line's own `.github/PULL_REQUEST_TEMPLATE.md`: the originals on the branch, each with how many commits it brought and which of them differ from the commit they were picked from by `git patch-id`, a resolution or a pick git had to fit, the deferred conflicts with the commit that failed and its files, the skipped pull requests counted by rule with the lists collapsed, the originals' release notes, and two blocks of JSON: the [line's record](#the-lines-record-and-reports) in `backport-batch`, which later runs, `check` and `notify` act on, and the run's report in `backport-batch-report`. The title is `fix(backport): batch for , `. A body that would pass GitHub's 65536-byte limit leaves out its longest skipped lists, and the run prints them on stderr instead; one that what it has to carry still puts over the limit is refused before anything is pushed. +The pull request body goes to stdout, built from the line's own `.github/PULL_REQUEST_TEMPLATE.md`: the originals on the branch, each with how many commits it brought and which of them differ from the commit they were picked from by `git patch-id`, a resolution or a pick git had to fit, the deferred conflicts with the commit that failed and its files, what is parked and why, the skipped pull requests counted by rule with the lists collapsed, the originals' release notes, and two blocks of JSON. An original whose unit differs from it, a pick whose patch-id differs from its commit's or a unit whose net patch-id differs from the original merge's, as when a first commit came out empty because the line already had that change, keeps its release note out of the release-note block: it goes under "Check these notes", with a `git range-diff --creation-factor=999 ^! ^!` for each pick that differs, which pairs the two however much the pick lost, and the `git diff` of the original's merge and of its unit, and the record and the report mark it `changed`. The two blocks of JSON are the [line's record](#the-lines-record-and-reports) in `backport-batch`, which later runs, `check` and `notify` act on, and the run's report in `backport-batch-report`. The title is `fix(backport): batch for , `. A body that would pass GitHub's 65536-byte limit leaves out its longest skipped lists, and the run prints them on stderr instead; one that what it has to carry still puts over the limit is refused before anything is pushed. `--onto SHA` continues a batch already pushed instead of starting a new one. SHA has to be the head the remote has for a batch branch of the line right now, read with `git ls-remote`, so a head someone pushed onto since is refused. What that branch carries stays, and only what it does not carry yet is picked on top, so the next push is a fast-forward. `--continue` takes SHA from the open batch pull request, and starts a new batch when none is open, which is what the scheduled run does. `--no-workflow-files` is for a token that may not push a change to `.github/workflows/`, which the scheduled run's App token may not. An original that changes anything there is not picked: the body lists it under "Waiting for a maintainer" with those files, and the `backport-batch` block records it with `"workflows": true`, so `check`, `notify` and a run that continues the batch know it waits. It is never a conflict. A run without the flag, a maintainer's with their own token, picks it like any other original, onto the open batch when it continues one. +A clean pick can still carry only part of its original: git deletes a file the line never had without a word, and leaves alone a file the line lacks. So every path the original modifies or deletes, `M` or `D` in `git diff --name-status --no-renames M^1 M`, is judged at its counterpart on the line: the path itself, or the one a rename `git diff -M` finds between the line's fork point and `M^1` on `main`, between the fork point and the line, or both in turn. A modified path counts as left without effect when its counterpart exists neither before nor after the pick; when the pick leaves an existing file as it was, the file counts as there only when its blob is the original's, binary files included, or when `git merge-file` of the original's preimage, the line's file and the original's postimage merges cleanly back into the line's file. A deleted path counts as left without effect when its counterpart is absent before a pick that changes something, a deletion the line already made being no change of that pick, or still there after it, whatever the pick changed in it; when the original adds that path itself, as when the line made the same rename, it is judged as a modified file against what the original makes of that path. Picks that change nothing carry the original only when the line already has what it does to every file it modifies or deletes, so a change made where its branch merged `main`, which no pick brings, is still missing. What cannot be judged, a mode or type change, content `merge-file` refuses, or a rename git skipped looking for, is shown with the original and never counts against it. An original whose net merge onto the line, `git merge-tree --merge-base=M^1`, leaves something without effect is not tried, and one whose picks do is taken back off the branch: either is parked as partial, listed under "Parked" with those paths, and its release note stays out of the body. A conflict is never called partial. `backport/include` lifts no park. + +Before it tries an original, `run` looks at what the original [builds on](#what-an-original-builds-on) that the branch lacks, in merge order, so what became of each prerequisite in this run is known. A prerequisite the line or the open batch carries, or that this run picked or found empty, counts as there; once earlier picks moved the branch, the original is judged at the branch's head, so a line another unit brought counts too. Of what is left, a direct push, an original rules 2 to 4, the line's start, `backport/skip` or a drop keep out, and one that is itself parked never come: the original is parked, not tried, and listed under "Parked" with what it builds on. In the scheduled mode an original that changes a workflow, whose missing dependencies that never come are only tests or docs and whose workflow lines build on nothing missing, waits for a maintainer instead, with a note saying so. Prerequisites still to come, a conflict, a deferral, a wait for a maintainer, one rule 6 holds, make the original wait: it is listed under "Waiting on another original", recorded with `needs`, the prerequisites of the kind that decides, in the order not on the line, waiting for a maintainer, deferred, rule 6, and tried again by a later run. A reason given with `--defer` comes before all of this, and a partial park before any of it. + `--defer N[,N] --reason TEXT` takes originals out of the open batch pull request, usually the ones a failing check was pinned on. It builds the batch branch again off `origin/` without their units: while the batch still sits on the line's head, the units before the first one it drops stay as they are, and every later one, a pushed resolution included, is picked again in its own shape, a merge's picks one by one and then the merge with its message. When dropping one leaves a later commit that no longer applies, the run stops and names that commit's original, to defer as well. It adds no new original. It also takes out an original that got `backport/skip` after the batch picked it, which a run never removes on its own and the body lists as still on the branch. The body records each deferred original with the reason in the `backport-batch` block; a run that continues this batch keeps them out, and the next batch tries them again. A batch that carries a merge commit other than an original's, as GitHub's update-branch button makes, is refused, since picking such a merge again would lose what it resolved. When it takes out everything the batch carries, there is nothing left to push, and `--push` closes the batch pull request instead, as [the line's record](#the-lines-record-and-reports) describes. `--drop N[,N] --reason TEXT` drops originals from the line: where `--defer` holds one out of this batch, a drop keeps it out of every later batch of the line too, until `--undrop N[,N]` withdraws it. A drop is recorded in the [line's record](#the-lines-record-and-reports), so both need an open batch pull request that carries one, and exit 2 without it; once a batch merges, its drops are carried into the next batch's record. A dropped original the open batch carries is taken off it as `--defer` would, and the rebuild is pushed the same way; any other drop or undrop changes only the record, so the batch stays as it is even when the line moved since. An undropped original is missing until the next run tries it. A drop names the original's merge commit on `main` as well as its number, and a record whose drop does not match what that merge commit merged stops every command with exit 2. An original the line already carries cannot be dropped, and neither can a number that merged into `main` before the line branched. The order is the rules', then the drop: what `backport/skip`, rules 2 to 4 or the line itself decide comes first, with the drop kept on record beside it, and `backport/include` does not lift a drop. The body lists dropped originals under their own heading among the skipped ones, `check` prints a note for each, and `fold` refuses one. @@ -51,18 +57,26 @@ Exits 1 and prints the URL of every pull request the rules take that is neither An original with `backport/skip` that the open batch picked before the label came prints a warning in either mode, naming the batch and `run --defer`, which takes it out; it fails neither. +Every other original the policy takes that neither the line nor the open batch carries is decided in merge order, as [`run`](#run) decides it, with the open batch's found picks counting as there and the mode the open record saved, the scheduled run's when no batch is open, whose workflow waits on a closed record wait on nothing any more, so the original is judged as the scheduled run judges it: deferred for a reason, a warning; parked, a note naming what it builds on; in the scheduled mode a workflow original whose tests alone build on what never comes, waiting as the open batch lists it, or missing with none; and one that waits on other originals by the prerequisite that decides it: missing, which fails both modes, waiting for a maintainer, which warns and fails `--cut`, deferred, which warns, or held by rule 6, a note. A wait on one that waits itself inherits what decides that one. + +A unit on the open batch that leaves part of its original without effect, judged as [`run`](#run) judges a pick, at the unit's own first parent and whatever its original's rule, label or drop, prints a warning naming what it leaves out, and fails `--cut` ahead of any skip, drop or stale warning: take it out with `run --defer`. An original that is parked as partial prints as a note, `#N is parked: only part of it applies`, and fails neither. That includes one the open record parks as partial that the net merge at the line's head finds whole, since `run`'s picker found it after the picks before it: `check` and the operations that only change the record keep it parked while its net merge onto what the batch carries now still leaves something without effect, and call it missing once that went, as when the pick that made it partial was taken off. A park only the commit-by-commit replay sees, which no net merge shows, reads as missing in `check` until the next run. + ### fold -Checks a hand-made resolution of a deferred conflict before it is pushed onto the batch, in the shape the run gives an original. For a merged pull request `--commit` is a merge whose first parent is the head of the open batch pull request and whose message names the original in exactly one `Backported-from:` trailer; its second parent ends a straight line from that head of picks of the pull request's own commits, in their order, each ending in the `(cherry picked from commit …)` line of the commit it came from, none claiming another commit of the same original, a commit of another pull request never among them. For a squash-merged one it is a single commit on that head whose `(cherry picked from commit …)` line names the original's commit on `main`. Either has to change something, and only files the original changes. The [conflict task](../../docs/release/backport-tasks/resolve-conflict.md) gives the commands. fold prints `ok` and exits 0, or prints why not and exits 1. It never commits or pushes; push one checked resolution at a time, since the next one has to sit on the new head. +Checks a hand-made resolution of a deferred conflict before it is pushed onto the batch, in the shape the run gives an original. For a merged pull request `--commit` is a merge whose first parent is the head of the open batch pull request and whose message names the original in exactly one `Backported-from:` trailer; its second parent ends a straight line from that head of picks of the pull request's own commits, in their order, each ending in the `(cherry picked from commit …)` line of the commit it came from, none claiming another commit of the same original, a commit of another pull request never among them. For a squash-merged one it is a single commit on that head whose `(cherry picked from commit …)` line names the original's commit on `main`. An original parked for what it builds on folds like any other when the resolution is complete. Either has to change something, and only files the original changes or the paths a rename on `main` or on the line gave them, and has to leave nothing the original modifies or deletes without effect, judged as [`run`](#run) judges a pick: a resolution that ports what the line has and leaves out what it lacks is refused, since such a port goes to the line as a pull request of its own. The [conflict task](../../docs/release/backport-tasks/resolve-conflict.md) gives the commands. fold prints `ok` and exits 0, or prints why not and exits 1. It never commits or pushes; push one checked resolution at a time, since the next one has to sit on the new head. ### verify -Holds every unit of a batch to the shape fold checks for one, the clean picks as well as resolutions: it walks the batch's first-parent line from `origin/` to `--head`, or to the head of the open batch pull request, and requires each unit to be one original, merged into `main` after the line branched, in that shape, and no original twice. A commit that carries no original fails it, and so does a merge of the line into the batch. It prints one line per unit, and exits 0 when all pass, 1 when one does not, and 2 when it cannot tell. With `--head` it reads git alone, nothing from GitHub. fold runs only where someone resolved a conflict and chose to run it; [`pr-labeler.yaml`](../../.github/workflows/pr-labeler.yaml) runs verify on every push to a batch pull request, from a build of `main`, with the batch head fetched as data. +Holds every unit of a batch to the shape fold checks for one, the clean picks as well as resolutions: it walks the batch's first-parent line from `origin/` to `--head`, or to the head of the open batch pull request, and requires each unit to be one original, merged into `main` after the line branched, in that shape, and no original twice. A commit that carries no original fails it, and so does a merge of the line into the batch, and a unit that leaves part of its original without effect, judged as [`run`](#run) judges a pick. It prints one line per unit, and exits 0 when all pass, 1 when one does not, and 2 when it cannot tell. With `--head` it reads git alone, nothing from GitHub. fold runs only where someone resolved a conflict and chose to run it; [`pr-labeler.yaml`](../../.github/workflows/pr-labeler.yaml) runs verify on every push to a batch pull request, from a build of `main`, with the batch head fetched as data. ### notify Reads the [line's record](#the-lines-record-and-reports) first, as the other commands do, and exits 2 when it cannot read it or when it says the open batch is being closed, with a batch open or not. Then it reads the open batch pull request of the line, trusted as described below, and prints one line for each thing that waits on it: a batch open for three days or more, the shortest gap between two scheduled runs, so one the previous run already saw; the conflicts it deferred; and the originals waiting for a maintainer because they change workflows. Nothing open prints nothing. +An original the record has wait on others is listed once, as `#M, after #F`, where what decides its wait is listed: after a deferred conflict with the conflicts, after a workflow wait with the waits; one waiting on what is missing or held by rule 6 is not listed. A unit on the batch that carries only part of its original is listed apart, to be taken out before the cut. + +Each park the record of the open batch holds is named once. `run` marks a park `announced` when the line's record it found, open or closed, holds the same park announced: the same merge, kind, prerequisites, commits and files; a park that changed is news again. `notify` lists the parks not yet marked, and with `--push`, once the comment is written, marks them in the record with `gh pr edit --body-file`, only while the pull request is open and its body still carries the record `notify` read. A mark that fails after the comment exits 2, and the next `notify` names those parks once more. With no batch open nothing is named, and the next batch's first `notify` names what it parks. + With `--push` it writes the same into one comment on the batch pull request: the conflicts, each to be resolved as the [conflict task](../../docs/release/backport-tasks/resolve-conflict.md) describes, and the originals that wait because they change workflows. The comment starts with `` and is one per pull request: a later run edits it in place rather than adding another, and once nothing waits it says so. Only a comment whose author the batch trusts counts as it, so one someone else wrote with the same first line is neither edited nor believed. It mentions `--mention`, an `org/team` or a login, with or without `@`, only while something waits. A failed batch CI is not among what it reports: the run has no trigger on the batch's CI finishing. @@ -105,6 +119,15 @@ Only the line's own commits count, the ones after the point where it branched fr Titles and bodies are read from the merge commits on `main`, which hold them as a maintainer merged them; the live pull request stays editable by its author. +## What an original builds on + +An original the policy takes can build on code its line does not have: another pull request merged into `main` after the line branched that the line never got, or a commit pushed to `main` directly. For each original the policy takes that neither the line nor the open batch carries, `run` reads what it builds on from git alone, in two ways: + +- A file it modifies or deletes whose counterpart, as [`run`](#run) follows renames, the line does not have is named by the first-parent commit on `main` that added it since the line branched: `git --literal-pathspecs log --first-parent --diff-merges=first-parent --no-renames --diff-filter=A -1 ..M^1 -- `. A file nothing added since the fork point is shown with the original, since no other pull request brought it. +- A line it removes or changes in a file the line has is given by `git blame --porcelain -w`, at `M^1` named by its full commit, to a commit, which counts through the first-parent commit on `main` that brought it: each commit of `main`'s first-parent line since the fork point brings itself and every commit of the range reachable from its other parents that no earlier one brought. A commit from before the fork point, which the line has, and the original's own count for nothing. The commit counts only while one of its lines, with its edges trimmed and nothing else, is absent from the line's version of the file, so a hand backport under another commit clears it. A file the line already has as the original makes it, the same blob or a clean `git merge-file` back into the line's file, builds on nothing. + +Lines a tool wrote are not blamed: a generated file whole, `zz_generated*`, one with a `Code generated … DO NOT EDIT` marker in its first 100 lines, `go.sum` and the package-manager lock files, and in an ApplicationDefinition under `packages/system/*-rd/cozyrds/` the keys `hack/update-crd.sh` writes, found by their indentation. Controller-gen CRDs and `values.schema.json` are blamed like any other file. Each `git log`, `git diff` and `git blame` the detector and the rename detection run has two minutes, and rename detection that runs out of time counts as skipped, and an original has at most 256 files, missing or blamed, 20 000 blamed lines and 8 MiB per file read for it. Evidence it cannot read, past one of those bounds, a binary file, a symlink, a submodule, a commit blame names that did not come through `main`, a path that is not UTF-8, or a rename git skipped looking for, is shown with the original, whatever becomes of it, under its line in the body and in the report's `diag`. Output of git the tool cannot read whole stops it with exit 2, and is never taken for an absent dependency. A dependency carried only by lines an original adds is not seen. `run` prints how long reading the line took on stderr. What follows from a dependency, a park or a wait, is in [`run`](#run) and [`check`](#check). + ## The line's record and reports The line's record is the `backport-batch` block of its newest batch pull request: the newest pull request against the line from a `backport-batch/-` branch of this repository, opened by an author the batch trusts as above, whether it is open, merged or closed. `run`, `check`, `fold` and `notify` find it in GitHub's list of the line's pull requests, newest first, reading at most ten pages of 100. A batch pull request a person opened without the block is passed over. A block whose author the batch does not trust, a bot's batch without its block, a block that cannot be read whole, a closed record newer than the open batch, or ten pages with no record stop the command with exit 2, naming the pull request, rather than fall back to an older record. @@ -113,7 +136,7 @@ The record is written only while its pull request is open: a merged or closed ba The record is only as trustworthy as the body it sits in. Anyone with write access can edit any body, the bot's included, and neither the bot's authorship nor anything in the block proves who edited it last. A hand edit that leaves a readable record is believed; one that breaks it stops every command with exit 2, naming the pull request. A reason or a file name in the record counts only in the form a run writes it, one line with no control, zero-width or bidi character, so text someone typed into the body never reaches a terminal, a body or a comment as they typed it. When the newest batch pull request is deleted and no later one exists, nothing shows it was there: the older record governs, and what was recorded since is lost. -The record holds only what later commands act on. Every run that classifies the line also makes a report, one line of JSON in a `backport-batch-report` block: every original merged into `main` since the line branched in exactly one category, picked, missing, deferred with what it was deferred for, or skipped with the rule that skipped it, each drop the record keeps with the merge commit it is bound to, its reason and the batch pull request that recorded it, the commits on `main` that merged no pull request, the labels, and what the classification was made of: the heads of `main`, the line and the newer line, the policy, the batch head it describes, the tool's commit, the Actions run and the flags. Its ID is a hash of all that but the time and the run, so two runs that find the same make the same report, and it names no pull request, so a new batch's report is whole before GitHub numbers the pull request. The body carries the latest report beside the record, and a comment on the open batch pull request keeps each one: `` and the same block, folded away. A report already posted is not posted again, and the tool never edits or deletes such a comment, so a report the next run replaces in the body is still read back by `report`, also after the batch merged. A comment that looks like one but whose author the batch does not trust is not read. When a run stops after writing the body but before posting its comment, the next `run --push` on the open batch posts that report first, byte for byte; once the batch is closed nothing is written to it, and the report stays in its body. A run that picks nothing while no batch is open keeps no report. +The record holds only what later commands act on. Version 3, which this tool writes, adds what is parked and why, with the merge commit each park is bound to, the units on the batch that do only part of their original, the picks that differ from theirs, and what each wait on another original waits on; it reads versions 1 to 3, and a tool that reads only 1 and 2 refuses version 3 with exit 2. Every run that classifies the line also makes a report, one line of JSON in a `backport-batch-report` block: every original merged into `main` since the line branched in exactly one category, picked, missing, deferred with what it was deferred for, or skipped with the rule that skipped it, each drop the record keeps with the merge commit it is bound to, its reason and the batch pull request that recorded it, the commits on `main` that merged no pull request, the labels, and what the classification was made of: the heads of `main`, the line and the newer line, the policy, the batch head it describes, the tool's commit, the Actions run and the flags. Its ID is a hash of all that but the time and the run, so two runs that find the same make the same report, and it names no pull request, so a new batch's report is whole before GitHub numbers the pull request. The body carries the latest report beside the record, and a comment on the open batch pull request keeps each one: `` and the same block, folded away. A report already posted is not posted again, and the tool never edits or deletes such a comment, so a report the next run replaces in the body is still read back by `report`, also after the batch merged. A comment that looks like one but whose author the batch does not trust is not read. When a run stops after writing the body but before posting its comment, the next `run --push` on the open batch posts that report first, byte for byte; once the batch is closed nothing is written to it, and the report stays in its body. A run that picks nothing while no batch is open keeps no report. A `--defer` that leaves the batch carrying nothing closes it, and writes everything while the pull request is still open: the report comment, then the body, whose record keeps the deferrals and says the batch is closing, with its number, branch and the head the rebuild started from. It then reads the pull request and the branch again, closes the pull request while both are at that head, and deletes the branch with `--force-with-lease` on it, printing `closed `. The next batch tries what was deferred again. A close that stops part way is finished by the next `run --push`, before it does anything else: it closes and deletes when the pull request is open at the frozen head, deletes the branch when the pull request is closed and the branch still at that head, and goes on as usual once both are gone. A run that finishes a close stops there, with exit 0, or with exit 2 when it was asked for `--defer`, `--drop` or `--undrop`, which it does not apply. It never takes a head it did not freeze: with the branch moved, it stops with exit 2 and names both heads. `run` without `--push`, `check` and `fold` stop with exit 2 while the pull request is still open; once it is closed they go on, warning about a branch the close left, and `notify` says nothing. A closed pull request is never written to, so its record says it was closing for good, and a closed pull request whose branch is gone counts as closed. diff --git a/cmd/backport-batch/apisurface.go b/cmd/backport-batch/apisurface.go index 706e4be62..6966bce74 100644 --- a/cmd/backport-batch/apisurface.go +++ b/cmd/backport-batch/apisurface.go @@ -99,7 +99,7 @@ func apiSnapshot(commit string) (apigate.Snapshot, error) { if err != nil { return nil, err } - defer os.RemoveAll(dir) + defer func() { _ = os.RemoveAll(dir) }() for name, blob := range paths { file := filepath.Join(dir, filepath.FromSlash(name)) if err := os.MkdirAll(filepath.Dir(file), 0o700); err != nil { diff --git a/cmd/backport-batch/assemble.go b/cmd/backport-batch/assemble.go index 7f36ff3f4..7f5529bcd 100644 --- a/cmd/backport-batch/assemble.go +++ b/cmd/backport-batch/assemble.go @@ -33,6 +33,8 @@ type pick struct { files []string // failed is the commit whose pick conflicted. failed string + // needs are the originals still to come it waits on. + needs []int // more counts the conflicting files a recorded conflict no longer names. more int reason string @@ -138,6 +140,7 @@ func (a *app) run(args []string) int { if err := fetch(); err != nil { return a.fail(err) } + started := time.Now() b, err := a.prepare(l) if err != nil { return a.fail(err) @@ -145,6 +148,12 @@ func (a *app) run(args []string) int { if goOn, code := a.resumeClose(b, *push, deferred != nil || drops != nil); !goOn { return code } + if err := b.analyze(); err != nil { + return a.fail(err) + } + // The scheduled job has half an hour for every line; this says where it + // goes. + _, _ = fmt.Fprintf(a.stderr, "prepared %s in %s, %s of it judging partial units and what originals build on\n", l.branch(), time.Since(started).Round(time.Millisecond), b.detecting.Round(time.Millisecond)) if drops != nil { return a.redrop(b, drops, reason, *undropArg != "", *push) } @@ -197,19 +206,62 @@ func (a *app) run(args []string) int { return a.fail(err) } } + // cls is what each original became in this run, to an original that + // builds on it; the rules decide the rest. + mode := analysisMode{noWorkflowFiles: *noWorkflows} + cls := map[*verdict]edgeClass{} + classOf := func(f *verdict) edgeClass { + if c, ok := b.ruleClass(f); ok { + return c + } + if c, ok := cls[f]; ok { + return c + } + return edgeMissing + } for _, v := range b.qualifying() { + on := false + if carried != nil { + if on, err = b.onBranch(carried, v); err != nil { + return a.fail(err) + } + } switch { - case carried != nil && found(carried, v): + case on: r.onBranch = append(r.onBranch, v) + cls[v] = edgeSatisfied continue case held[v.o.number].Reason != "": r.held = append(r.held, pick{v: v, status: "held", reason: held[v.o.number].Reason}) + cls[v] = edgeDeferred continue case v.onBatch != "": v.rule, v.reason = ruleOnLine, fmt.Sprintf("on the open batch pull request #%d", b.pr.Number) r.skipped[ruleOnLine] = append(r.skipped[ruleOnLine], v) continue } + if v.partial != nil { + // Judged at the line's head; what the batch picked since may + // supply what it lacked there. + if err := a.partialAtHead(b, v); err != nil { + return a.fail(err) + } + if v.partial != nil { + r.parked = append(r.parked, v) + cls[v] = edgeNever + continue + } + } + if len(v.deps) > 0 { + kind, err := a.dependsAt(b, r, v, mode, classOf) + if err != nil { + return a.fail(err) + } + if kind != edgeSatisfied { + cls[v] = kind + continue + } + } if *noWorkflows { files, err := workflowFiles(v.commits) if err != nil { @@ -217,18 +269,30 @@ func (a *app) run(args []string) int { } if len(files) > 0 { r.waiting = append(r.waiting, pick{v: v, status: "waiting", files: files}) + cls[v] = edgeWorkflow continue } } - p, err := pickOriginal(v) + p, err := a.pickUnit(v, func(base, head string) ([]string, error) { + omitted, diag, err := b.omittedEffects(v, base, head) + v.addDiag(diag...) + return omitted, err + }) if err != nil { return a.fail(err) } switch p.status { + case "partial": + v.partial = p.files + v.park = &parkedEntry{Original: v.o.number, Merge: v.o.merge, Kind: parkPartial, Files: sortedClean(p.files)} + r.parked = append(r.parked, v) + cls[v] = edgeNever case "picked": r.onBranch = append(r.onBranch, v) + cls[v] = edgeSatisfied case "conflict": r.deferred = append(r.deferred, p) + cls[v] = edgeDeferred case "empty": v.rule, v.reason = ruleOnLine, "the earlier picks already carry it" r.skipped[ruleOnLine] = append(r.skipped[ruleOnLine], v) @@ -246,10 +310,67 @@ func (a *app) run(args []string) int { if !descends { return a.fail(fmt.Errorf("%.12s does not descend from --onto %.12s", head, *onto)) } + if err := b.markPartial(r, carried.head); err != nil { + return a.fail(err) + } } return a.finish(b, r, branch, date, *push, false) } +// dependsAt decides, from what v builds on that the branch lacks, whether +// the run leaves it out: waiting for a maintainer in the scheduled mode +// when only tests or docs build on what never comes, parked when anything +// else never comes, or waiting on prerequisites still to come. What is +// there, the line's, the batch's, or what this run picked or found empty, +// counts for nothing; once earlier picks moved the branch, v is judged at +// its head. It returns what v became to an original that builds on it, +// edgeSatisfied when the run goes on to pick it. +func (a *app) dependsAt(b *batch, r *report, v *verdict, mode analysisMode, classOf func(*verdict) edgeClass) (edgeClass, error) { + deps := v.deps + head, err := resolve("HEAD") + if err != nil { + return 0, err + } + if head != b.lineHead { + if deps, err = b.depsAt(v, head); err != nil { + return 0, err + } + } + never, pending := edges(deps, classOf) + primary, err := workflowPrimary(mode, v, never, pending) + switch { + case err != nil: + return 0, err + case primary: + files, err := workflowFiles(v.commits) + if err != nil { + return 0, err + } + v.addDiag(testsOnly(v, never, b.line)) + r.waiting = append(r.waiting, pick{v: v, status: "waiting", files: files}) + return edgeWorkflow, nil + case len(never) > 0: + v.park = dependencyPark(v, never) + r.parked = append(r.parked, v) + return edgeNever, nil + case len(pending) > 0: + class, needs := strongest(pending) + r.needs = append(r.needs, pick{v: v, status: "waiting", needs: numbers(needs)}) + return class, nil + } + return edgeSatisfied, nil +} + +// partialAtHead judges v again where the run has got to, when earlier picks +// moved the branch past the line's head. +func (a *app) partialAtHead(b *batch, v *verdict) error { + head, err := resolve("HEAD") + if err != nil || head == b.lineHead { + return err + } + return b.partialAt(v, head) +} + // keptOut lists the originals with backport/skip, or dropped, that the batch // still carries: a run never takes a commit off the batch, --defer does. func keptOut(b *batch, carried *history) ([]*verdict, error) { @@ -261,7 +382,11 @@ func keptOut(b *batch, carried *history) ([]*verdict, error) { if err := b.load(v); err != nil { return nil, err } - if found(carried, v) { + on, err := b.onBranch(carried, v) + if err != nil { + return nil, err + } + if on { out = append(out, v) } } @@ -276,7 +401,7 @@ func found(h *history, v *verdict) bool { // newReport starts the report of a run with what the rules left out and what // the labels overrode. func newReport(b *batch) *report { - r := &report{line: b.line, newer: b.newer, skipped: map[int][]*verdict{}, other: len(b.other), start: b.start} + r := &report{line: b.line, newer: b.newer, skipped: map[int][]*verdict{}, other: len(b.other), start: b.start, carried: b.src.rec} for _, v := range b.verdicts { if v.rule != 0 { r.skipped[v.rule] = append(r.skipped[v.rule], v) @@ -324,20 +449,20 @@ func (a *app) finish(b *batch, r *report, branch string, date time.Time, push, r } body, left := r.body(template) p.body = body - fmt.Fprint(a.stdout, body) + _, _ = fmt.Fprint(a.stdout, body) skipped := 0 for _, items := range r.skipped { skipped += len(items) } for _, rule := range left { for _, v := range r.skipped[rule] { - fmt.Fprintf(a.stderr, "skipped by %s: %s %s (%s)\n", ruleName(rule), v.o.url(), terminalSafe(v.o.title), terminalSafe(v.reason)) + _, _ = fmt.Fprintf(a.stderr, "skipped by %s: %s %s (%s)\n", ruleName(rule), v.o.url(), terminalSafe(v.o.title), terminalSafe(v.reason)) } } for _, p := range r.waiting { - fmt.Fprintf(a.stderr, "waiting for a maintainer: %s %s changes %s\n", p.v.o.url(), terminalSafe(p.v.o.title), terminalSafe(strings.Join(p.files, ", "))) + _, _ = fmt.Fprintf(a.stderr, "waiting for a maintainer: %s %s changes %s\n", p.v.o.url(), terminalSafe(p.v.o.title), terminalSafe(strings.Join(p.files, ", "))) } - fmt.Fprintf(a.stderr, "%s at %.12s: %d on the branch, %d deferred, %d waiting for a maintainer, %d skipped\ntitle: %s\nreport: %s\n", + _, _ = fmt.Fprintf(a.stderr, "%s at %.12s: %d on the branch, %d deferred, %d waiting for a maintainer, %d skipped\ntitle: %s\nreport: %s\n", branch, head, len(r.onBranch), len(r.deferred)+len(r.held), len(r.waiting), skipped, p.title, doc.ID) if !push { return 0 @@ -346,8 +471,8 @@ func (a *app) finish(b *batch, r *report, branch string, date time.Time, push, r // Picking nothing is not a failure of the run; check still reports // whatever the line lacks. Merged and closed batches are never // written, so there is nowhere to keep what the run found. - fmt.Fprintf(a.stderr, "%s carries nothing beyond %s; nothing to push\n", branch, b.line.branch()) - fmt.Fprintf(a.stderr, "no batch pull request is open on %s; this run's classification is not retained\n", b.line.branch()) + _, _ = fmt.Fprintf(a.stderr, "%s carries nothing beyond %s; nothing to push\n", branch, b.line.branch()) + _, _ = fmt.Fprintf(a.stderr, "no batch pull request is open on %s; this run's classification is not retained\n", b.line.branch()) return 0 } if err := a.publish(b, p, rebuilt); err != nil { @@ -365,7 +490,8 @@ func (r *report) lineRecord(doc reportDoc) lineRecord { dropped = []droppedEntry{} } slices.SortFunc(dropped, func(a, b droppedEntry) int { return a.Original - b.Original }) - return lineRecord{Version: 2, Line: doc.Line, Built: &built, Report: doc.ID, Closing: r.closing, Deferred: slices.Clone(doc.Primary.Deferred), Dropped: dropped, Stale: slices.Clone(doc.Overlays.Stale)} + return lineRecord{Version: recordVersion, Line: doc.Line, Built: &built, Report: doc.ID, Closing: r.closing, Deferred: slices.Clone(doc.Primary.Deferred), Dropped: dropped, Stale: slices.Clone(doc.Overlays.Stale), + Parked: carryAnnounced(doc.Primary.Parked, r.carried), Partial: slices.Clone(doc.Overlays.Partial), Changed: slices.Clone(doc.Overlays.Changed)} } // branchDate is the date a batch branch was opened under. @@ -403,6 +529,9 @@ func cleanTree() error { // the move drops commits of and how to get them back: a clean tree says // nothing about commits only a local branch has. func (a *app) warnReset(branch, start string) error { + if err := a.mayAct(); err != nil { + return err + } old, err := lookup("refs/heads/" + branch) if err != nil || old == "" || old == start { return err @@ -411,7 +540,11 @@ func (a *app) warnReset(branch, start string) error { if err != nil || kept { return err } - fmt.Fprintf(a.stderr, "resetting the local %s from %s, dropping commits %.12s does not carry; git branch %s-%.12s %s restores them\n", branch, old, start, branch, old, old) + // The hint is the only way back to the dropped commits, so the reset does + // not run when it could not be written. + if _, err := fmt.Fprintf(a.stderr, "resetting the local %s from %s, dropping commits %.12s does not carry; git branch %s-%.12s %s restores them\n", branch, old, start, branch, old, old); err != nil { + return fmt.Errorf("not resetting the local %s: %w", branch, err) + } return nil } @@ -593,6 +726,9 @@ func (a *app) publish(b *batch, p publication, rebuilt bool) error { } } if pushed != p.head { + if err := a.mayAct(); err != nil { + return err + } if _, err := git(pushArgs(p.branch, p.head, lease)...); err != nil { return err } @@ -601,13 +737,13 @@ func (a *app) publish(b *batch, p publication, rebuilt bool) error { if err != nil { return err } - defer os.Remove(name) + defer func() { _ = os.Remove(name) }() if b.pr == nil { n, url, err := a.gh.createPull(b.line.branch(), p.branch, p.title, name) if err != nil { return err } - fmt.Fprintf(a.stderr, "opened draft %s\n", url) + _, _ = fmt.Fprintf(a.stderr, "opened draft %s\n", url) if err := a.postReport(n, p.report); err != nil { return fmt.Errorf("posting report %.12s on #%d: %w; the body keeps it, and the next run --push posts it", p.report.doc.ID, n, err) } @@ -619,7 +755,7 @@ func (a *app) publish(b *batch, p publication, rebuilt bool) error { if err := a.gh.editPull(b.pr.Number, p.title, name); err != nil { return err } - fmt.Fprintf(a.stderr, "updated %s\n", pullURL(b.pr)) + _, _ = fmt.Fprintf(a.stderr, "updated %s\n", pullURL(b.pr)) return nil } @@ -702,7 +838,7 @@ func (a *app) closeBatch(b *batch, p publication) error { if err != nil { return err } - defer os.Remove(name) + defer func() { _ = os.Remove(name) }() if err := a.stillOpen(c.PR); err != nil { return err } @@ -735,7 +871,7 @@ func (a *app) finishClose(l line, url string, c closing) error { if err := a.deleteBranch(l, c); err != nil { return err } - fmt.Fprintf(a.stderr, "closed %s: without what is deferred it carries nothing beyond %s; the next batch tries them again\n", url, l.branch()) + _, _ = fmt.Fprintf(a.stderr, "closed %s: without what is deferred it carries nothing beyond %s; the next batch tries them again\n", url, l.branch()) return nil } @@ -752,6 +888,9 @@ func (a *app) deleteBranch(l line, c closing) error { case h != c.Head: return movedBranch(c, h) } + if err := a.mayAct(); err != nil { + return err + } if _, err := git(deleteArgs(c.Branch, c.Head)...); err != nil { return fmt.Errorf("deleting %s, left by the close of #%d: %w; backport-batch run --line %s --push deletes it", c.Branch, c.PR, err, l.branch()) } @@ -807,7 +946,7 @@ func (a *app) resumeClose(b *batch, push, changing bool) (bool, int) { if err := a.deleteBranch(b.line, c); err != nil { return false, a.fail(err) } - fmt.Fprintf(a.stderr, "deleted %s, which the close of #%d left\n", c.Branch, c.PR) + _, _ = fmt.Fprintf(a.stderr, "deleted %s, which the close of #%d left\n", c.Branch, c.PR) return false, a.finished(c, changing) } diff --git a/cmd/backport-batch/body.go b/cmd/backport-batch/body.go index da4691450..dbb9d6050 100644 --- a/cmd/backport-batch/body.go +++ b/cmd/backport-batch/body.go @@ -176,6 +176,9 @@ type deferredEntry struct { More int `json:"more,omitempty"` Reason string `json:"reason,omitempty"` Workflows bool `json:"workflows,omitempty"` + // Needs, from version 3, are the originals still to come that it waits + // on, those of the kind that decides the wait. + Needs []int `json:"needs,omitempty"` } // How many conflicting files of one original the body names, in the list and @@ -194,7 +197,7 @@ type deferredRecord struct { // had a version. encoding/json leaves bidi and zero-width characters as they // are, so file names go through clean first, as the reason does. func deferredBlock(l line, conflicts, held, waiting []pick) string { - data, _ := json.Marshal(deferredRecord{Line: l.branch(), Deferred: deferredEntries(conflicts, held, waiting)}) + data, _ := json.Marshal(deferredRecord{Line: l.branch(), Deferred: deferredEntries(conflicts, held, waiting, nil)}) return deferredFence + "\n" + string(data) + "\n```" } @@ -231,6 +234,7 @@ type report struct { deferred []pick // conflicts held []pick // what --defer took out, with the reason waiting []pick // what changes workflows, left to a maintainer + needs []pick // what waits on other originals still to come skipped map[int][]*verdict other int // overrides are the originals a backport/ label kept out or let in, and @@ -238,8 +242,10 @@ type report struct { overrides []*verdict stale []*verdict start time.Time - // shapes says, for an original on the branch, what it brought. - shapes map[*verdict]string + // shapes says, for an original on the branch, what it brought, and + // partialFiles what a partial unit leaves without effect. + shapes map[*verdict]unitShape + partialFiles map[*verdict][]string // missing are originals the policy takes that the run neither tried nor // found anywhere. missing []*verdict @@ -253,6 +259,14 @@ type report struct { closing *closing // dropped are the maintainers' drops the record keeps. dropped []droppedEntry + // parked are what the batch leaves out because it cannot land whole, + // each with its park, and partial the originals on the branch whose unit + // does only part of what they do on main. + parked []*verdict + partial []*verdict + // carried is the line's record this run found, whose announcements the + // parks that did not change keep. + carried *lineRecord } // ruleName is how output names what kept an original out: the policy's @@ -299,16 +313,16 @@ func (r *report) ruleTitle(rule int) string { // in details written out. func (r *report) what(details map[int]bool) string { var b strings.Builder - fmt.Fprintf(&b, "Backports to %s from `main`, picked by the rules of the [backport policy](%s) in merge order. A merged pull request comes as its own commits, picked with `git cherry-pick -x` so each keeps its author, message and sign-offs, and a merge commit that carries the original's merge message with a `%s:` trailer naming it; a squash-merged one comes as one pick. `backport-batch verify` checks that shape. A pick listed as changed differs from the commit it came from; `git range-diff ^1..^2 ^1..^2` shows how. A conflict or a failing check here is a [backport task](%sREADME.md).\n\n", codeSpan(r.line.branch()), policyURL, batchTrailer, tasksURL) + fmt.Fprintf(&b, "Backports to %s from `main`, picked by the rules of the [backport policy](%s) in merge order. A merged pull request comes as its own commits, picked with `git cherry-pick -x` so each keeps its author, message and sign-offs, and a merge commit that carries the original's merge message with a `%s:` trailer naming it; a squash-merged one comes as one pick. `backport-batch verify` checks that shape. A pick listed as changed differs from the commit it came from, and its release note waits under Check these notes. A conflict or a failing check here is a [backport task](%sREADME.md).\n\n", codeSpan(r.line.branch()), policyURL, batchTrailer, tasksURL) if len(r.onBranch) == 0 { b.WriteString("Nothing is picked yet.\n") } for _, v := range r.onBranch { shape := "" - if s := r.shapes[v]; s != "" { - shape = ", " + s + if s, ok := r.shapes[v]; ok { + shape = ", " + s.String() } - fmt.Fprintf(&b, "- %s %s%s\n", prLink(v.o.number), titleSpan(v.o.title), shape) + fmt.Fprintf(&b, "- %s %s%s\n%s", prLink(v.o.number), titleSpan(v.o.title), shape, diagItems(v)) } if len(r.deferred) > 0 { fmt.Fprintf(&b, "\n### Deferred conflicts\n\nThese do not pick cleanly onto this branch. Each names the first of its commits that conflicted and the files it conflicted in, and nothing of it is on the branch. Each moves to the next batch, unless a resolution checked with `backport-batch fold` is pushed here first, as [resolving a conflict](%sresolve-conflict.md) describes.\n\n", tasksURL) @@ -324,13 +338,13 @@ func (r *report) what(details map[int]bool) string { if p.failed != "" { at = " at " + codeSpan(p.failed[:12]) } - fmt.Fprintf(&b, "- %s %s%s: %s\n", prLink(p.v.o.number), titleSpan(p.v.o.title), at, strings.Join(files, ", ")) + fmt.Fprintf(&b, "- %s %s%s: %s\n%s", prLink(p.v.o.number), titleSpan(p.v.o.title), at, strings.Join(files, ", "), diagItems(p.v)) } } if len(r.held) > 0 { b.WriteString("\n### Deferred to the next batch\n\nTaken out of this batch with `backport-batch run --defer`, each for the reason given. This pull request keeps them out; the next batch picks them again.\n\n") for _, p := range r.held { - fmt.Fprintf(&b, "- %s %s: %s\n", prLink(p.v.o.number), titleSpan(p.v.o.title), titleSpan(p.reason)) + fmt.Fprintf(&b, "- %s %s: %s\n%s", prLink(p.v.o.number), titleSpan(p.v.o.title), titleSpan(p.reason), diagItems(p.v)) } } if len(r.waiting) > 0 { @@ -343,7 +357,41 @@ func (r *report) what(details map[int]bool) string { if more := len(p.files) - len(files) + p.more; more > 0 { files = append(files, fmt.Sprintf("%d more", more)) } - fmt.Fprintf(&b, "- %s %s: %s\n", prLink(p.v.o.number), titleSpan(p.v.o.title), strings.Join(files, ", ")) + fmt.Fprintf(&b, "- %s %s: %s\n%s", prLink(p.v.o.number), titleSpan(p.v.o.title), strings.Join(files, ", "), diagItems(p.v)) + } + } + if changed := r.changed(); len(changed) > 0 { + b.WriteString("\n### Check these notes\n\nThese picks differ from their originals, so their release notes are left out of the release-note block until someone checks that each still holds for this line. Each comes with what to compare: every pick that differs against the commit it came from, and the original's change against the unit's.\n") + for _, v := range changed { + s := r.shapes[v] + fmt.Fprintf(&b, "\n%s %s, carried by %s:\n\n", prLink(v.o.number), titleSpan(v.o.title), codeSpan(s.unit[:12])) + if n := releaseNote(v.o.body); n != "" { + fmt.Fprintf(&b, "```text\n%s\n```\n\n", neutralNote(n)) + } else { + b.WriteString("It has no release note.\n\n") + } + for _, p := range s.pairs { + fmt.Fprintf(&b, "- %s\n", codeSpan(fmt.Sprintf("git range-diff --creation-factor=999 %s^! %s^!", p.source[:12], p.pick[:12]))) + } + fmt.Fprintf(&b, "- %s\n- %s\n", codeSpan(fmt.Sprintf("git diff %s^1 %s", v.o.merge[:12], v.o.merge[:12])), codeSpan(fmt.Sprintf("git diff %s^1 %s", s.unit[:12], s.unit[:12]))) + } + } + if len(r.partial) > 0 { + fmt.Fprintf(&b, "\n### Still on this branch: only part of it applies\n\nEach of these is on this branch but leaves files of its original as they were on %s, so it carries only part of what it does on `main`. Take each out with `backport-batch run --defer` before the cut; what applies of it goes as a pull request of its own.\n\n", codeSpan(r.line.branch())) + for _, v := range r.partial { + fmt.Fprintf(&b, "- %s %s: leaves %s without effect\n", prLink(v.o.number), titleSpan(v.o.title), fileSpans(r.partialFiles[v])) + } + } + if len(r.parked) > 0 { + fmt.Fprintf(&b, "\n### Parked\n\nThese cannot land on %s whole, so the batch does not pick them: each would leave part of it without effect there, or builds on a change that never reaches the line. A maintainer ports what applies as a pull request of its own, or drops the original with `backport-batch run --drop`; `backport/include` does not lift a park.\n\n", codeSpan(r.line.branch())) + for _, v := range r.parked { + fmt.Fprintf(&b, "- %s %s: %s\n%s", prLink(v.o.number), titleSpan(v.o.title), parkReason(*v.park), diagItems(v)) + } + } + if len(r.needs) > 0 { + fmt.Fprintf(&b, "\n### Waiting on another original\n\nThese build on another original that has not reached %s yet, so the batch does not try them; each names the one it waits on, and the batch tries it again once that one is there.\n\n", codeSpan(r.line.branch())) + for _, p := range r.needs { + fmt.Fprintf(&b, "- %s %s: after %s\n%s", prLink(p.v.o.number), titleSpan(p.v.o.title), prLinks(p.needs), diagItems(p.v)) } } if len(r.overrides) > 0 { @@ -384,6 +432,8 @@ func (r *report) what(details map[int]bool) string { // outcome is what became of an original a label overrode. func (r *report) outcome(v *verdict) string { switch { + case slices.Contains(r.partial, v): + return "still on this branch, but only part of it applies; take it out with `backport-batch run --defer`" case slices.Contains(r.stale, v) && v.rule == ruleDropped: return "dropped, but still on this branch; take it out with `backport-batch run --defer`" case slices.Contains(r.stale, v): @@ -398,15 +448,83 @@ func (r *report) outcome(v *verdict) string { return "deferred to the next batch" case slices.ContainsFunc(r.waiting, func(p pick) bool { return p.v == v }): return "waiting for a maintainer: it changes " + codeSpan(workflowsDir) + case slices.Contains(r.parked, v): + return "parked: " + parkReason(*v.park) + case slices.ContainsFunc(r.needs, func(p pick) bool { return p.v == v }): + i := slices.IndexFunc(r.needs, func(p pick) bool { return p.v == v }) + return "waiting on " + prLinks(r.needs[i].needs) case v.rule != 0: return "skipped by " + ruleName(v.rule) + ", " + titleSpan(v.reason) } return "not picked" } +// diagItems lists, under an original's line, what could not be judged about +// it or what it was seen to build on, each as a code span. +func diagItems(v *verdict) string { + var b strings.Builder + for _, d := range v.diag { + fmt.Fprintf(&b, " - %s\n", titleSpan(d)) + } + return b.String() +} + +// fileSpans lists files as code spans, the first few of them. +func fileSpans(files []string) string { + var out []string + for _, f := range files[:min(len(files), filesListed)] { + out = append(out, codeSpan(f)) + } + if more := len(files) - len(out); more > 0 { + out = append(out, fmt.Sprintf("%d more", more)) + } + return strings.Join(out, ", ") +} + +// prLinks links to pull requests by number. +func prLinks(numbers []int) string { + var out []string + for _, n := range numbers { + out = append(out, prLink(n)) + } + return strings.Join(out, ", ") +} + +// parkReason says why a park leaves its original out, for the body. +func parkReason(p parkedEntry) string { + if p.Kind == parkPartial { + return "only part of it applies; it would leave " + fileSpans(p.Files) + " without effect" + } + var needs []string + for _, n := range p.Needs { + needs = append(needs, prLink(n)) + } + for _, c := range p.Commits { + needs = append(needs, codeSpan(c[:12])) + } + return "builds on " + strings.Join(needs, ", ") + ", which never reaches this line" +} + +// changed lists the originals on the branch whose unit differs from them, in +// merge order. +func (r *report) changed() []*verdict { + var out []*verdict + for _, v := range r.onBranch { + if s, ok := r.shapes[v]; ok && s.changed() { + out = append(out, v) + } + } + return out +} + +// notes are the release notes of what the branch carries as its originals +// made it; a changed pick's note waits under Check these notes. func (r *report) notes() string { var notes []string for _, v := range r.onBranch { + if s, ok := r.shapes[v]; ok && s.changed() { + continue + } if n := releaseNote(v.o.body); n != "" { notes = append(notes, neutralNote(n)) } diff --git a/cmd/backport-batch/check.go b/cmd/backport-batch/check.go index ae5395d45..2101d251f 100644 --- a/cmd/backport-batch/check.go +++ b/cmd/backport-batch/check.go @@ -20,6 +20,7 @@ import ( "fmt" "slices" "strconv" + "strings" ) // check says whether every original the policy takes is on the line, on the @@ -50,7 +51,13 @@ func (a *app) check(args []string) int { if err := a.warnLeftBranch(b); err != nil { return a.fail(err) } - disp, err := b.dispositions(*cut) + // The scheduled run's mode, as the open batch saved it; with none open, + // the scheduled run's, which is what the next batch is built in. + mode := analysisMode{noWorkflowFiles: true} + if b.src.open { + mode.noWorkflowFiles = b.src.savedNoWorkflowFiles() + } + disp, err := b.dispositions(mode, *cut) if err != nil { return a.fail(err) } @@ -66,14 +73,14 @@ func (a *app) check(args []string) int { case d.kind == outcomeWarning: a.warn("%s", d.text) case d.kind == outcomeNote: - fmt.Fprintf(a.stderr, "note: %s\n", d.text) + _, _ = fmt.Fprintf(a.stderr, "note: %s\n", d.text) } } if len(missing)+len(failing) == 0 { if *cut { - fmt.Fprintf(a.stdout, "%s: no change the policy takes is missing before the cut\n", l.branch()) + _, _ = fmt.Fprintf(a.stdout, "%s: no change the policy takes is missing before the cut\n", l.branch()) } else { - fmt.Fprintf(a.stdout, "%s: every change the policy takes is on the line or on the batch\n", l.branch()) + _, _ = fmt.Fprintf(a.stdout, "%s: every change the policy takes is on the line or on the batch\n", l.branch()) } return 0 } @@ -82,13 +89,13 @@ func (a *app) check(args []string) int { where = "nor on batch pull request #" + strconv.Itoa(b.pr.Number) } if len(missing) > 0 { - fmt.Fprintf(a.stdout, "%s: %d changes the policy takes are not on the line, %s:\n", l.branch(), len(missing), where) + _, _ = fmt.Fprintf(a.stdout, "%s: %d changes the policy takes are not on the line, %s:\n", l.branch(), len(missing), where) for _, text := range missing { - fmt.Fprintf(a.stdout, " %s\n", text) + _, _ = fmt.Fprintf(a.stdout, " %s\n", text) } } for _, text := range failing { - fmt.Fprintf(a.stdout, "%s: %s\n", l.branch(), text) + _, _ = fmt.Fprintf(a.stdout, "%s: %s\n", l.branch(), text) } return 1 } @@ -137,8 +144,11 @@ type disposition struct { // dispositions says what check reports about each original that needs a line // of its own; the others pass in silence. The text is what follows the line's -// name, or the original alone for one that is missing. -func (b *batch) dispositions(cut bool) (map[*verdict]disposition, error) { +// name, or the original alone for one that is missing. Originals are taken in +// merge order, each by the first of: a unit on the batch that does only part +// of it, a reason a person deferred it for, a park, a wait for a maintainer +// or on other originals, and what the record says of it. +func (b *batch) dispositions(mode analysisMode, cut bool) (map[*verdict]disposition, error) { if c := b.src.closingOpen(); c != nil { return nil, fmt.Errorf("batch pull request #%d is being closed; backport-batch run --line %s --push finishes it", c.PR, b.line.branch()) } @@ -158,11 +168,45 @@ func (b *batch) dispositions(cut bool) (map[*verdict]disposition, error) { on += " (" + prState(c) + ")" } } + if err := b.analyze(); err != nil { + return nil, err + } + if b.pr != nil { + if err := b.keepRecordedPartials(b.prHead); err != nil { + return nil, err + } + } + held := map[int]string{} + for n, e := range deferred { + if e.Reason != "" { + held[n] = e.Reason + } + } + states, err := b.classify(mode, held, deferred, b.src.open) + if err != nil { + return nil, err + } + // waitsOn names what an original waits on, as check says it. + waitsOn := func(s *state) string { + var names []string + for _, f := range s.needs { + names = append(names, fmt.Sprintf("#%d", f.o.number)) + } + return strings.Join(names, " and ") + } out := map[*verdict]disposition{} for _, v := range b.verdicts { named := v.o.url() + " " + terminalSafe(v.o.title) e, isDeferred := deferred[v.o.number] + s := states[v] + if s == nil { + s = &state{kind: stOwn} + } switch { + case b.partialUnits[v] != nil && cut: + out[v] = disposition{outcomeUnmerged, fmt.Sprintf("#%d is on batch #%d, but only part of it applies to %s: it leaves %s without effect; take it out with backport-batch run --defer before the cut: %s", v.o.number, b.pr.Number, b.line.branch(), quotedFiles(b.partialUnits[v]), named)} + case b.partialUnits[v] != nil: + out[v] = disposition{outcomeWarning, fmt.Sprintf("#%d is on batch #%d, but only part of it applies to %s: it leaves %s without effect; take it out with backport-batch run --defer: %s", v.o.number, b.pr.Number, b.line.branch(), quotedFiles(b.partialUnits[v]), named)} case slices.Contains(stale, v): out[v] = disposition{outcomeWarning, fmt.Sprintf("#%d is still on batch #%d; take it out with backport-batch run --defer: %s", v.o.number, b.pr.Number, named)} case v.rule == ruleDropped: @@ -173,15 +217,41 @@ func (b *batch) dispositions(cut bool) (map[*verdict]disposition, error) { if cut { out[v] = disposition{outcomeUnmerged, fmt.Sprintf("#%d is only on open batch PR #%d, merge it before the cut: %s", v.o.number, b.pr.Number, named)} } + case s.kind == stHeld: + out[v] = disposition{outcomeWarning, fmt.Sprintf("deferred on %s to the next batch: %s (%s)", on, named, terminalSafe(e.Reason))} + case s.kind == stPartial: + out[v] = disposition{outcomeNote, fmt.Sprintf("#%d is parked: only part of it applies to %s, which would leave %s without effect; port what applies as a pull request of its own: %s", v.o.number, b.line.branch(), quotedFiles(v.park.Files), named)} + case s.kind == stWorkflowPrimary && s.class == edgeMissing: + out[v] = disposition{outcomeMissing, named} + case s.kind == stParked: + out[v] = disposition{outcomeNote, fmt.Sprintf("#%d is parked: it builds on %s, which never reaches %s; port what applies as a pull request of its own, or drop it: %s", v.o.number, parkNames(*v.park), b.line.branch(), named)} + case s.kind == stNeeds && s.wait == edgeMissing: + out[v] = disposition{outcomeMissing, fmt.Sprintf("%s (it waits on %s, which is not on the line either)", named, waitsOn(s))} + case s.kind == stNeeds && s.wait == edgeWorkflow && cut: + out[v] = disposition{outcomeWaiting, fmt.Sprintf("#%d waits on %s, which changes %s and waits on batch PR %s for a maintainer; bring it over, or defer both, before the cut: %s", v.o.number, waitsOn(s), workflowsDir, on, named)} + case s.kind == stNeeds && s.wait == edgeWorkflow: + out[v] = disposition{outcomeWarning, fmt.Sprintf("#%d waits on %s, which waits on %s for a maintainer: %s", v.o.number, waitsOn(s), on, named)} + case s.kind == stNeeds && s.wait == edgeDeferred: + what := "a deferred conflict on " + on + if f := s.needs[0]; deferred[f.o.number].Reason != "" { + what = "deferred on " + on + " to the next batch" + } else if states[f] != nil && states[f].kind == stNeeds { + what = "which waits itself" + } + out[v] = disposition{outcomeWarning, fmt.Sprintf("#%d waits on %s, %s: %s", v.o.number, waitsOn(s), what, named)} + case s.kind == stNeeds: + newer := "the newer line" + if b.newer != nil { + newer = b.newer.branch() + } + out[v] = disposition{outcomeNote, fmt.Sprintf("#%d waits on %s, which is not on %s yet: %s", v.o.number, waitsOn(s), newer, named)} case isDeferred && e.Workflows && !b.src.open: out[v] = disposition{outcomeMissing, named} case isDeferred && e.Workflows && cut: out[v] = disposition{outcomeWaiting, fmt.Sprintf("#%d changes %s and waits on batch PR %s for a maintainer, bring it over or defer it before the cut: %s", v.o.number, workflowsDir, on, named)} case isDeferred && e.Workflows: out[v] = disposition{outcomeWarning, fmt.Sprintf("waiting on %s for a maintainer, since it changes %s: %s", on, workflowsDir, named)} - case isDeferred && e.Reason != "": - out[v] = disposition{outcomeWarning, fmt.Sprintf("deferred on %s to the next batch: %s (%s)", on, named, terminalSafe(e.Reason))} - case isDeferred: + case isDeferred && len(e.Needs) == 0: out[v] = disposition{outcomeWarning, fmt.Sprintf("deferred conflict on %s: %s", on, named)} default: out[v] = disposition{outcomeMissing, named} @@ -190,6 +260,30 @@ func (b *batch) dispositions(cut bool) (map[*verdict]disposition, error) { return out, nil } +// parkNames names what a park waits on for a terminal. +func parkNames(p parkedEntry) string { + var out []string + for _, n := range p.Needs { + out = append(out, fmt.Sprintf("#%d", n)) + } + for _, c := range p.Commits { + out = append(out, c[:12]) + } + return strings.Join(out, ", ") +} + +// quotedFiles names files for a terminal, each quoted, the first few of them. +func quotedFiles(files []string) string { + var out []string + for _, f := range files[:min(len(files), filesListed)] { + out = append(out, strconv.Quote(f)) + } + if more := len(files) - len(out); more > 0 { + out = append(out, fmt.Sprintf("%d more", more)) + } + return strings.Join(out, ", ") +} + // terminalSafe keeps a title from carrying escape sequences into a terminal. func terminalSafe(s string) string { return capped(oneLine(clean(s, false)), titleLimit) } @@ -232,10 +326,10 @@ func (a *app) fold(args []string) int { return a.fail(err) } if why != "" { - fmt.Fprintf(a.stdout, "refused: %s\n", why) + _, _ = fmt.Fprintf(a.stdout, "refused: %s\n", why) return 1 } - fmt.Fprintln(a.stdout, "ok") + _, _ = fmt.Fprintln(a.stdout, "ok") return 0 } @@ -289,5 +383,9 @@ func refuseFold(b *batch, n int, commit string) (string, error) { case carried != v: return fmt.Sprintf("%.12s carries #%d, not #%d", commit, carried.o.number, n), nil } - return "", nil + omitted, _, err := b.omittedEffects(v, u.parents[0], u.commit) + if err != nil || len(omitted) == 0 { + return "", err + } + return leftOut(v, b.line, omitted), nil } diff --git a/cmd/backport-batch/defer.go b/cmd/backport-batch/defer.go index 52fa8cf62..84e05d5e8 100644 --- a/cmd/backport-batch/defer.go +++ b/cmd/backport-batch/defer.go @@ -78,7 +78,7 @@ func (a *app) rebuild(b *batch, numbers []int, reason string, push bool) int { switch v := b.find(n); { case v == nil: return a.fail(fmt.Errorf("#%d did not merge into main after %s branched", n, b.line.branch())) - case v.rule != 0 && !slices.Contains(stale, v): + case v.rule != 0 && !slices.Contains(stale, v) && b.partialUnits[v] == nil: return a.fail(fmt.Errorf("#%d is not one the batch takes: %s, %s", n, ruleName(v.rule), terminalSafe(v.reason))) } hold[n] = reason @@ -102,7 +102,7 @@ func (a *app) reassemble(b *batch, out, hold map[int]string, drops []droppedEntr return a.fail(err) } for _, u := range replay { - failed, files, err := replayUnit(u) + failed, files, err := a.replayNext(u) switch { case err != nil: return a.fail(err) @@ -138,16 +138,50 @@ func (a *app) reclassify(b *batch, picks *history, hold map[int]string, drops [] if r.stale, err = keptOut(b, picks); err != nil { return a.fail(err) } + // What the rebuild took off is no longer there for what builds on it. + units, err := b.unitsOn(picks.head) + if err != nil { + return a.fail(err) + } + for _, v := range b.verdicts { + v.batched = units[v] + } + on := map[*verdict]bool{} + for _, v := range b.qualifying() { + if on[v], err = b.onBranch(picks, v); err != nil { + return a.fail(err) + } + if !on[v] { + v.onBatch = "" + } + } + if err := b.keepRecordedPartials(picks.head); err != nil { + return a.fail(err) + } + states, err := b.classify(analysisMode{noWorkflowFiles: r.opts.NoWorkflowFiles}, hold, recorded, true) + if err != nil { + return a.fail(err) + } for _, v := range b.qualifying() { e := recorded[v.o.number] + s := states[v] + if s == nil { + s = &state{kind: stOwn} + } switch { - case found(picks, v): + case on[v]: r.onBranch = append(r.onBranch, v) case hold[v.o.number] != "": r.held = append(r.held, pick{v: v, status: "held", reason: hold[v.o.number]}) + case s.kind == stPartial || s.kind == stParked: + r.parked = append(r.parked, v) + case s.kind == stNeeds: + r.needs = append(r.needs, pick{v: v, status: "waiting", needs: numbers(s.needs)}) + case s.kind == stWorkflowPrimary && !e.Workflows: + r.missing = append(r.missing, v) case e.Workflows: r.waiting = append(r.waiting, pick{v: v, status: "waiting", files: e.Files, more: e.More}) - case len(e.Files) > 0: + case len(e.Files) > 0 && len(e.Needs) == 0: r.deferred = append(r.deferred, pick{v: v, status: "conflict", files: e.Files, more: e.More, failed: e.Commit}) default: // Merged into main since the batch was assembled, or recorded @@ -155,6 +189,9 @@ func (a *app) reclassify(b *batch, picks *history, hold map[int]string, drops [] r.missing = append(r.missing, v) } } + if err := b.markPartial(r, picks.head); err != nil { + return a.fail(err) + } return a.finish(b, r, b.pr.Head.Ref, date, push, true) } diff --git a/cmd/backport-batch/deps.go b/cmd/backport-batch/deps.go new file mode 100644 index 000000000..895897709 --- /dev/null +++ b/cmd/backport-batch/deps.go @@ -0,0 +1,912 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "path" + "regexp" + "slices" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +// What an original builds on that its line lacks is read from git alone, in +// two ways: a file it modifies or deletes that the line does not have, named +// by the first-parent commit on main that added it (V1), and a line it +// removes or changes that git blame gives to a commit the line lacks, kept +// only while that line's text is absent from the line's file (V2c). Either +// names the first-parent commit that brought the evidence to main: the merge +// of another original, or a commit that merged no pull request. + +// Bounds per original, past which the evidence is incomplete rather than +// read at any cost. +const ( + maxDepFiles = 256 + maxBlamedLines = 20000 + maxDepBlob = 8 << 20 + gitDeadline = 2 * time.Minute +) + +// dependency is a first-parent commit on main an original builds on, with +// the original's paths the evidence is in. +type dependency struct { + merge string + // v is the original merge merged, nil for a commit that merged none. + v *verdict + paths []string + // added says some of the evidence is a file the line lacks (V1). + added bool +} + +// errDeadline is a git call that ran out of time: incomplete evidence, not +// a failure. +var errDeadline = errors.New("git took longer than " + gitDeadline.String()) + +// gitTimed runs git under the deadline the detector gives each call. +func gitTimed(args ...string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), gitDeadline) + defer cancel() + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Env = append(cmd.Environ(), commandEnv...) + var out, errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errOut + err := cmd.Run() + switch { + case ctx.Err() != nil: + return "", errDeadline + case err != nil: + return "", fmt.Errorf("git %s: %w: %s", quotedArgs(args), err, terminalSafe(errOut.String())) + } + return out.String(), nil +} + +// quotedArgs names a git command's arguments for an error, each quoted: a +// file name can hold a newline, and the error reaches a terminal and a CI +// log, which would take the rest for a line of their own. +func quotedArgs(args []string) string { + out := make([]string, len(args)) + for i, a := range args { + out[i] = strconv.Quote(a) + } + return strings.Join(out, " ") +} + +// gitStatusTimed is gitStatus under the detector's deadline: a non-zero +// exit is an answer, and running out of time is errDeadline. +func gitStatusTimed(args ...string) (stdout, stderr string, code int, err error) { + ctx, cancel := context.WithTimeout(context.Background(), gitDeadline) + defer cancel() + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Env = append(cmd.Environ(), commandEnv...) + var out, errOut bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errOut + err = cmd.Run() + var ee *exec.ExitError + switch { + case ctx.Err() != nil: + return "", "", -1, errDeadline + case errors.As(err, &ee): + return out.String(), errOut.String(), ee.ExitCode(), nil + case err != nil: + return "", "", -1, fmt.Errorf("git %s: %w", quotedArgs(args), err) + } + return out.String(), errOut.String(), 0, nil +} + +// broughtBy maps every commit in base..mainHead to the first-parent commit +// that brought it to main: each first-parent commit claims itself, then +// every commit of the range reachable from its other parents that no +// earlier one claimed. A commit outside the map is an ancestor of base. +func broughtBy(base, mainHead string) (map[string]string, error) { + first, err := revList("--first-parent", "--reverse", "--end-of-options", base+".."+mainHead) + if err != nil { + return nil, err + } + out, err := git("rev-list", "--parents", "--topo-order", "--end-of-options", base+".."+mainHead) + if err != nil { + return nil, err + } + parents := map[string][]string{} + for ln := range strings.Lines(out) { + f := strings.Fields(ln) + if len(f) == 0 { + continue + } + for _, sha := range f { + if checkSHA(sha) != nil { + return nil, fmt.Errorf("rev-list: %q is not a commit", sha) + } + } + parents[f[0]] = f[1:] + } + by := map[string]string{} + for _, fp := range first { + if _, ok := parents[fp]; !ok { + return nil, fmt.Errorf("rev-list: %s is on main's first-parent line but not in its range", fp) + } + by[fp] = fp + stack := slices.Clone(parents[fp][min(1, len(parents[fp])):]) + for len(stack) > 0 { + c := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if _, inRange := parents[c]; !inRange || by[c] != "" { + continue + } + by[c] = fp + stack = append(stack, parents[c]...) + } + } + return by, nil +} + +// hunk is an old range of a diff, a,+b in git blame's terms. +type hunk struct{ start, count int } + +var hunkRE = regexp.MustCompile(`^@@ -(\d+)(?:,(\d+))? \+\d+(?:,\d+)? @@`) + +// oldRanges reads the removed or changed line ranges of a -U0 diff of one +// file, and whether git called it binary. A header it cannot read is an +// error, never evidence of absence. +func oldRanges(diff string) ([]hunk, bool, error) { + var out []hunk + for ln := range strings.Lines(diff) { + ln = strings.TrimSuffix(ln, "\n") + switch { + case strings.HasPrefix(ln, "Binary files ") && strings.HasSuffix(ln, " differ"): + return nil, true, nil + case strings.HasPrefix(ln, "@@"): + m := hunkRE.FindStringSubmatch(ln) + if m == nil { + return nil, false, fmt.Errorf("unreadable hunk header %q", terminalSafe(ln)) + } + start, _ := strconv.Atoi(m[1]) + count := 1 + if m[2] != "" { + count, _ = strconv.Atoi(m[2]) + } + if count > 0 { + out = append(out, hunk{start, count}) + } + } + } + return out, false, nil +} + +// blameArgs blames ranges of file at rev, a full commit, which git takes for +// a revision whatever follows: blame refuses --end-of-options. +func blameArgs(rev, file string, ranges []hunk) ([]string, error) { + if err := checkSHA(rev); err != nil { + return nil, err + } + args := []string{"blame", "--porcelain", "-w"} + for _, r := range ranges { + args = append(args, "-L", fmt.Sprintf("%d,+%d", r.start, r.count)) + } + return append(args, rev, "--", file), nil +} + +// blamedLine is one line git blame gives to a commit. +type blamedLine struct{ commit, text string } + +// parseBlame reads git blame --porcelain: a header per line, the commit's +// fields the first time it appears, and the line itself after a tab. +// Anything else is an error, never evidence of absence. +func parseBlame(out string) ([]blamedLine, error) { + var lines []blamedLine + commit := "" + for ln := range strings.Lines(out) { + ln = strings.TrimSuffix(ln, "\n") + switch { + case strings.HasPrefix(ln, "\t"): + if commit == "" { + return nil, errors.New("git blame: a line with no header") + } + lines = append(lines, blamedLine{commit: commit, text: ln[1:]}) + commit = "" + case commit == "": + f := strings.Fields(ln) + if len(f) < 3 || len(f) > 4 || checkSHA(f[0]) != nil { + return nil, fmt.Errorf("git blame: unreadable header %q", terminalSafe(ln)) + } + for _, n := range f[1:] { + if v, err := strconv.Atoi(n); err != nil || v <= 0 { + return nil, fmt.Errorf("git blame: unreadable header %q", terminalSafe(ln)) + } + } + commit = f[0] + } + } + if commit != "" { + return nil, errors.New("git blame: a header with no line") + } + return lines, nil +} + +var generatedMarkerRE = regexp.MustCompile(`Code generated .* DO NOT EDIT`) + +// generatedFiles are written whole by a tool, whatever they hold. +var generatedFiles = []string{"go.sum", "package-lock.json", "yarn.lock", "pnpm-lock.yaml"} + +// cozyRDGenerated are the keys hack/update-crd.sh writes into an +// ApplicationDefinition, the rest of which people write. +var cozyRDGenerated = []string{ + "apiVersion", "kind", "metadata.name", "spec.application.openAPISchema", "spec.release.prefix", + "spec.release.chartRef.kind", "spec.release.chartRef.name", "spec.release.chartRef.namespace", + "spec.dashboard.description", "spec.dashboard.icon", "spec.dashboard.keysOrder", +} + +var yamlKeyRE = regexp.MustCompile(`^( *)([A-Za-z0-9_.-]+):(?:\s|$)`) + +// generatedLines says which lines of a file at a commit a tool wrote: all of +// them for a generated file, the keys hack/update-crd.sh sets for an +// ApplicationDefinition, found by indentation, and none otherwise. ok is +// false for an ApplicationDefinition the scan cannot read, whose lines are +// then not blamed at all. +func generatedLines(file string, blob []byte) (skip map[int]bool, whole, ok bool) { + base := path.Base(file) + if strings.HasPrefix(base, "zz_generated") || slices.Contains(generatedFiles, base) { + return nil, true, true + } + lines := strings.Split(string(blob), "\n") + for _, ln := range lines[:min(len(lines), 100)] { + if generatedMarkerRE.MatchString(ln) { + return nil, true, true + } + } + if rd, _ := path.Match(cozyRDGlob, file); !rd { + return nil, false, true + } + skip = map[int]bool{} + type key struct { + indent int + name string + } + var stack []key + genIndent := -1 + for i, ln := range lines { + n := i + 1 + trimmed := strings.TrimLeft(ln, " ") + indent := len(ln) - len(trimmed) + switch { + case strings.HasPrefix(trimmed, "\t") || strings.HasPrefix(ln, "---") && i > 0: + return nil, false, false + case trimmed == "" || strings.HasPrefix(trimmed, "#"): + if genIndent >= 0 { + skip[n] = true + } + continue + case genIndent >= 0 && indent > genIndent: + skip[n] = true + continue + } + genIndent = -1 + m := yamlKeyRE.FindStringSubmatch(ln) + if m == nil { + continue + } + for len(stack) > 0 && stack[len(stack)-1].indent >= indent { + stack = stack[:len(stack)-1] + } + stack = append(stack, key{indent, m[2]}) + var names []string + for _, k := range stack { + names = append(names, k.name) + } + if slices.Contains(cozyRDGenerated, strings.Join(names, ".")) { + skip[n] = true + genIndent = indent + } + } + return skip, false, true +} + +// trimmedLines is the set of a blob's lines with their edges trimmed, which +// is all a reformat of indentation or trailing space changes. +func trimmedLines(blob []byte) map[string]bool { + set := map[string]bool{} + for ln := range strings.Lines(string(blob)) { + set[strings.TrimSpace(ln)] = true + } + return set +} + +// shown is a path as a diagnostic names it. +func shown(p string) string { return strconv.Quote(p) } + +// incomplete is a diagnostic about evidence the detector could not read. +func incomplete(format string, args ...any) string { + return oneLineText("incomplete evidence: " + fmt.Sprintf(format, args...)) +} + +// dependencies reads what v builds on that target lacks, with what it could +// not read. V1 looks at the paths whose counterpart target lacks, V2c at +// the ones it has. +func (b *batch) dependencies(v *verdict, target string) ([]dependency, []string, error) { + if err := b.load(v); err != nil { + return nil, nil, err + } + if b.brought == nil { + var err error + if b.brought, err = broughtBy(b.base, b.mainHead); err != nil { + return nil, nil, err + } + } + corr, unknown, err := b.correspondence(v, target) + if err != nil { + return nil, nil, err + } + var diag []string + for _, p := range unknown { + diag = append(diag, incomplete("git skipped rename detection for %s", shown(p))) + } + var at []string + for _, c := range corr { + at = append(at, c) + } + have, err := treeEntries(target, at) + if err != nil { + return nil, nil, err + } + parent, err := resolve(v.o.merge + "^1") + if err != nil { + return nil, nil, err + } + byMerge := map[string]*dependency{} + add := func(merge, p string, added bool) { + d := byMerge[merge] + if d == nil { + d = &dependency{merge: merge, v: b.findMerge(merge)} + byMerge[merge] = d + } + if !slices.Contains(d.paths, p) { + d.paths = append(d.paths, p) + } + d.added = d.added || added + } + files, blamed := 0, 0 + for _, c := range v.changes { + cp, ok := corr[c.path] + if !ok || c.status != 'M' && c.status != 'D' { + continue + } + if !utf8.ValidString(c.path) { + diag = append(diag, incomplete("a path that is not UTF-8, %s", shown(c.path))) + continue + } + if files++; files > maxDepFiles { + diag = append(diag, incomplete("more than %d files to read", maxDepFiles)) + break + } + cur, present := have[cp] + if !present { + out, err := gitTimed("--literal-pathspecs", "log", "--first-parent", "--diff-merges=first-parent", "--no-renames", "--diff-filter=A", "--format=%H", "--no-patch", "-1", "--end-of-options", b.base+".."+parent, "--", c.path) + switch { + case errors.Is(err, errDeadline): + diag = append(diag, incomplete("%s, finding what added %s", err, shown(c.path))) + continue + case err != nil: + return nil, nil, err + } + adder := strings.TrimSpace(out) + if adder == "" { + diag = append(diag, oneLineText(fmt.Sprintf("%s is not on the line, and nothing merged into main since it branched added it", shown(c.path)))) + continue + } + if checkSHA(adder) != nil { + return nil, nil, fmt.Errorf("git log for %s: %q is not a commit", shown(c.path), terminalSafe(adder)) + } + add(adder, c.path, true) + continue + } + found, more, err := b.blamed(v, parent, c.path, cur, target, maxBlamedLines-blamed) + if err != nil { + return nil, nil, err + } + blamed += more.lines + diag = append(diag, more.diag...) + for merge := range found { + add(merge, c.path, false) + } + if more.capped { + break + } + } + var deps []dependency + for _, d := range byMerge { + slices.Sort(d.paths) + deps = append(deps, *d) + // The rules decide before the line is looked at, so whether the + // line carries a prerequisite they keep out is asked here. + if f := d.v; f != nil && f.rule != 0 && f.rule != ruleOnLine && f.rule != ruleNewer && b.lineHist != nil { + if _, asked := b.lineCarries[f]; !asked { + if err := b.load(f); err != nil { + return nil, nil, err + } + how, err := b.lineHist.carries(f.o, f.commits) + if err != nil { + return nil, nil, err + } + if b.lineCarries == nil { + b.lineCarries = map[*verdict]bool{} + } + b.lineCarries[f] = how != "" + } + } + } + slices.SortFunc(deps, func(x, y dependency) int { return strings.Compare(x.merge, y.merge) }) + return deps, diag, nil +} + +// blameResult is what blaming one file read: how many lines, what it could +// not read, and whether the original's line budget ran out. +type blameResult struct { + lines int + diag []string + capped bool +} + +// blamed gives each line of file that v removes or changes, at parent, to +// the first-parent commit that brought it, and keeps the commits one of +// whose lines target's counterpart of the file, cur, lacks. +func (b *batch) blamed(v *verdict, parent, file string, cur treeEntry, target string, budget int) (map[string]bool, blameResult, error) { + var res blameResult + entries, err := treeEntries(parent, []string{file}) + if err != nil { + return nil, res, err + } + pre := entries[file] + switch { + case pre.mode == "120000" || cur.mode == "120000": + res.diag = append(res.diag, incomplete("%s is a symlink", shown(file))) + return nil, res, nil + case pre.kind == "commit" || cur.kind == "commit": + res.diag = append(res.diag, incomplete("%s is a submodule", shown(file))) + return nil, res, nil + case !pre.regular() || !cur.regular(): + res.diag = append(res.diag, incomplete("%s is no file git can blame", shown(file))) + return nil, res, nil + } + // A file the line already has as the original makes it builds on + // nothing the line lacks, whoever wrote the lines it replaced. + if v.o.merge != "" && cur.oid != pre.oid { + post, err := treeEntries(v.o.merge, []string{file}) + if err != nil { + return nil, res, err + } + if p1, ok := post[file]; ok && p1.regular() { + if cur.oid == p1.oid { + return nil, res, nil + } + applied, _, err := b.alreadyApplied(pre.oid, cur.oid, p1.oid) + if err != nil { + return nil, res, err + } + if applied { + return nil, res, nil + } + } + } + for _, oid := range []string{pre.oid, cur.oid} { + out, err := git("cat-file", "-s", oid) + if err != nil { + return nil, res, err + } + size, err := strconv.Atoi(strings.TrimSpace(out)) + if err != nil { + return nil, res, fmt.Errorf("git cat-file -s %s: %q is no size", oid, terminalSafe(out)) + } + if size > maxDepBlob { + res.diag = append(res.diag, incomplete("%s is larger than %d bytes", shown(file), maxDepBlob)) + return nil, res, nil + } + } + diff, err := gitTimed("--literal-pathspecs", "diff", "--no-ext-diff", "--no-textconv", "--no-renames", "-U0", "--end-of-options", parent, v.o.merge, "--", file) + if errors.Is(err, errDeadline) { + res.diag = append(res.diag, incomplete("%s, diffing %s", err, shown(file))) + return nil, res, nil + } else if err != nil { + return nil, res, err + } + ranges, binary, err := oldRanges(diff) + switch { + case err != nil: + return nil, res, fmt.Errorf("the diff of %s in #%d: %w", shown(file), v.o.number, err) + case binary: + res.diag = append(res.diag, incomplete("%s is binary", shown(file))) + return nil, res, nil + case len(ranges) == 0: + return nil, res, nil + } + blobs, err := readObjects([]string{pre.oid, cur.oid}, "blob") + if err != nil { + return nil, res, err + } + skip, whole, ok := generatedLines(file, blobs[pre.oid]) + switch { + case whole: + return nil, res, nil + case !ok: + res.diag = append(res.diag, incomplete("%s, whose generated keys could not be told apart", shown(file))) + return nil, res, nil + } + var kept []hunk + for _, r := range ranges { + for n := r.start; n < r.start+r.count; n++ { + if skip[n] { + continue + } + if l := len(kept) - 1; l >= 0 && kept[l].start+kept[l].count == n { + kept[l].count++ + } else { + kept = append(kept, hunk{n, 1}) + } + res.lines++ + } + } + if res.lines > budget { + res.diag = append(res.diag, incomplete("more than %d lines to blame", maxBlamedLines)) + res.capped = true + return nil, res, nil + } + if len(kept) == 0 { + return nil, res, nil + } + args, err := blameArgs(parent, file, kept) + if err != nil { + return nil, res, err + } + out, err := gitTimed(args...) + if errors.Is(err, errDeadline) { + res.diag = append(res.diag, incomplete("%s, blaming %s", err, shown(file))) + return nil, res, nil + } else if err != nil { + return nil, res, err + } + lines, err := parseBlame(out) + if err != nil { + return nil, res, fmt.Errorf("blaming %s for #%d: %w", shown(file), v.o.number, err) + } + onTarget := trimmedLines(blobs[cur.oid]) + found := map[string]bool{} + unmapped := false + for _, ln := range lines { + merge, inRange := b.brought[ln.commit] + switch { + case !inRange: + // An ancestor of the line's fork point, which the line has, or + // a commit main does not have at all. + before, err := b.beforeBase(ln.commit) + if err != nil { + return nil, res, err + } + unmapped = unmapped || !before + case merge == v.o.merge: + case !onTarget[strings.TrimSpace(ln.text)]: + found[merge] = true + } + } + if unmapped { + res.diag = append(res.diag, incomplete("git blame gives lines of %s to a commit that did not come through main", shown(file))) + } + return found, res, nil +} + +// timed adds what has passed since start to the time the detector took. +func (b *batch) timed(start time.Time) { b.detecting += time.Since(start) } + +// beforeBase says whether commit is an ancestor of the line's fork point, +// once per commit. +func (b *batch) beforeBase(commit string) (bool, error) { + if ok, seen := b.ancestors[commit]; seen { + return ok, nil + } + ok, err := isAncestor(commit, b.base) + if err == nil { + if b.ancestors == nil { + b.ancestors = map[string]bool{} + } + b.ancestors[commit] = ok + } + return ok, err +} + +// analyze reads what every original the policy takes and nothing carries +// builds on that the line lacks, and keeps it with the original, for the +// parks and waits that follow from it. +func (b *batch) analyze() error { + defer b.timed(time.Now()) + for _, v := range b.qualifying() { + if v.onBatch != "" { + continue + } + deps, diag, err := b.dependencies(v, b.lineHead) + if err != nil { + return err + } + v.deps = deps + v.addDiag(diag...) + } + return nil +} + +// analysisMode is how a classification treats an original that changes a +// workflow: as the scheduled run does, whose token cannot push one, or as a +// maintainer's run. +type analysisMode struct{ noWorkflowFiles bool } + +// edgeClass is what a prerequisite is to an original that builds on it, +// from what never matters to what decides first. +type edgeClass int + +const ( + edgeSatisfied edgeClass = iota + // edgeNewer is held by rule 6 on this line. + edgeNewer + // edgeDeferred is a deferred conflict, or deferred for a reason. + edgeDeferred + // edgeWorkflow waits on the open batch for a maintainer. + edgeWorkflow + // edgeMissing is on neither the line, the batch nor the record. + edgeMissing + // edgeNever never reaches the line. + edgeNever +) + +// edge is a dependency whose prerequisite has not reached the line, with +// what that prerequisite is. +type edge struct { + dep dependency + class edgeClass +} + +// ruleClass is what the rules alone make of a prerequisite, or ok false when +// it depends on what this classification makes of it: one the line or the +// open batch carries is there, whatever its label, drop or rule says, and a +// direct push and an original the rules keep out never come. +func (b *batch) ruleClass(f *verdict) (edgeClass, bool) { + switch { + case f == nil: + return edgeNever, true + case f.rule == ruleOnLine || f.onBatch != "" || f.batched != "" || b.lineCarries[f]: + return edgeSatisfied, true + case f.rule == ruleNewer: + return edgeNewer, true + case f.rule != 0: + return edgeNever, true + } + return 0, false +} + +// edges sorts dependencies by what classOf says of each prerequisite into +// those that never come and those still to come, dropping the ones there. +func edges(deps []dependency, classOf func(*verdict) edgeClass) (never, pending []edge) { + for _, d := range deps { + switch c := classOf(d.v); c { + case edgeSatisfied: + case edgeNever: + never = append(never, edge{d, c}) + default: + pending = append(pending, edge{d, c}) + } + } + return never, pending +} + +// strongest is the class of prerequisite that decides a wait, and the +// prerequisites of that class, in merge order. +func strongest(pending []edge) (edgeClass, []*verdict) { + top := edgeSatisfied + for _, e := range pending { + top = max(top, e.class) + } + var out []*verdict + for _, e := range pending { + if e.class == top && !slices.Contains(out, e.dep.v) { + out = append(out, e.dep.v) + } + } + return top, out +} + +// auxiliary says whether a path holds tests or documentation, which an +// original can change without building on what it needs to work. +func auxiliary(p string) bool { + parts := strings.Split(p, "/") + base := parts[len(parts)-1] + for _, d := range parts[:len(parts)-1] { + if d == "test" || d == "tests" || d == "testdata" { + return true + } + } + switch { + case strings.HasSuffix(base, "_test.go"), strings.Contains(base, ".test."), strings.Contains(base, ".spec."), strings.HasSuffix(base, ".bats"): + return true + case len(parts) > 1 && parts[0] == "docs": + return true + } + return len(parts) == 1 && strings.HasSuffix(base, ".md") +} + +// workflowPrimary says whether v waits for a maintainer rather than parks: +// in the scheduled mode, an original that changes a workflow, all of whose +// missing dependencies that never come are tests or docs, and whose +// workflow lines build on nothing still missing. The workflow is what it is +// for, and a maintainer brings over what applies. Only what is still +// missing counts: never and pending are what is left once what is there was +// taken out. +func workflowPrimary(mode analysisMode, v *verdict, never, pending []edge) (bool, error) { + if !mode.noWorkflowFiles || len(never) == 0 { + return false, nil + } + for _, e := range never { + if slices.ContainsFunc(e.dep.paths, func(p string) bool { return !auxiliary(p) }) { + return false, nil + } + } + for _, e := range append(slices.Clone(never), pending...) { + if slices.ContainsFunc(e.dep.paths, func(p string) bool { return strings.HasPrefix(p, workflowsDir) }) { + return false, nil + } + } + files, err := workflowFiles(v.commits) + return len(files) > 0, err +} + +// testsOnly is the note a workflow original gets that waits for a +// maintainer although its tests build on what never comes. +func testsOnly(v *verdict, never []edge, l line) string { + var who []string + for _, e := range never { + if e.dep.v != nil { + who = append(who, fmt.Sprintf("#%d", e.dep.v.o.number)) + } else { + who = append(who, e.dep.merge[:12]) + } + } + slices.Sort(who) + return oneLineText(fmt.Sprintf("only its tests or docs build on %s, which never reaches %s", strings.Join(slices.Compact(who), ", "), l.branch())) +} + +// dependencyPark is the park of v for prerequisites that never come. +func dependencyPark(v *verdict, never []edge) *parkedEntry { + p := &parkedEntry{Original: v.o.number, Merge: v.o.merge, Kind: parkDependency} + for _, e := range never { + if e.dep.v != nil { + p.Needs = append(p.Needs, e.dep.v.o.number) + } else { + p.Commits = append(p.Commits, e.dep.merge) + } + } + slices.Sort(p.Needs) + p.Needs = slices.Compact(p.Needs) + slices.Sort(p.Commits) + p.Commits = slices.Compact(p.Commits) + return p +} + +// depsAt is what v builds on that target lacks, read once per target: the +// run asks again at its head once earlier picks moved the branch. +func (b *batch) depsAt(v *verdict, target string) ([]dependency, error) { + key := target + " " + v.o.merge + if d, ok := b.depCache[key]; ok { + return d, nil + } + deps, diag, err := b.dependencies(v, target) + if err != nil { + return nil, err + } + v.addDiag(diag...) + if b.depCache == nil { + b.depCache = map[string][]dependency{} + } + b.depCache[key] = deps + return deps, nil +} + +// stateKind is what a classification makes of an original the policy takes +// that nothing carries, in the order that decides it. +type stateKind int + +const ( + // stHeld is deferred for a reason a person gave. + stHeld stateKind = iota + 1 + // stPartial would leave part of it without effect on the line. + stPartial + // stWorkflowPrimary waits for a maintainer, though its tests build on + // what never comes. + stWorkflowPrimary + // stParked builds on what never comes. + stParked + // stNeeds waits on prerequisites still to come. + stNeeds + // stOwn is what the record says of it, or missing. + stOwn +) + +// state is a classification of one original: what it is, what it is to an +// original that builds on it, and for a wait the class that decides it and +// the prerequisites of that class. +type state struct { + kind stateKind + class edgeClass + wait edgeClass + needs []*verdict +} + +// classify decides, in merge order, what becomes of every original the +// policy takes that neither the line nor the open batch carries, as an +// operation that picks nothing sees it: check, and the ones that only +// change the record. held are the reasons people deferred originals for, +// recorded what the record defers, and listed says the record's workflow +// waits still wait on an open batch. Parks it finds go into each verdict. +func (b *batch) classify(mode analysisMode, held map[int]string, recorded map[int]deferredEntry, listed bool) (map[*verdict]*state, error) { + states := map[*verdict]*state{} + classOf := func(f *verdict) edgeClass { + if c, ok := b.ruleClass(f); ok { + return c + } + if s := states[f]; s != nil { + return s.class + } + return edgeMissing + } + for _, v := range b.qualifying() { + if v.onBatch != "" { + continue + } + s := &state{} + states[v] = s + e := recorded[v.o.number] + never, pending := edges(v.deps, classOf) + primary, err := workflowPrimary(mode, v, never, pending) + if err != nil { + return nil, err + } + switch { + case held[v.o.number] != "": + s.kind, s.class = stHeld, edgeDeferred + case v.park != nil && v.park.Kind == parkPartial: + s.kind, s.class = stPartial, edgeNever + case primary: + s.kind, s.class = stWorkflowPrimary, edgeMissing + if listed && e.Workflows { + s.class = edgeWorkflow + } + case len(never) > 0: + v.park = dependencyPark(v, never) + s.kind, s.class = stParked, edgeNever + case len(pending) > 0: + s.kind = stNeeds + s.wait, s.needs = strongest(pending) + s.class = s.wait + case listed && e.Workflows: + s.kind, s.class = stOwn, edgeWorkflow + case !e.Workflows && len(e.Needs) == 0 && (e.Commit != "" || len(e.Files) > 0): + s.kind, s.class = stOwn, edgeDeferred + default: + s.kind, s.class = stOwn, edgeMissing + } + } + return states, nil +} diff --git a/cmd/backport-batch/deps_test.go b/cmd/backport-batch/deps_test.go new file mode 100644 index 000000000..d10088e4b --- /dev/null +++ b/cmd/backport-batch/deps_test.go @@ -0,0 +1,329 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "slices" + "strings" + "testing" +) + +// depsOf reads what original n builds on that the line lacks. +func depsOf(t testing.TB, f *fixture, n int) ([]dependency, []string) { + t.Helper() + f.git("fetch", "-q", "origin") + b, err := shapeOf(line11) + if err != nil { + t.Fatal(err) + } + deps, diag, err := b.dependencies(b.find(n), b.lineHead) + if err != nil { + t.Fatal(err) + } + return deps, diag +} + +// depNames lists dependencies as #N or a short commit, with their paths. +func depNames(deps []dependency) []string { + var out []string + for _, d := range deps { + who := d.merge[:12] + if d.v != nil { + who = fmt.Sprintf("#%d", d.v.o.number) + } + out = append(out, fmt.Sprintf("%s %v %v", who, d.paths, d.added)) + } + return out +} + +// Every commit main gained since the line branched is brought by one commit +// of its first-parent line: a pull request's own commits by its merge, a +// direct push by itself, and a commit from before the fork point by none. +func TestBroughtBy(t *testing.T) { + f := newFixture(t) + f.lines() + base := f.git("rev-parse", "origin/release-1.1") + mc3 := f.prCommits(3, "feat(app): three", "", + change{files: map[string]string{"app/3.txt": "3\n"}}, + change{mergeMain: true}, + change{files: map[string]string{"app/3.txt": "3b\n"}}) + f.write("app/push.txt", "pushed\n") + f.git("add", "-A") + f.git("commit", "-q", "-m", "chore: a direct push") + f.git("push", "-q", "origin", "main") + push := f.git("rev-parse", "HEAD") + mc5 := f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + by, err := broughtBy(base, mc5) + if err != nil { + t.Fatal(err) + } + moved := f.git("rev-parse", mc3+"^1") + for _, c := range strings.Fields(f.git("rev-list", mc3+"^1.."+mc3+"^2")) { + if c == moved { + continue + } + if by[c] != mc3 { + t.Errorf("%.12s, a commit of #3's branch, is brought by %.12s, want #3's merge", c, by[c]) + } + } + for c, want := range map[string]string{mc3: mc3, moved: moved, push: push, mc5: mc5, f.git("rev-parse", mc5+"^2"): mc5} { + if by[c] != want { + t.Errorf("%.12s is brought by %.12s, want %.12s", c, by[c], want) + } + } + if _, ok := by[base]; ok { + t.Errorf("the fork point %.12s is in the map", base) + } +} + +// V1: a file the original changes that the line does not have names the +// first-parent commit on main that added it, a merge or a direct push. +func TestDependenciesV1(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(3, "feat(app): x", "", map[string]string{"app/x.txt": appFile}) + f.write("app/y.txt", appFile) + f.git("add", "-A") + f.git("commit", "-q", "-m", "chore: y, pushed") + f.git("push", "-q", "origin", "main") + push := f.git("rev-parse", "HEAD") + f.pr(5, "fix(app): fix x and y", "", map[string]string{"app/x.txt": "fixed\n", "app/y.txt": "fixed\n"}) + deps, diag := depsOf(t, f, 5) + want := []string{push[:12] + " [app/y.txt] true", "#3 [app/x.txt] true"} + slices.Sort(want) + got := depNames(deps) + slices.Sort(got) + if !slices.Equal(got, want) || len(diag) != 0 { + t.Errorf("dependencies = %q, %q; want %q", got, diag, want) + } +} + +// v2World gives the line app/a.txt as the base had it, and main a feat that +// changes its second line and a fix of that same line. +func v2World(t testing.TB, onLine string) *fixture { + f := newFixture(t) + f.squash(2, "chore: a", "", map[string]string{"app/a.txt": appFile}) + f.lines() + if onLine != "" { + f.onLine("release-1.1", "fix(app): a, by hand", map[string]string{"app/a.txt": onLine}) + } + f.pr(3, "feat(app): a loud two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(5, "fix(app): an excited two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + return f +} + +// V2c: a line the original changes that blame gives to another original is +// a dependency only while the line's file lacks that line's text. +func TestDependenciesV2c(t *testing.T) { + if deps, diag := depsOf(t, v2World(t, ""), 5); !slices.Equal(depNames(deps), []string{"#3 [app/a.txt] false"}) || len(diag) != 0 { + t.Errorf("dependencies = %q, %q; want #3 through app/a.txt", depNames(deps), diag) + } + if deps, _ := depsOf(t, v2World(t, strings.Replace(appFile, "two", "TWO", 1)), 5); len(deps) != 0 { + t.Errorf("dependencies with #3's line on the line by hand = %q; want none", depNames(deps)) + } +} + +// The content check trims a line's edges and nothing else: a reindented line +// is there, a line with a space inside it is not. +func TestDependenciesTrimOnlyEdges(t *testing.T) { + if deps, _ := depsOf(t, v2World(t, strings.Replace(appFile, "two", " TWO\t", 1)), 5); len(deps) != 0 { + t.Errorf("dependencies with #3's line reindented = %q; want none", depNames(deps)) + } + if deps, _ := depsOf(t, v2World(t, strings.Replace(appFile, "two", "T WO", 1)), 5); !slices.Equal(depNames(deps), []string{"#3 [app/a.txt] false"}) { + t.Errorf("dependencies with a space inside #3's line = %q; want #3", depNames(deps)) + } +} + +// Lines a tool wrote are not blamed: a generated file whole, and the keys +// hack/update-crd.sh writes into an ApplicationDefinition, while the keys a +// person writes there are. +func TestDependenciesSkipGeneratedRanges(t *testing.T) { + const rd = "packages/system/app-rd/cozyrds/app.yaml" + rdFile := func(schema, category string) string { + return "apiVersion: cozystack.io/v1alpha1\nkind: ApplicationDefinition\nmetadata:\n name: app\nspec:\n application:\n kind: App\n openAPISchema: |-\n {\"v\":" + schema + "}\n dashboard:\n category: " + category + "\n keysOrder: [[\"spec\", \"" + schema + "\"]]\n" + } + world := func(t *testing.T, before, feat, fix map[string]string) *fixture { + f := newFixture(t) + f.squash(2, "chore: generated files", "", before) + f.lines() + f.pr(3, "feat(api): regenerate", "", feat) + f.pr(5, "fix(api): regenerate again", "", fix) + return f + } + gen := "// Code generated by controller-gen. DO NOT EDIT.\n\npackage v1\n\nvar a = 1\n" + f := world(t, + map[string]string{"api/zz_generated.deepcopy.go": "package v1\n\nvar a = 1\n", "api/types_gen.go": gen, rd: rdFile("1", "A")}, + map[string]string{"api/zz_generated.deepcopy.go": "package v1\n\nvar a = 2\n", "api/types_gen.go": strings.Replace(gen, "= 1", "= 2", 1), rd: rdFile("2", "A")}, + map[string]string{"api/zz_generated.deepcopy.go": "package v1\n\nvar a = 3\n", "api/types_gen.go": strings.Replace(gen, "= 1", "= 3", 1), rd: rdFile("3", "A")}) + if deps, diag := depsOf(t, f, 5); len(deps) != 0 || len(diag) != 0 { + t.Errorf("dependencies through generated lines = %q, %q; want none", depNames(deps), diag) + } + f = world(t, map[string]string{rd: rdFile("1", "A")}, map[string]string{rd: rdFile("2", "B")}, map[string]string{rd: rdFile("3", "C")}) + if deps, _ := depsOf(t, f, 5); !slices.Equal(depNames(deps), []string{"#3 [" + rd + "] false"}) { + t.Errorf("dependencies through a key a person writes = %q; want #3", depNames(deps)) + } +} + +// Evidence the detector cannot read, a binary file or a symlink, is a +// diagnostic and never a dependency. +func TestDependenciesIncompleteNeverParks(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: an image", "", map[string]string{"app/a.bin": "a\x00one"}) + f.lines() + f.pr(3, "feat(app): a new image", "", map[string]string{"app/a.bin": "a\x00two"}) + f.pr(5, "fix(app): a fixed image", "", map[string]string{"app/a.bin": "a\x00three"}) + if deps, diag := depsOf(t, f, 5); len(deps) != 0 || !slices.Equal(diag, []string{`incomplete evidence: "app/a.bin" is binary`}) { + t.Errorf("dependencies = %q, %q; want none and a diagnostic", depNames(deps), diag) + } +} + +// Output of git the detector cannot read whole is an error, never taken +// for an absent dependency. +func TestDependenciesMalformedOutputIsAnError(t *testing.T) { + sha := strings.Repeat("a", 40) + for name, out := range map[string]string{ + "a short header": sha + " 1\n\tline\n", + "a header of no commit": "aaaa 1 1 1\n\tline\n", + "a header of no number": sha + " x 1 1\n\tline\n", + "a line with no header": "\tline\n", + "a header with no line": sha + " 1 1 1\nauthor x\n", + } { + if _, err := parseBlame(out); err == nil { + t.Errorf("%s: parseBlame accepted %q", name, out) + } + } + if lines, err := parseBlame(sha + " 2 2 1\nauthor x\nfilename a\n\ttwo\n" + sha + " 4 4\n\tfour\n"); err != nil || len(lines) != 2 || lines[1].text != "four" { + t.Errorf("parseBlame of sound output = %+v, %v", lines, err) + } + if _, _, err := oldRanges("@@ -x +1 @@\n"); err == nil { + t.Error("oldRanges accepted an unreadable hunk header") + } +} + +// Paths go to git as they are: a name a pathspec would read as a pattern +// means only itself. +func TestDependenciesLiteralPaths(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(3, "feat(app): a bracketed file", "", map[string]string{"app/[x].txt": appFile}) + f.pr(4, "feat(app): x", "", map[string]string{"app/x.txt": appFile}) + f.pr(5, "fix(app): fix the bracketed file", "", map[string]string{"app/[x].txt": "fixed\n"}) + if deps, _ := depsOf(t, f, 5); !slices.Equal(depNames(deps), []string{"#3 [app/[x].txt] true"}) { + t.Errorf("dependencies = %q; want #3 alone", depNames(deps)) + } +} + +// A file with no final newline blames its last line like any other. +func TestDependenciesNoFinalNewline(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a", "", map[string]string{"app/a.txt": "one\ntwo"}) + f.lines() + f.pr(3, "feat(app): a loud two", "", map[string]string{"app/a.txt": "one\nTWO"}) + f.pr(5, "fix(app): an excited two", "", map[string]string{"app/a.txt": "one\nTWO!"}) + if deps, diag := depsOf(t, f, 5); !slices.Equal(depNames(deps), []string{"#3 [app/a.txt] false"}) || len(diag) != 0 { + t.Errorf("dependencies = %q, %q; want #3", depNames(deps), diag) + } +} + +// Blame takes the revision as a full commit name, which git cannot read as +// an option, and refuses --end-of-options, so the tool passes none. +func TestBlameInvocation(t *testing.T) { + sha := strings.Repeat("a", 40) + args, err := blameArgs(sha, "-x", []hunk{{2, 3}, {9, 1}}) + if want := []string{"blame", "--porcelain", "-w", "-L", "2,+3", "-L", "9,+1", sha, "--", "-x"}; err != nil || !slices.Equal(args, want) { + t.Errorf("blameArgs = %q, %v; want %q", args, err, want) + } + for _, rev := range []string{"HEAD", "--output=x", strings.Repeat("a", 39)} { + if _, err := blameArgs(rev, "a", nil); err == nil { + t.Errorf("blameArgs took %q for a revision", rev) + } + } +} + +// What the detector could not read shows with the original whatever became +// of it: picked, waiting for a maintainer, or a deferred conflict, in the +// body and in the report. +func TestDiagnosticsShowOnEveryOutcome(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: images", "", map[string]string{"app/p.bin": "p\x00one", "app/w.bin": "w\x00one", "app/c.bin": "c\x00one", "app/c.txt": "old\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): c, on the line", map[string]string{"app/c.txt": "line\n"}) + f.pr(5, "fix(app): picked", "", map[string]string{"app/p.bin": "p\x00two"}) + f.pr(6, "fix(ci): waiting", "", map[string]string{"app/w.bin": "w\x00two", ".github/workflows/six.yaml": "on: push\n"}) + f.pr(7, "fix(app): conflicting", "", map[string]string{"app/c.bin": "c\x00two", "app/c.txt": "seven\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1", "--no-workflow-files") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + for _, s := range []struct{ section, file string }{{"[#5]", "p.bin"}, {"### Deferred conflicts", "c.bin"}, {"### Waiting for a maintainer", "w.bin"}} { + _, after, ok := strings.Cut(stdout, s.section) + if want := " - `incomplete evidence: \"app/" + s.file + "\" is binary`"; !ok || !strings.Contains(after, want) { + t.Errorf("the body does not show %q after %s:\n%s", want, s.section, stdout) + } + } + for n, file := range map[int]string{5: "p.bin", 6: "w.bin", 7: "c.bin"} { + if want := fmt.Sprintf(`"%d":["incomplete evidence: \"app/%s\" is binary"]`, n, file); !strings.Contains(stdout, want) { + t.Errorf("the report lacks %s:\n%s", want, stdout) + } + } +} + +// BenchmarkAnalyze reads what forty originals build on, over a history +// where half of them build on a feat the line lacks. +func BenchmarkAnalyze(b *testing.B) { + f := newFixture(b) + f.squash(2, "chore: files", "", map[string]string{"app/a.txt": appFile}) + f.lines() + for n := 10; n < 50; n++ { + file := fmt.Sprintf("app/f%d.txt", n) + if n%2 == 0 { + f.pr(n, "feat(app): "+file, "", map[string]string{file: appFile}) + continue + } + prev := fmt.Sprintf("app/f%d.txt", n-1) + f.pr(n, "fix(app): "+prev, "", map[string]string{prev: strings.Replace(appFile, "two", fmt.Sprint(n), 1), "app/a.txt": strings.Replace(appFile, "three", fmt.Sprint(n), 1)}) + } + f.git("fetch", "-q", "origin") + b.ResetTimer() + for range b.N { + bt, err := shapeOf(line11) + if err != nil { + b.Fatal(err) + } + if err := bt.analyze(); err != nil { + b.Fatal(err) + } + } +} + +// Files the line lacks count toward the files an original may have read +// for it, like the ones blamed. +func TestDependenciesCapMissingFiles(t *testing.T) { + f := newFixture(t) + f.lines() + added, fixed := map[string]string{}, map[string]string{} + for i := range maxDepFiles + 1 { + added[fmt.Sprintf("app/f%03d.txt", i)] = "v1\n" + fixed[fmt.Sprintf("app/f%03d.txt", i)] = "v2\n" + } + f.pr(3, "feat(app): many files", "", added) + f.pr(5, "fix(app): all of them", "", fixed) + if _, diag := depsOf(t, f, 5); !slices.ContainsFunc(diag, func(d string) bool { return strings.Contains(d, fmt.Sprintf("more than %d files", maxDepFiles)) }) { + t.Errorf("diag = %q, want the file cap", diag) + } +} diff --git a/cmd/backport-batch/drop.go b/cmd/backport-batch/drop.go index b374081cf..1af2e5b3a 100644 --- a/cmd/backport-batch/drop.go +++ b/cmd/backport-batch/drop.go @@ -117,6 +117,9 @@ func (a *app) redrop(b *batch, numbers []int, reason string, undrop bool, push b if err := b.load(v); err != nil { return a.fail(err) } + if err := b.rejudged(v); err != nil { + return a.fail(err) + } if undrop { continue } @@ -159,6 +162,38 @@ func (b *batch) bindDrops(merges []*original) (map[int]droppedEntry, error) { }) } +// waitOrder refuses a record whose waits or parks need an original merged +// after the one that needs it: a dependency is always on main first. +func (b *batch) waitOrder(merges []*original) error { + if b.src.rec == nil { + return nil + } + at := map[int]int{} + for i, o := range merges { + at[o.number] = i + } + check := func(n int, needs []int) error { + i, ok := at[n] + for _, f := range needs { + if j, known := at[f]; ok && known && j >= i { + return fmt.Errorf("batch pull request #%d has #%d wait on #%d, which merged after it", b.src.carrier.Number, n, f) + } + } + return nil + } + for _, d := range b.src.rec.Deferred { + if err := check(d.Original, d.Needs); err != nil { + return err + } + } + for _, p := range b.src.rec.Parked { + if err := check(p.Original, p.Needs); err != nil { + return err + } + } + return nil +} + // bindEntries binds drops to main: one whose original is among merges, the // originals main merged since the line branched, has to name that // original's merge exactly; any other has to be a merge of its number on diff --git a/cmd/backport-batch/effects.go b/cmd/backport-batch/effects.go new file mode 100644 index 000000000..6388c42ae --- /dev/null +++ b/cmd/backport-batch/effects.go @@ -0,0 +1,625 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" +) + +// A clean pick can still carry only part of an original: git deletes a file +// the line never had without a word, and leaves alone a file the line lacks. +// What follows tells such a unit from a whole one, by what it leaves without +// effect of the files the original modifies or deletes. + +// treeEntry is one record of git ls-tree. +type treeEntry struct{ mode, kind, oid string } + +// regular says whether git can merge the entry as text: a file, not a +// symlink, a submodule or a tree. +func (e treeEntry) regular() bool { + return e.kind == "blob" && (e.mode == "100644" || e.mode == "100755") +} + +// treeEntries reads the entries paths have at treeish, from full ls-tree +// records: a path that is a tree there, or absent, has none. +func treeEntries(treeish string, paths []string) (map[string]treeEntry, error) { + out := map[string]treeEntry{} + want := map[string]bool{} + for _, p := range paths { + want[p] = true + } + const chunk = 500 + for len(paths) > 0 { + n := min(chunk, len(paths)) + args := append([]string{"--literal-pathspecs", "ls-tree", "-r", "-z", "--full-tree", "--end-of-options", treeish, "--"}, paths[:n]...) + raw, stderr, code, err := gitStatus(args...) + switch { + case err != nil: + return nil, fmt.Errorf("git ls-tree %s did not run", terminalSafe(treeish)) + case code != 0: + // The paths stay out of the error: they are file names someone + // chose, and the error reaches a terminal. + return nil, fmt.Errorf("git ls-tree %s: exit %d: %s", terminalSafe(treeish), code, terminalSafe(stderr)) + } + for _, rec := range splitNUL(raw) { + meta, path, ok := strings.Cut(rec, "\t") + f := strings.Fields(meta) + if !ok || len(f) != 3 || checkSHA(f[2]) != nil { + return nil, fmt.Errorf("git ls-tree %s: unreadable record %q", treeish, terminalSafe(rec)) + } + if want[path] { + out[path] = treeEntry{mode: f[0], kind: f[1], oid: f[2]} + } + } + paths = paths[n:] + } + return out, nil +} + +// renameSet is what git diff -M found renamed between two commits, old path +// to new, and whether it looked at every candidate. +type renameSet struct { + moved map[string]string + complete bool +} + +// renames reads the renames between from and to, once per pair. git says on +// stderr when it skipped inexact detection for too many files; complete is +// then false, and no path may be called absent for want of a rename. +func (b *batch) renames(from, to string) (renameSet, error) { + key := from + ".." + to + if s, ok := b.renameCache[key]; ok { + return s, nil + } + // -l0 lifts git's own limit, which the picker's merge does not share: + // a rename the pick follows has to be one the checks of its shape see. + stdout, stderr, code, err := gitStatusTimed("diff", "--no-ext-diff", "--no-textconv", "-M", "-l"+strconv.Itoa(b.renameLimit), "--name-status", "-z", "--diff-filter=R", "--end-of-options", from, to) + switch { + case errors.Is(err, errDeadline): + // Out of time is rename detection skipped: no path is called + // absent for want of it. + s := renameSet{moved: map[string]string{}} + if b.renameCache == nil { + b.renameCache = map[string]renameSet{} + } + b.renameCache[key] = s + return s, nil + case err != nil: + return renameSet{}, err + case code != 0: + return renameSet{}, fmt.Errorf("git diff -M %s %s: exit %d: %s", from, to, code, strings.TrimSpace(stderr)) + } + f := splitNUL(stdout) + if len(f)%3 != 0 { + return renameSet{}, fmt.Errorf("unreadable renames between %s and %s", from, to) + } + s := renameSet{moved: map[string]string{}, complete: !strings.Contains(stderr, "detection was skipped")} + for i := 0; i < len(f); i += 3 { + if !strings.HasPrefix(f[i], "R") { + return renameSet{}, fmt.Errorf("unreadable renames between %s and %s: %q", from, to, terminalSafe(f[i])) + } + s.moved[f[i+1]] = f[i+2] + } + if b.renameCache == nil { + b.renameCache = map[string]renameSet{} + } + b.renameCache[key] = s + return s, nil +} + +// effectPaths are the paths v modifies or deletes; what it adds has nothing +// to correspond to. +func effectPaths(v *verdict) []string { + var out []string + for _, c := range v.changes { + if c.status == 'M' || c.status == 'D' { + out = append(out, c.path) + } + } + return out +} + +// correspondence maps a path of the original (at M^1) to the path holding the +// same file on target, through a main-side rename base..M^1, a line-side +// rename base..target, or both composed through the base path. A path with +// no counterpart maps to itself, and one git skipped rename detection for is +// left out and named in unknown: it is not known to be absent. +func (b *batch) correspondence(v *verdict, target string) (map[string]string, []string, error) { + paths := effectPaths(v) + have, err := treeEntries(target, paths) + if err != nil { + return nil, nil, err + } + corr := map[string]string{} + var missing []string + for _, p := range paths { + if _, ok := have[p]; ok { + corr[p] = p + } else { + missing = append(missing, p) + } + } + if len(missing) == 0 { + return corr, nil, nil + } + onMain, err := b.renames(b.base, v.o.merge+"^1") + if err != nil { + return nil, nil, err + } + onLine, err := b.renames(b.base, target) + if err != nil { + return nil, nil, err + } + fromBase := map[string]string{} + for old, now := range onMain.moved { + fromBase[now] = old + } + candidates := map[string]string{} + var names []string + for _, p := range missing { + q := p + if old, ok := fromBase[p]; ok { + q = old + } + r := q + if now, ok := onLine.moved[q]; ok { + r = now + } + if r != p { + candidates[p] = r + names = append(names, r) + } + } + found, err := treeEntries(target, names) + if err != nil { + return nil, nil, err + } + var unknown []string + for _, p := range missing { + if r, ok := candidates[p]; ok { + if _, ok := found[r]; ok { + corr[p] = r + continue + } + } + if !onMain.complete || !onLine.complete { + unknown = append(unknown, p) + continue + } + corr[p] = p + } + return corr, unknown, nil +} + +// omittedEffects lists the paths v modifies or deletes that the change from +// before to after leaves without effect, each judged at its corresponding +// path c(p) on before. A modified path is omitted when c(p) exists on neither +// side, applied when the change touches it, and when the change leaves an +// existing file alone, satisfied only if the line already has what the +// original makes of it. A deleted path is omitted when c(p) is absent before: +// a deletion the line already did is no change this unit makes. diag says +// what could not be judged, which is never counted as omitted. +func (b *batch) omittedEffects(v *verdict, before, after string) ([]string, []string, error) { + if err := b.load(v); err != nil { + return nil, nil, err + } + corr, unknown, err := b.correspondence(v, before) + if err != nil { + return nil, nil, err + } + var diag []string + for _, p := range unknown { + diag = append(diag, "git skipped rename detection, so where "+strconv.Quote(p)+" is on the line is not known") + } + added := map[string]bool{} + for _, c := range v.changes { + if c.status == 'A' { + added[c.path] = true + } + } + // A deleted path whose counterpart the original adds itself is a rename + // the line already made the same way: it is judged as a modified file, + // against what the original makes of that path. + var at, own, moved []string + movedTo := map[string]string{} + for _, c := range v.changes { + cp, ok := corr[c.path] + if !ok { + continue + } + at = append(at, cp) + switch { + case c.status == 'M': + own = append(own, c.path) + case c.status == 'D' && cp != c.path && added[cp]: + own, moved = append(own, c.path), append(moved, cp) + movedTo[c.path] = cp + } + } + bef, err := treeEntries(before, at) + if err != nil { + return nil, nil, err + } + aft, err := treeEntries(after, at) + if err != nil { + return nil, nil, err + } + pre, err := treeEntries(v.o.merge+"^1", own) + if err != nil { + return nil, nil, err + } + post, err := treeEntries(v.o.merge, append(slices.Clone(own), moved...)) + if err != nil { + return nil, nil, err + } + // A change that makes none is judged by its trees, whatever names them: + // a commit and the tree a net merge wrote can be the same change. + same := before == after + if !same { + bt, err := resolveTree(before) + if err != nil { + return nil, nil, err + } + at, err := resolveTree(after) + if err != nil { + return nil, nil, err + } + same = bt == at + } + var omitted []string + // modified judges a path whose counterpart cp should end up as the + // original's postimage p1 of it. + modified := func(path, cp string, p0, p1 treeEntry) error { + be, inBefore := bef[cp] + ae, inAfter := aft[cp] + switch { + case !inBefore && !inAfter: + omitted = append(omitted, path) + return nil + case inBefore != inAfter || be != ae: + return nil + case be.oid == p1.oid && (p0.mode == p1.mode || be.mode == p1.mode): + return nil + case !be.regular() || !p0.regular() || !p1.regular() || p0.mode != p1.mode: + diag = append(diag, "the mode or type of "+strconv.Quote(path)+" changes, which only a person can judge") + return nil + } + applied, unsupported, err := b.alreadyApplied(p0.oid, be.oid, p1.oid) + switch { + case err != nil: + return fmt.Errorf("judging %s of #%d: %w", strconv.Quote(path), v.o.number, err) + case unsupported: + diag = append(diag, "git cannot merge "+strconv.Quote(path)+" as text, so whether the line has its change is not known") + case !applied: + omitted = append(omitted, path) + } + return nil + } + for _, c := range v.changes { + cp, ok := corr[c.path] + if !ok { + continue + } + switch { + case c.status == 'D' && movedTo[c.path] != "": + if err := modified(c.path, cp, pre[c.path], post[cp]); err != nil { + return nil, nil, err + } + case c.status == 'D': + // Absent before: the line already made the deletion, which is + // no change of a unit that makes others, and is all of one + // that makes none. Still there after: the deletion did not + // happen, whatever the unit changed in the file; keeping what + // the line made of it is a port of its own. + _, inBefore := bef[cp] + _, inAfter := aft[cp] + if !inBefore && !same || inAfter { + omitted = append(omitted, c.path) + } + case c.status == 'M': + if err := modified(c.path, cp, pre[c.path], post[c.path]); err != nil { + return nil, nil, err + } + } + } + for i := range diag { + diag[i] = oneLineText(diag[i]) + } + return omitted, diag, nil +} + +// alreadyApplied says whether cur, a file the unit leaves alone, already +// holds the original's change from pre to post: it is post itself, or git +// merge-file of the three merges cleanly back into cur. unsupported is +// content merge-file refuses, such as differing binary files, which says +// nothing either way. +func (b *batch) alreadyApplied(pre, cur, post string) (applied, unsupported bool, err error) { + if cur == post { + return true, false, nil + } + b.mergeFiles++ + blobs, err := readObjects([]string{pre, cur, post}, "blob") + if err != nil { + return false, false, err + } + dir, err := os.MkdirTemp("", "backport-batch-merge-file-*") + if err != nil { + return false, false, err + } + defer func() { _ = os.RemoveAll(dir) }() + names := map[string]string{} + for _, f := range []struct{ name, oid string }{{"cur", cur}, {"pre", pre}, {"post", post}} { + names[f.name] = filepath.Join(dir, f.name) + if err := os.WriteFile(names[f.name], blobs[f.oid], 0o600); err != nil { + return false, false, err + } + } + stdout, stderr, code, err := gitStatus("merge-file", "-p", names["cur"], names["pre"], names["post"]) + switch { + case err != nil: + return false, false, err + case code == 0: + return stdout == string(blobs[cur]), false, nil + case code > 0 && code < 128: + // The number of conflicts. + return false, false, nil + case strings.Contains(stderr, "Cannot merge binary files"): + return false, true, nil + } + return false, false, fmt.Errorf("git merge-file: exit %d: %s", code, strings.TrimSpace(stderr)) +} + +// unitsOn maps each original the batch head carries a unit of to that unit, +// by its Backported-from trailer or appended -x line: found needs every +// commit of a merged original, and misses one whose first pick came out +// empty because the line already had that change. +func (b *batch) unitsOn(head string) (map[*verdict]string, error) { + if m, ok := b.unitCache[head]; ok { + return m, nil + } + units, err := batchUnits(head, b.lineHead) + if err != nil { + return nil, err + } + m := map[*verdict]string{} + for _, u := range units { + if v := b.unitOriginal(u); v != nil && m[v] == "" { + m[v] = u.commit + } + } + if b.unitCache == nil { + b.unitCache = map[string]map[*verdict]string{} + } + b.unitCache[head] = m + return m, nil +} + +// onBranch says whether the batch head h carries v. +func (b *batch) onBranch(h *history, v *verdict) (bool, error) { + if found(h, v) { + return true, nil + } + m, err := b.unitsOn(h.head) + return m[v] != "", err +} + +// partialStates finds what applies only in part on the line. Every unit on +// the open batch is checked against its own first parent, whatever the rules +// now say of its original: one that leaves an effect out fails the cut until +// it is taken out. Every original the policy takes and nothing carries is +// checked against the tree its clean net merge onto the line writes, so a +// partial one is found with no record and no adder; a conflict is never +// called partial. +func (b *batch) partialStates() error { + defer b.timed(time.Now()) + b.partialUnits = map[*verdict][]string{} + if b.pr != nil { + var err error + if b.partialUnits, err = b.unitsPartial(b.prHead); err != nil { + return err + } + on, err := b.unitsOn(b.prHead) + if err != nil { + return err + } + for v, commit := range on { + v.batched = commit + if v.rule == 0 && v.onBatch == "" { + v.onBatch = commit + } + } + } + for _, v := range b.qualifying() { + if v.onBatch != "" { + continue + } + if err := b.partialAt(v, b.lineHead); err != nil { + return err + } + } + return nil +} + +// unitsPartial maps each original whose unit between the line's head and +// head leaves part of it without effect to what it leaves. +func (b *batch) unitsPartial(head string) (map[*verdict][]string, error) { + units, err := batchUnits(head, b.lineHead) + if err != nil { + return nil, err + } + out := map[*verdict][]string{} + for _, u := range units { + v := b.unitOriginal(u) + if v == nil || len(u.parents) == 0 { + continue + } + omitted, diag, err := b.omittedEffects(v, u.parents[0], u.commit) + if err != nil { + return nil, err + } + v.addDiag(diag...) + if len(omitted) > 0 { + out[v] = append(out[v], omitted...) + } + } + return out, nil +} + +// markPartial puts every original on the branch at head whose unit there +// does only part of it among the partial units, and so among the stale +// ones: still on the batch, and to be taken out before the cut. +func (b *batch) markPartial(r *report, head string) error { + units := b.partialUnits + if b.pr == nil || head != b.prHead { + var err error + if units, err = b.unitsPartial(head); err != nil { + return err + } + } + r.partialFiles = map[*verdict][]string{} + for _, v := range b.verdicts { + files := units[v] + if len(files) == 0 { + continue + } + r.partial = append(r.partial, v) + r.partialFiles[v] = files + if !slices.Contains(r.stale, v) { + r.stale = append(r.stale, v) + } + } + return nil +} + +// keepRecordedPartials judges again, at head, what the batch carries now, +// each original the open record parks as partial that the line's head +// judges whole: the run's picker found it partial after the picks before +// it, which check and an operation that picks nothing do not replay. It +// stays parked while its net merge onto head still leaves something +// without effect, and is the line's again once that went, as when the pick +// that made it partial was taken off. +func (b *batch) keepRecordedPartials(head string) error { + if !b.src.open || b.src.rec == nil { + return nil + } + for _, p := range b.src.rec.Parked { + v := b.find(p.Original) + if p.Kind != parkPartial || v == nil || v.rule != 0 || v.onBatch != "" || v.park != nil || v.o.merge != p.Merge { + continue + } + if err := b.partialAt(v, head); err != nil { + return err + } + } + return nil +} + +// rejudged reads again what partialStates and analyze read for v once its +// rules changed, as a withdrawn drop changes them: it is back where the +// line puts it. +func (b *batch) rejudged(v *verdict) error { + if b.pr != nil { + on, err := b.unitsOn(b.prHead) + if err != nil { + return err + } + v.batched = on[v] + if v.rule == 0 && v.onBatch == "" { + v.onBatch = on[v] + } + } + if v.rule != 0 || v.onBatch != "" { + return nil + } + if err := b.partialAt(v, b.lineHead); err != nil { + return err + } + deps, diag, err := b.dependencies(v, b.lineHead) + if err != nil { + return err + } + v.deps = deps + v.addDiag(diag...) + return nil +} + +// partialAt judges v's clean net merge onto head, and parks it, or clears +// the park, by what that merge leaves without effect. +func (b *batch) partialAt(v *verdict, head string) error { + tree, err := b.netTree(head, v.o) + if err != nil { + return err + } + v.partial = nil + if v.park != nil && v.park.Kind == parkPartial { + v.park = nil + } + if tree == "" { + return nil + } + omitted, diag, err := b.omittedEffects(v, head, tree) + if err != nil { + return err + } + v.addDiag(diag...) + if len(omitted) > 0 { + v.partial = omitted + v.park = &parkedEntry{Original: v.o.number, Merge: v.o.merge, Kind: parkPartial, Files: sortedClean(omitted)} + } + return nil +} + +// netTree is the tree a clean net merge of o onto head writes, "" for a +// conflict; the line's head reuses what judging the originals merged. +func (b *batch) netTree(head string, o *original) (string, error) { + if head == b.lineHead && b.lineHist != nil { + return b.lineHist.netTree(o) + } + tree, _, err := noopPick(head, o) + return tree, err +} + +// addDiag keeps what could not be judged of v, each line once. +func (v *verdict) addDiag(lines ...string) { + for _, ln := range lines { + if ln != "" && !slices.Contains(v.diag, ln) { + v.diag = append(v.diag, ln) + } + } +} + +func sortedClean(files []string) []string { + out := cleanFiles(files) + slices.Sort(out) + return slices.Compact(out) +} + +// leftOut is the refusal for a unit that leaves paths of v without effect. +func leftOut(v *verdict, l line, omitted []string) string { + quoted := make([]string, len(omitted)) + for i, p := range omitted { + quoted[i] = strconv.Quote(p) + } + return fmt.Sprintf("#%d leaves %s without effect on %s; port only what the line has as a pull request of its own", v.o.number, strings.Join(quoted, ", "), l.branch()) +} diff --git a/cmd/backport-batch/effects_test.go b/cmd/backport-batch/effects_test.go new file mode 100644 index 000000000..9dfe0de95 --- /dev/null +++ b/cmd/backport-batch/effects_test.go @@ -0,0 +1,363 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +const appFile = "one\ntwo\nthree\nfour\nfive\n" + +// renameWorld is the sign-off's rename history: the line keeps app/old.txt, +// main renames it to app/new.txt with #3, a feat, and #5 fixes app/new.txt. +func renameWorld(t *testing.T) (*fixture, string) { + t.Helper() + f := newFixture(t) + f.squash(2, "chore: an app file", "", map[string]string{"app/old.txt": appFile}) + f.lines() + f.pr(3, "feat(app): rename the app file", "", map[string]string{"app/old.txt": "", "app/new.txt": appFile}) + mc5 := f.pr(5, "fix(app): fix the app file", "", map[string]string{"app/new.txt": strings.Replace(appFile, "three", "THREE", 1)}) + return f, mc5 +} + +// emptyBatch pushes the line's head as the batch branch and opens a batch +// pull request on it with body, for a fold to land on. +func emptyBatch(f *fixture, gh *fakeGH, body string) *foldWorld { + f.t.Helper() + f.git("fetch", "-q", "origin") + f.git("push", "-q", "origin", "origin/release-1.1:refs/heads/"+batchBranch1) + f.git("fetch", "-q", "origin") + gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, body)} + return &foldWorld{f: f, gh: gh, head: f.git("rev-parse", "origin/"+batchBranch1)} +} + +// A fix of a file main renamed picks onto the file's old name on the line, +// and every check of the batch's shape takes the old name as the same file: +// verify, a hand fold, and a rebuild by --defer. +func TestRenamePickThroughFoldAndVerify(t *testing.T) { + f, _ := renameWorld(t) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + head := openBatch(t, f, gh) + if got := f.subjects("origin/release-1.1", head); !slices.Equal(got, []string{"fix(app): fix the app file (#5)", "fix(app): six (#6)"}) { + t.Fatalf("the batch carries %q, want #5 and #6", got) + } + if got := f.git("show", head+":app/old.txt"); !strings.Contains(got, "THREE") { + t.Fatalf("#5 did not land on app/old.txt: %q", got) + } + if code, out, stderr := f.exec(gh, "verify", "--line", "release-1.1", "--head", head); code != 0 { + t.Errorf("verify = %d\n%s\n%s", code, out, stderr) + } + t.Setenv("GIT_COMMITTER_DATE", "2026-10-27T06:00:00Z") + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "6", "--reason", "fails the e2e", "--push"); code != 0 { + t.Fatalf("run --defer 6 --push = %d\n%s", code, stderr) + } + if code, out, stderr := f.exec(gh, "verify", "--line", "release-1.1", "--head", f.remoteHead(batchBranch1)); code != 0 { + t.Errorf("verify after the rebuild = %d\n%s\n%s", code, out, stderr) + } + + f, mc5 := renameWorld(t) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/old.txt": strings.Replace(appFile, "three", "THREE", 1)}) + fold := w.merge(w.head, pick, "fix(app): fix the app file (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 0 || out != "ok\n" { + t.Errorf("fold of #5 onto app/old.txt = %d, %q\n%s", code, out, stderr) + } +} + +// shaped reads the line from the fixture's remote as verify does. +func shaped(t *testing.T, f *fixture) *batch { + t.Helper() + f.git("fetch", "-q", "origin") + b, err := shapeOf(line11) + if err != nil { + t.Fatal(err) + } + return b +} + +// commitOn writes files onto parent and returns the commit, leaving every +// branch where it was. +func commitOn(f *fixture, parent string, files map[string]string) string { + f.t.Helper() + f.git("switch", "-q", "--detach", parent) + f.apply(files) + f.git("commit", "-q", "--allow-empty", "-m", "a unit") + sha := f.git("rev-parse", "HEAD") + f.git("switch", "-q", "main") + return sha +} + +// A path of the original is followed to where the line keeps the same file: +// through a rename on main, on the line, or both, composed through the path +// at the base. Where git skipped rename detection the path is unknown, never +// absent. +func TestCorrespondence(t *testing.T) { + corr := func(t *testing.T, f *fixture, n int, limit ...int) (map[string]string, []string) { + t.Helper() + b := shaped(t, f) + if len(limit) > 0 { + b.renameLimit = limit[0] + } + v := b.find(n) + if err := b.load(v); err != nil { + t.Fatal(err) + } + c, unknown, err := b.correspondence(v, b.lineHead) + if err != nil { + t.Fatal(err) + } + return c, unknown + } + t.Run("renamed on main", func(t *testing.T) { + f, _ := renameWorld(t) + if c, unknown := corr(t, f, 5); c["app/new.txt"] != "app/old.txt" || len(unknown) != 0 { + t.Errorf("correspondence = %v, %v; want app/new.txt on app/old.txt", c, unknown) + } + }) + t.Run("renamed on the line", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: an app file", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.onLine("release-1.1", "fix(app): the line renames it", map[string]string{"app/a.txt": "", "app/line.txt": appFile}) + f.pr(5, "fix(app): fix it", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + if c, unknown := corr(t, f, 5); c["app/a.txt"] != "app/line.txt" || len(unknown) != 0 { + t.Errorf("correspondence = %v, %v; want app/a.txt on app/line.txt", c, unknown) + } + }) + t.Run("renamed on both", func(t *testing.T) { + f, _ := renameWorld(t) + f.onLine("release-1.1", "fix(app): the line renames it its own way", map[string]string{"app/old.txt": "", "app/release.txt": appFile}) + if c, unknown := corr(t, f, 5); c["app/new.txt"] != "app/release.txt" || len(unknown) != 0 { + t.Errorf("correspondence = %v, %v; want app/new.txt on app/release.txt", c, unknown) + } + }) + t.Run("not renamed, absent", func(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(3, "feat(app): x", "", map[string]string{"app/x.txt": appFile}) + f.pr(5, "fix(app): fix x", "", map[string]string{"app/x.txt": strings.Replace(appFile, "two", "TWO", 1)}) + if c, unknown := corr(t, f, 5); c["app/x.txt"] != "app/x.txt" || len(unknown) != 0 { + t.Errorf("correspondence = %v, %v; want app/x.txt on itself", c, unknown) + } + }) + t.Run("rename detection skipped", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: two app files", "", map[string]string{"app/a.txt": appFile, "app/b.txt": appFile + "six\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): the line renames both, changed", map[string]string{"app/a.txt": "", "app/b.txt": "", "app/x.txt": appFile + "x\n", "app/y.txt": appFile + "six\ny\n"}) + f.pr(5, "fix(app): fix a", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + // Past the limit, git says it skipped looking; a configured limit + // of its own does not decide for the tool. + f.git("config", "diff.renameLimit", "1") + if c, unknown := corr(t, f, 5, 0); c["app/a.txt"] != "app/x.txt" || len(unknown) != 0 { + t.Errorf("correspondence under diff.renameLimit=1 = %v, %v; want app/a.txt on app/x.txt", c, unknown) + } + if c, unknown := corr(t, f, 5, 1); !slices.Equal(unknown, []string{"app/a.txt"}) || c["app/a.txt"] != "" { + t.Errorf("correspondence past the limit = %v, %v; want app/a.txt unknown", c, unknown) + } + }) +} + +// What a change leaves without effect: a modified file the line lacks on both +// sides of it, a deleted file the line never had. A mode change that only a +// person can judge is a diagnostic, not an omission. +func TestOmittedEffects(t *testing.T) { + omitted := func(t *testing.T, f *fixture, n int, files map[string]string) ([]string, []string) { + t.Helper() + b := shaped(t, f) + after := commitOn(f, b.lineHead, files) + om, diag, err := b.omittedEffects(b.find(n), b.lineHead, after) + if err != nil { + t.Fatal(err) + } + return om, diag + } + t.Run("a modified file missing before and after", func(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(3, "feat(app): x", "", map[string]string{"app/x.txt": appFile}) + f.pr(5, "fix(app): fix x and five", "", map[string]string{"app/x.txt": "fixed\n", "app/five.txt": "5\n"}) + if om, diag := omitted(t, f, 5, map[string]string{"app/five.txt": "5\n"}); !slices.Equal(om, []string{"app/x.txt"}) || len(diag) != 0 { + t.Errorf("omitted = %q, %q; want app/x.txt", om, diag) + } + }) + t.Run("a deleted file absent before", func(t *testing.T) { + f, _ := policyWorld(t) + if om, _ := omitted(t, f, 5, map[string]string{"chart/tests/values_test.yaml": "suite: values\n"}); !slices.Equal(om, []string{"chart/templates/policy.yaml", "chart/tests/policy_test.yaml"}) { + t.Errorf("omitted = %q; want both policy files", om) + } + }) + t.Run("a mode change", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a script", "", map[string]string{"app/run.sh": "#!/bin/sh\n", "app/b.txt": "old\n"}) + f.lines() + f.git("switch", "-q", "-c", "pr-5") + if err := os.Chmod(filepath.Join(f.dir, "app/run.sh"), 0o755); err != nil { + t.Fatal(err) + } + f.write("app/b.txt", "fixed\n") + f.git("add", "-A") + f.git("commit", "-q", "-m", "run it") + f.git("switch", "-q", "main") + f.git("merge", "-q", "--no-ff", "-m", "fix(app): make the script runnable (#5)", "pr-5") + f.git("push", "-q", "origin", "main") + om, diag := omitted(t, f, 5, map[string]string{"app/b.txt": "fixed\n"}) + if len(om) != 0 || len(diag) != 1 || !strings.Contains(diag[0], `the mode or type of "app/run.sh" changes`) { + t.Errorf("omitted = %q, %q; want a diagnostic for app/run.sh and nothing omitted", om, diag) + } + }) +} + +// What git says when it fails names file names as text, never as lines a +// terminal or a CI log would take for its own. +func TestGitErrorsQuoteFileNames(t *testing.T) { + newFixture(t) + const evil = "x\n::warning::forged" + if _, err := gitTimed("log", "--no-such-option", "--", evil); err == nil || strings.Contains(err.Error(), "\n::warning") { + t.Errorf("gitTimed error = %v, want the path quoted", err) + } + if _, err := treeEntries("no-such-revision", []string{evil}); err == nil || strings.Contains(err.Error(), "\n::warning") { + t.Errorf("treeEntries error = %v, want the path quoted", err) + } +} + +// fixedWorld is the sign-off's two-file case: the base has a and b as old, +// #5 fixes both, and the line already has a as onLine says, if anything. +func fixedWorld(t *testing.T, a, old, onLine, fixed string) (*fixture, string) { + t.Helper() + f := newFixture(t) + f.squash(2, "chore: a and b", "", map[string]string{a: old, "app/b.txt": "old\n"}) + f.lines() + if onLine != "" { + f.onLine("release-1.1", "fix(app): a, on the line", map[string]string{a: onLine}) + } + mc5 := f.pr(5, "fix(app): a and b", "", map[string]string{a: fixed, "app/b.txt": "fixed\n"}) + return f, mc5 +} + +// The line can already have what an original does to a file: then a pick +// that leaves the file alone carries the original whole. The check-time +// pass, the picker, fold and verify all take it, binary files included, +// without asking git merge-file what blob equality settles. A different fix +// on the line, or a deletion of what the line never had, is still left out. +func TestAlreadyAppliedFileIsNotOmitted(t *testing.T) { + const textPost = "one\nfixed\nthree\nfour\nfive\n" + accepted := func(t *testing.T, a, old, onLine, fixed string) { + t.Helper() + f, _ := fixedWorld(t, a, old, onLine, fixed) + if code, stdout, stderr := f.exec(nil, "check", "--line", "release-1.1"); code != 1 || !strings.Contains(stdout, "/pull/5 ") || strings.Contains(stderr, "parked") { + t.Errorf("check = %d, want #5 missing and not parked:\n%s\n%s", code, stdout, stderr) + } + gh := &fakeGH{} + head := openBatch(t, f, gh) + if got := f.subjects("origin/release-1.1", head); !slices.Equal(got, []string{"fix(app): a and b (#5)"}) { + t.Errorf("picked %q, want #5", got) + } + if code, out, stderr := f.exec(gh, "verify", "--line", "release-1.1", "--head", head); code != 0 { + t.Errorf("verify = %d\n%s\n%s", code, out, stderr) + } + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || strings.Contains(stdout, "only part of it applies") { + t.Errorf("check --cut = %d, want #5 only on the batch, whole:\n%s\n%s", code, stdout, stderr) + } + f, mc5 := fixedWorld(t, a, old, onLine, fixed) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/b.txt": "fixed\n"}) + fold := w.merge(w.head, pick, "fix(app): a and b (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 0 { + t.Errorf("fold = %d, %q\n%s", code, out, stderr) + } + } + t.Run("the line has the original's file", func(t *testing.T) { + accepted(t, "app/a.txt", "old\n", "fixed\n", "fixed\n") + }) + t.Run("the line has the fix beside lines of its own", func(t *testing.T) { + accepted(t, "app/a.txt", strings.Replace(textPost, "fixed", "old", 1), textPost+"line only\n", textPost) + }) + t.Run("the line has the original's binary file", func(t *testing.T) { + accepted(t, "app/a.bin", "old\x00payload", "fixed\x00payload", "fixed\x00payload") + f, _ := fixedWorld(t, "app/a.bin", "old\x00payload", "fixed\x00payload", "fixed\x00payload") + b := shaped(t, f) + om, diag, err := b.omittedEffects(b.find(5), b.lineHead, commitOn(f, b.lineHead, map[string]string{"app/b.txt": "fixed\n"})) + if err != nil || len(om)+len(diag) != 0 || b.mergeFiles != 0 { + t.Errorf("omitted = %q, %q, %v, merge-file ran %d times; want nothing, and merge-file never asked", om, diag, err, b.mergeFiles) + } + }) + t.Run("an earlier unit of the batch brought the file", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a and b", "", map[string]string{"app/a.txt": "old\n", "app/b.txt": "old\n"}) + f.lines() + f.pr(4, "fix(app): a", "", map[string]string{"app/a.txt": "fixed\n"}) + f.squash(40, "chore: put a back for now", "", map[string]string{"app/a.txt": "old\n"}) + f.pr(5, "fix(app): a and b", "", map[string]string{"app/a.txt": "fixed\n", "app/b.txt": "fixed\n"}) + gh := &fakeGH{} + head := openBatch(t, f, gh) + if got := f.subjects("origin/release-1.1", head); !slices.Equal(got, []string{"fix(app): a (#4)", "fix(app): a and b (#5)"}) { + t.Errorf("picked %q, want #4 and #5", got) + } + if code, out, stderr := f.exec(gh, "verify", "--line", "release-1.1", "--head", head); code != 0 { + t.Errorf("verify = %d\n%s\n%s", code, out, stderr) + } + }) + t.Run("the line has a different fix", func(t *testing.T) { + f, mc5 := fixedWorld(t, "app/a.txt", "old\n", "other\n", "fixed\n") + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/b.txt": "fixed\n"}) + fold := w.merge(w.head, pick, "fix(app): a and b (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 1 || !strings.Contains(out, `leaves "app/a.txt" without effect`) { + t.Errorf("fold = %d, %q, want it refused for app/a.txt\n%s", code, out, stderr) + } + }) + t.Run("the line has a different binary file", func(t *testing.T) { + f, _ := fixedWorld(t, "app/a.bin", "old\x00payload", "other\x00payload", "fixed\x00payload") + b := shaped(t, f) + om, diag, err := b.omittedEffects(b.find(5), b.lineHead, commitOn(f, b.lineHead, map[string]string{"app/b.txt": "fixed\n"})) + if err != nil || len(om) != 0 || len(diag) != 1 || !strings.Contains(diag[0], `cannot merge "app/a.bin" as text`) { + t.Errorf("omitted = %q, %q, %v; want a diagnostic for app/a.bin, nothing omitted", om, diag, err) + } + }) + t.Run("a deletion of what the line never had", func(t *testing.T) { + f, mc5 := policyWorld(t) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"chart/tests/values_test.yaml": "suite: values\n"}) + fold := w.merge(w.head, pick, "fix(chart): remove the policy a rebase brought back (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 1 || !strings.Contains(out, "without effect") { + t.Errorf("fold = %d, %q, want it refused\n%s", code, out, stderr) + } + }) +} + +// A change that makes none, whatever names its two sides, leaves a deletion +// the line already made as it is: the original is there, not partial. +func TestDeletionAlreadyMadeByNoChange(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a policy", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): the line drops the policy", map[string]string{"app/policy.yaml": ""}) + f.pr(5, "fix(app): drop the policy", "", map[string]string{"app/policy.yaml": ""}) + b := shaped(t, f) + tree := f.git("rev-parse", b.lineHead+"^{tree}") + for _, after := range []string{b.lineHead, tree} { + if om, _, err := b.omittedEffects(b.find(5), b.lineHead, after); err != nil || len(om) != 0 { + t.Errorf("omitted against %.12s = %q, %v; want nothing", after, om, err) + } + } +} diff --git a/cmd/backport-batch/fixture_test.go b/cmd/backport-batch/fixture_test.go index 4ae5731b6..08aa5f2ac 100644 --- a/cmd/backport-batch/fixture_test.go +++ b/cmd/backport-batch/fixture_test.go @@ -33,7 +33,7 @@ import ( // main and release lines shaped the way GitHub leaves them: every pull request // lands as a merge commit whose message is its title, its number and its body. type fixture struct { - t *testing.T + t testing.TB dir string remote string now time.Time @@ -41,7 +41,7 @@ type fixture struct { tool string } -func newFixture(t *testing.T) *fixture { +func newFixture(t testing.TB) *fixture { t.Helper() // A hook or an outer repository can export these into the test process. for _, key := range []string{"GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR", "GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES"} { @@ -458,6 +458,22 @@ func (g *fakeGH) editPull(number int, title, bodyFile string) error { return nil } +func (g *fakeGH) editBody(number int, bodyFile string) error { + if err := g.write(number, "a body edit"); err != nil { + return err + } + if err := g.inject("editBody"); err != nil { + return err + } + if err := g.readBody(bodyFile); err != nil { + return err + } + if pr := g.find(number); pr != nil { + pr.Body = g.bodies[len(g.bodies)-1] + } + return nil +} + func (g *fakeGH) closePull(number int) error { if err := g.write(number, "a close"); err != nil { return err @@ -506,15 +522,6 @@ func (f *fixture) batchPR(n int, base, branch, body string) pullRequest { return pr } -// botBatchPR is batchPR opened by the bot the scheduled batch runs as. -func (f *fixture) botBatchPR(gh *fakeGH, n int, base, branch, body string) pullRequest { - f.t.Helper() - gh.bot = defaultBatchBot - pr := f.batchPR(n, base, branch, body) - pr.User.Login, pr.User.Type, pr.AuthorAssociation = defaultBatchBot, "Bot", "CONTRIBUTOR" - return pr -} - // mergeBatch merges the open batch into release-1.1 with a merge commit, as // GitHub does, and deletes its branch, as the repository's // delete_branch_on_merge does. The pull request stays, merged, with its body diff --git a/cmd/backport-batch/fold_test.go b/cmd/backport-batch/fold_test.go index 89797d0d5..f783802b2 100644 --- a/cmd/backport-batch/fold_test.go +++ b/cmd/backport-batch/fold_test.go @@ -348,3 +348,75 @@ func TestFoldRefusesADroppedOriginal(t *testing.T) { t.Errorf("fold = %d, %q, %s; want a refusal naming the drop", code, stdout, stderr) } } + +// A resolution that ports what the line has of #5 and leaves alone a file +// the line lacks is no fold: what #5 does to that file would be lost, and a +// line-only port is a pull request of its own. +func TestFoldRefusesAnOmittedEffect(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(3, "feat(app): a feature the line never gets", "", map[string]string{"app/feature2.txt": "v1\n"}) + mc5 := f.pr(5, "fix(app): five", "", map[string]string{"app/feature2.txt": "v2\n", "app/five.txt": "5\n"}) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/five.txt": "5\n"}) + fold := w.merge(w.head, pick, "fix(app): five (#5)\n\n"+batchTrailer+": "+mc5) + code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold) + if code != 1 || out != "refused: #5 leaves \"app/feature2.txt\" without effect on release-1.1; port only what the line has as a pull request of its own\n" { + t.Errorf("fold = %d, %q, want it refused for app/feature2.txt\n%s", code, out, stderr) + } +} + +// A resolution that keeps a file the original deletes leaves the deletion +// without effect. +func TestFoldRefusesAKeptDeletion(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: x", "", map[string]string{"app/x.txt": "x\n"}) + f.lines() + mc5 := f.pr(5, "fix(app): five, and x goes", "", map[string]string{"app/x.txt": "", "app/five.txt": "5\n"}) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/five.txt": "5\n"}) + fold := w.merge(w.head, pick, "fix(app): five, and x goes (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 1 || !strings.Contains(out, `leaves "app/x.txt" without effect`) { + t.Errorf("fold that keeps app/x.txt = %d, %q\n%s", code, out, stderr) + } +} + +// An original parked for what it builds on can still land as a complete +// hand adaptation: every file it changes, changed, in the batch's shape. +func TestFoldAcceptsACompleteAdaptationOfAParkedOriginal(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.pr(3, "feat(app): a loud two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + mc5 := f.pr(5, "fix(app): an excited two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/a.txt": strings.Replace(appFile, "two", "two!", 1)}) + fold := w.merge(w.head, pick, "fix(app): an excited two (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 0 || out != "ok\n" { + t.Errorf("fold of an adaptation of #5 = %d, %q\n%s", code, out, stderr) + } +} + +// A fold that keeps a file the original deletes leaves the deletion without +// effect, whatever it changes in the file: a port that keeps what the line +// made of it, or the file with a line added, is a pull request of its own. +func TestFoldRefusesAChangedKeptDeletion(t *testing.T) { + for name, kept := range map[string]string{ + "the line's own content": "line only\n", + "the file, one line more": "feature\nline only\n# reviewed\n", + } { + t.Run(name, func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a feature", "", map[string]string{"app/x.txt": "feature\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): a line-only setting beside the feature", map[string]string{"app/x.txt": "feature\nline only\n"}) + mc5 := f.pr(5, "fix(app): drop the feature, and five", "", map[string]string{"app/x.txt": "", "app/five.txt": "5\n"}) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/x.txt": kept, "app/five.txt": "5\n"}) + fold := w.merge(w.head, pick, "fix(app): drop the feature, and five (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 1 || !strings.Contains(out, `leaves "app/x.txt" without effect`) { + t.Errorf("fold = %d, %q, want it refused\n%s", code, out, stderr) + } + }) + } +} diff --git a/cmd/backport-batch/github.go b/cmd/backport-batch/github.go index 072673a77..0e549b1f6 100644 --- a/cmd/backport-batch/github.go +++ b/cmd/backport-batch/github.go @@ -127,6 +127,8 @@ type github interface { // createPull opens the draft and returns its number and URL. createPull(base, head, title, bodyFile string) (int, string, error) editPull(number int, title, bodyFile string) error + // editBody replaces the body of pull request number and nothing else. + editBody(number int, bodyFile string) error closePull(number int) error comments(number int) ([]issueComment, error) createComment(number int, bodyFile string) error @@ -135,6 +137,54 @@ type github interface { batchBot() string } +// guardedGitHub refuses every GitHub write once the run's output was lost. +type guardedGitHub struct { + github + a *app +} + +func (g guardedGitHub) createPull(base, head, title, bodyFile string) (int, string, error) { + if err := g.a.mayAct(); err != nil { + return 0, "", err + } + return g.github.createPull(base, head, title, bodyFile) +} + +func (g guardedGitHub) editPull(number int, title, bodyFile string) error { + if err := g.a.mayAct(); err != nil { + return err + } + return g.github.editPull(number, title, bodyFile) +} + +func (g guardedGitHub) editBody(number int, bodyFile string) error { + if err := g.a.mayAct(); err != nil { + return err + } + return g.github.editBody(number, bodyFile) +} + +func (g guardedGitHub) closePull(number int) error { + if err := g.a.mayAct(); err != nil { + return err + } + return g.github.closePull(number) +} + +func (g guardedGitHub) createComment(number int, bodyFile string) error { + if err := g.a.mayAct(); err != nil { + return err + } + return g.github.createComment(number, bodyFile) +} + +func (g guardedGitHub) editComment(id int64, bodyFile string) error { + if err := g.a.mayAct(); err != nil { + return err + } + return g.github.editComment(id, bodyFile) +} + type ghCLI struct{ bot string } func (g ghCLI) batchBot() string { return g.bot } @@ -260,6 +310,11 @@ func (ghCLI) editPull(number int, title, bodyFile string) error { return err } +func (ghCLI) editBody(number int, bodyFile string) error { + _, err := capture("gh", "pr", "edit", strconv.Itoa(number), "--body-file", bodyFile) + return err +} + func (ghCLI) closePull(number int) error { _, err := capture("gh", "pr", "close", strconv.Itoa(number)) return err diff --git a/cmd/backport-batch/history.go b/cmd/backport-batch/history.go index 74f4c66af..a530aa244 100644 --- a/cmd/backport-batch/history.go +++ b/cmd/backport-batch/history.go @@ -205,6 +205,9 @@ type history struct { picks []pickRef batches map[string]string backports map[int]string + // trees caches what a net merge of an original onto head writes, by its + // merge commit: "" for a conflict. + trees map[string]string } func readHistory(head, base string, textUntil time.Time) (*history, error) { @@ -225,7 +228,7 @@ func readHistory(head, base string, textUntil time.Time) (*history, error) { // readPicks reads only the messages of head's commits outside base, which is // all a batch branch is asked about; textUntil is the batch-only day. func readPicks(head, base string, textUntil time.Time) (*history, error) { - h := &history{head: head, reachable: map[string]bool{}, appended: map[string]string{}, batches: map[string]string{}, backports: map[int]string{}} + h := &history{head: head, reachable: map[string]bool{}, appended: map[string]string{}, batches: map[string]string{}, backports: map[int]string{}, trees: map[string]string{}} oids, err := revList("--end-of-options", head, "^"+base) if err != nil { return nil, err @@ -332,34 +335,54 @@ func (h *history) carries(o *original, commits []string) (string, error) { if c, ok := h.backports[o.number]; ok { return `named by "Backport of" in ` + c[:12], nil } - noop, err := noopPick(h.head, o) + tree, err := h.netTree(o) if err != nil { return "", err } - if noop { + headTree, err := resolveTree(h.head) + if err != nil { + return "", err + } + if tree == headTree { return "a pick would change nothing", nil } return "", nil } -// noopPick reports whether cherry-picking the original onto head would leave -// head's tree as it is: the content is there, whatever names it. -func noopPick(head string, o *original) (bool, error) { +// netTree is noopPick's tree for o onto the branch's head, read once. +func (h *history) netTree(o *original) (string, error) { + if tree, ok := h.trees[o.merge]; ok { + return tree, nil + } + tree, _, err := noopPick(h.head, o) + if err == nil { + h.trees[o.merge] = tree + } + return tree, err +} + +// noopPick merges the original onto head as one change and returns the tree +// that writes for a clean merge, "" for a conflict, and whether the tree is +// head's own: the content is there, whatever names it. +func noopPick(head string, o *original) (string, bool, error) { stdout, stderr, code, err := gitStatus("merge-tree", "--write-tree", "--no-messages", "--merge-base="+o.merge+"^1", "--end-of-options", head, o.merge) switch { case err != nil: - return false, err + return "", false, err case code == 1: - return false, nil + return "", false, nil case code != 0: - return false, fmt.Errorf("git merge-tree for #%d: exit %d: %s", o.number, code, strings.TrimSpace(stderr)) + return "", false, fmt.Errorf("git merge-tree for #%d: exit %d: %s", o.number, code, strings.TrimSpace(stderr)) } tree, _, _ := strings.Cut(stdout, "\n") + if err := checkSHA(tree); err != nil { + return "", false, fmt.Errorf("git merge-tree for #%d: %w", o.number, err) + } headTree, err := resolveTree(head) if err != nil { - return false, err + return "", false, err } - return tree == headTree, nil + return tree, tree == headTree, nil } func resolveTree(commit string) (string, error) { diff --git a/cmd/backport-batch/injection_test.go b/cmd/backport-batch/injection_test.go index f1d12b004..a1f5abbe6 100644 --- a/cmd/backport-batch/injection_test.go +++ b/cmd/backport-batch/injection_test.go @@ -142,11 +142,11 @@ func TestOptionInjection(t *testing.T) { // no bidi override reorders it and no zero-width character hides in it. func TestDeferredBlockCleansFileNames(t *testing.T) { l, _ := parseLine("release-1.6") - files := []string{"app/‮txt.exe", "app/zero​width.yaml"} + files := []string{"app/\u202etxt.exe", "app/zero\u200bwidth.yaml"} conflict := pick{v: &verdict{o: &original{number: 5}}, files: files, failed: strings.Repeat("a", 40)} waiting := pick{v: &verdict{o: &original{number: 6}}, files: files} block := deferredBlock(l, []pick{conflict}, nil, []pick{waiting}) - if strings.ContainsAny(block, "‮​") { + if strings.ContainsAny(block, "\u202e\u200b") { t.Errorf("the block carries format characters: %q", block) } want := []string{"app/txt.exe", "app/zerowidth.yaml"} @@ -154,7 +154,7 @@ func TestDeferredBlockCleansFileNames(t *testing.T) { if err != nil || !slices.Equal(got[5].Files, want) || !slices.Equal(got[6].Files, want) { t.Errorf("parseDeferred = %+v, %v; want both originals with %q", got, err, want) } - if files[0] != "app/‮txt.exe" { + if files[0] != "app/\u202etxt.exe" { t.Errorf("deferredBlock rewrote the pick's own file names: %q", files) } } diff --git a/cmd/backport-batch/inputs_test.go b/cmd/backport-batch/inputs_test.go new file mode 100644 index 000000000..0632f80ef --- /dev/null +++ b/cmd/backport-batch/inputs_test.go @@ -0,0 +1,112 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "os/exec" + "strings" + "testing" +) + +const modulePath = "github.com/cozystack/cozystack/" + +// readTestInputs returns the entries of test-inputs.txt, the list CI reads to +// decide whether a pull request runs these tests. +func readTestInputs(t *testing.T) []string { + t.Helper() + data, err := os.ReadFile("test-inputs.txt") + if err != nil { + t.Fatal(err) + } + var entries []string + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + entries = append(entries, line) + } + if len(entries) == 0 { + t.Fatal("test-inputs.txt lists no path") + } + return entries +} + +// covered reports whether a change to path matches an entry the way the plan +// job matches it: a trailing slash is a directory prefix, anything else is +// one file. +func covered(entries []string, path string) bool { + for _, e := range entries { + if strings.HasSuffix(e, "/") && strings.HasPrefix(path, e) || path == e { + return true + } + } + return false +} + +// A pull request that changes an input of these tests has to run them, and CI +// knows the inputs only from test-inputs.txt. An import added here, or added to +// a package this tool imports, lands in a pull request that touches a listed +// path, so it runs this test, which fails until the new package is listed. +func TestInputsListed(t *testing.T) { + entries := readTestInputs(t) + + out, err := exec.Command("go", "list", "-deps", "-test", "-f", "{{.ImportPath}}", ".").Output() + if err != nil { + t.Fatalf("go list: %v", err) + } + var deps int + for _, pkg := range strings.Split(strings.TrimSpace(string(out)), "\n") { + // A test variant prints as "pkg [pkg.test]"; the synthesized main + // package as "pkg.test". Both are this directory. + pkg, _, _ = strings.Cut(pkg, " ") + pkg = strings.TrimSuffix(pkg, ".test") + rel, ok := strings.CutPrefix(pkg, modulePath) + if !ok { + continue + } + deps++ + if !covered(entries, rel+"/x.go") { + t.Errorf("these tests import %s, which test-inputs.txt does not list: add %s/", pkg, rel) + } + } + if deps == 0 { + t.Fatalf("go list named no package of %s, so nothing was checked:\n%s", modulePath, out) + } + + for _, page := range []string{policyPage, "docs/release/backport-tasks/x.md"} { + if !covered(entries, page) { + t.Errorf("the tests read %s, which test-inputs.txt does not cover", page) + } + } + for _, f := range []string{"go.mod", "go.sum"} { + if !covered(entries, f) { + t.Errorf("test-inputs.txt does not list %s", f) + } + } +} + +// A listed path that no longer exists makes the list read as broader than it +// is, and usually means a dependency moved and its new home is missing. +func TestInputsExist(t *testing.T) { + for _, e := range readTestInputs(t) { + if _, err := os.Stat("../../" + e); err != nil { + t.Errorf("test-inputs.txt lists %s: %v", e, err) + } + } +} diff --git a/cmd/backport-batch/main.go b/cmd/backport-batch/main.go index c02601240..c49fa951f 100644 --- a/cmd/backport-batch/main.go +++ b/cmd/backport-batch/main.go @@ -24,7 +24,9 @@ import ( "fmt" "io" "os" + "os/signal" "strings" + "syscall" "time" ) @@ -111,18 +113,42 @@ type app struct { policyRef string // tool is the commit backport-batch was built from, for the record. tool string + // unwritten is the first error writing to stdout or stderr. Output goes + // out before a push or a publish, so a run that lost it acts on nothing, + // and one that lost a confirmation afterwards fails. + unwritten error } func main() { + catchSIGPIPE() a := &app{gh: ghCLI{bot: batchBotLogin(os.LookupEnv)}, stdout: os.Stdout, stderr: os.Stderr, now: time.Now, tool: toolVersion()} os.Exit(a.main(os.Args[1:])) } +// catchSIGPIPE makes a write to a closed stdout or stderr fail with EPIPE like +// any other write, so the run stops before it pushes or publishes and says +// why, instead of dying silently. signal.Ignore would do the same here but +// would also leave SIGPIPE ignored in the git and gh children, across exec. +func catchSIGPIPE() { + signal.Notify(make(chan os.Signal, 1), syscall.SIGPIPE) +} + func (a *app) main(args []string) int { - a.stdout, a.stderr = textWriter{a.stdout}, textWriter{a.stderr} - a.gh = &permCache{github: a.gh, perms: map[string]string{}} + a.stdout, a.stderr = textWriter{w: a.stdout, failed: &a.unwritten}, textWriter{w: a.stderr, failed: &a.unwritten} + code := a.command(args) + if a.unwritten != nil { + _, _ = fmt.Fprintf(a.stderr, "backport-batch: output was lost: %v\n", a.unwritten) + if code == 0 { + code = 2 + } + } + return code +} + +func (a *app) command(args []string) int { + a.gh = guardedGitHub{github: &permCache{github: a.gh, perms: map[string]string{}}, a: a} if len(args) == 0 { - fmt.Fprint(a.stderr, usageText) + _, _ = fmt.Fprint(a.stderr, usageText) return 2 } if bot := a.gh.batchBot(); bot != "" && !botLoginRE.MatchString(bot) { @@ -144,20 +170,26 @@ func (a *app) main(args []string) int { case "lines": return a.lines(args[1:]) case "-h", "--help", "help": - fmt.Fprint(a.stdout, usageText) + _, _ = fmt.Fprint(a.stdout, usageText) return 0 } - fmt.Fprintf(a.stderr, "unknown command %q\n\n%s", args[0], usageText) + _, _ = fmt.Fprintf(a.stderr, "unknown command %q\n\n%s", args[0], usageText) return 2 } // textWriter keeps output fit for a terminal. Titles, file names and the // messages git relays come from other people, and a control sequence among // them would move the cursor or rewrite what was printed before. -type textWriter struct{ w io.Writer } +type textWriter struct { + w io.Writer + failed *error +} func (t textWriter) Write(p []byte) (int, error) { if _, err := io.WriteString(t.w, clean(string(p), true)); err != nil { + if t.failed != nil && *t.failed == nil { + *t.failed = err + } return 0, err } return len(p), nil @@ -166,7 +198,7 @@ func (t textWriter) Write(p []byte) (int, error) { func (a *app) flags(name string) *flag.FlagSet { fs := flag.NewFlagSet(name, flag.ContinueOnError) fs.SetOutput(a.stderr) - fs.Usage = func() { fmt.Fprint(a.stderr, usageText) } + fs.Usage = func() { _, _ = fmt.Fprint(a.stderr, usageText) } return fs } @@ -188,13 +220,24 @@ func parse(fs *flag.FlagSet, args []string) error { return nil } +// mayAct refuses any further change once output was lost: the run would act +// on what it printed while nobody read it. Every push and every move of a +// local batch branch go through it, and every GitHub write through +// guardedGitHub. +func (a *app) mayAct() error { + if a.unwritten != nil { + return fmt.Errorf("output was lost: %w; nothing more is pushed or written to GitHub", a.unwritten) + } + return nil +} + func (a *app) fail(err error) int { - fmt.Fprintf(a.stderr, "backport-batch: %v\n", err) + _, _ = fmt.Fprintf(a.stderr, "backport-batch: %v\n", err) return 2 } func (a *app) warn(format string, args ...any) { - fmt.Fprintf(a.stderr, "warning: "+format+"\n", args...) + _, _ = fmt.Fprintf(a.stderr, "warning: "+format+"\n", args...) } // verdict is what the policy says about one original for the line. @@ -207,10 +250,22 @@ type verdict struct { // and by load for anything else. commits []string changes []fileChange - // onBatch is the commit of the open batch pull request that picks it. + // onBatch is the commit of the open batch pull request that picks it, + // and batched the unit of the open batch that carries it whatever the + // rules now say of it. onBatch string + batched string // label is the maintainer override the original carries, if any. label string + // partial lists what its clean net merge onto the line would leave + // without effect, and park why the batch leaves it out, if it does. + partial []string + park *parkedEntry + // deps is what it builds on that the line lacks. + deps []dependency + // diag is what could not be judged about it, shown whatever became of + // it. + diag []string } // batch is everything read about a line before anything is written. @@ -242,6 +297,27 @@ type batch struct { // textUntil is the batch-only day: what a commit's text says counts as // evidence only on commits from before it. textUntil time.Time + // partialUnits are the originals whose unit on the open batch leaves + // what it lists without effect. + partialUnits map[*verdict][]string + // What one command read once and asks again: renames by commit pair, + // units by batch head, and how often git merge-file ran. + renameCache map[string]renameSet + unitCache map[string]map[*verdict]string + mergeFiles int + // renameLimit is the -l rename detection runs with; 0, git's widest. + renameLimit int + // brought maps main's commits since the fork point to the first-parent + // commit that brought each, and ancestors says which commits blame + // named are older than the fork point. + brought map[string]string + ancestors map[string]bool + // detecting is how long judging partial units and dependencies took. + detecting time.Duration + depCache map[string][]dependency + // lineCarries says, for a prerequisite the rules keep out, whether the + // line carries it all the same. + lineCarries map[*verdict]bool } // fetch moves to the top of the working tree first: every path the batch @@ -348,6 +424,9 @@ func (a *app) prepare(l line) (*batch, error) { if b.drops, err = b.bindDrops(merges); err != nil { return nil, err } + if err := b.waitOrder(merges); err != nil { + return nil, err + } for _, o := range merges { v := &verdict{o: o, label: labels[o.number]} if err := b.judge(v, b.drops); err != nil { @@ -355,6 +434,9 @@ func (a *app) prepare(l line) (*batch, error) { } b.verdicts = append(b.verdicts, v) } + if err := b.partialStates(); err != nil { + return nil, err + } return b, nil } diff --git a/cmd/backport-batch/notes_test.go b/cmd/backport-batch/notes_test.go new file mode 100644 index 000000000..35f6ccbb4 --- /dev/null +++ b/cmd/backport-batch/notes_test.go @@ -0,0 +1,99 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os/exec" + "regexp" + "slices" + "strings" + "testing" +) + +// releaseNoteBlock is the release-note block of a body. +func releaseNoteBlock(t *testing.T, body string) string { + t.Helper() + m := regexp.MustCompile("(?s)```release-note\n(.*?)\n```").FindStringSubmatch(body) + if m == nil { + t.Fatalf("no release-note block in:\n%s", body) + } + return m[1] +} + +// A pick that differs from its original keeps its release note out of the +// release-note block, under Check these notes with what to compare: one git +// had to fit to a line that already had part of the change, and one whose +// first commit came out empty because the line had resized the value itself. +func TestChangedOriginalNoteIsSetApart(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.onLine("release-1.1", "fix(app): two, on the line", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.onLine("release-1.1", "fix(chart): two replicas, on the line", map[string]string{"chart/values.yaml": "replicas: 2\nimage: app:1\n"}) + f.pr(5, "fix(app): two and four", "```release-note\nfix(app): two and four are loud.\n```\n", map[string]string{"app/a.txt": strings.NewReplacer("two", "TWO", "four", "FOUR").Replace(appFile)}) + f.prCommits(6, "fix(chart): two replicas and six", "```release-note\nfix(chart): two replicas by default.\n```\n", + change{files: map[string]string{"chart/values.yaml": "replicas: 2\nimage: app:1\n"}}, + change{files: map[string]string{"app/six.txt": "6\n"}}) + f.pr(7, "fix(app): seven", "```release-note\nfix(app): seven is calm.\n```\n", map[string]string{"app/seven.txt": "7\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, []string{"fix(app): two and four (#5)", "fix(chart): two replicas and six (#6)", "fix(app): seven (#7)"}) { + t.Fatalf("picked %q, want #5, #6 and #7", got) + } + notes := releaseNoteBlock(t, stdout) + if !strings.Contains(notes, "seven is calm") || strings.Contains(notes, "two and four are loud") || strings.Contains(notes, "two replicas by default") { + t.Errorf("the release-note block keeps a changed pick's note, or loses #7's:\n%s", notes) + } + _, check, ok := strings.Cut(stdout, "### Check these notes") + check, _, _ = strings.Cut(check, "\n#") + if !ok || !strings.Contains(check, "two and four are loud") || !strings.Contains(check, "two replicas by default") || strings.Contains(check, "seven is calm") { + t.Errorf("Check these notes does not list #5 and #6 alone:\n%s", stdout) + } + if !strings.Contains(stdout, `"changed":[5,6]`) { + t.Errorf("the record or the report does not mark #5 and #6 changed:\n%s", stdout) + } +} + +// The command printed for a changed pick pairs the pick with the commit it +// came from, however much the pick lost: the default creation factor leaves +// a pick that carries only #5088's test unpaired. +func TestRangeDiffCommandPairsThePick(t *testing.T) { + f, mc5 := policyWorld(t) + gh := &fakeGH{} + head := handBatch(f, gh, mc5) + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue") + if code != 0 { + t.Fatalf("run --continue = %d\n%s", code, stderr) + } + m := regexp.MustCompile("`git range-diff --creation-factor=999 ([0-9a-f]{12})\\^! ([0-9a-f]{12})\\^!`").FindStringSubmatch(stdout) + if m == nil { + t.Fatalf("the body prints no range-diff for #5's pick:\n%s", stdout) + } + src, pick := f.git("rev-parse", mc5+"^2"), f.git("rev-parse", head+"^2") + if !strings.HasPrefix(src, m[1]) || !strings.HasPrefix(pick, m[2]) { + t.Errorf("the range-diff names %s and %s, want #5's commit %.12s and its pick %.12s", m[1], m[2], src, pick) + } + out, err := exec.Command("git", "range-diff", "--no-color", "--creation-factor=999", m[1]+"^!", m[2]+"^!").Output() + if want := "1: " + f.git("rev-parse", "--short", src) + " ! 1: " + f.git("rev-parse", "--short", pick); err != nil || !strings.HasPrefix(string(out), want) { + t.Errorf("the printed range-diff gives %q, %v; want it to start %q", out, err, want) + } + if out := f.git("range-diff", "--no-color", mc5+"^1.."+mc5+"^2", head+"^1.."+head+"^2"); !strings.Contains(out, " < ") { + t.Errorf("the default creation factor pairs #5's pick after all, so the test shows nothing:\n%s", out) + } +} diff --git a/cmd/backport-batch/notify.go b/cmd/backport-batch/notify.go index 7191f42b4..b2a8f7c18 100644 --- a/cmd/backport-batch/notify.go +++ b/cmd/backport-batch/notify.go @@ -87,51 +87,186 @@ func (a *app) notify(args []string) int { return a.fail(err) } deferred := map[int]deferredEntry{} + var partial []int + var parks []parkedEntry if src.open { - deferred = src.rec.deferred() + deferred, partial = src.rec.deferred(), src.rec.Partial + for _, p := range src.rec.Parked { + if !p.Announced { + parks = append(parks, p) + } + } } + // An original that waits on others is listed once, after what decides + // its wait, where that one is listed. var conflicts, waiting []int + after := map[int][]int{} for n, e := range deferred { - switch { - case e.Workflows: + switch placement(deferred, n, map[int]bool{}) { + case "workflow": waiting = append(waiting, n) - case e.Reason == "": + case "conflict": conflicts = append(conflicts, n) } + if len(e.Needs) > 0 { + after[n] = e.Needs + } } slices.Sort(conflicts) slices.Sort(waiting) + item := func(n int) string { + if needs := after[n]; len(needs) > 0 { + return fmt.Sprintf("\n- %s, after %s", prLink(n), prLinks(needs)) + } + return "\n- " + prLink(n) + } + count := func(ns []int) int { + c := 0 + for _, n := range ns { + if len(after[n]) == 0 { + c++ + } + } + return c + } today := a.now().UTC().Truncate(24 * time.Hour) days := int(today.Sub(opened).Hours() / 24) stalled := days >= stalledAfter var text []string if stalled { - fmt.Fprintf(a.stdout, "%s: batch pull request #%d has been open since %s, %d days\n", l.branch(), pr.Number, opened.Format(time.DateOnly), days) + _, _ = fmt.Fprintf(a.stdout, "%s: batch pull request #%d has been open since %s, %d days\n", l.branch(), pr.Number, opened.Format(time.DateOnly), days) text = append(text, fmt.Sprintf("This batch has been open since %s, %d days, through at least one scheduled run. Merge it, or close it and the next run starts a new one; until then each run adds what the rules take to it.", opened.Format(time.DateOnly), days)) } - if len(conflicts) > 0 { - fmt.Fprintf(a.stdout, "%s: %d deferred conflicts on #%d wait for a maintainer\n", l.branch(), len(conflicts), pr.Number) + if len(partial) > 0 { + _, _ = fmt.Fprintf(a.stdout, "%s: %d units on #%d carry only part of their originals\n", l.branch(), len(partial), pr.Number) var b strings.Builder - fmt.Fprintf(&b, "These conflicts wait for a maintainer, as [resolving a conflict](%sresolve-conflict.md) describes; unresolved, each goes to the next batch:\n", tasksURL) + b.WriteString("These are on the batch but carry only part of their originals on this line; take each out with `backport-batch run --defer` before the cut, and port what applies as a pull request of its own:\n") + for _, n := range partial { + b.WriteString(item(n)) + } + text = append(text, b.String()) + } + if len(conflicts) > 0 { + _, _ = fmt.Fprintf(a.stdout, "%s: %d deferred conflicts on #%d wait for a maintainer\n", l.branch(), count(conflicts), pr.Number) + var b strings.Builder + fmt.Fprintf(&b, "These conflicts wait for a maintainer, as [resolving a conflict](%sresolve-conflict.md) describes; unresolved, each goes to the next batch, and so does what waits on one:\n", tasksURL) for _, n := range conflicts { - fmt.Fprintf(&b, "\n- %s", prLink(n)) + b.WriteString(item(n)) } text = append(text, b.String()) } if len(waiting) > 0 { - fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s and wait for a maintainer\n", l.branch(), len(waiting), pr.Number, workflowsDir) + if count(waiting) == len(waiting) { + _, _ = fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s and wait for a maintainer\n", l.branch(), len(waiting), pr.Number, workflowsDir) + } else { + _, _ = fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s or wait on one that does\n", l.branch(), len(waiting), pr.Number, workflowsDir) + } var b strings.Builder - fmt.Fprintf(&b, "These change `%s`, which the batch's token may not push, and wait for a maintainer to bring them over, as [changes to workflows](%s#changes-to-workflows) describes:\n", workflowsDir, policyURL) + fmt.Fprintf(&b, "These change `%s`, which the batch's token may not push, and wait for a maintainer to bring them over, as [changes to workflows](%s#changes-to-workflows) describes, or wait on one that does:\n", workflowsDir, policyURL) for _, n := range waiting { - fmt.Fprintf(&b, "\n- %s", prLink(n)) + b.WriteString(item(n)) + } + text = append(text, b.String()) + } + if len(parks) > 0 { + _, _ = fmt.Fprintf(a.stdout, "%s: %d parked originals on #%d, named once\n", l.branch(), len(parks), pr.Number) + var b strings.Builder + fmt.Fprintf(&b, "These cannot land on this line whole, so the batch parked them; each is named here once. A maintainer ports what applies as a pull request of its own, as [fixes that exist only on a release line](%s#fixes-that-exist-only-on-a-release-line) describes, or drops the original with `backport-batch run --drop`:\n", policyURL) + for _, p := range parks { + fmt.Fprintf(&b, "\n- %s: %s", prLink(p.Original), parkReason(p)) } text = append(text, b.String()) } if !*push { return 0 } - return a.comment(pr, *mention, today, text) + if code := a.comment(pr, *mention, today, text); code != 0 || len(parks) == 0 { + return code + } + if err := a.announced(pr, src.rec, parks); err != nil { + return a.fail(fmt.Errorf("%w; the next notify names them again", err)) + } + return 0 +} + +// announced marks parks as named in the record of the open batch pull +// request, which the comment just named: only while it is open and its body +// still carries the record notify read, so a run that rewrote the body +// meanwhile loses nothing; the next notify names those parks again. +func (a *app) announced(pr *pullRequest, rec *lineRecord, parks []parkedEntry) error { + marked := *rec + marked.Parked = slices.Clone(rec.Parked) + for i, p := range marked.Parked { + if slices.ContainsFunc(parks, func(q parkedEntry) bool { return q.Original == p.Original && parkEvent(q) == parkEvent(p) }) { + marked.Parked[i].Announced = true + } + } + now, err := a.gh.pull(pr.Number) + if err != nil { + return err + } + if st := prState(&now); st != "open" { + return fmt.Errorf("batch pull request #%d is %s now; nothing is written to it", pr.Number, st) + } + lines, err := fencedLines(now.Body, deferredFence) + if err != nil || len(lines) != 1 { + return fmt.Errorf("batch pull request #%d no longer carries the record notify read", pr.Number) + } + read, err := parseRecord(now.Body, notifyLineOf(rec)) + if err != nil || recordBlock(*read) != recordBlock(*rec) { + return fmt.Errorf("batch pull request #%d no longer carries the record notify read", pr.Number) + } + // GitHub keeps the CRLF line ends a body edited on the web has; the + // record is found, and the body written back, with LF ones. + body := strings.ReplaceAll(now.Body, "\r\n", "\n") + block := deferredFence + "\n" + lines[0] + "\n```" + if strings.Count(body, block) != 1 { + return fmt.Errorf("batch pull request #%d carries the record notify read, but not as one block it can mark", pr.Number) + } + body = strings.Replace(body, block, recordBlock(marked), 1) + name, err := writeTemp("backport-batch-body-*.md", body) + if err != nil { + return err + } + defer func() { _ = os.Remove(name) }() + return a.gh.editBody(pr.Number, name) +} + +// notifyLineOf is the line a record is for, which it was read against. +func notifyLineOf(rec *lineRecord) line { + l, _ := parseLine(rec.Line) + return l +} + +// placement says under which list notify names original n of the record: +// workflow for a wait for a maintainer, conflict for a deferred conflict, "" +// for neither, a deferral for a reason among them. One that waits on others +// goes where what decides its wait goes, followed down the record; one that +// is not in the record is held by rule 6 or missing, and waits on nobody. +func placement(deferred map[int]deferredEntry, n int, seen map[int]bool) string { + e, ok := deferred[n] + switch { + case !ok || seen[n]: + return "" + case e.Workflows: + return "workflow" + case e.Reason != "": + return "" + case len(e.Needs) == 0: + return "conflict" + } + seen[n] = true + place := "" + for _, f := range e.Needs { + switch p := placement(deferred, f, seen); p { + case "workflow": + return p + case "conflict": + place = p + } + } + return place } // comment writes what waits into the one notify comment on the batch pull @@ -174,9 +309,9 @@ func (a *app) comment(pr *pullRequest, mention string, today time.Time, text []s if err != nil { return a.fail(err) } - defer os.Remove(f.Name()) + defer func() { _ = os.Remove(f.Name()) }() if _, err := f.WriteString(b.String()); err != nil { - f.Close() + _ = f.Close() return a.fail(err) } if err := f.Close(); err != nil { @@ -219,7 +354,7 @@ func (a *app) lines(args []string) int { return a.fail(err) } for _, l := range slices.Backward(lines) { - fmt.Fprintln(a.stdout, l.branch()) + _, _ = fmt.Fprintln(a.stdout, l.branch()) } return 0 } diff --git a/cmd/backport-batch/notify_test.go b/cmd/backport-batch/notify_test.go index 4bfb650e9..7f870baee 100644 --- a/cmd/backport-batch/notify_test.go +++ b/cmd/backport-batch/notify_test.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "errors" "slices" "strings" "testing" @@ -334,3 +335,162 @@ func TestNotifyWritesNothingOnABatchClosedMeanwhile(t *testing.T) { }) } } + +// parkedBody is an open batch body whose version 3 record parks p. +func parkedBody(p ...parkedEntry) string { + sha := strings.Repeat("a", 40) + rec := lineRecord{Version: recordVersion, Line: "release-1.1", Report: strings.Repeat("b", 64), + Built: &builtFrom{Main: sha, Line: sha, Policy: sha, Batch: sha, At: "2026-10-26T06:00:00Z", Options: runOptions{Mode: modeContinue}}, + Deferred: []deferredEntry{}, Dropped: []droppedEntry{}, Stale: []int{}, Parked: p} + return "batch\n\n" + recordBlock(rec) + "\n" +} + +// notifyAt runs notify at a time of day, with gh faked. +func notifyAt(gh *fakeGH, at time.Time, args ...string) (int, string, string) { + var stdout, stderr bytes.Buffer + a := &app{gh: gh, stdout: &stdout, stderr: &stderr, now: func() time.Time { return at }} + code := a.main(append([]string{"notify", "--line", "release-1.1"}, args...)) + return code, stdout.String(), stderr.String() +} + +// A park is announced once on the open batch: notify names it in its +// comment and marks it announced in the record; a later notify is silent +// about it. A mark that fails to land repeats the announcement once, the +// next day. With no batch open nothing is announced, and the first notify of +// the next batch does it. +func TestNotifyAnnouncesAParkOnce(t *testing.T) { + park := parkedEntry{Original: 5, Merge: strings.Repeat("c", 40), Kind: parkDependency, Needs: []int{3}} + parked := func(gh *fakeGH) []parkedEntry { + t.Helper() + rec, err := parseRecord(gh.prs[0].Body, notifyLine) + if err != nil { + t.Fatal(err) + } + return rec.Parked + } + gh := &fakeGH{prs: []pullRequest{notifyPR("20261026", parkedBody(park))}} + code, stdout, stderr := notifyRun(gh, "--push") + if code != 0 || !strings.Contains(stdout, "release-1.1: 1 parked originals on #90, named once") || len(gh.issueComments) != 1 || !strings.Contains(gh.issueComments[0].Body, "- [#5](https://github.com/cozystack/cozystack/pull/5): builds on [#3](") { + t.Fatalf("notify = %d, want #5 announced:\n%s\n%s\n%v", code, stdout, stderr, gh.issueComments) + } + if p := parked(gh); len(p) != 1 || !p[0].Announced { + t.Errorf("the record parks %+v, want #5 marked announced", p) + } + if code, stdout, _ = notifyRun(gh, "--push"); code != 0 || strings.Contains(stdout, "parked") || strings.Contains(gh.issueComments[0].Body, "[#5]") { + t.Errorf("notify again = %d, want nothing about #5:\n%s\n%s", code, stdout, gh.issueComments[0].Body) + } + + gh = &fakeGH{prs: []pullRequest{notifyPR("20261026", parkedBody(park))}, fail: map[string]error{"editBody": errors.New("HTTP 502")}} + if code, _, _ := notifyRun(gh, "--push"); code != 2 || len(gh.issueComments) != 1 || parked(gh)[0].Announced { + t.Fatalf("notify with a failing mark = %d, want 2 after the comment, the record unmarked", code) + } + code, _, stderr = notifyAt(gh, time.Date(2026, 10, 27, 0, 5, 0, 0, time.UTC), "--push") + if code != 0 || !strings.Contains(gh.issueComments[0].Body, "[#5](") || !parked(gh)[0].Announced { + t.Errorf("notify the next day = %d, want #5 named once more and marked:\n%s\n%s", code, stderr, gh.issueComments[0].Body) + } + + // Closed, or rewritten by a run, between the comment and the mark: the + // mark is not written, and the next notify names #5 again. + gh = &fakeGH{prs: []pullRequest{notifyPR("20261026", parkedBody(park))}} + gh.after = map[string]func(){"createComment": func() { gh.prs[0].State = "closed" }} + if code, _, _ := notifyRun(gh, "--push"); code != 2 || len(gh.closedWrites) != 0 || parked(gh)[0].Announced { + t.Errorf("notify on a batch closed meanwhile = %d, writes %q; want 2 and no mark", code, gh.closedWrites) + } + gh = &fakeGH{prs: []pullRequest{notifyPR("20261026", parkedBody(park))}} + rewritten := parkedBody(park, parkedEntry{Original: 6, Merge: strings.Repeat("d", 40), Kind: parkPartial, Files: []string{"x"}}) + gh.after = map[string]func(){"createComment": func() { gh.prs[0].Body = rewritten }} + if code, _, _ := notifyRun(gh, "--push"); code != 2 || gh.prs[0].Body != rewritten || len(gh.bodies) != 0 { + t.Errorf("notify on a body rewritten meanwhile = %d; want 2 and the new body left as it is", code) + } + + // A body someone edited on the web comes back with CRLF line ends; the + // mark lands all the same. + gh = &fakeGH{prs: []pullRequest{notifyPR("20261026", strings.ReplaceAll(parkedBody(park), "\n", "\r\n"))}} + if code, _, stderr := notifyRun(gh, "--push"); code != 0 || !parked(gh)[0].Announced { + t.Errorf("notify on a CRLF body = %d, want #5 marked:\n%s", code, stderr) + } + + closed := notifyPR("20261026", parkedBody(park)) + at := "2026-10-26T12:00:00Z" + closed.State, closed.MergedAt = "closed", &at + gh = &fakeGH{prs: []pullRequest{closed}} + if code, stdout, _ := notifyRun(gh, "--push"); code != 0 || stdout != "" || len(gh.issueComments) != 0 || len(gh.bodies) != 0 { + t.Errorf("notify with no batch open = %d, %q, %d comments, %d edits; want nothing", code, stdout, len(gh.issueComments), len(gh.bodies)) + } + next := notifyPR("20261027", parkedBody(park)) + next.Number = 91 + gh.prs = append(gh.prs, next) + if code, _, _ := notifyRun(gh, "--push"); code != 0 || len(gh.issueComments) != 1 || !strings.Contains(gh.issueComments[0].Body, "[#5](") { + t.Errorf("the next batch's notify = %d, want #5 announced on it: %v", code, gh.issueComments) + } +} + +// A park carries its announcement into the next record only while it is +// the same park: the same merge, kind, prerequisites, commits and files. +func TestParkEventIdentity(t *testing.T) { + a, b := strings.Repeat("a", 40), strings.Repeat("b", 40) + was := parkedEntry{Original: 5, Merge: a, Kind: parkDependency, Commits: []string{a}, Announced: true} + carrier := &lineRecord{Parked: []parkedEntry{was}} + for name, tc := range map[string]struct { + now parkedEntry + announced bool + }{ + "the same park": {parkedEntry{Original: 5, Merge: a, Kind: parkDependency, Commits: []string{a}}, true}, + "another unnumbered commit": {parkedEntry{Original: 5, Merge: a, Kind: parkDependency, Commits: []string{b}}, false}, + "another kind": {parkedEntry{Original: 5, Merge: a, Kind: parkPartial, Files: []string{"x"}}, false}, + "another prerequisite": {parkedEntry{Original: 5, Merge: a, Kind: parkDependency, Commits: []string{a}, Needs: []int{3}}, false}, + "another merge": {parkedEntry{Original: 5, Merge: b, Kind: parkDependency, Commits: []string{a}}, false}, + "another original": {parkedEntry{Original: 6, Merge: a, Kind: parkDependency, Commits: []string{a}}, false}, + } { + got := carryAnnounced([]parkedEntry{tc.now}, carrier) + if got[0].Announced != tc.announced { + t.Errorf("%s: announced = %v, want %v", name, got[0].Announced, tc.announced) + } + } + if got := carryAnnounced([]parkedEntry{was}, nil); got[0].Announced { + t.Error("a park with no carrier came out announced") + } + unannounced := was + unannounced.Announced = false + if got := carryAnnounced([]parkedEntry{was}, &lineRecord{Parked: []parkedEntry{unannounced}}); got[0].Announced { + t.Error("a park the carrier holds unannounced came out announced") + } +} + +// A park keeps its announcement through a continuation and into the next +// batch, so notify does not name it again. +func TestRolloverKeepsTheAnnouncement(t *testing.T) { + f := featWorld(t) + gh := &fakeGH{} + openBatch(t, f, gh) + if code, _, stderr := f.exec(gh, "notify", "--line", "release-1.1", "--push"); code != 0 || !strings.Contains(notifyComment(gh), "[#5](") { + t.Fatalf("notify = %d, want #5 announced:\n%s", code, stderr) + } + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 0 || !strings.Contains(stdout, `"needs":[3],"announced":true`) { + t.Fatalf("run --continue = %d, want #5's park still announced:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + mergeBatch(f, gh) + f.pr(8, "fix(app): eight", "", map[string]string{"app/eight.txt": "8\n"}) + gh.nextPR = 91 + f.now = f.now.Add(24 * time.Hour) + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--push"); code != 0 || !strings.Contains(stdout, `"needs":[3],"announced":true`) { + t.Fatalf("the next batch's run = %d, want #5's park announced from the merged batch:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + gh.issueComments = nil + if code, stdout, _ := f.exec(gh, "notify", "--line", "release-1.1", "--push"); code != 0 || strings.Contains(stdout, "parked") || strings.Contains(notifyComment(gh), "[#5](") { + t.Errorf("notify on the next batch = %d, want nothing about #5:\n%s\n%s", code, stdout, notifyComment(gh)) + } +} + +// notifyComment is the notify comment on the fake, or "". +func notifyComment(gh *fakeGH) string { + for _, c := range slices.Backward(gh.issueComments) { + if strings.HasPrefix(c.Body, notifyMarker) { + return c.Body + } + } + return "" +} diff --git a/cmd/backport-batch/onto_test.go b/cmd/backport-batch/onto_test.go index ee4d52d33..994059235 100644 --- a/cmd/backport-batch/onto_test.go +++ b/cmd/backport-batch/onto_test.go @@ -41,12 +41,12 @@ func TestOntoContinues(t *testing.T) { f.lines() head := pushed(t, f) f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) - code, stdout, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head, "--date", "2026-10-29") + code, _, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head, "--date", "2026-10-29") if code != 2 || !strings.Contains(stderr, "not the head of backport-batch/release-1.1-20261029") { t.Errorf("--onto with another date = %d (%s), want a refusal", code, stderr) } f.now = f.now.AddDate(0, 0, 3) - code, stdout, stderr = f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head) + code, stdout, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head) if code != 0 { t.Fatalf("run --onto = %d\n%s", code, stderr) } diff --git a/cmd/backport-batch/output_test.go b/cmd/backport-batch/output_test.go new file mode 100644 index 000000000..a094d4b75 --- /dev/null +++ b/cmd/backport-batch/output_test.go @@ -0,0 +1,385 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "io" + "os" + "os/exec" + "reflect" + "strings" + "testing" + "time" +) + +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("broken pipe") } + +// Output that never reached its reader fails the command, and stderr says why. +func TestUnwritableOutputFailsTheCommand(t *testing.T) { + var stderr strings.Builder + a := &app{gh: &fakeGH{}, stdout: failingWriter{}, stderr: &stderr} + if code := a.main([]string{"help"}); code != 2 { + t.Fatalf("help with an unwritable stdout = %d, want 2", code) + } + if !strings.Contains(stderr.String(), "broken pipe") { + t.Errorf("stderr does not name the write failure:\n%s", stderr.String()) + } +} + +// A command that already failed keeps its own exit code when its output is +// lost as well: check still says something is missing. +func TestUnwritableOutputKeepsTheCommandsFailure(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + var stderr strings.Builder + a := &app{gh: &fakeGH{remoteHead: f.remoteHead}, stdout: failingWriter{}, stderr: &stderr, now: func() time.Time { return f.now }, tool: f.tool} + if code := a.main([]string{"check", "--line", "release-1.1"}); code != 1 { + t.Fatalf("check with #6 missing and an unwritable stdout = %d, want 1\n%s", code, stderr.String()) + } + if !strings.Contains(stderr.String(), "broken pipe") { + t.Errorf("stderr does not name the write failure:\n%s", stderr.String()) + } +} + +// A closed pipe on stdout does not kill the run before it can say what +// already happened: the write fails like any other, and the run exits 2. +func TestClosedStdoutPipeIsReported(t *testing.T) { + if os.Getenv("BACKPORT_BATCH_CLOSED_STDOUT_HELPER") == "1" { + os.Args = []string{"backport-batch", "help"} + main() + return + } + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + _ = r.Close() + t.Cleanup(func() { _ = w.Close() }) + cmd := exec.Command(os.Args[0], "-test.run=^TestClosedStdoutPipeIsReported$") + cmd.Env = append(os.Environ(), "BACKPORT_BATCH_CLOSED_STDOUT_HELPER=1") + var stderr strings.Builder + cmd.Stdout, cmd.Stderr = w, &stderr + err = cmd.Run() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 2 { + t.Fatalf("help into a closed pipe = %v, want exit 2\nstderr:\n%s", err, stderr.String()) + } + if !strings.Contains(stderr.String(), "output was lost") { + t.Errorf("stderr does not say the output was lost:\n%s", stderr.String()) + } +} + +// failOn fails every write that contains want, and takes every other one. +type failOn struct { + want string + buf strings.Builder +} + +func (w *failOn) Write(p []byte) (int, error) { + if strings.Contains(string(p), w.want) { + return 0, errors.New("broken pipe") + } + return w.buf.Write(p) +} + +func runWith(f *fixture, gh *fakeGH, stdout, stderr io.Writer, args ...string) int { + f.t.Helper() + if gh.remoteHead == nil { + gh.remoteHead = f.remoteHead + } + a := &app{gh: gh, stdout: stdout, stderr: stderr, now: func() time.Time { return f.now }, tool: f.tool} + return a.main(args) +} + +// Output is written before the push; when it is lost, on either stream, +// nothing is pushed and no pull request is opened. +func TestLostOutputPushesNothing(t *testing.T) { + for _, stream := range []string{"stdout", "stderr"} { + t.Run(stream, func(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{} + var stdout, stderr io.Writer = &strings.Builder{}, &strings.Builder{} + if stream == "stdout" { + stdout = failingWriter{} + } else { + stderr = failingWriter{} + } + if code := runWith(f, gh, stdout, stderr, "run", "--line", "release-1.1", "--push"); code != 2 { + t.Fatalf("run --push with a failing %s = %d, want 2", stream, code) + } + if head := f.remoteHead(batchBranch1); head != "" { + t.Errorf("the batch was pushed at %.12s", head) + } + if len(gh.prs) != 0 { + t.Errorf("a pull request was opened: %+v", gh.prs) + } + }) + } +} + +// A confirmation lost after the push fails the run: the pull request was +// opened, and the run does not exit 0 without having said so. +func TestLostConfirmationFailsTheRun(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{} + if code := runWith(f, gh, &strings.Builder{}, &failOn{want: "opened draft"}, "run", "--line", "release-1.1", "--push"); code != 2 { + t.Fatalf("run --push losing its confirmation = %d, want 2", code) + } + if len(gh.prs) != 1 { + t.Errorf("%d pull requests, want the one the run opened", len(gh.prs)) + } + if len(gh.issueComments) != 0 { + t.Errorf("the report was posted after the confirmation was lost: %+v", gh.issueComments) + } +} + +// notify writes what it found before it comments; when that is lost, it +// does not comment. +func TestLostOutputCommentsNothing(t *testing.T) { + gh := &fakeGH{prs: []pullRequest{notifyPR("20261019", "")}} + var stderr strings.Builder + a := &app{gh: gh, stdout: failingWriter{}, stderr: &stderr, now: func() time.Time { return time.Date(2026, 10, 26, 6, 23, 0, 0, time.UTC) }} + if code := a.main([]string{"notify", "--line", "release-1.1", "--push"}); code != 2 { + t.Fatalf("notify --push with a failing stdout = %d, want 2\n%s", code, stderr.String()) + } + if len(gh.issueComments) != 0 { + t.Errorf("notify commented: %+v", gh.issueComments) + } +} + +// writesGitHub records the GitHub writes that reach it. +type writesGitHub struct { + github + writes []string +} + +func (g *writesGitHub) createPull(string, string, string, string) (int, string, error) { + g.writes = append(g.writes, "createPull") + return 1, "", nil +} +func (g *writesGitHub) editPull(int, string, string) error { + g.writes = append(g.writes, "editPull") + return nil +} +func (g *writesGitHub) editBody(int, string) error { + g.writes = append(g.writes, "editBody") + return nil +} +func (g *writesGitHub) closePull(int) error { + g.writes = append(g.writes, "closePull") + return nil +} +func (g *writesGitHub) createComment(int, string) error { + g.writes = append(g.writes, "createComment") + return nil +} +func (g *writesGitHub) editComment(int64, string) error { + g.writes = append(g.writes, "editComment") + return nil +} + +// Every GitHub write is refused once output was lost. Every method of +// github is either a read or in this table, so a write added later without +// a guard fails here. +func TestEveryGitHubWriteIsGuarded(t *testing.T) { + reads := map[string]bool{"openPulls": true, "recentPulls": true, "permission": true, "labelled": true, "pull": true, "comments": true, "batchBot": true} + inner := &writesGitHub{} + g := guardedGitHub{github: inner, a: &app{unwritten: errors.New("broken pipe")}} + writes := map[string]func() error{ + "createPull": func() error { _, _, err := g.createPull("b", "h", "t", "f"); return err }, + "editPull": func() error { return g.editPull(1, "t", "f") }, + "editBody": func() error { return g.editBody(1, "f") }, + "closePull": func() error { return g.closePull(1) }, + "createComment": func() error { return g.createComment(1, "f") }, + "editComment": func() error { return g.editComment(1, "f") }, + } + iface := reflect.TypeFor[github]() + for i := range iface.NumMethod() { + name := iface.Method(i).Name + if _, ok := writes[name]; !ok && !reads[name] { + t.Errorf("github.%s is neither a known read nor a guarded write", name) + } + } + for name, write := range writes { + if err := write(); err == nil || !strings.Contains(err.Error(), "output was lost") { + t.Errorf("%s with output lost = %v, want a refusal", name, err) + } + } + if len(inner.writes) != 0 { + t.Errorf("writes reached GitHub after output was lost: %q", inner.writes) + } +} + +// A warning lost while the open batch is being updated stops the update: +// nothing is pushed, posted or edited. +func TestLostWarningLeavesTheOpenBatchAlone(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{} + mustRun(t, f, gh, "run", "--line", "release-1.1", "--push") + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + bad := issueComment{ID: 8, Body: reportMarker + "names none\n"} + bad.User.Login, bad.User.Type = "maintainer", "User" + gh.issueComments = append(gh.issueComments, bad) + head, comments, body := f.remoteHead(batchBranch1), len(gh.issueComments), gh.find(90).Body + f.now = f.now.AddDate(0, 0, 1) + if code := runWith(f, gh, &strings.Builder{}, &failOn{want: "does not count as a report"}, "run", "--line", "release-1.1", "--continue", "--push"); code != 2 { + t.Fatalf("run --continue --push losing a warning = %d, want 2", code) + } + if got := f.remoteHead(batchBranch1); got != head { + t.Errorf("the batch moved to %.12s", got) + } + if len(gh.issueComments) != comments { + t.Errorf("a comment was posted: %d comments, want %d", len(gh.issueComments), comments) + } + if gh.find(90).Body != body { + t.Error("the batch pull request was edited") + } +} + +// A warning lost while a batch is being closed leaves it open, with no +// closing record half written. +func TestLostWarningLeavesTheBatchUnclosed(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + gh := &fakeGH{} + mustRun(t, f, gh, "run", "--line", "release-1.1", "--push") + bad := issueComment{ID: 8, Body: reportMarker + "names none\n"} + bad.User.Login, bad.User.Type = "maintainer", "User" + gh.issueComments = append(gh.issueComments, bad) + body := gh.find(90).Body + if code := runWith(f, gh, &strings.Builder{}, &failOn{want: "does not count as a report"}, "run", "--line", "release-1.1", "--defer", "5", "--reason", "x", "--push"); code != 2 { + t.Fatalf("run --defer losing a warning = %d, want 2", code) + } + if gh.find(90).Body != body { + t.Error("the batch pull request was edited") + } + if !gh.isOpen(90) { + t.Error("the batch pull request was closed") + } +} + +// The git and gh children keep the default SIGPIPE: catching it here must not +// leave it ignored in them, as signal.Ignore would through exec. +func TestChildrenKeepTheDefaultSIGPIPE(t *testing.T) { + catchSIGPIPE() + out, err := exec.Command("sh", "-c", "kill -PIPE $$; echo survived").Output() + if err == nil || strings.Contains(string(out), "survived") { + t.Fatalf("a child survived SIGPIPE: %q, %v", out, err) + } +} + +// The branch a close left is not deleted once output was lost: a deletion +// is a push like any other. +func TestLostOutputDeletesNoBranch(t *testing.T) { + f, gh, h1 := closeWorld(t) + f.git("switch", "-q", "main") + l, err := parseLine("release-1.1") + if err != nil { + t.Fatal(err) + } + a := &app{gh: gh, unwritten: errors.New("broken pipe")} + if err := a.deleteBranch(l, closing{PR: 90, Branch: batchBranch1, Head: h1}); err == nil || !strings.Contains(err.Error(), "output was lost") { + t.Fatalf("deleteBranch with output lost = %v, want a refusal", err) + } + if got := f.remoteHead(batchBranch1); got != h1 { + t.Errorf("the branch is at %q, want it left at %.12s", got, h1) + } +} + +// A park line lost on stdout stops notify before it writes: no comment, and +// the record is not marked as announced. +func TestLostParkLineWritesNothing(t *testing.T) { + park := parkedEntry{Original: 5, Merge: strings.Repeat("c", 40), Kind: parkDependency, Needs: []int{3}} + gh := &fakeGH{prs: []pullRequest{notifyPR("20261026", parkedBody(park))}} + body := gh.prs[0].Body + var stderr strings.Builder + a := &app{gh: gh, stdout: &failOn{want: "parked originals"}, stderr: &stderr, now: func() time.Time { return time.Date(2026, 10, 26, 6, 23, 0, 0, time.UTC) }} + if code := a.main([]string{"notify", "--line", "release-1.1", "--push"}); code != 2 { + t.Fatalf("notify --push losing the park line = %d, want 2\n%s", code, stderr.String()) + } + if len(gh.issueComments) != 0 { + t.Errorf("notify commented: %+v", gh.issueComments) + } + if gh.prs[0].Body != body { + t.Error("notify marked the park as announced") + } +} + +// The hint naming the commits a reset drops is the only way back to them, so +// the local branch is not reset when the hint is lost. +func TestLostResetHintKeepsTheBranch(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.git("switch", "-q", "-C", batchBranch1, "origin/release-1.1") + f.write("app/mine.txt", "unpushed\n") + f.git("add", "-A") + f.git("commit", "-q", "-m", "an unpushed resolution") + tip := f.git("rev-parse", "HEAD") + f.git("switch", "-q", "main") + if code := runWith(f, &fakeGH{}, &strings.Builder{}, &failOn{want: "resetting the local"}, "run", "--line", "release-1.1"); code != 2 { + t.Fatalf("run losing the reset hint = %d, want 2", code) + } + if got := f.git("rev-parse", batchBranch1); got != tip { + t.Errorf("the local %s moved to %.12s, want it kept at %.12s", batchBranch1, got, tip) + } +} + +// Once output is lost the run moves no local branch, even one a reset would +// only fast-forward. +func TestLostOutputMovesNoLocalBranch(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.git("branch", "-f", batchBranch1, "origin/release-1.1^") + before := f.git("rev-parse", batchBranch1) + if code := runWith(f, &fakeGH{}, &strings.Builder{}, &failOn{want: "prepared"}, "run", "--line", "release-1.1"); code != 2 { + t.Fatalf("run losing an early line = %d, want 2", code) + } + if got := f.git("rev-parse", batchBranch1); got != before { + t.Errorf("the local %s moved to %.12s, want it kept at %.12s", batchBranch1, got, before) + } +} + +// No new unit starts once output is lost: neither a pick nor a replay touches +// the branch. +func TestLostOutputStartsNoUnit(t *testing.T) { + f := newFixture(t) + f.lines() + before := f.git("rev-parse", "HEAD") + a := &app{unwritten: errors.New("broken pipe")} + if _, err := a.pickUnit(&verdict{}, nil); err == nil || !strings.Contains(err.Error(), "output was lost") { + t.Errorf("pickUnit with output lost = %v, want a refusal", err) + } + if _, _, err := a.replayNext(unit{}); err == nil || !strings.Contains(err.Error(), "output was lost") { + t.Errorf("replayNext with output lost = %v, want a refusal", err) + } + if got := f.git("rev-parse", "HEAD"); got != before { + t.Errorf("HEAD moved to %.12s", got) + } +} diff --git a/cmd/backport-batch/park_test.go b/cmd/backport-batch/park_test.go new file mode 100644 index 000000000..94a06ada5 --- /dev/null +++ b/cmd/backport-batch/park_test.go @@ -0,0 +1,796 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "slices" + "strings" + "testing" +) + +var line11, _ = parseLine("release-1.1") + +// handBatch pushes a batch carrying merges in the batch's shape, as a run +// made one before the partial guard, and opens its pull request with a +// record that defers nothing. +func handBatch(f *fixture, gh *fakeGH, merges ...string) string { + f.t.Helper() + f.git("fetch", "-q", "origin") + f.git("switch", "-q", "--detach", "origin/release-1.1") + for _, mc := range merges { + if parents := strings.Fields(f.git("rev-list", "--parents", "-n", "1", mc)); len(parents) == 2 { + f.git("cherry-pick", "-x", mc) + continue + } + base := f.git("rev-parse", "HEAD") + for _, c := range strings.Fields(f.git("rev-list", "--reverse", "--no-merges", mc+"^1.."+mc+"^2")) { + f.git("cherry-pick", "-x", c) + } + merge := f.joined(base, "HEAD", f.git("log", "-1", "--format=%B", mc)+"\n\n"+batchTrailer+": "+mc) + f.git("update-ref", "HEAD", merge) + } + f.git("push", "-q", "origin", "HEAD:refs/heads/"+batchBranch1) + f.git("switch", "-q", "main") + gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, "batch\n\n"+deferredBlock(line11, nil, nil, nil)+"\n")} + return f.remoteHead(batchBranch1) +} + +// policyWorld is #5088's shape: #3, a feat the line never gets, adds a +// policy and its test; #5 deletes both again and adds a test of its own. +func policyWorld(t *testing.T) (*fixture, string) { + t.Helper() + f := newFixture(t) + f.lines() + f.pr(3, "feat(chart): a network policy", "", map[string]string{"chart/templates/policy.yaml": "kind: NetworkPolicy\n", "chart/tests/policy_test.yaml": "suite: policy\n"}) + mc5 := f.pr(5, "fix(chart): remove the policy a rebase brought back", "```release-note\nfix(chart): the stray network policy is gone.\n```\n", + map[string]string{"chart/templates/policy.yaml": "", "chart/tests/policy_test.yaml": "", "chart/tests/values_test.yaml": "suite: values\n"}) + return f, mc5 +} + +// A clean pick that would delete files the line never had, and so carry only +// #5's test, is withheld: #5 is parked as partial, with the paths it would +// leave without effect, and its release note stays out of the body. +func TestPartialPickIsWithheld(t *testing.T) { + f, mc5 := policyWorld(t) + f.pr(6, "fix(app): six", "```release-note\nfix(app): six works.\n```\n", map[string]string{"app/six.txt": "6\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, []string{"fix(app): six (#6)"}) { + t.Errorf("picked %q, want only #6", got) + } + want := `"parked":[{"original":5,"merge":"` + mc5 + `","kind":"partial","files":["chart/templates/policy.yaml","chart/tests/policy_test.yaml"]}]` + if !strings.Contains(stdout, want) { + t.Errorf("the record does not park #5 as partial (%s):\n%s", want, stdout) + } + if !strings.Contains(stdout, "### Parked") || !strings.Contains(stdout, "`chart/templates/policy.yaml`") { + t.Errorf("the body does not say #5 is parked and why:\n%s", stdout) + } + if strings.Contains(stdout, "stray network policy") || !strings.Contains(stdout, "six works") { + t.Errorf("the release notes are not #6's alone:\n%s", stdout) + } +} + +// A file the line removed itself, which no original added after the line +// branched, still leaves #5's deletion without effect: check finds it with no +// record and parks it, and run picks nothing. +func TestPartialWithoutAnAdder(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: add a policy", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): the line drops the policy", map[string]string{"app/policy.yaml": ""}) + f.pr(5, "fix(app): drop the policy", "", map[string]string{"app/policy.yaml": "", "app/policy_test.txt": "gone\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); len(got) != 0 { + t.Errorf("picked %q, want nothing", got) + } + if !strings.Contains(stdout, "### Parked") { + t.Errorf("the body does not park #5:\n%s", stdout) + } + f.git("switch", "-q", "main") + code, stdout, stderr = f.exec(nil, "check", "--line", "release-1.1") + if code != 0 || !strings.Contains(stderr, "note: #5 is parked: only part of it applies to release-1.1") { + t.Errorf("check = %d, want 0 with a parked-partial note:\nstdout:\n%s\nstderr:\n%s", code, stdout, stderr) + } +} + +// The picker judges what its picks did, not what the original's merge did: +// #5's first commit deletes a file the line no longer has and comes out +// empty, its second adds a test, and the unit left would be the test alone. +func TestPartialByAnEmptyOwnCommit(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: add a policy", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): the line drops the policy", map[string]string{"app/policy.yaml": ""}) + f.prCommits(5, "fix(app): drop the policy and test it", "```release-note\nfix(app): no more policy.\n```\n", + change{files: map[string]string{"app/policy.yaml": ""}}, + change{files: map[string]string{"app/policy_test.txt": "gone\n"}}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); len(got) != 0 { + t.Errorf("picked %q, want #5 withheld", got) + } + if strings.Contains(stdout, "no more policy") || !strings.Contains(stdout, "### Parked") { + t.Errorf("#5's note is in the body, or #5 is not parked:\n%s", stdout) + } +} + +// A batch built before the guard can already carry a partial unit. It fails +// the cut and verify whatever its label says, and --defer takes it off, +// closing a batch it was the last unit of. +func TestExistingPartialUnitFailsTheCut(t *testing.T) { + f, mc5 := policyWorld(t) + gh := &fakeGH{} + head := handBatch(f, gh, mc5) + code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut") + if code != 1 || !strings.Contains(stdout, "#5 is on batch #90, but only part of it applies to release-1.1") { + t.Errorf("check --cut = %d, want 1 naming the partial unit:\nstdout:\n%s\nstderr:\n%s", code, stdout, stderr) + } + code, stdout, stderr = f.exec(gh, "verify", "--line", "release-1.1", "--head", head) + if code != 1 || !strings.Contains(stdout, `#5 leaves "chart/templates/policy.yaml", "chart/tests/policy_test.yaml" without effect on release-1.1`) { + t.Errorf("verify = %d, want 1 naming what #5 leaves out:\n%s\n%s", code, stdout, stderr) + } + gh.labels = map[string][]int{labelSkip: {5}} + code, stdout, stderr = f.exec(gh, "check", "--line", "release-1.1", "--cut") + if code != 1 || !strings.Contains(stdout, "#5 is on batch #90, but only part of it applies to release-1.1") { + t.Errorf("check --cut with backport/skip = %d, want the partial unit to still fail it:\nstdout:\n%s\nstderr:\n%s", code, stdout, stderr) + } + code, _, stderr = f.exec(gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "only part of it applies", "--push") + if code != 0 || prState(gh.find(90)) != "closed" || f.remoteHead(batchBranch1) != "" { + t.Errorf("run --defer 5 --push = %d, #90 %s, branch at %q; want the batch closed and its branch gone:\n%s", code, prState(gh.find(90)), f.remoteHead(batchBranch1), stderr) + } +} + +// An original whose first pick came out empty, because the line already had +// that change, is on the batch all the same: check does not call it missing, +// a continuation keeps it picked, and so does a rebuild without another +// original. +func TestOnBatchAfterAnEmptyFirstPick(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a and b", "", map[string]string{"app/a.txt": "old\n", "app/b.txt": "old\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): a, on the line first", map[string]string{"app/a.txt": "fixed\n"}) + f.prCommits(5, "fix(app): a and b", "", + change{files: map[string]string{"app/a.txt": "fixed\n"}}, + change{files: map[string]string{"app/b.txt": "fixed\n"}}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + head := openBatch(t, f, gh) + if got := f.subjects("origin/release-1.1", head); !slices.Equal(got, []string{"fix(app): a and b (#5)", "fix(app): six (#6)"}) { + t.Fatalf("the batch carries %q, want #5 and #6", got) + } + code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1") + if code != 0 { + t.Errorf("check = %d, want 0 with #5 on the batch:\nstdout:\n%s\nstderr:\n%s", code, stdout, stderr) + } + code, stdout, _ = f.exec(gh, "check", "--line", "release-1.1", "--cut") + if code != 1 || !strings.Contains(stdout, "#5 is only on open batch PR #90") { + t.Errorf("check --cut = %d, want #5 only on the batch:\n%s", code, stdout) + } + code, stdout, stderr = f.exec(gh, "run", "--line", "release-1.1", "--continue") + if code != 0 || !strings.Contains(stdout, `"picked":[5,6]`) { + t.Errorf("run --continue = %d, want #5 and #6 picked:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + code, stdout, stderr = f.exec(gh, "run", "--line", "release-1.1", "--defer", "6", "--reason", "fails the e2e") + if code != 0 || !strings.Contains(stdout, `"picked":[5]`) || !strings.Contains(stdout, `"missing":[]`) { + t.Errorf("run --defer 6 = %d, want #5 still picked:\n%s\n%s", code, stdout, stderr) + } +} + +// featWorld is #5071's shape: #3, a feat the line never gets, adds a file, +// and #5 fixes that file; #6 is a plain fix beside it. +func featWorld(t *testing.T) *fixture { + t.Helper() + f := newFixture(t) + f.lines() + f.pr(3, "feat(app): tap", "", map[string]string{"app/tap.go": appFile}) + f.pr(5, "fix(app): fix the tap", "```release-note\nfix(app): the tap works.\n```\n", map[string]string{"app/tap.go": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + return f +} + +// A fix of a file a feat added on main, which the line never gets, is +// parked, not tried, and check takes the park as a note. +func TestParkAFixOfAFeatureTheLineLacks(t *testing.T) { + f := featWorld(t) + gh := &fakeGH{} + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push") + if code != 0 { + t.Fatalf("run --push = %d\n%s", code, stderr) + } + if strings.Contains(stdout, "### Deferred conflicts") || !strings.Contains(stdout, `"parked":[{"original":5,`) || !strings.Contains(stdout, `"kind":"dependency","needs":[3]`) { + t.Errorf("#5 is not parked on #3:\n%s", stdout) + } + if !strings.Contains(stdout, "### Parked") || !strings.Contains(stdout, "builds on [#3]") { + t.Errorf("the body does not say #5 builds on #3:\n%s", stdout) + } + f.git("switch", "-q", "main") + reopen(f, gh) + code, stdout, stderr = f.exec(gh, "check", "--line", "release-1.1") + if code != 0 || !strings.Contains(stderr, "note: #5 is parked: it builds on #3, which never reaches release-1.1") { + t.Errorf("check = %d, want 0 with a note that #5 is parked:\n%s\n%s", code, stdout, stderr) + } +} + +// backport/include lifts no park. +func TestIncludeDoesNotLiftThePark(t *testing.T) { + f := featWorld(t) + code, stdout, stderr := f.exec(&fakeGH{labels: map[string][]int{labelInclude: {5}}}, "run", "--line", "release-1.1") + if code != 0 || !strings.Contains(stdout, `"parked":[{"original":5,`) || !strings.Contains(stdout, "`backport/include`, parked: builds on") { + t.Errorf("run with backport/include on #5 = %d, want #5 still parked:\n%s\n%s", code, stdout, stderr) + } +} + +// A prerequisite that is itself parked, but adapted onto the batch by hand, +// satisfies what builds on it: #3 is parked on #2, a feat, and #5 changes a +// line #3 wrote. +func TestAdaptedPrerequisiteSatisfiesItsDependent(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a and b", "", map[string]string{"app/a.txt": appFile, "app/b.txt": appFile}) + f.lines() + f.pr(20, "feat(app): a loud two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + mc3 := f.pr(3, "fix(app): an excited two, a loud three", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1), "app/b.txt": strings.Replace(appFile, "three", "THREE", 1)}) + f.pr(5, "fix(app): an excited three", "", map[string]string{"app/b.txt": strings.Replace(appFile, "three", "THREE!", 1)}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || !strings.Contains(stdout, `"kind":"dependency","needs":[3]`) { + t.Fatalf("run = %d, want #5 parked as long as #3 is:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + w := emptyBatch(f, &fakeGH{}, "batch\n\n"+deferredBlock(line11, nil, nil, nil)+"\n") + // #3 adapted by hand: every file it changes, changed, in the batch's + // shape. + pick := w.pick(w.head, f.git("rev-parse", mc3+"^2"), map[string]string{"app/a.txt": strings.Replace(appFile, "two", "two!", 1), "app/b.txt": strings.Replace(appFile, "three", "THREE", 1)}) + fold := w.merge(w.head, pick, "fix(app): an excited two, a loud three (#3)\n\n"+batchTrailer+": "+mc3) + f.git("push", "-q", "origin", fold+":refs/heads/"+batchBranch1) + w.gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, w.gh.prs[0].Body)} + code, stdout, stderr := f.exec(w.gh, "check", "--line", "release-1.1") + if code != 1 || strings.Contains(stderr, "#5 is parked") || strings.Contains(stdout+stderr, "waits on") || !strings.Contains(stdout, "/pull/5 ") { + t.Errorf("check = %d, want #5 missing, not parked:\n%s\n%s", code, stdout, stderr) + } + code, stdout, stderr = f.exec(w.gh, "run", "--line", "release-1.1", "--continue") + if code != 0 || !strings.Contains(stdout, `"picked":[3,5]`) { + t.Errorf("run --continue = %d, want #5 picked after #3's adaptation:\n%s\n%s", code, stdout, stderr) + } +} + +// What an earlier unit of the batch brought counts: #3 writes the line #8 +// changes, a feat takes it back and another feat, #6, writes it again, so +// blame names #6, which never comes; once #3 is picked the branch has the +// line all the same. +func TestContentFromAnotherUnitSatisfies(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.pr(3, "fix(app): a loud two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(4, "feat(app): a quiet two", "", map[string]string{"app/a.txt": appFile}) + f.pr(6, "feat(app): a loud two again", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(8, "fix(app): an excited two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 { + t.Fatalf("run = %d\n%s", code, stderr) + } + if got := f.subjects("origin/release-1.1", "HEAD"); !slices.Equal(got, []string{"fix(app): a loud two (#3)", "fix(app): an excited two (#8)"}) { + t.Errorf("picked %q, want #3 and #8:\n%s", got, stdout) + } +} + +// auxWorld is #5142's shape: #3, a feat, adds a bats test; #5 fixes a +// workflow and that test, so all it builds on that never comes is a test; +// #6 is a plain fix. +func auxWorld(t *testing.T) *fixture { + t.Helper() + f := newFixture(t) + f.squash(2, "chore: a workflow", "", map[string]string{".github/workflows/release.yaml": "on: push\njobs: {}\n"}) + f.lines() + f.pr(3, "feat(ci): test the changelog", "", map[string]string{"hack/changelog.bats": "@test x {\n true\n}\n"}) + f.pr(5, "fix(ci): run the release on tags", "", map[string]string{".github/workflows/release.yaml": "on: [push, tag]\njobs: {}\n", "hack/changelog.bats": "@test x {\n false\n}\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + return f +} + +// A workflow original whose only missing dependency is a test waits for a +// maintainer in the scheduled run, whose token cannot push workflows, and +// check holds the cut on it. A maintainer's own run parks it. With no batch +// open, or one whose body carries no record, it is missing; a record from +// before the mode was saved says the scheduled mode by listing a wait. +func TestWorkflowModeCarriesToCheck(t *testing.T) { + f := auxWorld(t) + gh := &fakeGH{} + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push"); code != 0 || strings.Contains(stdout, `"parked":[{`) || !strings.Contains(stdout, `{"original":5,"files":[".github/workflows/release.yaml"],"workflows":true}`) { + t.Fatalf("scheduled run = %d, want #5 waiting, not parked:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "#5 changes .github/workflows/ and waits on batch PR #90") { + t.Errorf("check --cut = %d, want #5 waiting:\n%s\n%s", code, stdout, stderr) + } + v1 := "batch\n\n" + deferredBlock(line11, nil, nil, []pick{{v: &verdict{o: &original{number: 5}}, files: []string{".github/workflows/release.yaml"}}}) + "\n" + gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, v1)} + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "#5 changes .github/workflows/ and waits on batch PR #90") { + t.Errorf("check --cut with a version 1 record = %d, want #5 waiting:\n%s\n%s", code, stdout, stderr) + } + gh.prs = []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, "a batch someone wrote")} + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 1 || !strings.Contains(stdout, "/pull/5 ") || strings.Contains(stderr, "parked") { + t.Errorf("check under a batch with no record = %d, want #5 missing:\n%s\n%s", code, stdout, stderr) + } + + f = auxWorld(t) + if code, stdout, stderr := f.exec(nil, "check", "--line", "release-1.1"); code != 1 || !strings.Contains(stdout, "/pull/5 ") || strings.Contains(stderr, "parked") { + t.Errorf("check with no batch = %d, want #5 missing:\n%s\n%s", code, stdout, stderr) + } + gh = &fakeGH{} + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--push"); code != 0 || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Fatalf("a maintainer's run = %d, want #5 parked:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 || !strings.Contains(stderr, "note: #5 is parked") { + t.Errorf("check after a maintainer's run = %d, want 0 with #5 parked:\n%s\n%s", code, stdout, stderr) + } +} + +// Evidence found only in a test file still parks an original that changes +// no workflow: #4973's shape. +func TestTestOnlyEvidenceParksOtherOriginals(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: nodes", "", map[string]string{"app/nodes.go": "package app\n", "app/nodes_test.go": "package app\n\nfunc a() {}\n"}) + f.lines() + f.pr(3, "feat(app): proxmox nodes", "", map[string]string{"app/nodes_test.go": "package app\n\nfunc a() { proxmox() }\n"}) + f.pr(5, "fix(app): nodes", "", map[string]string{"app/nodes.go": "package app\n\nvar fixed = true\n", "app/nodes_test.go": "package app\n\nfunc a() { proxmox(); fixed() }\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 || strings.Contains(stdout, "### Deferred conflicts") || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Errorf("run = %d, want #5 parked on #3:\n%s\n%s", code, stdout, stderr) + } +} + +// What an original builds on through lines it adds is not seen: #4973's +// $isProxmox. The original is picked; CI on the batch is what catches it. +func TestAddedLinesDependencyIsNotSeen(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: nodes", "", map[string]string{"app/nodes.tpl": "a\nb\n"}) + f.lines() + f.pr(3, "feat(app): define isProxmox", "", map[string]string{"app/proxmox.tpl": "isProxmox\n"}) + f.pr(5, "fix(app): use isProxmox", "", map[string]string{"app/nodes.tpl": "a\nif isProxmox\nb\n"}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || strings.Contains(stdout, `"parked":[{`) || !strings.Contains(stdout, `"picked":[5]`) { + t.Errorf("run = %d, want #5 picked:\n%s\n%s", code, stdout, stderr) + } +} + +// With no batch open, check reads a closed record whatever its version and +// whatever mode it saved as the scheduled run's: the workflow original it +// left waiting is missing, not parked, while its drops and its other +// deferrals still count. +func TestClosedRecordModeFallback(t *testing.T) { + for _, version := range []int{2, 3} { + for _, saved := range []bool{false, true} { + t.Run(fmt.Sprintf("version %d, noWorkflowFiles %v", version, saved), func(t *testing.T) { + f := auxWorld(t) + mc6 := f.git("log", "--first-parent", "--format=%H", "-1", "--grep", "(#6)$", "main") + mc7 := f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + sha := f.git("rev-parse", "origin/main") + rec := lineRecord{Version: version, Line: "release-1.1", Report: strings.Repeat("b", 64), + Built: &builtFrom{Main: sha, Line: sha, Policy: sha, Batch: sha, At: "2026-10-26T06:00:00Z", Options: runOptions{NoWorkflowFiles: saved, Mode: modeContinue}}, + Deferred: []deferredEntry{{Original: 5, Files: []string{".github/workflows/release.yaml"}, Workflows: true}, {Original: 7, Commit: f.git("rev-parse", mc7+"^2"), Files: []string{"app/seven.txt"}}}, + Dropped: []droppedEntry{{Original: 6, Merge: mc6, Reason: "not for this line", On: 90, Date: "2026-10-26"}}, Stale: []int{}} + pr := f.batchPR(90, "release-1.1", batchBranch1, "batch\n\n"+recordBlock(rec)+"\n") + at := "2026-10-26T12:00:00Z" + pr.State, pr.MergedAt = "closed", &at + code, stdout, stderr := f.exec(&fakeGH{prs: []pullRequest{pr}}, "check", "--line", "release-1.1") + if code != 1 || !strings.Contains(stdout, "/pull/5 ") || strings.Contains(stderr, "#5 is parked") { + t.Errorf("check = %d, want #5 missing and not parked:\n%s\n%s", code, stdout, stderr) + } + if !strings.Contains(stderr, "note: #6 is dropped") || !strings.Contains(stderr, "warning: deferred conflict on #90 (merged): https://github.com/cozystack/cozystack/pull/7 ") { + t.Errorf("check lost the closed record's drop or deferral:\n%s", stderr) + } + }) + } + } +} + +// Withdrawing a drop puts the original back where the rules and the line +// put it: a partial original is parked again, in the record as in check. +func TestUndropRestoresThePark(t *testing.T) { + for name, world := range map[string]func(*testing.T) *fixture{ + "partial": func(t *testing.T) *fixture { + f, _ := policyWorld(t) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + return f + }, + "dependency": featWorld, + } { + t.Run(name, func(t *testing.T) { + f := world(t) + gh := &fakeGH{} + openBatch(t, f, gh) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--drop", "5", "--reason", "not for 1.1", "--push") + reopen(f, gh) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--undrop", "5", "--push") + reopen(f, gh) + if rec := recordOf(t, gh); len(rec.Parked) != 1 || rec.Parked[0].Original != 5 || rec.Parked[0].Kind != name { + t.Errorf("after the undrop the record parks %+v, want #5 parked %s", rec.Parked, name) + } + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 || !strings.Contains(stderr, "note: #5 is parked") { + t.Errorf("check = %d, want #5 parked:\n%s\n%s", code, stdout, stderr) + } + }) + } +} + +// A prerequisite the open batch carries is there for what builds on it, +// whatever its label now says: check and run agree that #7 is not parked. +func TestPrerequisiteStillOnTheBatchSatisfies(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: files", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + gh := &fakeGH{} + openBatch(t, f, gh) + f.pr(7, "fix(app): seven", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + gh.labels = map[string][]int{labelSkip: {5}} + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 1 || strings.Contains(stderr, "#7 is parked") || !strings.Contains(stdout, "/pull/7 ") { + t.Errorf("check = %d, want #7 missing, not parked:\n%s\n%s", code, stdout, stderr) + } + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue"); code != 0 || !strings.Contains(stdout, `"picked":[7]`) { + t.Errorf("run --continue = %d, want #7 picked:\n%s\n%s", code, stdout, stderr) + } +} + +// The picker judges what its picks did where the run has got to: #4 +// deletes the file #5 also deletes, so after #4 #5 would bring only its +// test, which the net merge at the line's head cannot see. +func TestPartialByAnEarlierPick(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a policy", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.lines() + f.pr(4, "fix(app): drop the policy", "", map[string]string{"app/policy.yaml": "", "app/four.txt": "4\n"}) + f.pr(40, "feat(app): the policy again", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.pr(5, "fix(app): drop the policy and test it", "", map[string]string{"app/policy.yaml": "", "app/policy_test.txt": "gone\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 || !strings.Contains(stdout, `"picked":[4]`) || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Errorf("run = %d, want #4 picked and #5 parked as partial:\n%s\n%s", code, stdout, stderr) + } +} + +// What an earlier pick brought clears a partial park the line's head +// gave: the line removed the policy, #4 brings it back, and #5's deletion +// then has something to delete. +func TestPartialClearedByAnEarlierPick(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a policy", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): the line drops the policy", map[string]string{"app/policy.yaml": ""}) + f.pr(3, "feat(app): no policy", "", map[string]string{"app/policy.yaml": ""}) + f.pr(4, "fix(app): the policy, back", "", map[string]string{"app/policy.yaml": "allow: some\n"}) + f.pr(5, "fix(app): drop the policy and test it", "", map[string]string{"app/policy.yaml": "", "app/policy_test.txt": "gone\n"}) + code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1") + if code != 0 || !strings.Contains(stdout, `"picked":[4,5]`) { + t.Errorf("run = %d, want #4 and #5 picked:\n%s\n%s", code, stdout, stderr) + } +} + +// A reason a person deferred an original for comes before its partial +// park. +func TestReasonComesBeforeThePartialPark(t *testing.T) { + f, _ := policyWorld(t) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + mustRun(t, f, gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "waits for the policy rework", "--push") + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 || !strings.Contains(stderr, "warning: deferred on #90 to the next batch: https://github.com/cozystack/cozystack/pull/5 ") || strings.Contains(stderr, "#5 is parked") { + t.Errorf("check = %d, want #5 deferred for its reason:\n%s\n%s", code, stdout, stderr) + } +} + +// The workflow exception holds only while what never comes is tests or +// docs, and while the workflow lines build on nothing missing: a +// workflow original that also fixes a file a feat added, or whose workflow +// line another original still to come wrote, parks in the scheduled run too. +func TestWorkflowExceptionNeedsTestsOnly(t *testing.T) { + t.Run("a never dependency outside tests", func(t *testing.T) { + f := auxWorld(t) + f.pr(20, "feat(app): tap", "", map[string]string{"app/tap.go": appFile}) + f.pr(7, "fix(ci): test the tap, and fix it", "", map[string]string{".github/workflows/tap.yaml": "on: push\n", "app/tap.go": strings.Replace(appFile, "two", "TWO", 1)}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1", "--no-workflow-files"); code != 0 || !strings.Contains(stdout, `"parked":[{"original":7,`) { + t.Errorf("scheduled run = %d, want #7 parked:\n%s\n%s", code, stdout, stderr) + } + }) + t.Run("a workflow line still to come", func(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a workflow", "", map[string]string{".github/workflows/release.yaml": "on: push\njobs: {}\n", "app/four.txt": "old\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): four, the line's way", map[string]string{"app/four.txt": "line\n"}) + f.pr(3, "feat(ci): test the changelog", "", map[string]string{"hack/changelog.bats": "@test x {\n true\n}\n"}) + f.pr(4, "fix(ci): four", "", map[string]string{"app/four.txt": "four\n", ".github/workflows/release.yaml": "on: [push]\njobs: {}\n"}) + f.pr(5, "fix(ci): run the release on tags", "", map[string]string{".github/workflows/release.yaml": "on: [push, tag]\njobs: {}\n", "hack/changelog.bats": "@test x {\n false\n}\n"}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1", "--no-workflow-files"); code != 0 || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Errorf("scheduled run = %d, want #5 parked:\n%s\n%s", code, stdout, stderr) + } + }) +} + +// What a rebuild takes off the batch is no longer there for what builds on +// it: deferring #5 makes #7, which merged after the batch was built, wait +// on it. +func TestRebuildTakesThePrerequisiteAway(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: files", "", map[string]string{"app/a.txt": appFile}) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + f.pr(7, "fix(app): seven", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "fails the e2e"); code != 0 || !strings.Contains(stdout, `{"original":7,"files":[],"needs":[5]}`) { + t.Errorf("run --defer 5 = %d, want #7 waiting on #5:\n%s\n%s", code, stdout, stderr) + } +} + +// A file the line renamed the way the original does is the original's own +// rename, not a deletion left without effect. +func TestSameRenameOnBothSidesIsWhole(t *testing.T) { + const long = "l01\nl02\nl03\nl04\nl05\nl06\nl07\nl08\nl09\nl10\nl11\nl12\n" + f := newFixture(t) + f.squash(2, "chore: a file", "", map[string]string{"app/p.txt": long}) + f.lines() + f.onLine("release-1.1", "fix(app): the line renames p to q", map[string]string{"app/p.txt": "", "app/q.txt": long}) + f.pr(5, "fix(app): rename p to q, and fix l10", "", map[string]string{"app/p.txt": "", "app/q.txt": strings.Replace(long, "l10", "L10", 1)}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || !strings.Contains(stdout, `"picked":[5]`) { + t.Errorf("run = %d, want #5 picked whole:\n%s\n%s", code, stdout, stderr) + } +} + +// A partial park only the picker saw, where the net merge at the line's +// head looks whole, holds in check and in the operations that pick nothing +// as the open record keeps it: check does not call the original missing. +func TestPickerParkHoldsInCheck(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a policy", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.lines() + f.pr(4, "fix(app): drop the policy", "", map[string]string{"app/policy.yaml": "", "app/four.txt": "4\n"}) + f.pr(40, "feat(app): the policy again", "", map[string]string{"app/policy.yaml": "allow: all\n"}) + f.pr(5, "fix(app): drop the policy and test it", "", map[string]string{"app/policy.yaml": "", "app/policy_test.txt": "gone\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + gh := &fakeGH{} + openBatch(t, f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 0 || !strings.Contains(stderr, "note: #5 is parked: only part of it applies") { + t.Errorf("check = %d, want #5 parked as the record says:\n%s\n%s", code, stdout, stderr) + } + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "6", "--reason", "fails the e2e"); code != 0 || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Errorf("run --defer 6 = %d, want #5 still parked:\n%s\n%s", code, stdout, stderr) + } + // Deferring #4, which made #5 partial, ends the park: #5 applies whole + // on what the batch carries now. + f.git("switch", "-q", "main") + code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "4", "--reason", "fails the e2e", "--push") + if code != 0 || strings.Contains(stdout, `"parked":[{`) || !strings.Contains(stdout, `"missing":[5]`) { + t.Errorf("run --defer 4 = %d, want #5 missing, no longer parked:\n%s\n%s", code, stdout, stderr) + } + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 1 || strings.Contains(stderr, "#5 is parked") || !strings.Contains(stdout, "/pull/5 ") { + t.Errorf("check after deferring #4 = %d, want #5 missing:\n%s\n%s", code, stdout, stderr) + } +} + +// A workflow original that waits for a maintainer only as the open record +// lists it: one merged since the batch was built is missing, and so is +// what waits on it. +func TestUnlistedWorkflowWaitIsMissing(t *testing.T) { + f := auxWorld(t) + gh := &fakeGH{} + mustRun(t, f, gh, "run", "--line", "release-1.1", "--no-workflow-files", "--push") + f.git("switch", "-q", "main") + reopen(f, gh) + f.pr(8, "fix(ci): run the new release on tags", "", map[string]string{".github/workflows/new.yaml": "on: tag\n", "hack/changelog.bats": "@test x {\n false\n}\n# eight\n", "app/eight.txt": appFile + "TWO\n"}) + f.pr(9, "fix(app): nine", "", map[string]string{"app/eight.txt": appFile + "TWO!\n"}) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 1 || !strings.Contains(stdout, "/pull/9 fix(app): nine (it waits on #8, which is not on the line either)") { + t.Errorf("check = %d, want #9 missing, waiting on #8:\n%s\n%s", code, stdout, stderr) + } +} + +// The line's own rename stands for the original's only with the original's +// change in it: a fold that leaves the renamed file as it was is refused. +func TestSameRenameStillNeedsItsFix(t *testing.T) { + const long = "l01\nl02\nl03\nl04\nl05\nl06\nl07\nl08\nl09\nl10\nl11\nl12\n" + f := newFixture(t) + f.squash(2, "chore: a file", "", map[string]string{"app/p.txt": long}) + f.lines() + f.onLine("release-1.1", "fix(app): the line renames p to q", map[string]string{"app/p.txt": "", "app/q.txt": long}) + mc5 := f.pr(5, "fix(app): rename p to q, fix l10, and five", "", map[string]string{"app/p.txt": "", "app/q.txt": strings.Replace(long, "l10", "L10", 1), "app/five.txt": "5\n"}) + w := emptyBatch(f, &fakeGH{}, "") + pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/five.txt": "5\n"}) + fold := w.merge(w.head, pick, "fix(app): rename p to q, fix l10, and five (#5)\n\n"+batchTrailer+": "+mc5) + if code, out, stderr := f.exec(w.gh, "fold", "--line", "release-1.1", "--original", "5", "--commit", fold); code != 1 || !strings.Contains(out, `leaves "app/p.txt" without effect`) { + t.Errorf("fold without the fix in q = %d, %q\n%s", code, out, stderr) + } +} + +// A prerequisite the line carries is there, though the rules would keep it +// out: a feat carried by a hand adaptation does not park what builds on it. +func TestCarriedExcludedPrerequisiteSatisfies(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a", "", map[string]string{"app/a.txt": appFile}) + f.lines() + mc3 := f.pr(3, "feat(app): a loud two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.onLine("release-1.1", "feat(app): a loud two, adapted\n\n(cherry picked from commit "+mc3+")", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "Two", 1)}) + f.pr(5, "fix(app): an excited two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || strings.Contains(stdout, `"parked":[{`) { + t.Errorf("run = %d, want #5 not parked on the carried #3:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + if code, stdout, stderr := f.exec(nil, "check", "--line", "release-1.1"); code != 1 || strings.Contains(stderr, "#5 is parked") { + t.Errorf("check = %d, want #5 missing, not parked:\n%s\n%s", code, stdout, stderr) + } +} + +// With no batch open, a workflow wait a closed record keeps waits on +// nothing: the original is judged as the scheduled run judges it, so one +// that also builds on a feat the line lacks is parked, in check as in run. +func TestClosedWorkflowWaitIsJudgedAsRunJudgesIt(t *testing.T) { + f := auxWorld(t) + f.pr(20, "feat(app): tap", "", map[string]string{"app/tap.go": appFile}) + f.pr(7, "fix(ci): test the tap, and fix it", "", map[string]string{".github/workflows/tap.yaml": "on: push\n", "app/tap.go": strings.Replace(appFile, "two", "TWO", 1)}) + sha := f.git("rev-parse", "origin/main") + rec := lineRecord{Version: 2, Line: "release-1.1", Report: strings.Repeat("b", 64), + Built: &builtFrom{Main: sha, Line: sha, Policy: sha, Batch: sha, At: "2026-10-26T06:00:00Z", Options: runOptions{NoWorkflowFiles: true, Mode: modeContinue}}, + Deferred: []deferredEntry{{Original: 7, Files: []string{".github/workflows/tap.yaml"}, Workflows: true}}, Dropped: []droppedEntry{}, Stale: []int{}} + pr := f.batchPR(90, "release-1.1", batchBranch1, "batch\n\n"+recordBlock(rec)+"\n") + at := "2026-10-26T12:00:00Z" + pr.State, pr.MergedAt = "closed", &at + gh := &fakeGH{prs: []pullRequest{pr}} + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut"); strings.Contains(stdout, "/pull/7 ") || !strings.Contains(stderr, "note: #7 is parked: it builds on #20") { + t.Errorf("check --cut = %d, want #7 parked:\n%s\n%s", code, stdout, stderr) + } + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--no-workflow-files"); code != 0 || !strings.Contains(stdout, `"parked":[{"original":7,`) { + t.Errorf("scheduled run = %d, want #7 parked:\n%s\n%s", code, stdout, stderr) + } +} + +// A pick that comes out empty is judged too: #5's own commit did half its +// fix, which the line has, and a merge of main into its branch did the +// rest, which no pick carries; #5 is not on the line. +func TestEmptyPicksStillJudged(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: b", "", map[string]string{"app/b.txt": "old\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): b, half of it", map[string]string{"app/b.txt": "middle\n"}) + f.git("switch", "-q", "-c", "pr-5", "main") + f.apply(map[string]string{"app/b.txt": "middle\n"}) + f.git("commit", "-q", "-m", "b, half of it") + f.git("switch", "-q", "main") + f.apply(map[string]string{"app/main.txt": "moved\n"}) + f.git("commit", "-q", "-m", "chore: move main along") + f.git("switch", "-q", "pr-5") + f.git("merge", "-q", "--no-ff", "--no-commit", "main") + f.apply(map[string]string{"app/b.txt": "fixed\n"}) + f.git("commit", "-q", "-m", "Merge branch 'main' into pr-5") + f.git("switch", "-q", "main") + f.git("merge", "-q", "--no-ff", "-m", "fix(app): b (#5)", "pr-5") + f.git("push", "-q", "origin", "main") + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || strings.Contains(stdout, `"on-line":[5]`) || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Errorf("run = %d, want #5 parked as partial, not on the line:\n%s\n%s", code, stdout, stderr) + } +} + +// --defer takes off a unit that does only part of its original whatever +// rule now keeps that original out, as check tells it to. +func TestDeferTakesOffAPartialExcludedUnit(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(3, "feat(chart): a network policy", "", map[string]string{"chart/templates/policy.yaml": "kind: NetworkPolicy\n"}) + mc5 := f.pr(5, "feat(chart): remove the policy, and test values", "", map[string]string{"chart/templates/policy.yaml": "", "chart/tests/values_test.yaml": "suite: values\n"}) + gh := &fakeGH{} + handBatch(f, gh, mc5) + if code, stdout, _ := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "only part of it applies") { + t.Fatalf("check --cut = %d, want the partial unit to fail it:\n%s", code, stdout) + } + if code, _, stderr := f.exec(gh, "run", "--line", "release-1.1", "--defer", "5", "--reason", "only part of it applies", "--push"); code != 0 || prState(gh.find(90)) != "closed" { + t.Errorf("run --defer 5 --push = %d, #90 %s; want the unit taken off and the batch closed:\n%s", code, prState(gh.find(90)), stderr) + } +} + +// A stored wait on an original merged after the one that waits cannot be +// what a run wrote. +func TestForwardWaitIsRefused(t *testing.T) { + f := newFixture(t) + f.lines() + f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"}) + f.pr(7, "fix(app): seven", "", map[string]string{"app/seven.txt": "7\n"}) + sha := f.git("rev-parse", "origin/main") + rec := lineRecord{Version: recordVersion, Line: "release-1.1", Report: strings.Repeat("b", 64), + Built: &builtFrom{Main: sha, Line: sha, Policy: sha, Batch: sha, At: "2026-10-26T06:00:00Z", Options: runOptions{Mode: modeContinue}}, + Deferred: []deferredEntry{{Original: 5, Files: []string{}, Needs: []int{7}}}, Dropped: []droppedEntry{}, Stale: []int{}} + f.git("push", "-q", "origin", "origin/release-1.1:refs/heads/"+batchBranch1) + gh := &fakeGH{prs: []pullRequest{f.batchPR(90, "release-1.1", batchBranch1, "batch\n\n"+recordBlock(rec)+"\n")}} + if code, _, stderr := f.exec(gh, "check", "--line", "release-1.1"); code != 2 || !strings.Contains(stderr, "#5 wait on #7") { + t.Errorf("check = %d, want 2 for a wait on a later original:\n%s", code, stderr) + } +} + +// The workflow exception looks only at what still keeps the original from +// the line: a workflow line its prerequisite wrote, which the open batch +// carries though that prerequisite is labelled backport/skip, does not +// stop it, in check as in run. +func TestWorkflowExceptionIgnoresWhatIsThere(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a workflow", "", map[string]string{".github/workflows/release.yaml": "on: push\njobs: {}\n"}) + f.lines() + f.pr(3, "feat(ci): test the changelog", "", map[string]string{"hack/changelog.bats": "@test x {\n true\n}\n"}) + f.pr(4, "fix(ci): run the release on pushes", "", map[string]string{".github/workflows/release.yaml": "on: [push]\njobs: {}\n"}) + f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"}) + f.pr(7, "fix(ci): run the release on tags too", "", map[string]string{".github/workflows/release.yaml": "on: [push, tag]\njobs: {}\n", "hack/changelog.bats": "@test x {\n false\n}\n"}) + gh := &fakeGH{} + mustRun(t, f, gh, "run", "--line", "release-1.1", "--push") + f.git("switch", "-q", "main") + reopen(f, gh) + gh.labels = map[string][]int{labelSkip: {4}} + if code, stdout, stderr := f.exec(gh, "run", "--line", "release-1.1", "--continue", "--no-workflow-files", "--push"); code != 0 || !strings.Contains(stdout, `{"original":7,"files":[".github/workflows/release.yaml"],"workflows":true}`) { + t.Fatalf("scheduled run = %d, want #7 waiting for a maintainer:\n%s\n%s", code, stdout, stderr) + } + f.git("switch", "-q", "main") + reopen(f, gh) + if code, stdout, stderr := f.exec(gh, "check", "--line", "release-1.1", "--cut"); code != 1 || !strings.Contains(stdout, "#7 changes .github/workflows/ and waits on batch PR #90") { + t.Errorf("check --cut = %d, want #7 waiting, as run has it:\n%s\n%s", code, stdout, stderr) + } +} + +// An empty pick leaves a deletion undone that only a merge of main into the +// original's branch made: the original is not on the line. +func TestEmptyPickLeavesADeletionUndone(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a and b", "", map[string]string{"app/a.txt": "old feature\n", "app/b.txt": "old\n"}) + f.lines() + f.onLine("release-1.1", "fix(app): b, already", map[string]string{"app/b.txt": "fixed\n"}) + f.git("switch", "-q", "-c", "pr-5", "main") + f.apply(map[string]string{"app/b.txt": "fixed\n"}) + f.git("commit", "-q", "-m", "b") + f.git("switch", "-q", "main") + f.apply(map[string]string{"app/main.txt": "moved\n"}) + f.git("commit", "-q", "-m", "chore: move main along") + f.git("switch", "-q", "pr-5") + f.git("merge", "-q", "--no-ff", "--no-commit", "main") + f.git("rm", "-q", "app/a.txt") + f.git("commit", "-q", "-m", "Merge branch 'main' into pr-5") + f.git("switch", "-q", "main") + f.git("merge", "-q", "--no-ff", "-m", "fix(app): b, and a goes (#5)", "pr-5") + f.git("push", "-q", "origin", "main") + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || strings.Contains(stdout, `"on-line":[5]`) || !strings.Contains(stdout, `"parked":[{"original":5,`) { + t.Errorf("run = %d, want #5 parked as partial, not on the line:\n%s\n%s", code, stdout, stderr) + } +} + +// A file the line already has as the original makes it builds on nothing +// the line lacks: #5 fixes a.txt, which the line fixed the same way, and +// b.txt, so it picks whole and is not parked on the feat that wrote a.txt. +func TestAlreadyAppliedFileBuildsOnNothing(t *testing.T) { + f := newFixture(t) + f.squash(2, "chore: a and b", "", map[string]string{"app/a.txt": appFile, "app/b.txt": "old\n"}) + f.lines() + f.pr(3, "feat(app): a loud two", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO", 1)}) + f.onLine("release-1.1", "fix(app): two, the line's way", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1)}) + f.pr(5, "fix(app): an excited two, and b", "", map[string]string{"app/a.txt": strings.Replace(appFile, "two", "TWO!", 1), "app/b.txt": "fixed\n"}) + if code, stdout, stderr := f.exec(nil, "run", "--line", "release-1.1"); code != 0 || strings.Contains(stdout, `"parked":[{`) || !strings.Contains(stdout, `"picked":[5]`) { + t.Errorf("run = %d, want #5 picked, not parked:\n%s\n%s", code, stdout, stderr) + } +} diff --git a/cmd/backport-batch/policy_test.go b/cmd/backport-batch/policy_test.go index 920c18a0b..2cd7d20f3 100644 --- a/cmd/backport-batch/policy_test.go +++ b/cmd/backport-batch/policy_test.go @@ -176,7 +176,7 @@ func TestPolicyPageMatchesCode(t *testing.T) { t.Errorf("the page's block gives %d released lines, want 2 written out", pol.released) } - rule2 := regexp.MustCompile("(?m)^2\\. Its title type is (.*)$").FindStringSubmatch(page) + rule2 := regexp.MustCompile(`(?m)^2\. Its title type is (.*)$`).FindStringSubmatch(page) if rule2 == nil { t.Fatal("the page has no rule 2") } @@ -187,7 +187,7 @@ func TestPolicyPageMatchesCode(t *testing.T) { if !slices.Equal(types, takenTypes) { t.Errorf("rule 2 on the page takes %v, the code %v", types, takenTypes) } - never := regexp.MustCompile("(?m)^### What waits for the next minor\\n\\n(.*)$").FindStringSubmatch(page) + never := regexp.MustCompile(`(?m)^### What waits for the next minor\n\n(.*)$`).FindStringSubmatch(page) if never == nil { t.Fatal("the page has no list of what never goes into a patch release") } diff --git a/cmd/backport-batch/record.go b/cmd/backport-batch/record.go index 4ab43933e..37dd7946a 100644 --- a/cmd/backport-batch/record.go +++ b/cmd/backport-batch/record.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "maps" "regexp" "slices" "strings" @@ -34,7 +35,8 @@ import ( // carries a record of its own. // lineRecord is the one line of JSON in the block. Version 1, the block -// before it had a version, holds only line and deferred. +// before it had a version, holds only line and deferred; version 3 adds what +// is parked and which units on the batch do only part of their original. type lineRecord struct { Version int `json:"version"` Line string `json:"line"` @@ -44,6 +46,115 @@ type lineRecord struct { Deferred []deferredEntry `json:"deferred"` Dropped []droppedEntry `json:"dropped"` Stale []int `json:"stale"` + Parked []parkedEntry `json:"parked,omitempty"` + Partial []int `json:"partial,omitempty"` + // Changed are the picks that differ from their originals. + Changed []int `json:"changed,omitempty"` +} + +// recordVersion is the version a run writes. +const recordVersion = 3 + +// lineRecordV2 is version 2, which has none of version 3's keys. +type lineRecordV2 struct { + Version int `json:"version"` + Line string `json:"line"` + Built *builtFrom `json:"built,omitempty"` + Report string `json:"report,omitempty"` + Closing *closing `json:"closing"` + Deferred []deferredEntry `json:"deferred"` + Dropped []droppedEntry `json:"dropped"` + Stale []int `json:"stale"` +} + +// The kinds of park: an original that builds on what never reaches the +// line, and one whose clean pick would leave part of it without effect. +const ( + parkDependency = "dependency" + parkPartial = "partial" +) + +// parkedEntry is an original the batch leaves out because it cannot land on +// the line whole: what it needs that never comes, or the paths it would +// leave without effect. announced says notify has named it on the open +// batch, for exactly this entry. +type parkedEntry struct { + Original int `json:"original"` + Merge string `json:"merge"` + Kind string `json:"kind"` + Needs []int `json:"needs,omitempty"` + Commits []string `json:"commits,omitempty"` + Files []string `json:"files,omitempty"` + Announced bool `json:"announced,omitempty"` +} + +// parkEvent is what makes a park the same park from one record to the +// next: its merge, kind, prerequisites, commits and files. A park whose +// event changed is news, and notify names it again. +func parkEvent(p parkedEntry) string { + data, _ := json.Marshal(struct { + Merge string `json:"merge"` + Kind string `json:"kind"` + Needs []int `json:"needs"` + Commits []string `json:"commits"` + Files []string `json:"files"` + }{p.Merge, p.Kind, p.Needs, p.Commits, p.Files}) + return string(data) +} + +// carryAnnounced marks each park announced that the carrier's record holds +// announced as the same event, a closed carrier's too, so a new batch does +// not name again what an earlier one named. +func carryAnnounced(parked []parkedEntry, carrier *lineRecord) []parkedEntry { + out := slices.Clone(parked) + for i := range out { + out[i].Announced = false + } + if carrier == nil { + return out + } + for i := range out { + out[i].Announced = slices.ContainsFunc(carrier.Parked, func(c parkedEntry) bool { + return c.Announced && c.Original == out[i].Original && parkEvent(c) == parkEvent(out[i]) + }) + } + return out +} + +// validate holds a parked entry to what a run writes. Its errors read after +// "the block" or "report ". +func (p parkedEntry) validate() error { + sortedUnique := func(n int, less func(i int) bool) bool { + for i := 1; i < n; i++ { + if !less(i) { + return false + } + } + return true + } + switch { + case checkSHA(p.Merge) != nil: + return fmt.Errorf("parks #%d by merge %q", p.Original, p.Merge) + case !sortedUnique(len(p.Needs), func(i int) bool { return p.Needs[i-1] < p.Needs[i] }) || slices.ContainsFunc(p.Needs, func(n int) bool { return n <= 0 || n == p.Original }): + return fmt.Errorf("parks #%d for originals %v, which are not what a run writes", p.Original, p.Needs) + case !sortedUnique(len(p.Commits), func(i int) bool { return p.Commits[i-1] < p.Commits[i] }) || slices.ContainsFunc(p.Commits, func(c string) bool { return checkSHA(c) != nil }): + return fmt.Errorf("parks #%d for commits %q, which are not what a run writes", p.Original, p.Commits) + case !sortedUnique(len(p.Files), func(i int) bool { return p.Files[i-1] < p.Files[i] }) || !slices.Equal(p.Files, cleanFiles(p.Files)): + return fmt.Errorf("parks #%d for files %q, which are not what a run writes", p.Original, p.Files) + } + switch p.Kind { + case parkPartial: + if len(p.Files) == 0 || len(p.Needs)+len(p.Commits) > 0 { + return fmt.Errorf("parks #%d as partial with no files it leaves without effect, or with what it needs", p.Original) + } + case parkDependency: + if len(p.Needs)+len(p.Commits) == 0 || len(p.Files) > 0 { + return fmt.Errorf("parks #%d for a dependency it does not name, or with files", p.Original) + } + default: + return fmt.Errorf("parks #%d as %q, which is no kind of park", p.Original, p.Kind) + } + return nil } // builtFrom is what a classification was made of; batch is the head it @@ -173,9 +284,16 @@ func parseRecord(body string, l line) (*lineRecord, error) { err = decodeStrict(blocks[0], &v1) rec = lineRecord{Version: 1, Line: v1.Line, Deferred: v1.Deferred} case 2: + var v2 lineRecordV2 + err = decodeStrict(blocks[0], &v2) + rec = lineRecordWithout(v2) + case 3: err = decodeStrict(blocks[0], &rec) default: - return nil, fmt.Errorf("the %s block is version %d, which this backport-batch does not read; it reads 1 and 2", deferredFence, version) + return nil, fmt.Errorf("the %s block is version %d, which this backport-batch does not read; it reads 1 to %d", deferredFence, version, recordVersion) + } + if err == nil && version < 3 && slices.ContainsFunc(rec.Deferred, func(d deferredEntry) bool { return d.Needs != nil }) { + err = fmt.Errorf("version %d has no waits on other originals", version) } if err != nil { return nil, fmt.Errorf("reading the %s block: %w", deferredFence, err) @@ -213,7 +331,7 @@ func (rec *lineRecord) validate(l line) error { // A conflict names the commit that failed and its files, a deferral // its reason, and a wait its workflow files. kinds := 0 - for _, is := range []bool{d.Commit != "" || len(d.Files) > 0 && !d.Workflows && d.Reason == "", d.Reason != "", d.Workflows} { + for _, is := range []bool{d.Commit != "" || len(d.Files) > 0 && !d.Workflows && d.Reason == "" && len(d.Needs) == 0, d.Reason != "", d.Workflows, len(d.Needs) > 0} { if is { kinds++ } @@ -231,6 +349,13 @@ func (rec *lineRecord) validate(l line) error { return fmt.Errorf("names files of #%d that are not as a run writes them: %q", d.Original, d.Files) case d.More < 0: return fmt.Errorf("counts %d more files for #%d", d.More, d.Original) + case len(d.Needs) > 0 && (len(d.Files) > 0 || d.More != 0): + return fmt.Errorf("has #%d wait on other originals with files of its own", d.Original) + } + for i, n := range d.Needs { + if n <= 0 || n == d.Original || i > 0 && d.Needs[i-1] >= n { + return fmt.Errorf("has #%d wait on %v, which is not what a run writes", d.Original, d.Needs) + } } } for _, d := range rec.Dropped { @@ -244,6 +369,14 @@ func (rec *lineRecord) validate(l line) error { return fmt.Errorf("drops #%d for %q, which is no reason a run writes", d.Original, d.Reason) } } + for _, p := range rec.Parked { + if err := once(p.Original); err != nil { + return err + } + if err := p.validate(); err != nil { + return err + } + } staleSeen := map[int]bool{} for _, n := range rec.Stale { if n <= 0 || staleSeen[n] { @@ -251,6 +384,42 @@ func (rec *lineRecord) validate(l line) error { } staleSeen[n] = true } + changedSeen := map[int]bool{} + for _, n := range rec.Changed { + if n <= 0 || changedSeen[n] { + return fmt.Errorf("lists %d as a changed pick more than once or not at all", n) + } + changedSeen[n] = true + } + if err := rec.waitsHold(); err != nil { + return err + } + // What is on the batch is neither deferred nor parked. + left := map[int]bool{} + for _, d := range rec.Deferred { + left[d.Original] = true + } + for _, p := range rec.Parked { + left[p.Original] = true + if slices.Contains(p.Commits, p.Merge) { + return fmt.Errorf("parks #%d for its own merge", p.Original) + } + } + for _, list := range [][]int{rec.Partial, rec.Changed} { + for _, n := range list { + if left[n] { + return fmt.Errorf("has #%d on the batch and left off it at once", n) + } + } + } + partialSeen := map[int]bool{} + for _, n := range rec.Partial { + // A unit that does only part of its original is still on the batch. + if !staleSeen[n] || partialSeen[n] { + return fmt.Errorf("lists #%d as a partial unit more than once, or not as still on the batch", n) + } + partialSeen[n] = true + } if rec.Version < 2 { return nil } @@ -297,6 +466,79 @@ func (b *builtFrom) validate() error { var runURLRE = regexp.MustCompile(`^https://[^/\s]+/[^/\s]+/[^/\s]+/actions/runs/[0-9]+(/attempts/[0-9]+)?$`) +// waitsHold holds what waits and parks need to what the record says of the +// originals they need: a wait needs ones still to come, so none parked or +// dropped, a park ones that never come, so none deferred, and no wait ends +// up waiting on itself. Its errors read after "the block". +func (rec *lineRecord) waitsHold() error { + deferred, never := map[int][]int{}, map[int]bool{} + for _, d := range rec.Deferred { + deferred[d.Original] = d.Needs + } + for _, p := range rec.Parked { + never[p.Original] = true + } + for _, d := range rec.Dropped { + never[d.Original] = true + } + for _, d := range rec.Deferred { + for _, n := range d.Needs { + if never[n] { + return fmt.Errorf("has #%d wait on #%d, which never comes", d.Original, n) + } + } + } + for _, p := range rec.Parked { + for _, n := range p.Needs { + if _, ok := deferred[n]; ok { + return fmt.Errorf("parks #%d for #%d, which is still to come", p.Original, n) + } + } + } + needs := map[int][]int{} + for _, d := range rec.Deferred { + needs[d.Original] = d.Needs + } + for _, p := range rec.Parked { + needs[p.Original] = p.Needs + } + return waitCycle(needs) +} + +// waitCycle refuses waits that lead back to where they started, which no +// run writes: a prerequisite merged before what needs it. +func waitCycle(needs map[int][]int) error { + state := map[int]int{} // 1 on the path, 2 done + var visit func(n int) error + visit = func(n int) error { + switch state[n] { + case 1: + return fmt.Errorf("has #%d wait on itself through what it waits on", n) + case 2: + return nil + } + state[n] = 1 + for _, f := range needs[n] { + if err := visit(f); err != nil { + return err + } + } + state[n] = 2 + return nil + } + for _, n := range slices.Sorted(maps.Keys(needs)) { + if err := visit(n); err != nil { + return err + } + } + return nil +} + +// lineRecordWithout is a version 2 record as the newest version holds it. +func lineRecordWithout(v2 lineRecordV2) lineRecord { + return lineRecord{Version: v2.Version, Line: v2.Line, Built: v2.Built, Report: v2.Report, Closing: v2.Closing, Deferred: v2.Deferred, Dropped: v2.Dropped, Stale: v2.Stale} +} + // recordBlock is the block that holds rec. One line of JSON: encoding/json // escapes newlines and <, > and &, so nothing in a file name or a reason can // end the fence or open HTML. diff --git a/cmd/backport-batch/record_test.go b/cmd/backport-batch/record_test.go index 13ab6cc79..4bbddc58c 100644 --- a/cmd/backport-batch/record_test.go +++ b/cmd/backport-batch/record_test.go @@ -18,6 +18,7 @@ package main import ( "fmt" + "slices" "strings" "testing" ) @@ -272,7 +273,7 @@ func TestRecordNamesItsInputs(t *testing.T) { want := builtFrom{Main: f.git("rev-parse", "origin/main"), Line: f.git("rev-parse", "origin/release-1.0"), Newer: f.git("rev-parse", "origin/release-1.1"), Policy: f.git("rev-parse", "origin/main"), Batch: f.remoteHead(branch), Tool: f.tool, Run: "https://github.com/cozystack/cozystack/actions/runs/123/attempts/2", At: "2026-10-26T06:00:00Z", Options: runOptions{NoWorkflowFiles: true, Mode: modeNew}} - if rec.Version != 2 || *rec.Built != want || bodyReportOf(t, stdout, l).doc.Built != want || rec.Report != bodyReportOf(t, stdout, l).doc.ID { + if rec.Version != recordVersion || *rec.Built != want || bodyReportOf(t, stdout, l).doc.Built != want || rec.Report != bodyReportOf(t, stdout, l).doc.ID { t.Errorf("record = %+v, built %+v; want built %+v and the report it names", rec, rec.Built, want) } @@ -366,3 +367,67 @@ func TestPRStateNamesOnlyKnownStates(t *testing.T) { } } } + +// v3 is v2 as version 3, with extra after the stale list. +func v3(extra string) string { + return strings.Replace(v2(extra), `"version":2`, `"version":3`, 1) +} + +// Version 3 adds what is parked and which units do only part of their +// original, each held to what a run writes, and version 2 has neither. +func TestRecordVersion3(t *testing.T) { + l, _ := parseLine("release-1.1") + block := func(json string) string { return "text\n\n" + deferredFence + "\n" + json + "\n```\n" } + sha, other := strings.Repeat("a", 40), strings.Repeat("c", 40) + parked := `,"parked":[{"original":10,"merge":"` + sha + `","kind":"partial","files":["a","b"]},{"original":11,"merge":"` + sha + `","kind":"dependency","needs":[3,4],"commits":["` + other + `"],"announced":true}],"partial":[9]` + waits := strings.Replace(v3(parked), `{"original":6,"files":[],"reason":"fails the e2e"}`, `{"original":6,"files":[],"needs":[3,5]}`, 1) + rec, err := parseRecord(block(waits), l) + if err != nil || rec.Version != 3 || len(rec.Parked) != 2 || !rec.Parked[1].Announced || !slices.Equal(rec.Partial, []int{9}) || !slices.Equal(rec.deferred()[6].Needs, []int{3, 5}) { + t.Fatalf("parseRecord of a whole v3 record = %+v, %v", rec, err) + } + for name, json := range map[string]string{ + "version 1 with a wait": `{"line":"release-1.1","deferred":[{"original":6,"files":[],"needs":[3,5]}]}`, + "a wait with files": strings.Replace(waits, `"files":[],"needs":[3,5]`, `"files":["a"],"needs":[3,5]`, 1), + "a wait that is a conflict": strings.Replace(waits, `"files":[],"needs":[3,5]`, `"commit":"`+sha+`","files":[],"needs":[3,5]`, 1), + "a wait on itself": strings.Replace(waits, `"needs":[3,5]`, `"needs":[3,6]`, 1), + "an unsorted wait": strings.Replace(waits, `"needs":[3,5]`, `"needs":[5,3]`, 1), + "a wait on nothing": strings.Replace(waits, `"needs":[3,5]`, `"needs":[0]`, 1), + "a wait cycle": strings.Replace(strings.Replace(waits, `"needs":[3,5]`, `"needs":[12]`, 1), `"deferred":[`, `"deferred":[{"original":12,"files":[],"needs":[6]},`, 1), + "a wait on a parked one": strings.Replace(waits, `"needs":[3,5]`, `"needs":[10]`, 1), + "a wait on a dropped one": strings.Replace(waits, `"needs":[3,5]`, `"needs":[8]`, 1), + "a park on a deferred one": strings.Replace(waits, `"needs":[3,4]`, `"needs":[3,5]`, 1), + "a partial unit also deferred": strings.Replace(strings.Replace(waits, `"stale":[9]`, `"stale":[9,5]`, 1), `"partial":[9]`, `"partial":[9,5]`, 1), + "a changed pick also parked": v3(parked + `,"changed":[10]`), + "a changed pick also deferred": v3(parked + `,"changed":[5]`), + "a park on its own merge": strings.Replace(waits, `"commits":["`+other+`"]`, `"commits":["`+sha+`"]`, 1), + "a park cycle": strings.Replace(strings.Replace(waits, `"needs":[3,4]`, `"needs":[3,13]`, 1), `"parked":[`, `"parked":[{"original":13,"merge":"`+sha+`","kind":"dependency","needs":[11]},`, 1), + } { + if _, err := parseRecord(block(json), l); err == nil { + t.Errorf("%s: parseRecord accepted it", name) + } + } + for name, json := range map[string]string{ + "version 2 with a park": v2(parked), + "version 4": strings.Replace(v3(""), `"version":3`, `"version":4`, 1), + "a park of a deferred original": v3(strings.Replace(parked, `"original":10`, `"original":5`, 1)), + "a park of a dropped original": v3(strings.Replace(parked, `"original":10`, `"original":8`, 1)), + "a partial park with no files": v3(strings.Replace(parked, `"files":["a","b"]`, `"files":[]`, 1)), + "a partial park that needs": v3(strings.Replace(parked, `"files":["a","b"]`, `"files":["a"],"needs":[3]`, 1)), + "unsorted files": v3(strings.Replace(parked, `"files":["a","b"]`, `"files":["b","a"]`, 1)), + "a file that drives a terminal": v3(strings.Replace(parked, `"files":["a","b"]`, `"files":["a\u001b[2J"]`, 1)), + "a dependency park with no need": v3(strings.Replace(strings.Replace(parked, `"needs":[3,4],`, ``, 1), `"commits":["`+other+`"],`, ``, 1)), + "unsorted needs": v3(strings.Replace(parked, `"needs":[3,4]`, `"needs":[4,3]`, 1)), + "a park that needs itself": v3(strings.Replace(parked, `"needs":[3,4]`, `"needs":[3,11]`, 1)), + "a short commit": v3(strings.Replace(parked, `"commits":["`+other+`"]`, `"commits":["abc"]`, 1)), + "a short merge": v3(strings.Replace(parked, `"merge":"`+sha+`","kind":"partial"`, `"merge":"abc","kind":"partial"`, 1)), + "an unknown kind": v3(strings.Replace(parked, `"kind":"partial"`, `"kind":"later"`, 1)), + "a partial unit not on the batch": v3(strings.Replace(parked, `"partial":[9]`, `"partial":[10]`, 1)), + "a partial unit twice": v3(strings.Replace(parked, `"partial":[9]`, `"partial":[9,9]`, 1)), + "a changed pick twice": v3(parked + `,"changed":[4,4]`), + "a changed pick that is none": v3(parked + `,"changed":[0]`), + } { + if _, err := parseRecord(block(json), l); err == nil { + t.Errorf("%s: parseRecord accepted it", name) + } + } +} diff --git a/cmd/backport-batch/report.go b/cmd/backport-batch/report.go index 46e37c955..27fd8860c 100644 --- a/cmd/backport-batch/report.go +++ b/cmd/backport-batch/report.go @@ -23,6 +23,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "os" "regexp" "runtime/debug" @@ -43,7 +44,9 @@ import ( const ( reportFence = "```backport-batch-report" reportMarker = " + +# Cozystack v1.7.1 + +Cozystack v1.7.1 is a patch release on the v1.7 line. It fixes the COSI driver bug that could leave freshly granted S3 keys rejected for good, makes marketplace taps digest-pinned, makes MariaDB, PostgreSQL and ClickHouse backups report their real outcome, and repairs several tenant-facing issues: tenant cluster metrics, SecurityGroups on Kubernetes workers, Redis with a single replica, the Proxmox cloud-controller-manager and the tcp-balancer. Read the Migration and Upgrades section before upgrading. + +## Fixes + +* **SeaweedFS COSI driver v0.4.0**: bucket users granted together could get S3 keys that never work (`403 InvalidAccessKeyId`) because the v0.3.1 driver rewrote a legacy IAM file the S3 gateways were migrating. The driver is bumped to v0.4.0, which writes each grant to a per-identity IAM file. Keys already lost on running clusters stay broken: remove the user from the Bucket, wait until its BucketAccess is gone, and add it back ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5284, backport #5290). +* **Marketplace taps are pinned to a digest**: a tapped repository no longer follows a mutable tag, so moving the tag upstream cannot change what runs in the cluster. `tap connect` accepts `oci://...@sha256:...` references and an optional keyless cosign verify policy, anchored to an exact identity and preserved across re-connects; with the verify flags the CLI verifies the signature and pins the verified digest at tap time. Community index entries can record a digest too ([**@IvanHunters**](https://github.com/IvanHunters) in #5082, backport #5246). +* **Tenant cluster metrics reach the tenant's Monitoring**: the vmagent of tenant Kubernetes clusters now writes to the tenant's vminsert and labels series with the owning tenant instead of `cozy-monitoring`; before, no tenant cluster metric reached the tenant's Monitoring ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5024, backport #5246). +* **SecurityGroups apply to Kubernetes worker nodes**: the virt-launcher pods of Kubernetes worker nodes are now resolved to their Kubernetes application and marked as managed by Cozystack, so a SecurityGroup attached to a Kubernetes cluster applies to its workers, not only to its control plane ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5124, backport #5246). +* **Membership-only SecurityGroups**: a SecurityGroup with attachments and no rules no longer leaves an invalid CiliumNetworkPolicy behind; its backing policy is accepted by Cilium and still allows or denies nothing ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5133, backport #5246). +* **Proxmox CCM and CSI images**: the Proxmox cloud-controller-manager runs v0.16.1 and the Proxmox CCM and CSI driver images are pinned by digest. The v0.15.0 tag was re-pushed upstream to a build that panics on start, which left every new Proxmox worker with the uninitialized taint ([**@themoriarti**](https://github.com/themoriarti) in #4917, backport #5246). +* **Autoscaler capacity of KubernetesNodes pools**: on the kubevirt substrate the cluster-autoscaler capacity annotation is planned on the cores the VM actually gets, so a leading-zero `resources.cpu` such as `"010"` (decoded as YAML, eight cores) no longer yields a wrong capacity. No pool rolls ([**@themoriarti**](https://github.com/themoriarti) in #4974, backport #5246). +* **Tenant quota admission counts extended resources**: admission now reads the parent pool's usage of extended resources and hugepages under the `requests.*` key the controller counts, so a child tenant can no longer get a share the parent has already used ([**@lexfrei**](https://github.com/lexfrei) in #5184, backport #5246). +* **Postgres backup jobs fail as soon as the CloudNativePG Backup fails**: a Postgres `BackupJob` now fails with the recorded error instead of after 30 minutes, and reports a `WaitingForCNPGBackup` condition with the Backup's phase while it waits ([**@scooby87**](https://github.com/scooby87) in #5211, backport #5246). +* **MariaDB backups and restores report real failures**: a `BackupJob` or `RestoreJob` whose operator Job failed now ends Failed instead of Succeeded, and a failed backup no longer creates a Ready Backup without an archive ([**@scooby87**](https://github.com/scooby87) in #5207, backport #5246). +* **ClickHouse restores use the right archive**: a `RestoreJob` restores the archive its Backup was taken as, not the newest one. New Backups record the archive name; Backups taken before this release are restored best effort from the newest archive stamped no later than their `takenAt`, with a warning in the restore pod log. A ClickHouse `BackupJob` created under a name whose Backup still exists now fails instead of reporting that Backup as its result ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5208, backport #5246). +* **Backup S3 endpoint changes are followed**: the backup controller re-renders the Velero BSL, the default backup strategies and the platform bucket's credentials Secret when their S3 endpoint is stale, so after the platform bucket's BucketAccess is re-granted for a new `publishing.host`, backups move to the new endpoint without forcing releases by hand ([**@scooby87**](https://github.com/scooby87) in #5230, backport #5246). +* **Backup clients run on a dedicated image**: the ClickHouse (Altinity) and RabbitMQ backup strategy Pods now run a small `backup-http-client` image instead of the 349 MB `platform-migrations` image, so a platform migration change no longer re-digests the backup clients. Air-gapped mirrors need the new `ghcr.io/cozystack/cozystack/backup-http-client` repository ([**@androndo**](https://github.com/androndo) in #4756, backport #5246). +* **PostgreSQL system databases**: `databases` refuses `postgres`, `template0` and `template1`, and the init job never runs `DROP DATABASE` on them, which used to fail the init job on every upgrade and leave the HelmRelease in `UpgradeFailed` ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5155, backport #5246). +* **PostgreSQL init script is stable**: the declared databases render in a stable order, so an upgrade that leaves the databases unchanged no longer rewrites the init-script Secret ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5091, backport #5246). +* **Redis with a single replica**: an instance with `replicas: 1` runs one Sentinel instead of three, without a Sentinel PodDisruptionBudget, so node drains are no longer blocked by it; instances with two replicas or more are unchanged ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5151, backport #5246). +* **MariaDB operator on a custom cluster domain**: the operator uses the platform's `networking.clusterDomain` instead of a hard-coded `cozy.local`, so replication and User/Database/Grant reconciliation work on clusters with a non-default domain ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5123, backport #5246). +* **tcp-balancer no longer OOMs**: HAProxy takes its fd and thread caps from the replica's own memory and CPU limits instead of the node, so a `t1.nano` balancer is no longer OOMKilled on nodes with many CPUs ([**@yankawai**](https://github.com/yankawai) in #5005, backport #5246). +* **etcd alert noise**: `etcdHighNumberOfFailedGRPCRequests` counts only the gRPC codes the etcd mixin counts, so the Canceled watch streams of a stopping Kubernetes API server no longer fire it on every member ([**@yankawai**](https://github.com/yankawai) in #4985, backport #5246). +* **Cilium IPAM hook on larger clusters**: the hook passes the CiliumNode list to `jq` on stdin and no longer fails with "Argument list too long" on clusters with more than a few nodes, which blocked the cilium upgrade and every package depending on networking ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5212, backport #5246). +* **OIDC bootstrap Job with many users**: the Job no longer fails to start for a tenant cluster with about 240 or more `oidc.users`; the bindings now come from a ConfigMap the Job mounts ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5213, backport #5246). +* **Dashboard tenant list links to the tenant page**: the tenant name in the tenant list opens its detail page, where Delete lives, and top-level tenants get their Edit button back ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #4004, backport #5246). +* **Dead `registries` platform value removed**: the platform chart no longer has the `registries` value or the `patch-containerd` Secret, which nothing read. Mirrors for tenant workers are the per-pool `talos.registryMirrors` of `KubernetesNodes` ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5190, backport #5246). + +## Security + +* **Marketplace digest pinning and signature verification**: see the marketplace entry under Fixes (#5082). + +## Development, Testing, and CI/CD + +* **Chainsaw asserts survive transient API errors**: an e2e assert keeps polling through an error such as an etcd leader change instead of failing the suite early ([**@lexfrei**](https://github.com/lexfrei) in #4867, backport #5246). +* **Proxmox e2e hygiene**: the Proxmox tenant kubeconfig is kept per run and removed afterwards, the HelmRelease guard reads the `KubernetesNodes` pool release, and the leak allowlist is validated against every suite layout ([**@themoriarti**](https://github.com/themoriarti) in #4971, #4970 and #4972, backports #5246). +* **Frozen-line builds start with `GITHUB_TOKEN`**: the initial packages build dispatched when a release line is frozen now runs ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #4888, backport #5246). +* **Unit tests after the release image restamp**: two unit tests that read fields the release build rewrites stay green on `release-1.7`; this is a release-branch-only fix ([**@themoriarti**](https://github.com/themoriarti) in #5179). + +## Documentation + +* **[website] Documentation**: the v1.7 docs gain the guides that lived only in the cozystack repository, the pending v1.7 docs PRs, a DRBD note for the replicated storage class, a refreshed Chainsaw e2e guide, restored legacy URLs and a grouping of older doc versions ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#763, cozystack/website#782, cozystack/website#675, cozystack/website#611, cozystack/website#775 and cozystack/website#767). +* **[website] Blog and demo**: a full-stack observability post for v1.7, the Cozystack 1.7 announcement, a three-board Raspberry Pi 5 post, and a collapsible navbar with a closable hint on the live demo page ([**@tym83**](https://github.com/tym83) in cozystack/website#790, cozystack/website#801 and cozystack/website#802; [**@kvaps**](https://github.com/kvaps) in cozystack/website#789). +* **[website] Pages deploy**: the deploy workflow drops the Dart Sass install ([**@kvaps**](https://github.com/kvaps) in cozystack/website#798). +* **[ansible-cozystack] Installer v1.7.0**: tracks the Cozystack installer image at v1.7.0, released in ansible-cozystack v1.7.0 ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#89). + +## Migration and Upgrades + +Platform migrations do not change in this release: `migrations.targetVersion` stays at 61. + +* **Remove system databases from PostgreSQL `databases` before upgrading (#5155).** A release that declares `postgres`, `template0` or `template1` under `databases` is refused at render; remove the entry first. A system database a previous version took over is handed back to the superuser on the next upgrade: the event trigger is removed, ownership and the `public` schema are restored, and its `_admin` and `_readonly` roles are dropped. +* **Tenant quota updates can be rejected after upgrading (#5184).** Admission now counts the parent's usage of extended resources and hugepages, so an update to an existing tenant can be rejected when its quota for these resources exceeds the parent's remaining budget, including an update that leaves the quota unchanged. +* **Air-gapped mirrors need `backup-http-client` (#4756).** Mirror `ghcr.io/cozystack/cozystack/backup-http-client` before upgrading, or ClickHouse and RabbitMQ backup Pods cannot pull their image. +* **tcp-balancer pods roll once (#5005).** The new HAProxy `global` section changes the rendered config; `maxconn` follows the memory limit at about 16 connections per MiB (2035 on `t1.nano`). +* **Redis scaled to one replica loses its Sentinel PDB (#5151).** A post-upgrade hook deletes the Sentinel PodDisruptionBudget the operator created earlier for an instance with `replicas: 1`; scaling an instance between 1 and 2 replicas scales its Sentinel Deployment between 1 and 3. +* **Backups created by failed MariaDB runs are not cleaned up (#5207).** Backups made by failed runs before this fix are not removed; restoring from one can fail or select another dump from the shared prefix. +* **Keys lost to the COSI grant bug stay lost (#5284).** Re-add the affected user to the Bucket as described in the Fixes entry. +* **`registries` in the platform Package is ignored (#5190).** A Package that still sets it keeps installing; the `patch-containerd` Secret in `cozy-system` is deleted by the release. +* **SecurityGroups now reach Kubernetes workers (#5124).** A SecurityGroup attached to a `Kubernetes` application starts applying to its worker nodes after the upgrade, so review attached groups before upgrading. + +## Contributors + +We'd like to thank all contributors who made this release possible: + +* [**@androndo**](https://github.com/androndo) +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@kvaps**](https://github.com/kvaps) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@scooby87**](https://github.com/scooby87) +* [**@themoriarti**](https://github.com/themoriarti) +* [**@tym83**](https://github.com/tym83) +* [**@yankawai**](https://github.com/yankawai) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.7.0...v1.7.1 diff --git a/docs/release/backport-mechanics.md b/docs/release/backport-mechanics.md index 70da94e85..3474bffea 100644 --- a/docs/release/backport-mechanics.md +++ b/docs/release/backport-mechanics.md @@ -14,7 +14,7 @@ go build ./cmd/backport-batch ./backport-batch check --line release-X.Y --cut ``` -[`backport-batch check`](../../cmd/backport-batch/README.md) applies the rules to every pull request merged into `main` since the line branched, and looks for each one they take on the line and then on the open batch pull request: by picks whose `-x` lines name its merge commit or every commit of its own, by a pick that would change nothing, and, on commits from before the policy's batch-only day, by what their text says, such as `Backport of #N`. With `--cut` a change found only on the open batch pull request fails the check too, since a cut tags the line and would not carry it: merge the batch pull request first. A conflict the batch deferred, or an original deferred to the next batch, prints as a warning without failing the check, also once that batch merged or closed. An original that waits on the open batch for a maintainer fails the cut. A change that reached the line by hand counts like any other, by what reached the line. +[`backport-batch check`](../../cmd/backport-batch/README.md) applies the rules to every pull request merged into `main` since the line branched, and looks for each one they take on the line and then on the open batch pull request: by picks whose `-x` lines name its merge commit or every commit of its own, by a pick that would change nothing, and, on commits from before the policy's batch-only day, by what their text says, such as `Backport of #N`. With `--cut` a change found only on the open batch pull request fails the check too, since a cut tags the line and would not carry it: merge the batch pull request first. A conflict the batch deferred, or an original deferred to the next batch, prints as a warning without failing the check, also once that batch merged or closed. An original that waits on the open batch for a maintainer fails the cut, and so does a unit on the open batch that carries only part of its original. An original parked because it builds on a change the line never gets, or would land only in part, prints as a note; one that waits on another original fails, warns or notes by what it waits on. A change that reached the line by hand counts like any other, by what reached the line. ### Backporting code that touches monotonic counters diff --git a/docs/release/backport-tasks/README.md b/docs/release/backport-tasks/README.md index e4395480e..dbbb536ad 100644 --- a/docs/release/backport-tasks/README.md +++ b/docs/release/backport-tasks/README.md @@ -10,7 +10,7 @@ The [backport batch](../backports.md) is built by [`backport-batch`](../../../cm ### Rules for every task - Push nothing but the branch of the open batch pull request, `backport-batch/-` in this repository, and only what the task's check accepted. Never push to `main` or a `release-*` branch, and never open a second batch pull request: the scheduled run continues the open one. -- Change no file the original pull request does not change. A conflict that cannot be resolved inside those files is not resolved: the original stays deferred. +- Change no file the original pull request does not change, or the path a rename on `main` or on the line gave it. A conflict that cannot be resolved inside those files is not resolved: the original stays deferred. - Never rewrite the batch branch by hand. A plain push to it is a fast-forward; only `backport-batch run --defer --push` replaces it, leased on the head it rebuilt. - Leave the batch pull request's title and body to the tool: it rewrites both on every run, and a hand edit is lost or, inside the `backport-batch` block, makes `run`, `check`, `fold` and `notify` refuse the body, and inside the `backport-batch-report` block makes `run --push` and `backport-batch report` refuse it. Leave the tool's report comments as they are too: each keeps one classification of the line, and an edited one makes `backport-batch report` refuse it. - When the task cannot be done within these rules, push nothing and leave a comment on the batch pull request saying why, so the next maintainer to look finds it. Nothing is lost by waiting: a red batch does not merge, and a deferred original goes to the next batch. diff --git a/docs/release/backport-tasks/resolve-conflict.md b/docs/release/backport-tasks/resolve-conflict.md index 46e24cf8d..885e568d4 100644 --- a/docs/release/backport-tasks/resolve-conflict.md +++ b/docs/release/backport-tasks/resolve-conflict.md @@ -34,7 +34,7 @@ git add git -c core.hooksPath=/dev/null cherry-pick --continue --no-edit ``` -When the original builds on something the line does not have, a helper, a value or another pull request, the resolution would have to bring that along, which this task does not allow: abort with `git cherry-pick --abort`, go back with `git switch --detach "$base"`, and comment on the batch pull request which change the original depends on; it stays deferred until that change is on the line. When that change will never reach the line, drop the original there instead, with `backport-batch run --line --drop N --reason "" --push`, so later batches stop trying it. A commit whose change the line already has stops the pick as empty; skip it with `git cherry-pick --skip`. +When the original builds on something the line does not have, a helper, a value or another pull request, the resolution would have to bring that along, which this task does not allow: abort with `git cherry-pick --abort`, go back with `git switch --detach "$base"`, and comment on the batch pull request which change the original depends on; it stays deferred until that change is on the line. An original the batch parked because it builds on such a change is the same task, and lands only as a complete adaptation: every file it changes changed, unless the line already has that file as the original makes it. When that change will never reach the line, drop the original there instead, with `backport-batch run --line --drop N --reason "" --push`, so later batches stop trying it. A commit whose change the line already has stops the pick as empty; skip it with `git cherry-pick --skip`. A resolution that ports only what the line has and leaves a file of the original as it was, one the line does not have for instance, is refused by `fold`: such a port goes to the line as a pull request of its own, as [fixes that exist only on a release line](../backports.md#fixes-that-exist-only-on-a-release-line) describes, and the original is dropped there. For a merged pull request, join the picks to the batch head with the merge commit: diff --git a/docs/release/backports.md b/docs/release/backports.md index b0cefdd17..ad7424c40 100644 --- a/docs/release/backports.md +++ b/docs/release/backports.md @@ -25,6 +25,10 @@ The batch reads the pull requests off `main`'s first-parent history, one per mer A pull request the rules take for every line can still be wrong for one of them, for instance because it builds on code that line does not have. A maintainer drops it from that line with `./backport-batch run --line --drop --reason --push`: the batch takes it off, if it carries it, and no later batch for that line picks it again. The drop is kept in the [line's record](#the-lines-record-and-reports), so it needs an open batch pull request to record it on; before a line's first batch pull request exists there is nothing to record it on, and the command refuses. `--undrop ` withdraws it, and the next batch tries the pull request again. A pull request the line already carries cannot be dropped. To land a hand-made port instead, open it as a pull request of its own, as [fixes that exist only on a release line](#fixes-that-exist-only-on-a-release-line) describes, and drop the original. +The batch also leaves a pull request out of a line on its own when it cannot land there whole. A pick that would leave part of what the pull request does on `main` without effect on the line, a change to a file the line does not have, or the deletion of a file the line never had or already removed, is not made: the pull request is parked, the batch pull request lists it under "Parked" with those files, and `check` prints it as a note without failing. A file the line already has as the pull request makes it counts as there, and so does a file `main` or the line renamed, under its other name. A unit already on the open batch that does only part of its original fails `check --cut` and `backport-batch verify`, whatever its label says, until `backport-batch run --defer` takes it out. What applies of a parked pull request goes to the line as a pull request of its own, as [fixes that exist only on a release line](#fixes-that-exist-only-on-a-release-line) describes. + +A pull request is parked too when it builds on a change the line never gets: it changes a file a `feat` added on `main`, say, or a line such a change wrote, and the line has neither. `backport-batch` reads that from git, as [its README](../../cmd/backport-batch/README.md#what-an-original-builds-on) describes, and names the change in the batch pull request. One that builds on a change that is still to come, a deferred conflict, one waiting for a maintainer, one rule 6 holds or one not tried yet, waits for it instead, listed under "Waiting on another original", and the batch tries it again once that change is on the line; one picked earlier in the same batch counts as there. A complete hand adaptation of a parked pull request, every file it changes changed, still lands on the batch through [resolving a conflict](./backport-tasks/resolve-conflict.md). + ### What waits for the next minor `feat`, `chore` and `docs` pull requests, and dependency bumps (scope `deps`), ship with the next minor, not in a patch release; only a maintainer's `backport/include`, below, takes one. If a class of fixes should also wait for a minor, it gets a new rule on this page; a single pull request that the rules get wrong is what the overrides below are for. @@ -33,7 +37,7 @@ A pull request the rules take for every line can still be wrong for one of them, A maintainer overrides the rules for one pull request with a label on the merged pull request on `main`: -- `backport/include` takes it although rules 2 to 4 keep it out, a `feat` for instance, or although it merged before its line started under this policy. It still has to pick cleanly or have its conflict resolved, pass CI on the batch, and, on an older line, already be on the newer one, and a line that already carries it does not take it again. +- `backport/include` takes it although rules 2 to 4 keep it out, a `feat` for instance, or although it merged before its line started under this policy. It still has to pick cleanly or have its conflict resolved, pass CI on the batch, and, on an older line, already be on the newer one, and a line that already carries it does not take it again. It lifts no park, as [what the batch leaves out on one line](#what-the-batch-leaves-out-on-one-line) describes. - `backport/skip` keeps it out although the rules take it, and `backport-batch check` stops asking for it. An open batch that picked it before the label came keeps it until `backport-batch run --defer` takes it out; the batch pull request says so meanwhile. A label decides for every line. To keep a pull request out of one line only, drop it there with `backport-batch run --drop`, as [what the batch leaves out on one line](#what-the-batch-leaves-out-on-one-line) describes; `backport/include` does not lift a drop. @@ -65,7 +69,7 @@ released-lines: 2 ### Fixes that exist only on a release line -Some fixes go to a release line as a pull request of their own. Such pull requests are occasional, and maintainers open them: for a fix with no counterpart on `main`, for one that on `main` came as part of another pull request, or for one written for `main` rather than for the line. When the counterpart on `main` is a pull request of its own that the batch would take, the batch skips it as already on the line once picking it would change nothing there; otherwise a maintainer labels it `backport/skip`. +Some fixes go to a release line as a pull request of their own. Such pull requests are occasional, and maintainers open them: for a fix with no counterpart on `main`, for one that on `main` came as part of another pull request, or for one written for `main` rather than for the line. When the counterpart on `main` is a pull request of its own that the batch would take, the batch skips it as already on the line once picking it would change nothing there; otherwise a maintainer labels it `backport/skip`. A port of only what a line has of a pull request from `main`, one that leaves out what the line lacks, is such a pull request as well: the batch carries no unit that does part of its original, and `backport-batch fold` refuses one. A pull request opened against a `release-*` branch is closed automatically, with a comment that links to this page, when its author is a user account, not a bot, that GitHub names neither the repository's owner nor a member nor a collaborator, and whose access to the repository GitHub reports as read or none. Anyone with triage access or more keeps their pull request open. When GitHub's answer on the author's access cannot be read, the check closes nothing and fails, naming the read that failed. The check runs only when a pull request is opened, so a maintainer keeps a contributor's release-only fix by reopening it. [`hack/close-manual-backport.sh`](../../hack/close-manual-backport.sh) does this from [`pr-labeler.yaml`](../../.github/workflows/pr-labeler.yaml); run it with `--pr --dry-run` to see what it would decide for a given pull request when it is opened. @@ -87,11 +91,11 @@ A run by hand trusts the open batch pull request the scheduled run opened as `co `./backport-batch lines` prints the lines, newest first. `run`, `check`, `fold` and `lines` read the policy block from this page on `main`, so until the page with its block is on `main` they need `--policy ` naming a ref that has it, and the scheduled run, which passes none, fails. -`run` picks what the rules take onto `backport-batch/-` and opens a draft pull request against the line. A pull request merged into `main` with a merge commit goes onto the batch as its own commits, oldest first, each picked with `git cherry-pick -x --signoff` so it keeps its author, its message and its sign-offs and gains the sign-off of whoever runs the batch; commits that merged `main` into the pull request's branch are left out. A merge commit joins them to the batch: it carries the original's merge message from `main` and ends in a `Backported-from: ` trailer. A pull request squash-merged into `main` goes on as one pick. When any commit of an original conflicts, nothing of that original stays on the batch. The batch pull request is merged into the line with a merge commit, so all of this reaches the line as it is. Its body lists the originals with how many commits each brought and which of them differ from the commit they came from, the conflicts it deferred with the commit that failed and its files, and what it skipped under which rule, and ends with the [line's record and the run's report](#the-lines-record-and-reports); `--push` prints the pull request's URL. While that pull request is open, the next run continues it, `--continue` reading its head, rather than opening another. CI on the batch pull request runs the line's build and tests, and a maintainer merges it. Since rule 6 reads the newer line itself, an older line takes a fix in the batch after the newer line's batch with that fix has merged. +`run` picks what the rules take onto `backport-batch/-` and opens a draft pull request against the line. A pull request merged into `main` with a merge commit goes onto the batch as its own commits, oldest first, each picked with `git cherry-pick -x --signoff` so it keeps its author, its message and its sign-offs and gains the sign-off of whoever runs the batch; commits that merged `main` into the pull request's branch are left out. A merge commit joins them to the batch: it carries the original's merge message from `main` and ends in a `Backported-from: ` trailer. A pull request squash-merged into `main` goes on as one pick. When any commit of an original conflicts, nothing of that original stays on the batch. The batch pull request is merged into the line with a merge commit, so all of this reaches the line as it is. Its body lists the originals with how many commits each brought and which of them differ from the commit they came from, whose release notes wait under "Check these notes" with what to compare, since a note may not hold for a pick that differs, the conflicts it deferred with the commit that failed and its files, what it parked and why, and what it skipped under which rule, and ends with the [line's record and the run's report](#the-lines-record-and-reports); `--push` prints the pull request's URL. While that pull request is open, the next run continues it, `--continue` reading its head, rather than opening another. CI on the batch pull request runs the line's build and tests, and a maintainer merges it. Since rule 6 reads the newer line itself, an older line takes a fix in the batch after the newer line's batch with that fix has merged. A deferred conflict waits for the next batch. To land it sooner, pick the original's commits onto the batch head with `git cherry-pick -x`, resolving each conflict, and join them to the batch head with a merge commit that names the original; an original squash-merged into `main` is one pick of its commit there, with no merge. Check that merge or that pick with `./backport-batch fold --line --original --commit `, and push it onto the batch branch, as [resolving a conflict](./backport-tasks/resolve-conflict.md) describes. CI runs `backport-batch verify` on every push to a batch pull request, which holds every original on it to the same shape. When CI on the batch pull request fails, the original it comes from is taken out with `./backport-batch run --line --defer --reason --push` and goes to the next batch, as [attributing a failed batch CI](./backport-tasks/failed-ci.md) describes; the rest of the batch merges without it. When it is all the batch carries, `--push` records why in the batch pull request, closes it and deletes its branch, and the next batch tries it again. `--drop` and `--undrop` change the record of the open batch pull request in the same way, and need one. -Before a patch release or an rc is cut from a line, `./backport-batch check --line --cut` has to exit 0. A cut tags the line, so the check fails while a pull request the rules take is not on the line, including one that is only on the open batch pull request: merge the batch pull request first. It prints the URLs, and prints what the batch deferred as warnings, since deferring is a maintainer's call; so is a pull request labelled `backport/skip` that the open batch still carries. An original that waits on the batch for a maintainer fails the cut, also when it is all the batch has left. A cut that passes says so. Without `--cut`, as the scheduled run uses it, a change on the open batch pull request counts as carried. +Before a patch release or an rc is cut from a line, `./backport-batch check --line --cut` has to exit 0. A cut tags the line, so the check fails while a pull request the rules take is not on the line, including one that is only on the open batch pull request: merge the batch pull request first. It prints the URLs, and prints what the batch deferred as warnings, since deferring is a maintainer's call; so is a pull request labelled `backport/skip` that the open batch still carries. An original that waits on the batch for a maintainer fails the cut, also when it is all the batch has left, and so does a unit on the batch that carries only part of its original, whatever its label. A cut that passes says so. Without `--cut`, as the scheduled run uses it, a change on the open batch pull request counts as carried. ### The line's record and reports @@ -101,7 +105,7 @@ Each run also reports where every pull request merged into `main` since the line ### Changes to workflows -The scheduled run's token may not push a commit that changes `.github/workflows/`, so the scheduled run leaves such an original out (`run --no-workflow-files`). The batch pull request lists it under "Waiting for a maintainer", with the workflow files it changes, and so does the batch's comment. It is no conflict. The policy still takes it: `check` warns about it while it waits, and `check --cut` fails on it. When nothing else is picked there is no batch pull request to list it on, and `check` reports it as missing. +The scheduled run's token may not push a commit that changes `.github/workflows/`, so the scheduled run leaves such an original out (`run --no-workflow-files`). That holds also for one whose only missing dependency that never comes is a test or documentation, which a maintainer's run, with the token that may push the workflow, parks instead: what lands of it then is a pull request of its own; `check` reads which of the two the open batch was assembled as from its record, and with no batch open takes the scheduled run's. The batch pull request lists it under "Waiting for a maintainer", with the workflow files it changes, and so does the batch's comment. It is no conflict. The policy still takes it: `check` warns about it while it waits, and `check --cut` fails on it. When nothing else is picked there is no batch pull request to list it on, and `check` reports it as missing. A maintainer brings it over with their own token, one that may push workflow changes (the Workflows permission, or the classic `workflow` scope), by continuing the open batch without the flag: @@ -113,7 +117,7 @@ That picks it onto the batch in the batch's shape, after what the batch carries, ### The scheduled run -[`backport-batch.yaml`](../../.github/workflows/backport-batch.yaml) runs [`hack/backport-batch-scheduled.sh`](../../hack/backport-batch-scheduled.sh) on Monday and Thursday. For each line the batch runs for, newest first, it continues the open batch pull request or opens one, runs `check`, and then `backport-batch notify`, which reports what waits on the batch. A batch open for three days or more, so one the previous run already saw, gets a comment on its pull request saying so and a line in the run's summary. The comment is one per pull request, updated in place on every run. It also lists the deferred conflicts, which a maintainer resolves as [resolving a conflict](./backport-tasks/resolve-conflict.md) describes, and the originals that wait because they change workflows. A failed batch CI waits for a maintainer too, as [attributing a failed batch CI](./backport-tasks/failed-ci.md) describes; the comment does not list it, and it shows on the batch pull request's checks. `hack/backport-batch-scheduled.sh --dry-run` does the same in the current worktree and changes nothing on GitHub. Its `check` sees the line without the batch the dry run just built, so it can exit 1 where the real run's check passes. +[`backport-batch.yaml`](../../.github/workflows/backport-batch.yaml) runs [`hack/backport-batch-scheduled.sh`](../../hack/backport-batch-scheduled.sh) on Monday and Thursday. For each line the batch runs for, newest first, it continues the open batch pull request or opens one, runs `check`, and then `backport-batch notify`, which reports what waits on the batch. A batch open for three days or more, so one the previous run already saw, gets a comment on its pull request saying so and a line in the run's summary. The comment is one per pull request, updated in place on every run. It also lists the deferred conflicts, which a maintainer resolves as [resolving a conflict](./backport-tasks/resolve-conflict.md) describes, and the originals that wait because they change workflows, and names each parked original once: a park named on one batch is not named again on the next while it stays the same park, and with no batch open nothing is named until the next batch opens. A failed batch CI waits for a maintainer too, as [attributing a failed batch CI](./backport-tasks/failed-ci.md) describes; the comment does not list it, and it shows on the batch pull request's checks. `hack/backport-batch-scheduled.sh --dry-run` does the same in the current worktree and changes nothing on GitHub. Its `check` sees the line without the batch the dry run just built, so it can exit 1 where the real run's check passes. The batch runs as the cozystack-ci GitHub App, which the release workflows already use, and needs no secret of its own. The job mints a token from `COZYSTACK_CI_APP_ID` and `COZYSTACK_CI_PRIVATE_KEY` for this repository alone, with Contents and Pull requests write and nothing more, and fails when either secret is missing. Its pushes start CI on the batch pull request, which a push with the job's own `GITHUB_TOKEN` would not. Its commits carry the App's bot, `cozystack-ci[bot]`, whose batch pull request and comment `backport-batch` trusts: `BACKPORT_BATCH_BOT`, which the job sets from the token, names it, and a run without the variable trusts `cozystack-ci[bot]` too. The App's installation has to grant this repository Contents and Pull requests read and write, which an organization admin sees among the organization's installed GitHub Apps. It has no Workflows permission, so a change to `.github/workflows/` waits for a maintainer, as [changes to workflows](#changes-to-workflows) describes. diff --git a/docs/release/changelogs.md b/docs/release/changelogs.md index f26ac56c5..e5d7db401 100644 --- a/docs/release/changelogs.md +++ b/docs/release/changelogs.md @@ -60,7 +60,7 @@ git diff --name-only $PREV..$NEW -- 'packages/*/charts/*/Chart.yaml' 'packages/* done # B. Out-of-Chart.yaml pins (Talos, tenant K8s, vendored images) -for path in packages/core/talos/images/talos/profiles/installer.yaml packages/core/installer/values.yaml packages/apps/kubernetes/values.yaml; do +for path in packages/core/talos/images/talos/profiles/installer.yaml packages/core/installer/values.yaml packages/apps/kubernetes/values.yaml packages/apps/kubernetes-nodes/values.yaml; do git diff $PREV..$NEW -- "$path" | head -40 done git diff --name-only $PREV..$NEW -- 'images/*/Dockerfile' diff --git a/docs/security/self-assessment.md b/docs/security/self-assessment.md index 6b3b2e076..ca49853ab 100644 --- a/docs/security/self-assessment.md +++ b/docs/security/self-assessment.md @@ -55,7 +55,7 @@ Actors are decomposed by **trust boundary**: two components are distinct actors | GitOps / Flux delivery layer | T1 — platform control plane | Reconciles HelmReleases/Kustomizations into arbitrary privileged manifests; effectively cluster-admin. | Tenants cannot create HelmReleases (RBAC); not tenant-network-reachable. | | `cozystack-api` aggregated API server | T1 — platform control plane | Converts tenant CRs into HelmReleases; can CRUD HelmReleases, Secrets, and Cilium policies cluster-wide. Acts as the policy-enforcing gateway. | Constrains tenants via fixed chart reference, name/quota/host validation, and admission. | | System operators / controllers | T1 — platform control plane | Domain-scoped cluster privileges (provision control planes, mint certs/secrets, program CNI). | Distinct ServiceAccounts; isolated from tenants by RBAC and network policy. | -| Tenant administrator (OIDC `-admin` / `-super-admin`) | T2 — tenant boundary | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` in its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) `super-admin` holds the `apps.cozystack.io/*` wildcard (and can create child `Tenant`s); `admin` has an enumerated allowlist excluding some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`). | Namespace-scoped RBAC + admission policies + quota; no HelmRelease or cross-tenant access. | +| Tenant administrator (OIDC `-admin` / `-super-admin`) | T2 — tenant boundary | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` in its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); `get` on persistentvolumeclaims and `create` on `upload.cdi.kubevirt.io/uploadtokenrequests`, the pair `virtctl image-upload` needs to upload into an existing VMDisk; no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) `super-admin` holds the `apps.cozystack.io/*` wildcard (and can create child `Tenant`s); `admin` has an enumerated allowlist excluding some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`). | Namespace-scoped RBAC + admission policies + quota; no HelmRelease or cross-tenant access. | | Tenant user (OIDC `-view` / `-use`) | T2 — tenant boundary | Read-only view plus operational verbs (VM console/vnc/portforward, start/stop/restart). Cannot create Application CRs. | Strictly dominated by tenant-admin; cannot escalate. | | Parent vs child (nested) tenant | T2 — tenant boundary | Parent ServiceAccounts bind into descendant namespaces; children do not bind upward. Quota hierarchically capped. | Ancestor→descendant trust is one-way. | | Tenant workloads / managed services | T3 — workload boundary | Run tenant-controlled code; no cluster-API rights beyond their ServiceAccount. | Namespace-scoped RBAC/identity isolate API objects; on the network, Cilium egress confines each tenant pod to its own subtree + platform system services + `world`, so unrelated tenants cannot reach each other over the pod network (enforced on egress; ingress is open to `cluster` — see Non-goals). | diff --git a/docs/security/threat-model.md b/docs/security/threat-model.md index 9790d9591..0a4ea0bb3 100644 --- a/docs/security/threat-model.md +++ b/docs/security/threat-model.md @@ -39,7 +39,7 @@ The actors below are decomposed **by trust boundary**, not by component: two com | GitOps / Flux delivery layer | T1 | Reconciles HelmReleases/Kustomizations into arbitrary privileged manifests; effectively cluster-admin. | Tenants cannot create HelmReleases/Kustomizations (RBAC); not tenant-network-reachable. | | `cozystack-api` aggregated API server (ServiceAccount) | T1 | CRUD HelmReleases, Secrets, and CiliumNetworkPolicies cluster-wide; reads Namespaces/RBAC/quotas/nodes. Acts as the policy-enforcing gateway that converts tenant CRs to HelmReleases. | A distinct, network-exposed attack surface from Flux; constrains tenants via fixed chart ref + name/quota/host validation + admission. | | System operators / controllers | T1 | Domain-scoped cluster privileges (provision control planes, mint certs/secrets, program CNI). | Distinct ServiceAccounts, but all cluster-admin-adjacent; isolated from tenants only by RBAC and network policy. | -| Tenant administrator (OIDC groups `-admin`, `-super-admin`) | T2 | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` within its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) The `-super-admin` group holds the `apps.cozystack.io/*` wildcard (and so can create child `Tenant`s); the `-admin` group has an enumerated allowlist that excludes some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`, `external-dns`, `bootbox`). | Namespace-scoped RoleBindings + admission policies + quota; no access to HelmReleases, arbitrary core writes, or other tenants. | +| Tenant administrator (OIDC groups `-admin`, `-super-admin`) | T2 | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` within its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); `get` on persistentvolumeclaims and `create` on `upload.cdi.kubevirt.io/uploadtokenrequests`, the pair `virtctl image-upload` needs to upload into an existing VMDisk; no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) The `-super-admin` group holds the `apps.cozystack.io/*` wildcard (and so can create child `Tenant`s); the `-admin` group has an enumerated allowlist that excludes some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`, `external-dns`, `bootbox`). | Namespace-scoped RoleBindings + admission policies + quota; no access to HelmReleases, arbitrary core writes, or other tenants. | | Tenant user (OIDC groups `-view`, `-use`) | T2 | Read-only view of Application CRs, pods, and services, plus operational verbs only (VM console/vnc/portforward, start/stop/restart). Cannot create Application CRs. | Strictly dominated by tenant-admin; cannot escalate to create/delete, so it is a distinct actor. | | Parent tenant vs child (nested) tenant | T2 | Parent ServiceAccounts are bound into descendant namespaces; child ServiceAccounts are not bound upward. Quota is hierarchically capped. | Ancestor→descendant only; compromising a parent compromises its children, not the reverse. | | Tenant workloads / managed services | T3 | Run tenant-controlled code; no cluster-API rights beyond their ServiceAccount. | Namespace-scoped RBAC/identity isolate their API objects. On the network, Cilium egress confines each tenant pod to its own tenant subtree, platform system services, and `world` (out-of-cluster), so unrelated tenants cannot reach each other over the pod network — enforced on the egress side (tenant-pod ingress is open to `cluster`; see Non-goals for residual caveats). | @@ -92,7 +92,7 @@ The following are explicitly out of scope for the platform's security guarantees The single most load-bearing isolation control is the combination of two facts: -1. The tenant ClusterRoles (`packages/system/cozystack-basics/templates/clusterroles.yaml`) grant **no** `helm.toolkit.fluxcd.io` access and **no** create/update of arbitrary core resources — only CRUD on `apps.cozystack.io` application kinds (super-admin holds the `*` wildcard; admin an enumerated allowlist) and `sdn.cozystack.io/securitygroups`, read access to pods/services/ingresses, and (for tenant-admins) `delete` on a fixed set of core objects (pods/services/endpoints/events; the `persistentvolumes` entry in the ClusterRole is inert under the namespace-scoped RoleBinding). Tenant isolation is enforced by the *absence* of privileged grants in RBAC, not by an admission policy that whitelists `apps.cozystack.io/*` writes. +1. The tenant ClusterRoles (`packages/system/cozystack-basics/templates/clusterroles.yaml`) grant **no** `helm.toolkit.fluxcd.io` access and **no** create/update of arbitrary core resources — only CRUD on `apps.cozystack.io` application kinds (super-admin holds the `*` wildcard; admin an enumerated allowlist) and `sdn.cozystack.io/securitygroups`, read access to pods/services/ingresses, and (for tenant-admins) `delete` on a fixed set of core objects (pods/services/endpoints/events; the `persistentvolumes` entry in the ClusterRole is inert under the namespace-scoped RoleBinding), `get` on persistentvolumeclaims and `create` on `upload.cdi.kubevirt.io/uploadtokenrequests` (for uploading an image into an existing VMDisk with `virtctl image-upload`). Tenant isolation is enforced by the *absence* of privileged grants in RBAC, not by an admission policy that whitelists `apps.cozystack.io/*` writes. 2. The Application-to-HelmRelease conversion (`pkg/registry/apps/application/rest.go`) takes the chart reference, release-name prefix, and `valuesFrom` from server-side configuration; the tenant controls only the Helm values (`app.Spec`). If a future `ApplicationDefinition`, RBAC change, or API change ever let a tenant influence the chart reference, `valuesFrom`, or write HelmReleases directly, the T2→T1 isolation would collapse. Changes touching tenant RBAC or the Application REST layer should be reviewed against this invariant. diff --git a/examples/backups/kubernetes/00-helpers.sh b/examples/backups/kubernetes/00-helpers.sh new file mode 100755 index 000000000..401b5edb8 --- /dev/null +++ b/examples/backups/kubernetes/00-helpers.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# Shared helpers for the tenant Kubernetes backup/restore walkthrough. +# Source this file in other scripts: source "$(dirname "$0")/00-helpers.sh" + +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export YELLOW='\033[1;33m' +export BLUE='\033[0;34m' +export MAGENTA='\033[0;35m' +export CYAN='\033[0;36m' +export NC='\033[0m' +export BOLD='\033[1m' + +export NAMESPACE="${NAMESPACE:-tenant-root}" +export CLUSTER_NAME="${CLUSTER_NAME:-backup-demo}" +export POOL_NAME="${POOL_NAME:-md0}" +export WORKER_DISK_SIZE="${WORKER_DISK_SIZE:-20Gi}" +export BACKUPCLASS_NAME="${BACKUPCLASS_NAME:-cozy-default}" +export BACKUPJOB_NAME="${BACKUPJOB_NAME:-kubernetes-backup-job}" +export RESTOREJOB_INPLACE_NAME="${RESTOREJOB_INPLACE_NAME:-kubernetes-restore-inplace}" +export RESTOREJOB_RECREATE_NAME="${RESTOREJOB_RECREATE_NAME:-kubernetes-restore-recreate}" +export TENANT_API_PORT="${TENANT_API_PORT:-16443}" +export MARKER_VALUE="${MARKER_VALUE:-roundtrip-7c2e91d4}" +export RELEASE="kubernetes-${CLUSTER_NAME}" + +log_info() { echo -e "${BLUE}i${NC} $*" >&2; } +log_success() { echo -e "${GREEN}OK${NC} $*" >&2; } +log_warning() { echo -e "${YELLOW}!${NC} $*" >&2; } +log_error() { echo -e "${RED}x${NC} $*" >&2; } +log_substep() { echo -e "${CYAN} -> $*${NC}" >&2; } + +print_header() { + echo -e "\n${MAGENTA}${BOLD}== $1 ==${NC}\n" >&2 +} + +# shellcheck source-path=SCRIPTDIR source=../_lib/wait-helpers.sh +source "$(dirname "${BASH_SOURCE[0]}")/../_lib/wait-helpers.sh" + +# The admin kubeconfig points at the in-cluster Service, so the walkthrough +# reaches the tenant API through a port-forward to it and keeps the TLS server +# name the certificate is issued for. +TENANT_KUBECONFIG="$(dirname "${BASH_SOURCE[0]}")/.tenant-kubeconfig" +export TENANT_KUBECONFIG + +tenant_connect() { + kubectl -n "$NAMESPACE" get secret "${RELEASE}-admin-kubeconfig" \ + -o jsonpath='{.data.super-admin\.svc}' | base64 -d > "$TENANT_KUBECONFIG" + chmod 600 "$TENANT_KUBECONFIG" + local cluster + cluster=$(kubectl --kubeconfig "$TENANT_KUBECONFIG" config view -o jsonpath='{.clusters[0].name}') + kubectl --kubeconfig "$TENANT_KUBECONFIG" config set-cluster "$cluster" \ + --server="https://127.0.0.1:${TENANT_API_PORT}" \ + --tls-server-name="${RELEASE}.${NAMESPACE}.svc" >/dev/null + pkill -f "port-forward svc/${RELEASE} ${TENANT_API_PORT}:6443" 2>/dev/null || true + kubectl -n "$NAMESPACE" port-forward "svc/${RELEASE}" "${TENANT_API_PORT}:6443" >/dev/null 2>&1 & + for _ in $(seq 1 30); do + tkubectl get --raw /readyz >/dev/null 2>&1 && return 0 + sleep 2 + done + log_error "tenant API of ${RELEASE} is not reachable through the port-forward" + return 1 +} + +tenant_disconnect() { + pkill -f "port-forward svc/${RELEASE} ${TENANT_API_PORT}:6443" 2>/dev/null || true +} + +tkubectl() { + kubectl --kubeconfig "$TENANT_KUBECONFIG" "$@" +} + +# The CA fingerprint is what proves a restore brought back the cluster rather +# than a new one: nodes, kubeconfigs and tokens all trust this CA. +cluster_ca_fingerprint() { + kubectl -n "$NAMESPACE" get secret "${RELEASE}-ca" -o jsonpath='{.data.ca\.crt}' \ + | base64 -d | openssl x509 -noout -fingerprint -sha256 | cut -d= -f2 +} + +# wait_tenant_marker checks both halves of the backup: the ConfigMap comes back +# with etcd, the file on the PVC with the volumes. +wait_tenant_marker() { + local timeout="${1:-900}" elapsed=0 cm data + while true; do + cm=$(tkubectl -n demo get configmap marker -o jsonpath='{.data.marker}' 2>/dev/null || true) + data=$(tkubectl -n demo exec deploy/writer -- cat /data/marker 2>/dev/null || true) + if [[ "$cm" == "$MARKER_VALUE" && "$data" == "$MARKER_VALUE" ]]; then + log_success "marker present in etcd (ConfigMap) and on the volume" + return 0 + fi + (( elapsed >= timeout )) && { log_error "marker not restored: configmap='${cm}' volume='${data}'"; return 1; } + sleep 10 + elapsed=$((elapsed + 10)) + done +} diff --git a/examples/backups/kubernetes/01-create-cluster.sh b/examples/backups/kubernetes/01-create-cluster.sh new file mode 100755 index 000000000..8577ec202 --- /dev/null +++ b/examples/backups/kubernetes/01-create-cluster.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Step 01: Create a tenant Kubernetes cluster with one worker pool and seed it +# with a marker held in etcd (a ConfigMap) and on a PersistentVolume. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 01: Create Kubernetes '${CLUSTER_NAME}' and seed a marker" + +kubectl apply -f - </dev/null 2>&1 \ + && [[ -n "$(tkubectl get nodes -o name)" ]] \ + && [[ -n "$(tkubectl get storageclass -o name)" ]]; then + kubectl_ok=1 + break + fi + sleep 15 +done +(( kubectl_ok )) || { log_error "worker node or StorageClass did not appear"; exit 1; } + +tkubectl apply -f - < /data/marker; sync; sleep infinity"] + volumeMounts: + - {name: data, mountPath: /data} + volumes: + - name: data + persistentVolumeClaim: {claimName: data} +EOF + +wait_tenant_marker 900 +cluster_ca_fingerprint > "$SCRIPT_DIR/.ca-fingerprint" +log_success "cluster CA $(cat "$SCRIPT_DIR/.ca-fingerprint")" +tenant_disconnect + +echo -e "\n${GREEN}${BOLD}Next:${NC} ./02-create-backupjob.sh" diff --git a/examples/backups/kubernetes/02-create-backupjob.sh b/examples/backups/kubernetes/02-create-backupjob.sh new file mode 100755 index 000000000..c290711c1 --- /dev/null +++ b/examples/backups/kubernetes/02-create-backupjob.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Step 02: Back the cluster up through the cozy-default BackupClass. The +# controller dumps the cluster's etcd prefix to the platform bucket with a +# Job, then backs up its control-plane Secrets and volumes with Velero. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +print_header "Step 02: Create BackupJob '${BACKUPJOB_NAME}'" + +kubectl apply -f - </dev/null || true) +kubectl -n "$NAMESPACE" delete backupjob "$BACKUPJOB_NAME" --ignore-not-found +[[ -n "$backup_ref" ]] && kubectl -n "$NAMESPACE" delete backup "$backup_ref" --ignore-not-found +kubectl -n "$NAMESPACE" delete kubernetesnodeses.apps.cozystack.io "${CLUSTER_NAME}-${POOL_NAME}" --ignore-not-found +kubectl -n "$NAMESPACE" delete kuberneteses.apps.cozystack.io "$CLUSTER_NAME" --ignore-not-found +rm -f "$TENANT_KUBECONFIG" "$SCRIPT_DIR/.ca-fingerprint" +log_success "done" diff --git a/examples/backups/kubernetes/run-all.sh b/examples/backups/kubernetes/run-all.sh new file mode 100755 index 000000000..731448b4d --- /dev/null +++ b/examples/backups/kubernetes/run-all.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# End-to-end tenant Kubernetes backup/restore walkthrough. It proves the +# restore brings back the same cluster: the CA fingerprint is unchanged, and a +# marker kept in etcd and one kept on a PersistentVolume both come back, first +# after rolling a live cluster back, then after rebuilding a deleted one. +# +# Needs the platform default backups (backupstrategy-controller with a Velero +# BSL and the system bucket), so cozy-default routes Kubernetes to +# cozy-default-kubernetes. +# +# Env knobs: see 00-helpers.sh. SKIP_RESTORE=1 stops after the backup. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/00-helpers.sh" + +kubectl wait --for=create kubernetes.strategy.backups.cozystack.io/cozy-default-kubernetes --timeout=15m >/dev/null \ + || { log_error "cozy-default-kubernetes strategy is missing: enable the platform default backups first"; exit 1; } + +steps=(01 02 03 04) +[[ "${SKIP_RESTORE:-0}" == "1" ]] && steps=(01 02) +for step in "${steps[@]}"; do + script="$(find "$SCRIPT_DIR" -maxdepth 1 -type f -name "${step}-*.sh" | sort | head -n1)" + [[ -n "$script" ]] || { echo "no script found for step $step" >&2; exit 1; } + echo "=== Running $(basename "$script") ===" + bash "$script" +done + +print_header "Walkthrough complete" diff --git a/hack/alert-severity-contract.bats b/hack/alert-severity-contract.bats index e1315eb96..44f197000 100644 --- a/hack/alert-severity-contract.bats +++ b/hack/alert-severity-contract.bats @@ -9,6 +9,10 @@ load test_helper # scope: they are re-vendored by `make update`, so a bad severity there is # fixed upstream or in patches/, not by editing the label. ALERT_SEVERITY_ALLOWED=" security critical major minor warning indeterminate informational normal ok cleared debug trace unknown " +# The alerts that exist only to be dropped, the one place `none` is right. +# InfoInhibitor is the source side of an Alertmanager inhibit rule, which the +# platform config does not have, so delivering it at any severity is noise. +ALERT_SEVERITY_BLACKHOLED=" InfoInhibitor " helm_actions_stripped() { awk ' @@ -104,6 +108,14 @@ alert_severity_contract() { break fi done + if [ "$severity" = "none" ]; then + for blackholed in $ALERT_SEVERITY_BLACKHOLED; do + if [ "$blackholed" = "$alert" ]; then + accepted=1 + break + fi + done + fi if [ "$accepted" -eq 0 ]; then echo "$file: $alert -> '$severity'" >&2 bad=$((bad + 1)) @@ -128,6 +140,83 @@ alert_severity_contract() { alert_severity_contract } +@test "InfoInhibitor is labelled for the blackhole route" { + rows=$(mktemp) || return 1 + if ! alert_severity_rows > "$rows"; then + rm -f "$rows" + return 1 + fi + tab=$(printf '\t') + found=$(grep -c "${tab}InfoInhibitor${tab}" "$rows" || true) + delivered=$(grep "${tab}InfoInhibitor${tab}" "$rows" | grep -vc "${tab}none\$" || true) + rm -f "$rows" + [ "$found" -gt 0 ] + [ "$delivered" -eq 0 ] +} + +@test "severity none passes only on an alert that exists to be dropped" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/alerts" + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: general' \ + ' rules:' \ + ' - alert: InfoInhibitor' \ + ' labels:' \ + ' severity: none' \ + > "$fixture/packages/system/example/alerts/00-inhibitor.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml + + inhibitor_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || inhibitor_succeeded=$? + + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: silent' \ + ' rules:' \ + ' - alert: SilentAlert' \ + ' labels:' \ + ' severity: none' \ + > "$fixture/packages/system/example/alerts/01-silent.yaml" + git -C "$fixture" add packages/system/example/alerts/01-silent.yaml + + silent_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || silent_succeeded=$? + rm -rf "$fixture" + [ "$inhibitor_succeeded" -eq 0 ] + [ "$silent_succeeded" -ne 0 ] +} + +@test "an alert that exists to be dropped still fails on a refused severity" { + fixture=$(mktemp -d) + mkdir -p "$fixture/packages/system/example/alerts" + printf '%s\n' \ + 'apiVersion: operator.victoriametrics.com/v1beta1' \ + 'kind: VMRule' \ + 'spec:' \ + ' groups:' \ + ' - name: general' \ + ' rules:' \ + ' - alert: InfoInhibitor' \ + ' labels:' \ + ' severity: info' \ + > "$fixture/packages/system/example/alerts/00-inhibitor.yaml" + git -C "$fixture" init -q + git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml + + contract_succeeded=0 + ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$? + rm -rf "$fixture" + [ "$contract_succeeded" -ne 0 ] +} + @test "a parser failure cannot leave the contract green" { fixture=$(mktemp -d) mkdir -p "$fixture/packages/system/example/alerts" diff --git a/hack/altinity-backup-strategy.bats b/hack/altinity-backup-strategy.bats index 0861da42d..070b6bad5 100644 --- a/hack/altinity-backup-strategy.bats +++ b/hack/altinity-backup-strategy.bats @@ -1,14 +1,28 @@ #!/usr/bin/env bats -# Behavioural tests for the shipped Altinity (ClickHouse) backup/restore -# strategy script, extracted from the rendered cozy-default-altinity Strategy -# CR and EXECUTED against a stubbed clickhouse-backup HTTP API, with the real -# jq doing the archive selection. Which archive a restore reads back is -# decided by that selection over list/remote, so it is exercised against -# several archives rather than string-matched. +# Behavioural tests for the shipped ClickHouse (Altinity) strategy script, +# extracted from the rendered cozy-default-altinity Strategy CR and EXECUTED +# against a stubbed clickhouse-backup sidecar API. # -# The script runs with `bash`, not `sh`, for the reason given in -# hack/redis-backup-strategy.bats: it opens `set -euo pipefail`, which the -# production image's busybox ash supports and the CI runner's dash rejects. +# The contract pinned here is the start gate: the sidecar serialises its +# operations and answers any new one with HTTP 423 while another is "in +# progress", so a strategy Pod that starts its operation without waiting for +# an idle sidecar burns a Job attempt in seconds. The gate has to fail the Pod +# rather than proceed when the sidecar never becomes idle. Once started, an +# operation the sidecar no longer lists (its action log lives in memory and a +# restart empties it) has to fail the Pod too, since the Job has no deadline. +# A Job retry must not start its operation twice: it takes over the +# restore_remote, or the create_remote of the archive the controller named, +# that the attempt before it left running, and a few lost polls do not end an +# attempt in the first place. +# +# Which archive a restore reads back is decided by the script's own jq +# selection over list/remote, so it is exercised against several archives +# rather than string-matched: exactly the archive the Backup recorded, or for +# a Backup that records none, the newest one stamped no later than its takenAt. +# +# Run from the repo root with make bats-unit-tests. The rendered script is run +# with `bash`, not `sh`: it opens `set -euo pipefail`, which the production +# image's busybox ash supports but the CI runner's /bin/sh (dash) rejects. load test_helper @@ -16,12 +30,13 @@ CHART=packages/system/backupstrategy-controller # Render the shipped strategy's in-Pod script into $1 and substitute the # controller-render placeholders the way the driver fills them: $2 mode, $3 -# app (release and source application), $4 the archive name recorded on the -# Backup (empty for a Backup that records none), $5 the Backup name, $6 its -# takenAt in the archive-name layout. An empty render fails early, as does a -# placeholder left unsubstituted. +# app (release and source application), $4 the archive name the controller +# chose or the Backup recorded (empty, the default, for none), $5 the Backup +# name, $6 its takenAt in the archive-name layout (default: later than every +# archive the stub lists). An empty render fails early, as does a placeholder +# left unsubstituted: these tests EXECUTE it. render_script() { - out=$1; mode=$2; app=$3; archive=$4; backup=$5; takenat=$6 + out=$1; mode=$2; app=$3; archive=${4-}; backup=${5-bkp}; takenat=${6-99991231T235959Z} helm template t "$CHART" -n cozy-backup-controller \ --set backupStorage.bucketNameOverride=test-bucket 2>/dev/null \ | yq eval 'select(.kind == "Altinity") | .spec.template.spec.containers[0].args[0]' - \ @@ -39,56 +54,118 @@ render_script() { fi } -# Stub the clickhouse-backup API behind curl, first on PATH. list/remote -# serves $state/remote; actions serves $state/actions plus one success row per -# operation the script has issued, and after its first read swaps in -# $state/actions.next when a test seeds one (an operation that finishes while -# the script waits). Every request is logged to $state/curl.log. sleep is -# stubbed to fail after a few calls, so a script that never sees its -# operation finish fails the test instead of hanging it. +# A curl stub standing in for the sidecar API, put first on PATH next to a +# no-op sleep. State lives in $1: +# unreachable present: every request fails as a refused connection (7) +# down a number: that many next requests fail the same way, as +# before the sidecar has started +# busy a number: that many GET /backup/actions answers still list +# an operation of someone else "in progress" +# op the command line of the sidecar's own operation, written +# by an accepted POST the way the sidecar logs it ("--resume" +# before the name of a create_remote asking for it), or +# seeded by a test as the operation +# an earlier attempt of the same Job started +# op_left how many more GET /backup/actions answers list op as +# "in progress" before it ends with op_result +# op_duration op_left given to an operation a POST starts (default 0) +# op_result the status op ends with (default success) +# prior extra action-log rows, verbatim, older than op +# remote present: the list/remote answer, verbatim, in place of the +# one archive listed by default +# restarted present: the action log no longer lists op, as after the +# sidecar restarted and lost its in-memory log +# fail_on_post a POST that is accepted makes that many following +# GET /backup/actions requests fail (curl exit 7) +# idle created by an answer that lists nothing in progress; a +# POST is refused (423, curl -f exit 22) unless one was seen +# since the last accepted POST, and while anything runs +# accepted one line per POST the sidecar accepted +# requests one line per request: method and URL +# runaway created when the script exceeds 200 requests; that request +# and every later one fails, so a polling loop that never ends +# stops the test instead of hanging it make_stubs() { bin=$1; state=$2 - mkdir -p "$bin" - : > "$state/remote" - : > "$state/actions" + mkdir -p "$bin" "$state" cat > "$bin/curl" <> "$state/curl.log" +echo "\$method \$url" >> "\$state/requests" +if [ "\$(wc -l < "\$state/requests")" -gt 200 ]; then touch "\$state/runaway"; exit 7; fi +if [ -e "\$state/unreachable" ]; then exit 7; fi +num() { cat "\$state/\$1" 2>/dev/null || echo 0; } +down=\$(num down) +if [ "\$down" -gt 0 ]; then echo \$((down - 1)) > "\$state/down"; exit 7; fi +running=0 +if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ] && [ "\$(num op_left)" -gt 0 ]; then running=1; fi case "\$method \$url" in - "GET "*/backup/status) echo '{}' ;; - "GET "*/backup/list/remote) cat "$state/remote" ;; + "POST "*) + if [ "\$(num busy)" -gt 0 ] || [ "\$running" = 1 ] || [ ! -e "\$state/idle" ]; then exit 22; fi + case "\$url" in + */create_remote\?name=*) + name=\${url#*name=}; name=\${name%%&*} + case "\$url" in *[?\&]resume=true*) cmd="create_remote --resume \$name" ;; *) cmd="create_remote \$name" ;; esac + ;; + */restore_remote/*) cmd="restore_remote \${url##*/}" ;; + *) exit 22 ;; + esac + echo "\$cmd" > "\$state/op" + echo "\$cmd" >> "\$state/accepted" + num op_duration > "\$state/op_left" + num fail_on_post > "\$state/fail_polls" + rm -f "\$state/idle" + ;; "GET "*/backup/actions) - cat "$state/actions" - if [ -f "$state/actions.done" ]; then cat "$state/actions.done"; fi - if [ -f "$state/actions.next" ]; then mv "$state/actions.next" "$state/actions"; fi + fail=\$(num fail_polls) + if [ "\$fail" -gt 0 ]; then echo \$((fail - 1)) > "\$state/fail_polls"; exit 7; fi + echo '{"command":"create_remote clickhouse-myapp-20260101T000000Z","status":"error","start":"2026-01-01 00:00:00","error":"old"}' + cat "\$state/prior" 2>/dev/null || true + inprog=0 + busy=\$(num busy) + if [ "\$busy" -gt 0 ]; then + echo \$((busy - 1)) > "\$state/busy" + echo '{"command":"upload --resumable=1 clickhouse-myapp-20260101T000000Z","status":"in progress","start":"2026-01-01 00:01:00"}' + inprog=1 + fi + if [ -s "\$state/op" ] && [ ! -e "\$state/restarted" ]; then + left=\$(num op_left) + if [ "\$left" -gt 0 ]; then + echo \$((left - 1)) > "\$state/op_left" + status="in progress" + inprog=1 + else + status=\$(cat "\$state/op_result" 2>/dev/null || echo success) + fi + echo "{\"command\":\"\$(cat "\$state/op")\",\"status\":\"\$status\",\"start\":\"2026-01-01 00:02:00\",\"error\":\"boom\"}" + fi + if [ "\$inprog" = 0 ]; then touch "\$state/idle"; fi ;; - "POST "*/backup/create_remote*) - name=\${url#*name=}; name=\${name%%&*} - printf '{"command":"create_remote --resume %s","status":"success","start":"2099-01-01 00:00:00"}\n' "\$name" >> "$state/actions.done" + "GET "*/backup/list/remote) + if [ -e "\$state/remote" ]; then cat "\$state/remote"; else + echo '{"name":"clickhouse-myapp-20260101T000000Z","created":"2026-01-01 00:00:00"}' + fi ;; - "POST "*/backup/restore_remote/*) - printf '{"command":"restore_remote %s","status":"success","start":"2099-01-01 00:00:00"}\n' "\${url##*/}" >> "$state/actions.done" - ;; - *) echo "unexpected request: \$method \$url" >&2; exit 22 ;; esac +exit 0 STUB - cat > "$bin/sleep" </dev/null || echo 0) -echo \$((n + 1)) > "$state/sleeps" -if [ "\$n" -ge 3 ]; then echo "sleep budget spent: the operation never finished" >&2; exit 1; fi + cat > "$bin/sleep" <<'STUB' +#!/bin/sh +exit 0 STUB chmod 0755 "$bin/curl" "$bin/sleep" - export API_USERNAME=u API_PASSWORD=p } # One list/remote row per argument, "name" or "name|desc"; created mirrors -# the name's timestamp, which is the order the pre-fix selection sorted by. +# the name's timestamp. remote_rows() { for row in "$@"; do name=${row%%|*}; desc="tar, regular" @@ -98,49 +175,216 @@ remote_rows() { done } +accepted() { + if [ -e "$1/state/accepted" ]; then wc -l < "$1/state/accepted" | tr -d ' '; else echo 0; fi +} + +run_script() { + tmp=$1 + API_USERNAME=u API_PASSWORD=p PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" +} + @test "an empty render fails the fixture instead of passing as a no-op" { tmp=$(mktemp -d) CHART=$tmp/no-such-chart - if render_script "$tmp/s.sh" restore myapp "" bkp 20260304T050622Z 2>/dev/null; then + if render_script "$tmp/s.sh" backup myapp 2>/dev/null; then echo "FAIL: render_script reported success with no chart" false fi - rm -rf "$tmp" +} + +@test "backup on an idle sidecar starts create_remote and waits for its success" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" backup myapp + run_script "$tmp" + grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests" +} + +@test "backup waits until the running operation finishes before create_remote" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 3 > "$tmp/state/busy" + render_script "$tmp/s.sh" backup myapp + run_script "$tmp" + grep -q '^create_remote --resume clickhouse-myapp-' "$tmp/state/accepted" + [ "$(cat "$tmp/state/busy")" = 0 ] +} + +@test "restore on an idle sidecar starts restore_remote of the listed archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + grep -q '^POST .*/backup/restore_remote/clickhouse-myapp-20260101T000000Z$' "$tmp/state/requests" +} + +@test "backup fails without starting create_remote when the sidecar stays busy" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 1000 > "$tmp/state/busy" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp"; then echo "FAIL: a busy sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no operation may start on a busy sidecar"; false; fi +} + +@test "backup fails without starting create_remote when the sidecar never answers" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + touch "$tmp/state/unreachable" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp"; then echo "FAIL: an unreachable sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no operation may start on an unreachable sidecar"; false; fi +} + +@test "restore fails without starting restore_remote when the sidecar never answers" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + touch "$tmp/state/unreachable" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp" 2> "$tmp/err"; then echo "FAIL: an unreachable sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no restore may start on an unreachable sidecar"; false; fi + grep -q 'list/remote failed 60 times in a row' "$tmp/err" + [ "$(grep -c '^GET .*/backup/list/remote$' "$tmp/state/requests")" = 60 ] +} + +@test "restore fails without starting restore_remote when the sidecar stays busy" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 1000 > "$tmp/state/busy" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp"; then echo "FAIL: a busy sidecar must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then echo "FAIL: no restore may start on a busy sidecar"; false; fi +} + +@test "backup fails instead of polling forever when the sidecar restarts after create_remote" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + touch "$tmp/state/restarted" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp" 2> "$tmp/err"; then echo "FAIL: an operation lost from the action log must fail the Pod"; false; fi + grep -q 'no longer in the sidecar action log' "$tmp/err" + grep -q '^POST .*/backup/create_remote?name=clickhouse-myapp-' "$tmp/state/requests" + if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling for an operation the sidecar no longer lists"; false; fi +} + +@test "restore waits for a sidecar that has not started yet" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 10 > "$tmp/state/down" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] +} + +@test "restore waits until another operation finishes, then starts its own" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 3 > "$tmp/state/busy" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] + grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted" +} + +@test "failed polls after restore_remote, short of six in a row, do not fail the attempt" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 3 > "$tmp/state/op_duration" + echo 5 > "$tmp/state/fail_on_post" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] +} + +@test "a retry after an attempt lost its polls joins the running restore instead of starting another" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 20 > "$tmp/state/op_duration" + echo 6 > "$tmp/state/fail_on_post" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp"; then echo "FAIL: six failed polls in a row must fail the attempt"; false; fi + [ "$(accepted "$tmp")" = 1 ] + echo 0 > "$tmp/state/fail_polls" + run_script "$tmp" + if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: the retry started restore_remote a second time"; false; fi + [ "$(cat "$tmp/state/op_left")" = 0 ] +} + +@test "restore does not join an operation running on another archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'restore_remote clickhouse-myapp-20251231T000000Z' > "$tmp/state/op" + echo 3 > "$tmp/state/op_left" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] + grep -qx 'restore_remote clickhouse-myapp-20260101T000000Z' "$tmp/state/accepted" +} + +# The sidecar never trims its action log, so a finished restore_remote row of +# the same archive may belong to an earlier RestoreJob; only a running one may +# be taken over. +@test "restore starts restore_remote again when the log holds a finished one of the same archive" { + for status in success error cancel; do + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo "{\"command\":\"restore_remote clickhouse-myapp-20260101T000000Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior" + render_script "$tmp/s.sh" restore myapp + run_script "$tmp" + if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row of an earlier restore must not stand in for this one"; false; fi + done +} + +@test "a restore that ends in error fails the Pod" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo error > "$tmp/state/op_result" + render_script "$tmp/s.sh" restore myapp + if run_script "$tmp"; then echo "FAIL: a failed restore must fail the Pod"; false; fi + [ "$(accepted "$tmp")" = 1 ] +} + +@test "a cancelled backup fails the Pod instead of polling forever" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo cancel > "$tmp/state/op_result" + render_script "$tmp/s.sh" backup myapp + if run_script "$tmp"; then echo "FAIL: a cancelled operation must fail the Pod"; false; fi + if [ -e "$tmp/state/runaway" ]; then echo "FAIL: the script kept polling a cancelled operation"; false; fi } @test "restore reads back the archive the Backup recorded, not the newest one" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" + make_stubs "$tmp/bin" "$tmp/state" render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \ - clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/remote" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/curl.log" - [ "$(grep -c '/backup/restore_remote/' "$tmp/curl.log")" -eq 1 ] - rm -rf "$tmp" + clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote" + run_script "$tmp" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests" + [ "$(accepted "$tmp")" = 1 ] } @test "restore fails naming the recorded archive when remote storage lacks it" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" + make_stubs "$tmp/bin" "$tmp/state" render_script "$tmp/s.sh" restore myapp clickhouse-myapp-20260304T050607Z bkp-a 20260304T050622Z - remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T060000Z > "$tmp/remote" + remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T060000Z > "$tmp/state/remote" out="" - if out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1); then + if out=$(run_script "$tmp" 2>&1); then echo "FAIL: restore succeeded without the recorded archive: $out" false fi printf '%s\n' "$out" | grep -qF 'archive clickhouse-myapp-20260304T050607Z recorded on Backup bkp-a is not in remote storage' - if grep -qF '/backup/restore_remote/' "$tmp/curl.log"; then - echo "FAIL: restored another archive: $(cat "$tmp/curl.log")" + if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then + echo "FAIL: restored another archive: $(cat "$tmp/state/requests")" false fi - rm -rf "$tmp" } @test "restore of a Backup without a recorded archive takes the newest one stamped by its takenAt, and warns" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" + make_stubs "$tmp/bin" "$tmp/state" render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z # Each decoy sorts after the expected archive and is ruled out by one # filter: broken, not a bare timestamp after the prefix, or stamped after @@ -148,71 +392,140 @@ remote_rows() { remote_rows clickhouse-myapp-20260304T040000Z clickhouse-myapp-20260304T050607Z \ "clickhouse-myapp-20260304T050700Z|broken (can't stat metadata.json)" \ clickhouse-myapp-20260304T050710Z-copy \ - clickhouse-myapp-20260304T060000Z > "$tmp/remote" - out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1) - grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/curl.log" - [ "$(grep -c '/backup/restore_remote/' "$tmp/curl.log")" -eq 1 ] + myapp-20260304050720-0f3c9a1e5b7d \ + clickhouse-myapp-20260304T060000Z > "$tmp/state/remote" + out=$(run_script "$tmp" 2>&1) + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/restore_remote/clickhouse-myapp-20260304T050607Z' "$tmp/state/requests" + [ "$(accepted "$tmp")" = 1 ] printf '%s\n' "$out" | grep -qF 'WARNING: Backup bkp-a records no archive name' - rm -rf "$tmp" } @test "restore of a Backup without a recorded archive fails when no archive is stamped by its takenAt" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" + make_stubs "$tmp/bin" "$tmp/state" render_script "$tmp/s.sh" restore myapp "" bkp-a 20260304T050800Z - remote_rows clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/remote" + remote_rows clickhouse-myapp-20260304T060000Z clickhouse-myapp-20260304T070000Z > "$tmp/state/remote" out="" - if out=$(PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" 2>&1); then + if out=$(run_script "$tmp" 2>&1); then echo "FAIL: restore succeeded with every archive newer than takenAt: $out" false fi printf '%s\n' "$out" | grep -qF 'no archive of myapp stamped no later than 20260304T050800Z' - if grep -qF '/backup/restore_remote/' "$tmp/curl.log"; then - echo "FAIL: restored an archive newer than takenAt: $(cat "$tmp/curl.log")" + if grep -qF '/backup/restore_remote/' "$tmp/state/requests"; then + echo "FAIL: restored an archive newer than takenAt: $(cat "$tmp/state/requests")" false fi - rm -rf "$tmp" } @test "backup writes the archive under the name the controller chose" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" - render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" "" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/curl.log" - rm -rf "$tmp" -} - -@test "a retried backup waits on the attempt the sidecar is still running" { - tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" - render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" "" - printf '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"in progress","start":"2026-03-04 05:06:08"}\n' > "$tmp/actions" - printf '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-03-04 05:06:08"}\n' > "$tmp/actions.next" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - # The sidecar answers a second operation with 423 while one runs. - if grep -qF 'POST ' "$tmp/curl.log"; then - echo "FAIL: started another operation over the running one: $(cat "$tmp/curl.log")" - false - fi - rm -rf "$tmp" -} - -@test "a retried backup after a failed attempt resumes the same archive" { - tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" - render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z "" "" - printf '{"command":"create_remote clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-03-04 05:06:08","error":"S3 PutObject 403"}\n' > "$tmp/actions" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/curl.log" - rm -rf "$tmp" + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests" } @test "backup falls back to a timestamped name when the controller supplies none" { tmp=$(mktemp -d) - make_stubs "$tmp/bin" "$tmp" - render_script "$tmp/s.sh" backup myapp "" "" "" - PATH="$tmp/bin:$PATH" bash "$tmp/s.sh" - grep -qE '^POST http://[^ ]*/backup/create_remote\?name=clickhouse-myapp-[0-9]{8}T[0-9]{6}Z&resume=true$' "$tmp/curl.log" - rm -rf "$tmp" + make_stubs "$tmp/bin" "$tmp/state" + render_script "$tmp/s.sh" backup myapp "" + run_script "$tmp" + grep -qE '^POST http://[^ ]*/backup/create_remote\?name=clickhouse-myapp-[0-9]{8}T[0-9]{6}Z&resume=true$' "$tmp/state/requests" +} + +# The sidecar answers a second operation with 423 while one runs, so a retry +# that POSTed over its predecessor's upload would burn every attempt left. +@test "a retried backup takes over the upload of its archive the sidecar is still running" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op" + echo 3 > "$tmp/state/op_left" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + if grep -q '^POST ' "$tmp/state/requests"; then + echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")" + false + fi + [ "$(cat "$tmp/state/op_left")" = 0 ] +} + +@test "a taken-over backup that ends in error fails the Pod" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'create_remote --resume clickhouse-myapp-20260304T050607Z' > "$tmp/state/op" + echo 2 > "$tmp/state/op_left" + echo error > "$tmp/state/op_result" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + if run_script "$tmp"; then echo "FAIL: a taken-over upload that failed must fail the Pod"; false; fi + if grep -q '^POST ' "$tmp/state/requests"; then + echo "FAIL: started another operation over the running one: $(cat "$tmp/state/requests")" + false + fi +} + +@test "a retried backup after a failed attempt resumes the same archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo '{"command":"create_remote clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30","error":"S3 PutObject 403"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + grep -qxF 'POST http://chi-clickhouse-myapp-clickhouse-0-0.tenant-x.svc:7171/backup/create_remote?name=clickhouse-myapp-20260304T050607Z&resume=true' "$tmp/state/requests" +} + +@test "backup does not take over an upload running for another archive" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo 'create_remote --resume clickhouse-myapp-20260304T040000Z' > "$tmp/state/op" + echo 3 > "$tmp/state/op_left" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] + grep -qx 'create_remote --resume clickhouse-myapp-20260304T050607Z' "$tmp/state/accepted" +} + +# The archive is named per BackupJob, so a success row of exactly this +# upload is an earlier attempt of the same BackupJob that finished while it +# could not see it. Uploading again with resume would re-upload over the +# finished archive and break it if that second upload dies halfway. +@test "a retried backup takes the upload of its archive an earlier attempt finished as its result" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + if grep -q '^POST ' "$tmp/state/requests"; then + echo "FAIL: uploaded again over a finished archive: $(cat "$tmp/state/requests")" + false + fi +} + +@test "a retried backup resumes an upload of its archive that ended in error or was cancelled" { + for status in error cancel; do + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo "{\"command\":\"create_remote --resume clickhouse-myapp-20260304T050607Z\",\"status\":\"$status\",\"start\":\"2026-01-01 00:01:30\"}" > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + if [ "$(accepted "$tmp")" != 1 ]; then echo "FAIL: a $status row must lead to a resumed upload"; false; fi + done +} + +@test "a retried backup resumes when its latest upload failed after an earlier one succeeded" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + printf '%s\n' \ + '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"success","start":"2026-01-01 00:01:00"}' \ + '{"command":"create_remote --resume clickhouse-myapp-20260304T050607Z","status":"error","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] +} + +@test "backup does not take another archive's finished upload as its own" { + tmp=$(mktemp -d) + make_stubs "$tmp/bin" "$tmp/state" + echo '{"command":"create_remote --resume clickhouse-myapp-20260304T040000Z","status":"success","start":"2026-01-01 00:01:30"}' > "$tmp/state/prior" + render_script "$tmp/s.sh" backup myapp clickhouse-myapp-20260304T050607Z + run_script "$tmp" + [ "$(accepted "$tmp")" = 1 ] } diff --git a/hack/backport-batch-tests-needed.bats b/hack/backport-batch-tests-needed.bats new file mode 100644 index 000000000..9f37e768f --- /dev/null +++ b/hack/backport-batch-tests-needed.bats @@ -0,0 +1,87 @@ +#!/usr/bin/env bats +# Unit tests for hack/backport-batch-tests-needed.sh, which decides whether a +# pull request runs `make test-backport-batch`, and for its wiring into the +# checks job of pull-requests.yaml. +# +# Test-level EXIT traps replace Bats' own handler and hide failing TAP results. +# Cleanup follows aborting assertions; see docs/agents/e2e-testing.md. + +load test_helper + +CLASSIFIER="hack/backport-batch-tests-needed.sh" +WF=".github/workflows/pull-requests.yaml" + +classify() { + list="$1" + shift + printf '%s\n' "$@" > "$list" + "$CLASSIFIER" "$list" +} + +@test "a change to the tool, an imported package, a page its tests read or the module runs the tests" { + tmp=$(mktemp) + [ "$(classify "$tmp" cmd/backport-batch/report_test.go)" = true ] + [ "$(classify "$tmp" internal/apigate/surface.go)" = true ] + [ "$(classify "$tmp" pkg/config/config.go)" = true ] + [ "$(classify "$tmp" docs/release/backports.md)" = true ] + [ "$(classify "$tmp" docs/release/backport-tasks/conflict.md)" = true ] + [ "$(classify "$tmp" go.sum)" = true ] + # One input among unrelated paths is enough. + [ "$(classify "$tmp" packages/apps/mariadb/values.yaml pkg/config/config.go)" = true ] + rm -f "$tmp" +} + +@test "a change that reaches none of its inputs does not run the tests" { + tmp=$(mktemp) + [ "$(classify "$tmp" packages/apps/mariadb/values.yaml)" = false ] + [ "$(classify "$tmp" cmd/api-gate/main.go)" = false ] + [ "$(classify "$tmp" hack/backport-batch-scheduled.sh)" = false ] + [ "$(classify "$tmp" docs/release.md docs/release/other.md)" = false ] + # A file entry is not a prefix: backports.md does not cover backports.md.bak, + # and a directory entry does not cover a sibling sharing its name as prefix. + [ "$(classify "$tmp" docs/release/backports.md.bak)" = false ] + [ "$(classify "$tmp" pkg/configx/x.go)" = false ] + [ "$(classify "$tmp" internal/apigate-old/x.go)" = false ] + rm -f "$tmp" +} + +@test "entries are trimmed as the Go test trims them" { + tmp=$(mktemp -d) + printf '%s\n' cmd/backport-batch/main.go > "$tmp/diff" + printf ' cmd/backport-batch/ \r\n' > "$tmp/inputs" + [ "$("$CLASSIFIER" "$tmp/diff" "$tmp/inputs")" = true ] + rm -rf "$tmp" +} + +@test "an inputs file with no entry is refused rather than read as never" { + tmp=$(mktemp -d) + printf '%s\n' cmd/backport-batch/main.go > "$tmp/diff" + printf '# only a comment\n\n' > "$tmp/inputs" + if "$CLASSIFIER" "$tmp/diff" "$tmp/inputs" >"$tmp/out" 2>"$tmp/err"; then + echo "an empty inputs file must fail, it printed: $(cat "$tmp/out")" >&2 + exit 1 + fi + grep -q 'lists no path' "$tmp/err" + rm -rf "$tmp" +} + +@test "the checks job runs the target on the classifier's answer alone" { + plan_run=$(yq -r '.jobs.plan.steps[] | select(.id == "p") | .run' "$WF") + printf '%s\n' "$plan_run" | grep -qF 'backport_batch="$(hack/backport-batch-tests-needed.sh /tmp/changed_norenames.txt)"' + [ "$(yq -r '.jobs.plan.outputs.backport_batch' "$WF")" = '${{ steps.p.outputs.backport_batch }}' ] + + step_if=$(yq -r '.jobs.checks.steps[] | select(.name == "Run backport-batch tests") | .if' "$WF") + step_run=$(yq -r '.jobs.checks.steps[] | select(.name == "Run backport-batch tests") | .run' "$WF") + # !cancelled(): a failed unit lane above must not skip this suite too. + [ "$step_if" = "\${{ !cancelled() && needs.plan.outputs.backport_batch == 'true' }}" ] + [ "$step_run" = 'make test-backport-batch' ] + + # And nowhere else: the unit lane every code PR runs must not carry it. + prereqs=$(sed -n 's/^unit-tests:[[:space:]]*//p' Makefile) + [ -n "$prereqs" ] + case " $prereqs " in + *" test-backport-batch "*) + echo "make unit-tests still runs test-backport-batch on every code PR: $prereqs" >&2 + exit 1 ;; + esac +} diff --git a/hack/backport-batch-tests-needed.sh b/hack/backport-batch-tests-needed.sh new file mode 100755 index 000000000..71803e730 --- /dev/null +++ b/hack/backport-batch-tests-needed.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Print whether a changed-path list touches an input of +# `go test ./cmd/backport-batch/...`, as listed in +# cmd/backport-batch/test-inputs.txt. An entry ending in / matches every path +# below it, any other entry matches that file alone. The caller supplies the +# rename-blind list so both sides of a rename are classified. +# +# Usage: hack/backport-batch-tests-needed.sh [] +set -eu + +inputs="${2:-cmd/backport-batch/test-inputs.txt}" +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ] || [ ! -r "$1" ] || [ ! -r "$inputs" ]; then + echo "usage: backport-batch-tests-needed.sh []" >&2 + exit 2 +fi + +# An inputs file with no entry would answer false for every diff and switch the +# tests off without a word; refuse instead. +if ! grep -qvE '^[[:space:]]*(#|$)' "$inputs"; then + echo "backport-batch-tests-needed: $inputs lists no path" >&2 + exit 2 +fi + +# Entries are trimmed the way TestInputsListed trims them, so a trailing space +# or a CR cannot make the Go test accept an entry this never matches. +awk ' + NR == FNR { + e = $0 + sub(/^[[:space:]]+/, "", e) + sub(/[[:space:]]+$/, "", e) + if (e != "" && e !~ /^#/) entries[++n] = e + next + } + { + for (i = 1; i <= n; i++) { + e = entries[i] + if ((substr(e, length(e)) == "/" && index($0, e) == 1) || $0 == e) { + found = 1 + exit + } + } + } + END { print (found ? "true" : "false") } +' "$inputs" "$1" diff --git a/hack/build-matrix.sh b/hack/build-matrix.sh index 95e894a7f..d15267631 100755 --- a/hack/build-matrix.sh +++ b/hack/build-matrix.sh @@ -67,7 +67,13 @@ if [ -z "$CHANGED" ] || [ "$CHANGED" = "FULL" ]; then exit 0 fi -if grep -qE "$full_rebuild_pattern" "$CHANGED"; then +# CI-only Go tools that no image builds. Dropped before the full-rebuild check, +# whose cmd/ would otherwise rebuild everything for an edit no image contains. +# hack/select-e2e.sh skips the same two, and a diff made only of them and docs/ +# never reaches this script's build jobs (the plan job's `code` is false). +tooling_pattern='^cmd/(backport-batch|api-gate)/' + +if grep -vE "$tooling_pattern" "$CHANGED" | grep -qE "$full_rebuild_pattern"; then emit_json "$units" exit 0 fi diff --git a/hack/build-matrix_test.bats b/hack/build-matrix_test.bats index 44a9f2c04..c11f2707f 100644 --- a/hack/build-matrix_test.bats +++ b/hack/build-matrix_test.bats @@ -97,6 +97,23 @@ load test_helper rm -f "$tmp" } +@test "the CI-only tools do not force the full matrix" { + tmp=$(mktemp) + # No image builds cmd/backport-batch or cmd/api-gate, so they select nothing + # and leave the rest of the diff to its own rules. + printf '%s\n' cmd/backport-batch/report_test.go cmd/api-gate/main.go > "$tmp" + out=$(hack/build-matrix.sh "$tmp") + [ "$out" = '[]' ] + printf '%s\n' cmd/backport-batch/main.go packages/apps/mariadb/Makefile > "$tmp" + out=$(hack/build-matrix.sh "$tmp") + [ "$out" = '["packages/apps/mariadb"]' ] + # Every other cmd/ directory is a shipped binary and still rebuilds all. + printf '%s\n' cmd/backport-batch/main.go cmd/cozystack-api/main.go > "$tmp" + out=$(hack/build-matrix.sh "$tmp") + [ "$out" = "$(hack/build-matrix.sh)" ] + rm -f "$tmp" +} + @test "a package with no image target selects nothing" { tmp=$(mktemp) # postgres ships no in-repo image build, so it is not a build unit. diff --git a/hack/check-kubernetes-nodes-cidr-agreement.bats b/hack/check-kubernetes-nodes-cidr-agreement.bats new file mode 100644 index 000000000..b337296de --- /dev/null +++ b/hack/check-kubernetes-nodes-cidr-agreement.bats @@ -0,0 +1,104 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# The tenant cluster's pod and service CIDRs are written by two charts: +# +# * packages/apps/kubernetes renders them into the CAPI Cluster's +# spec.clusterNetwork, which the control plane is built from; +# * packages/apps/kubernetes-nodes renders them into every worker's Talos +# machine config (cluster.network.podSubnets / serviceSubnets), inside the +# TalosConfigTemplate the talos-reconcile Job applies. +# +# Each chart carries its own literal, and neither render fails when the two +# disagree: the workers boot, join, and route pod and Service traffic over +# ranges the control plane does not use. This guard renders both charts and +# compares the lists, so a range moved on one side only fails here. +# +# The worker side is read the way the Job produces it: the heredoc is cut out +# of the Job's command, run through sh, and the TalosConfigTemplate it prints is +# parsed, then the machine config inside it. A text grep of the Job would also +# match a comment. +# +# Requires: helm, yq (mikefarah v4+). Runs under both `bats` and the in-repo +# cozytest.sh runner, so TMP is provisioned inline per test and cleaned without +# an EXIT trap. +# ----------------------------------------------------------------------------- + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" + +# $1: output directory. Writes pods and services, one CIDR per line. +cluster_cidrs() { + helm template kubernetes-myk8s "$REPO_ROOT/packages/apps/kubernetes" \ + --namespace tenant-test \ + -f "$REPO_ROOT/packages/apps/kubernetes/tests/values-ci.yaml" \ + > "$1/cluster.yaml" 2> "$1/cluster.err" \ + || { cat "$1/cluster.err" >&2; return 1; } + yq 'select(.kind == "Cluster" and .apiVersion == "cluster.x-k8s.io/v1beta1") | .spec.clusterNetwork.pods.cidrBlocks[]' \ + "$1/cluster.yaml" > "$1/cluster.pods" + yq 'select(.kind == "Cluster" and .apiVersion == "cluster.x-k8s.io/v1beta1") | .spec.clusterNetwork.services.cidrBlocks[]' \ + "$1/cluster.yaml" > "$1/cluster.services" +} + +# $1: output directory. Writes pool.pods and pool.services, one CIDR per line. +pool_cidrs() { + helm template kubernetes-nodes-myk8s-md0 "$REPO_ROOT/packages/apps/kubernetes-nodes" \ + --namespace tenant-test \ + --set cluster=myk8s \ + --set _cluster.cluster-domain=cozy.local \ + --set instanceType="" \ + --set resources.cpu=2 --set resources.memory=4Gi \ + --show-only templates/talos-reconcile-job.yaml \ + > "$1/pool.yaml" 2> "$1/pool.err" \ + || { cat "$1/pool.err" >&2; return 1; } + yq 'select(.kind == "Job") | .spec.template.spec.containers[0].command[2]' "$1/pool.yaml" > "$1/cmd.sh" + awk ' + /(^|="?\$\()cat < "$1/heredoc.sh" + sh "$1/heredoc.sh" > "$1/tct.yaml" 2> "$1/heredoc.err" \ + || { cat "$1/heredoc.err" >&2; return 1; } + yq 'select(.kind == "TalosConfigTemplate") | .spec.template.spec.data' "$1/tct.yaml" > "$1/machineconfig.yaml" + yq '.cluster.network.podSubnets[]' "$1/machineconfig.yaml" > "$1/pool.pods" + yq '.cluster.network.serviceSubnets[]' "$1/machineconfig.yaml" > "$1/pool.services" +} + +# $1: directory, $2: pods or services, $3: what to call it in a message. +assert_agree() { + cluster=$(cat "$1/cluster.$2") + pool=$(cat "$1/pool.$2") + # An empty read means a field moved or a render came out short; comparing + # "" with "" would pass on nothing. + if [ -z "$cluster" ] || [ -z "$pool" ]; then + echo "FAIL: could not read the $3 CIDRs from both renders." + echo " kubernetes Cluster.spec.clusterNetwork: '${cluster:-}'" + echo " kubernetes-nodes machine config cluster.network: '${pool:-}'" + return 1 + fi + if [ "$cluster" != "$pool" ]; then + echo "FAIL: the tenant $3 CIDRs differ between the two charts." + echo " packages/apps/kubernetes (CAPI Cluster): $cluster" + echo " packages/apps/kubernetes-nodes (worker machine config): $pool" + echo "Workers would route $3 traffic over a range the control plane does not" + echo "use. Change both charts together. Moving the worker side also renames" + echo "the content-hashed TalosConfigTemplate and rolls every worker." + return 1 + fi +} + +@test "kubernetes and kubernetes-nodes render the same pod CIDRs" { + TMP=$(mktemp -d) + cluster_cidrs "$TMP" + pool_cidrs "$TMP" + assert_agree "$TMP" pods pod + rm -rf "$TMP" +} + +@test "kubernetes and kubernetes-nodes render the same service CIDRs" { + TMP=$(mktemp -d) + cluster_cidrs "$TMP" + pool_cidrs "$TMP" + assert_agree "$TMP" services service + rm -rf "$TMP" +} diff --git a/hack/check-worker-image-catalog-defaults.bats b/hack/check-worker-image-catalog-defaults.bats index 261def920..eaf942112 100644 --- a/hack/check-worker-image-catalog-defaults.bats +++ b/hack/check-worker-image-catalog-defaults.bats @@ -78,19 +78,16 @@ CATALOG_VALUES="$REPO_ROOT/packages/system/kubernetes-worker-image/values.yaml" # ----------------------------------------------------------------------------- # Second family of default-drift guards on the same chart: the Talos <-> Kubernetes -# support matrix. The render guard keyed only on `talos.version`, so a pool could -# reach a different Talos release through `osImage.builtin.version` / -# `osImage.factory.version` and boot a kubelet outside its support window with no -# render failure. The table now lives once in templates/_helpers.tpl and both -# paths consult it. +# support matrix. A pool reaches a Talos release through `talos.version` or through +# `osImage.builtin.version` / `osImage.factory.version`, and the table in +# templates/_helpers.tpl has to be consulted for whichever one the pool boots. # -# The symmetry is structural, not observable from a render: the matrix ships one -# row (`v1.13`) whose Kubernetes list is a superset of `values.schema.json`'s -# `version` enum, so no schema-valid pairing can make either call site fail today. -# A helm-unittest case would have to put an out-of-enum `version` in, and -# helm-unittest validates the schema, so it cannot. These two guards cover what a -# render cannot: that both call sites exist, and that the table still means -# something for the defaults the chart ships. +# The matrix ships one row (`v1.13`) whose Kubernetes list is a superset of +# `values.schema.json`'s `version` enum, so no schema-valid pairing makes either +# call site fail today. hack/kubernetes-nodes-talos-matrix.bats renders the +# refusals with the schema skipped; these two guards cover what it does not: that +# both call sites stay wired, and that the table still means something for the +# defaults the chart ships. NODES_TEMPLATE="$REPO_ROOT/packages/apps/kubernetes-nodes/templates/nodegroup.yaml" NODES_SCHEMA="$REPO_ROOT/packages/apps/kubernetes-nodes/values.schema.json" @@ -148,30 +145,3 @@ NODES_SCHEMA="$REPO_ROOT/packages/apps/kubernetes-nodes/values.schema.json" esac done } - -# ----------------------------------------------------------------------------- -# Third guard, and the premise the first one rests on. The catalog is keyed off -# the kubernetes-nodes chart's `talos` defaults, but the parent kubernetes chart -# carries its own copy of the same block since the Phase 2 split, and its own -# values.yaml says to keep the two in sync. Nothing enforced that: the pool -# chart's `make update` copies files/versions.yaml and images/kubectl.tag from -# the parent and not the talos block, so a one-sided bump leaves the parent's -# support-matrix guard reasoning about one Talos release while every worker boots -# another — and the catalog check above would go on passing, because it only ever -# reads one of the two files. - -PARENT_VALUES="$REPO_ROOT/packages/apps/kubernetes/values.yaml" - -@test "the parent kubernetes chart and kubernetes-nodes agree on their Talos defaults" { - for key in version schematicID imageFactoryURL installerRepository registryMirrors; do - parent=$(yq ".talos.${key}" "$PARENT_VALUES") - pool=$(yq ".talos.${key}" "$APP_VALUES") - if [ "$parent" != "$pool" ]; then - echo "talos.${key} differs between the two charts:" >&2 - echo " packages/apps/kubernetes/values.yaml: $parent" >&2 - echo " packages/apps/kubernetes-nodes/values.yaml: $pool" >&2 - echo "both are declared 'keep in sync' and a worker pool follows the second while the parent's support-matrix guard reads the first" >&2 - return 1 - fi - done -} diff --git a/hack/cozyreport.sh b/hack/cozyreport.sh index 0f20b1261..0e6c12b15 100755 --- a/hack/cozyreport.sh +++ b/hack/cozyreport.sh @@ -121,7 +121,7 @@ # the number; do not expect to find it written here. # # TWO ceilings sit outside it, and they fail differently. The job is capped at -# `timeout-minutes: 215`, and reaching that cancels every step still to come -- +# `timeout-minutes: 265`, and reaching that cancels every step still to come -- # the artifact upload among them. The step running this script is capped at 30 of # its own, and reaching THAT ends only this step: the uploads after it still run, # and `continue-on-error` on that step keeps the overrun out of the job's result, diff --git a/hack/cozytest.sh b/hack/cozytest.sh index 02a40f383..257bdcd08 100755 --- a/hack/cozytest.sh +++ b/hack/cozytest.sh @@ -208,7 +208,7 @@ _cozy_on_exit() { # capture would read as a complete one listing fewer crash-loops than # actually occurred. This handler's whole worst case is ~1350s (330 + 390 # + 630, each leg including its 30s kill grace) against a job capped at - # 215 minutes with no per-step timeout — + # 265 minutes with no per-step timeout — # deliberate, for the same reason recorded in hack/e2e-chainsaw/.chainsaw.yaml: # a bounded, honest wait beats a kill that collects nothing. if [ "$_prev_rc" -ne 0 ]; then diff --git a/hack/downstream-trigger-map.bats b/hack/downstream-trigger-map.bats index c5ebf6af3..39e1d5433 100644 --- a/hack/downstream-trigger-map.bats +++ b/hack/downstream-trigger-map.bats @@ -35,8 +35,9 @@ # a human's job, and reviewers should not assume CI did it for them. # # A docs-only PR skips builds and E2E, but the `checks` job still runs the Bats -# lane. Otherwise editing this map, or another document audited by a unit -# contract, would exempt the changed input from its own guard. +# lane whenever a docs/ path changed. Otherwise editing this map, or another +# document audited by a unit contract, would exempt the changed input from its +# own guard. load test_helper @@ -117,9 +118,18 @@ map_section() { [ "$job_if" = "!contains(github.event.pull_request.labels.*.name, 'release')" ] [ "$code_if" = "needs.plan.outputs.code == 'true'" ] - [ "$docs_if" = "needs.plan.outputs.code == 'false'" ] + [ "$docs_if" = "needs.plan.outputs.code == 'false' && needs.plan.outputs.docs == 'true'" ] [ "$docs_run" = 'make bats-unit-tests' ] [ "$code_run" = 'make unit-tests test-controllers test-controllers-envtest -j4 -k --output-sync=target' ] + + # `code` is false for the CI-only tools too, so the docs lane is keyed on a + # docs/ path being present, not on `code` alone. Pin that the output means + # exactly that and that Bats gets installed for it. + plan_run=$(yq -r '.jobs.plan.steps[] | select(.id == "p") | .run' "$WORKFLOW") + printf '%s\n' "$plan_run" | grep -qF "if grep -qE '^docs/' /tmp/changed_norenames.txt; then" + [ "$(yq -r '.jobs.plan.outputs.docs' "$WORKFLOW")" = '${{ steps.p.outputs.docs }}' ] + toolchain_if=$(yq -r '.jobs.checks.steps[] | select(.name == "Set up test toolchain") | .if' "$WORKFLOW") + [ "$toolchain_if" = "needs.plan.outputs.code == 'true' || needs.plan.outputs.docs == 'true'" ] } @test "the trigger map and the PR-template checklist list the same repositories" { diff --git a/hack/e2e-chainsaw/.chainsaw.yaml b/hack/e2e-chainsaw/.chainsaw.yaml index 87aa95619..6748a91c6 100644 --- a/hack/e2e-chainsaw/.chainsaw.yaml +++ b/hack/e2e-chainsaw/.chainsaw.yaml @@ -66,14 +66,14 @@ spec: # script: it SIGKILLs the shell alone, orphaned children hold the pipe open, # and the op overruns (#4268 measured 606s under a 300s ceiling), so the # nominal ~5m was a figure a busy catch could exceed, not a cap. Tests run - # sequentially, and the E2E job is capped at 215 timeout-minutes. A cascade + # sequentially, and the E2E job is capped at 265 timeout-minutes. A cascade # reddening many suites can therefore spend a large share of the job in # diagnostics. That is deliberate — the old ceiling # bought its speed by collecting nothing at all — and it is bounded: both legs # only approach their ceilings on a genuinely degraded cluster, a systemic # failure usually reds the install gate and skips the E2E step entirely, and # the report-upload steps are `if: always()`. That last point does NOT extend - # to a job timeout: a 215-minute cap ends the job `cancelled`, and the ~4m48s + # to a job timeout: a 265-minute cap ends the job `cancelled`, and the ~4m48s # grace window is consumed by the earlier `Collect report` step, so both # cozyreport.tgz and the job log are lost (measured on runs 29557003429 and # 29568562337). If this ever does start truncating runs, bound the number of diff --git a/hack/e2e-chainsaw/_lib/run-kubernetes.sh b/hack/e2e-chainsaw/_lib/run-kubernetes.sh index 14e247e13..9aaf9f1f6 100644 --- a/hack/e2e-chainsaw/_lib/run-kubernetes.sh +++ b/hack/e2e-chainsaw/_lib/run-kubernetes.sh @@ -18,7 +18,7 @@ cozy_e2e_storage_class() { esac } -# talos_spec_block: emit the tenant Kubernetes CR `spec.talos` block with the Talos +# talos_spec_block: emit the tenant KubernetesNodes CR `spec.talos` block with the Talos # OS image cache (imageFactoryURL) when the in-sandbox cache is up. Prints nothing # when it falls back to the public default, so the chart default applies. # Indented for insertion directly under `spec:` in the heredoc below. @@ -5702,8 +5702,9 @@ YAML # Point worker DataVolume imports at the in-sandbox Talos OS image cache when it # is up, falling back to the public factory otherwise. Worker image pulls use - # the chart's public-registry default. Emitted right under spec: as - # `talos: { imageFactoryURL: ... }`, or nothing when the default applies. + # the chart's public-registry default. Emitted right under spec: of the + # KubernetesNodes pool below as `talos: { imageFactoryURL: ... }`, or nothing + # when the default applies. The Kubernetes CR takes no worker image field. local talos_block talos_block=$(talos_spec_block) @@ -5714,7 +5715,6 @@ metadata: name: "${test_name}" namespace: tenant-test spec: -${talos_block} ${oidc_block} addons: certManager: diff --git a/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml b/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml index 2a02dc06e..4e0c86535 100644 --- a/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/computeplane/chainsaw-test.yaml @@ -35,7 +35,7 @@ # and the teardown. This is the third suite in a full run that brings up a # tenant cluster, next to kubernetes-latest and kubernetes-previous. A failure # that stalls worker join everywhere reddens all three, and their red paths -# together can exceed the E2E job's 215-minute cap, which cancels the job and +# together can exceed the E2E job's 265-minute cap, which cancels the job and # loses its log and report. The cap is sized for the red path of the # kubernetes pair only; this suite is not counted in it, and that is an # accepted risk. diff --git a/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml b/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml index 75ee5a2fa..695425364 100644 --- a/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml +++ b/hack/e2e-chainsaw/kubernetes-latest/chainsaw-test.yaml @@ -80,7 +80,8 @@ spec: # wait, all on the passing path. Raising the ceiling to cover them is not # available -- the guard in hack/run-kubernetes-serial-console_test.bats # holds COZY_OP_CEILING equal to BOTH suites' `timeout:`, and two suites - # at 79m plus their 22m teardowns exceed the job's own 215-minute cap. + # at 79m plus their 22m teardowns take 202m, which with the install and + # the rest of a full run does not fit the job's own 265-minute cap. # What is available is a measurement: on the green run of this branch # (33724179053) the whole Test took 569.72s against this 4020s op, with # kubernetes-previous, which carries no OIDC fold, at 470.19s. The diff --git a/hack/e2e-install-cozystack.bats b/hack/e2e-install-cozystack.bats index 47315ac85..500fe68ee 100644 --- a/hack/e2e-install-cozystack.bats +++ b/hack/e2e-install-cozystack.bats @@ -49,10 +49,10 @@ # Best-effort: never fail the suite on the band-aid. Remove once tenant workers # no longer bulk-pull the OS image from the public internet in CI. local sid ver - sid=$(yq '.talos.schematicID' packages/apps/kubernetes/values.yaml 2>/dev/null) - ver=$(yq '.talos.version' packages/apps/kubernetes/values.yaml 2>/dev/null) + sid=$(yq '.talos.schematicID' packages/apps/kubernetes-nodes/values.yaml 2>/dev/null) + ver=$(yq '.talos.version' packages/apps/kubernetes-nodes/values.yaml 2>/dev/null) if [ -z "$sid" ] || [ "$sid" = "null" ] || [ -z "$ver" ] || [ "$ver" = "null" ]; then - echo "WARNING: could not read talos.schematicID/version from values.yaml — skipping mirror (fallback to public factory)" + echo "WARNING: could not read talos.schematicID/version from packages/apps/kubernetes-nodes/values.yaml — skipping mirror (fallback to public factory)" return 0 fi # The CiliumClusterwideNetworkPolicy in the manifest is intentionally NOT applied @@ -561,3 +561,40 @@ data: version: "${version}" EOF } + +@test "Tenant namespace delete policy refuses a direct delete and keeps the Tenant uninstall" { + if ! kubectl api-resources --api-group=admissionregistration.k8s.io \ + 2>/dev/null | grep -qw validatingadmissionpolicies; then + skip "ValidatingAdmissionPolicy API not available on this cluster" + fi + kubectl get validatingadmissionpolicy cozystack-tenant-namespace-delete-policy + kubectl get validatingadmissionpolicybinding cozystack-tenant-namespace-delete-policy + + kubectl apply -f - </dev/null 2>&1; do sleep 2; done' + kubectl wait hr/tenant-deleteprobe -n tenant-root --timeout=1m --for=condition=ready + + local selected output rc still_there + selected=$(kubectl get namespaces --selector tenant.cozystack.io/tenant-root --field-selector metadata.name=tenant-deleteprobe -o name) + [ "$selected" = "namespace/tenant-deleteprobe" ] + + output=$(kubectl delete namespace tenant-deleteprobe --dry-run=server 2>&1) && rc=0 || rc=$? + echo "kubectl delete exit=$rc, output=$output" + [ "$rc" -ne 0 ] + echo "$output" | grep -qF "ValidatingAdmissionPolicy 'cozystack-tenant-namespace-delete-policy'" + echo "$output" | grep -qF 'kubectl delete tenants.apps.cozystack.io deleteprobe --namespace tenant-root.' + + kubectl delete tenants.apps.cozystack.io deleteprobe -n tenant-root --timeout=5m + kubectl wait namespace/tenant-deleteprobe --for=delete --timeout=5m + if ! still_there=$(kubectl get namespace tenant-deleteprobe --ignore-not-found -o name); then + echo "FAIL: reading the tenant-deleteprobe namespace did not answer"; false + fi + if [ -n "$still_there" ]; then echo "FAIL: deleting the Tenant must remove its namespace"; false; fi +} diff --git a/hack/e2e-talos-image-cache.yaml b/hack/e2e-talos-image-cache.yaml index cac252d69..fe499d1bd 100644 --- a/hack/e2e-talos-image-cache.yaml +++ b/hack/e2e-talos-image-cache.yaml @@ -14,7 +14,7 @@ # This Deployment fetches the image ONCE at e2e-install time with an aggressive # curl retry loop (decoupling the one flaky external fetch from the many # per-worker imports), then serves it locally over plain HTTP with range support. -# Tenant `Kubernetes` CRs point `spec.talos.imageFactoryURL` at this Service +# Tenant `KubernetesNodes` CRs point `spec.talos.imageFactoryURL` at this Service # (see hack/e2e-chainsaw/_lib/run-kubernetes.sh) so every worker imports from the # warm local copy instead of the public internet. Tenant namespaces run under a # default-deny Cilium egress installed by the tenant chart, whose allow rules cover @@ -28,7 +28,7 @@ # mirror can only help, never make CI worse. # # The __SCHEMATIC_ID__ / __TALOS_VERSION__ placeholders are substituted at apply -# time by hack/e2e-install-cozystack.bats from packages/apps/kubernetes/values.yaml. +# time by hack/e2e-install-cozystack.bats from packages/apps/kubernetes-nodes/values.yaml. # # CI band-aid, not a product artifact. The durable fix is the overridable # `talos.imageFactoryURL` chart value this manifest exercises; a self-hosted diff --git a/hack/gate-concurrency-contract.bats b/hack/gate-concurrency-contract.bats index 5133a4cd8..0c4c90f15 100644 --- a/hack/gate-concurrency-contract.bats +++ b/hack/gate-concurrency-contract.bats @@ -266,24 +266,56 @@ resolve_silent_conclusions() { plan_block="$(job_block plan "$PULL_REQUESTS" | code_lines)" [ -n "$plan_block" ] - count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^docs/' /tmp/changed_norenames.txt" || true)" + count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^(.*)' /tmp/changed_norenames.txt" || true)" [ "${count:-0}" -eq 1 ] # The rename-detected list may still feed the build matrix, but never this. - count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^docs/' /tmp/changed.txt" || true)" + count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^(.*)' /tmp/changed.txt" || true)" [ "${count:-0}" -eq 0 ] # The fork half of the same agreement: both sides of a rename are inspected. - # Pinned on the docs-only expression itself — `previous_filename` also appears + # Pinned on the skip expression itself — `previous_filename` also appears # in the TIA file list a few lines down, so counting the identifier would stay # satisfied by that one with this check deleted. block="$(job_block resolve "$FORK" | code_lines)" [ -n "$block" ] count="$(printf '%s\n' "$block" \ - | grep -c 'f\.previous_filename && !isDocs(f\.previous_filename)' || true)" + | grep -c 'f\.previous_filename && !isNoRuntime(f\.previous_filename)' || true)" [ "${count:-0}" -eq 1 ] } +@test "both lanes skip e2e for the same path prefixes" { + # The rename test above pins HOW each lane reads its list; this one pins + # WHAT it skips. A prefix added to one lane only is a fork PR that one lane + # reads as nothing-to-build and the other as needing the pr-patch artifact, + # which nothing will ever produce. + plan_prefixes="$(job_block plan "$PULL_REQUESTS" | code_lines \ + | sed -nE "s,.*grep -qvE '\^\(([^)]*)\)' /tmp/changed_norenames\.txt.*,\1,p" \ + | tr '|' '\n' | sort)" + fork_prefixes="$(job_block resolve "$FORK" | code_lines \ + | sed -nE "s,.*const noRuntimePrefixes = \[([^]]*)\];.*,\1,p" \ + | tr ',' '\n' | tr -d " '" | sort)" + [ -n "$plan_prefixes" ] + printf '%s\n' "$plan_prefixes" | grep -qx 'docs/' + if [ "$plan_prefixes" != "$fork_prefixes" ]; then + echo "plan skips:" >&2; printf ' %s\n' $plan_prefixes >&2 + echo "fork skips:" >&2; printf ' %s\n' $fork_prefixes >&2 + exit 1 + fi + + # The tools in that list must also be the ones the build matrix and the suite + # selector leave alone in a mixed diff; otherwise a tool skipped on its own + # still rebuilds everything beside a one-line package change. + tools="$(printf '%s\n' "$plan_prefixes" | grep -vx 'docs/')" + [ -n "$tools" ] + matrix_tools="$(sed -nE "s,^tooling_pattern='\^cmd/\(([^)]*)\)/'$,\1,p" "$REPO_ROOT/hack/build-matrix.sh" \ + | tr '|' '\n' | sed 's,^,cmd/,; s,$,/,' | sort)" + select_tools="$(sed -nE 's,^ (cmd/[^)]*)\)$,\1,p' "$REPO_ROOT/hack/select-e2e.sh" \ + | tr '|' '\n' | sed 's,\*$,,' | sort)" + [ "$tools" = "$matrix_tools" ] + [ "$tools" = "$select_tools" ] +} + @test "same-repo gate: every publishing guard names the same label set" { # `plan`, `verify-release-candidate`, `e2e-report` and the concurrency key # each decide from the label name whether this run owns the status. Renaming diff --git a/hack/kubernetes-nodes-talos-matrix.bats b/hack/kubernetes-nodes-talos-matrix.bats new file mode 100644 index 000000000..33c2d0af6 --- /dev/null +++ b/hack/kubernetes-nodes-talos-matrix.bats @@ -0,0 +1,81 @@ +#!/usr/bin/env bats +# Unit test: the refusals of the Talos to Kubernetes support-matrix guard in +# packages/apps/kubernetes-nodes (kubernetes-nodes.assertTalosSupportsKubernetes, +# called from templates/nodegroup.yaml for talos.version and again for the +# release osImage resolves to). +# +# On the shipped chart a refusal is unreachable: the values schema offers only +# Kubernetes versions the v1.13 row lists, and files/versions.yaml, which another +# template fails on first, holds the same set. The guard is there for the day a +# Talos row stops listing a version the chart still offers. So this renders a +# copy of the chart with --skip-schema-validation and one extra versions.yaml row +# for a version the matrix does not list. The pairings the guard lets through +# are in the chart's own tests/talos_k8s_support_matrix_test.yaml. + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +CHART="$REPO_ROOT/packages/apps/kubernetes-nodes" + +# render_pool renders the pool chart copy with the given --set flags and +# prints helm's output, errors included. The copy lives in the test's own tmpdir. +render_pool() { + copy="$BATS_TEST_TMPDIR/chart" + if [ ! -d "$copy" ]; then + cp -RL "$CHART" "$copy" + printf '"v1.30": "v1.30.14"\n' >> "$copy/files/versions.yaml" + fi + helm template kubernetes-nodes-myk8s-md0 "$copy" --namespace tenant-test \ + --skip-schema-validation \ + --set cluster=myk8s \ + --set _cluster.cluster-domain=cozy.local \ + --set minReplicas=1 --set maxReplicas=3 \ + --set instanceType= --set resources.cpu=2 --set resources.memory=4Gi \ + --set diskSize=20Gi --set storageClass=replicated \ + "$@" 2>&1 +} + +@test "an in-window pairing renders on the same copy, so the refusals below are the guard's" { + out=$(render_pool --set version=v1.31) || { echo "$out" >&2; return 1; } + printf '%s\n' "$out" | grep -q '^kind: KubevirtMachineTemplate$' +} + +@test "a Kubernetes version outside the row of talos.version is refused" { + if out=$(render_pool --set version=v1.30); then + echo "render succeeded with Kubernetes v1.30 on the default Talos" >&2 + return 1 + fi + printf '%s\n' "$out" | grep -qF "Kubernetes v1.30 is not supported by Talos v1.13.6. Supported versions: [v1.31 v1.32 v1.33 v1.34 v1.35 v1.36]. Bump 'version' or set talos.version explicitly." \ + || { echo "$out" >&2; return 1; } +} + +@test "an older patch release of a listed Talos minor is checked against the same row" { + if out=$(render_pool --set version=v1.30 --set talos.version=v1.13.0); then + echo "render succeeded with Kubernetes v1.30 on Talos v1.13.0: the guard is not keyed on the minor" >&2 + return 1 + fi + printf '%s\n' "$out" | grep -qF 'Kubernetes v1.30 is not supported by Talos v1.13.0.' \ + || { echo "$out" >&2; return 1; } +} + +# talos.version names a minor the matrix does not list, so the first call lets the +# pool through and only the call fed the osImage release can refuse it. +@test "a Kubernetes version outside the row of osImage.factory.version is refused" { + if out=$(render_pool --set version=v1.30 --set talos.version=v1.12.0 --set osImage.factory.version=v1.13.0); then + echo "render succeeded with Kubernetes v1.30 on Talos v1.13.0 reached through osImage.factory" >&2 + return 1 + fi + printf '%s\n' "$out" | grep -qF 'Kubernetes v1.30 is not supported by Talos v1.13.0.' \ + && printf '%s\n' "$out" | grep -qF 'nodeGroup md0 reaches this Talos release through its osImage override' \ + || { echo "$out" >&2; return 1; } +} + +@test "a Kubernetes version outside the row of osImage.builtin.version is refused" { + if out=$(render_pool --set version=v1.30 --set talos.version=v1.12.0 --set osImage.builtin.version=v1.13.0); then + echo "render succeeded with Kubernetes v1.30 on Talos v1.13.0 reached through osImage.builtin" >&2 + return 1 + fi + printf '%s\n' "$out" | grep -qF 'Kubernetes v1.30 is not supported by Talos v1.13.0.' \ + && printf '%s\n' "$out" | grep -qF 'nodeGroup md0 reaches this Talos release through its osImage override' \ + || { echo "$out" >&2; return 1; } +} diff --git a/hack/linstor-error-report-alerts.bats b/hack/linstor-error-report-alerts.bats new file mode 100644 index 000000000..4c6124aad --- /dev/null +++ b/hack/linstor-error-report-alerts.bats @@ -0,0 +1,32 @@ +#!/usr/bin/env bats +# Replays the LINSTOR error-report alerts from +# packages/system/piraeus-operator/alerts/piraeus-datastore.yaml through +# vmalert-tool, pinned to the VictoriaMetrics release that the +# victoria-metrics-operator deploys by default. Its increase() counts the first +# samples of a series as new when they are small, and the report counts are +# small, so the cases pin that a series which starts over after a controller +# move, a label change or a disconnect does not report old error reports again. +# Only pulling the image touches the network. + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +VMALERT_TOOL_IMAGE="victoriametrics/vmalert-tool:v1.136.0@sha256:417346d7b6fdb66d8e3984887c2ad114535ec7d983eeef53381b56c776f333f1" + +@test "LINSTOR error-report alerts pass their vmalert-tool cases" { + docker info >/dev/null 2>&1 || { + echo "docker is required to run the cases in $VMALERT_TOOL_IMAGE" >&2 + return 1 + } + work=$(mktemp -d) + yq '.spec' "$REPO_ROOT/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml" > "$work/rules.yaml" + cp "$REPO_ROOT/hack/testdata/linstor-error-report-alerts/unittest.yaml" "$work/unittest.yaml" + rc=0 + docker run --rm --network none -v "$work:/w:ro" "$VMALERT_TOOL_IMAGE" \ + unittest --files /w/unittest.yaml > "$work/out" 2>&1 || rc=$? + cat "$work/out" + passed=$(grep -cx 'SUCCESS' "$work/out" || true) + rm -rf "$work" + [ "$rc" -eq 0 ] + [ "$passed" -eq 1 ] +} diff --git a/hack/monitoring-agents-vm-cleanup-hook.bats b/hack/monitoring-agents-vm-cleanup-hook.bats new file mode 100644 index 000000000..290a91314 --- /dev/null +++ b/hack/monitoring-agents-vm-cleanup-hook.bats @@ -0,0 +1,312 @@ +#!/usr/bin/env bats +# Behavioural tests for the monitoring-agents pre-delete hook: +# +# packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml +# +# The script is extracted from the rendered chart and RUN against a fake +# kubectl that keeps the VictoriaMetrics objects in a state file, with fake +# `date` and `sleep` so the 60s grace period passes in an instant. The +# helm-unittest suite in packages/system/monitoring-agents/tests/ +# uninstall_test.yaml matches the script text instead, and stayed green when the +# ownership condition was removed, when the finalizer patch was made a no-op and +# when the polling condition was inverted. Each of those turns this file red. +# +# What is pinned here: +# * only objects whose Helm annotations name this release, by name and +# namespace both, are deleted or patched: an object of another release, of +# a same-named release in another namespace, or one Helm does not own, is +# left alone; +# * with the operator gone, the hook waits before touching finalizers, then +# clears them, and nothing of this release is left behind; +# * with the operator alive, it finalizes the objects itself and the hook +# never patches a finalizer; +# * a kind whose CRD is gone, or a release with nothing left, exits 0, since a +# failing pre-delete hook would itself block the uninstall. +# +# Run from the repo root with make bats-unit-tests. Direct status checks +# retain compatibility with cozytest, which has no Bats run/setup helpers. +# Inline cleanup preserves Bats' TAP handler on failed assertions. + +load test_helper + +MA_CHART=packages/system/monitoring-agents +MA_RELEASE=monitoring-agents +MA_NAMESPACE=cozy-monitoring + +# render_hook +# Extract the named Job's shell script from the rendered chart into . The +# non-empty check must be an early `return 1`: hack/cozytest.sh rewrites every +# line matching ^}$ into `return 0` plus `}`, helpers included, so a check left +# in last-command position has its status discarded. +render_hook() { + helm template "$MA_RELEASE" "$MA_CHART" --namespace "$MA_NAMESPACE" \ + | yq eval "select(.kind == \"Job\" and .metadata.name == \"$1\") | .spec.template.spec.containers[0].command[2]" - > "$2" + [ -s "$2" ] || return 1 +} + +# write_fakes +# Emit fake `kubectl`, `date` and `sleep` into . +# +# kubectl reads and writes $STATE, one object per line as +# kind|release-name|release-namespace|namespace|name, where the two release +# fields are the Helm annotations (empty when Helm does not own the object). +# `get` prints this kind's objects in the hook's jsonpath format, or fails for a +# kind listed in $MISSING_KINDS as kubectl does when the CRD is gone. `delete` +# removes the object only when $OPERATOR_LIVE is 1, standing for an operator +# that finalizes it; otherwise the object stays, Terminating. `patch` removes it +# only when the payload actually clears the finalizers. Every call goes to $KLOG. +# +# date advances 1s per call from $CLOCK, so the 60s wait ends after sixty +# of polls, and records each time it returns in $KLOG; sleep records itself, +# and with $SLEEP_ADVANCES=1 also moves $CLOCK by its argument. The log thus +# shows how much fake time passed before each patch. +# No line below is a bare column-0 `}`, so cozytest.sh's parser leaves it alone. +write_fakes() { + cat > "$1/kubectl" <<'KEOF' +#!/bin/sh +echo "kubectl $*" >> "$KLOG" +drop() { + awk -F'|' -v k="$1" -v ns="$2" -v n="$3" '!($1 == k && $4 == ns && $5 == n)' "$STATE" > "$STATE.new" + mv "$STATE.new" "$STATE"; } +case "$1" in + get) + kind=${2%%.*} + case " ${MISSING_KINDS:-} " in + *" $kind "*) + echo "error: the server doesn't have a resource type \"$kind\"" >&2 + exit 1 ;; + esac + awk -F'|' -v k="$kind" '$1 == k { print $2 "|" $3 "|" $4 "|" $5 }' "$STATE" + exit 0 ;; + -n) + ns=$2 verb=$3 kind=${4%%.*} name=$5 + case "$verb" in + delete) + [ "${OPERATOR_LIVE:-0}" = 1 ] && drop "$kind" "$ns" "$name" + echo "$kind \"$name\" deleted" + exit 0 ;; + patch) + case "$*" in + *'"finalizers":null'*) drop "$kind" "$ns" "$name" ;; + esac + echo "$kind \"$name\" patched" + exit 0 ;; + esac ;; +esac +echo "fake kubectl: unexpected call: $*" >&2 +exit 64 +KEOF + cat > "$1/date" <<'KEOF' +#!/bin/sh +c=$(cat "$CLOCK" 2>/dev/null || echo 0) +echo $((c + 1)) > "$CLOCK" +echo "date $((1000000 + c))" >> "$KLOG" +echo $((1000000 + c)) +KEOF + cat > "$1/sleep" <<'KEOF' +#!/bin/sh +echo "sleep $*" >> "$KLOG" +if [ "${SLEEP_ADVANCES:-0}" = 1 ]; then + c=$(cat "$CLOCK" 2>/dev/null || echo 0) + echo $((c + $1)) > "$CLOCK" +fi +KEOF + chmod +x "$1/kubectl" "$1/date" "$1/sleep" +} + +# write_state +# Three objects of this release in two namespaces; a VMRule of another release; +# a VMRule whose release name matches but whose release namespace does not, so +# only the pair of annotations tells it apart; and a VMServiceScrape that +# carries no Helm annotations. +write_state() { + cat > "$1" < +# Run the rendered hook with the fakes first on PATH; its status lands in $rc. +run_hook() { + export KLOG="$1/calls" STATE="$1/state" CLOCK="$1/clock" + : > "$KLOG" + rc=0 + PATH="$1:$PATH" sh "$1/hook.sh" > "$1/out" 2> "$1/err" || rc=$? +} + +@test "a render that produces no script fails instead of passing as a no-op" { + # Guards the guard: every test below runs whatever render_hook produced, so an + # empty render would satisfy the exit-0 assertions without executing a line of + # the hook. + tmp=$(mktemp -d) + if render_hook no-such-job "$tmp/hook.sh"; then + echo "FAIL: render_hook reported success for a Job name the chart never emits" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook clears the finalizers of this release's objects only, after the grace period" { + tmp=$(mktemp -d) + write_fakes "$tmp" + write_state "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + export OPERATOR_LIVE=0 MISSING_KINDS="" + run_hook "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + for obj in "vmagents.operator.victoriametrics.com vmagent" \ + "vmrules.operator.victoriametrics.com alerts-node" \ + "vmservicescrapes.operator.victoriametrics.com operator"; do + if ! grep -q -- " delete $obj --wait=false" "$tmp/calls"; then + echo "FAIL: $obj was not deleted" + cat "$tmp/calls" + false + fi + done + # The objects of this release must be gone, which the fake only does for a + # patch that really clears the finalizers. + if grep -q "^[a-z]*|$MA_RELEASE|$MA_NAMESPACE|" "$tmp/state"; then + echo "FAIL: objects of this release are still there, Terminating" + cat "$tmp/state" "$tmp/calls" + false + fi + # Another release's object, one of a same-named release in another + # namespace, and an object Helm does not own are never touched. + if grep -E -q -- ' (delete|patch) [a-z.]+ (foreign-rule|foreign-ns-rule|unmanaged-scrape)( |$)' "$tmp/calls"; then + echo "FAIL: the hook touched an object this release does not own" + cat "$tmp/calls" + false + fi + if [ "$(grep -c '|' "$tmp/state")" -ne 3 ]; then + echo "FAIL: the three objects this release does not own should remain" + cat "$tmp/state" + false + fi + # The operator gets its chance first: the hook polls and sleeps before the + # first patch, and does not stop waiting before the 60s are up. + first_patch=$(grep -n ' patch ' "$tmp/calls" | head -1 | cut -d: -f1) + if [ -z "$first_patch" ]; then + echo "FAIL: no finalizer was cleared" + cat "$tmp/calls" + false + fi + # The first date call sets the deadline; the last one before the first patch + # is the time the hook gave up waiting. + start=$(grep '^date ' "$tmp/calls" | head -1 | cut -d' ' -f2) + waited_until=$(head -n "$first_patch" "$tmp/calls" | grep '^date ' | tail -1 | cut -d' ' -f2) + if [ -z "$start" ] || [ $((waited_until - start)) -lt 60 ]; then + echo "FAIL: finalizers cleared after $((waited_until - start))s; the 60s grace period was cut short" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook clears the finalizers before its Job's deadline" { + # The grace-period test above keeps sleep free so a shortened wait shows; here + # sleep costs its argument, so a poll interval long enough to overshoot the + # Job's activeDeadlineSeconds fails instead of passing. + tmp=$(mktemp -d) + write_fakes "$tmp" + write_state "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + job_deadline=$(helm template "$MA_RELEASE" "$MA_CHART" --namespace "$MA_NAMESPACE" \ + | yq eval 'select(.kind == "Job" and .metadata.name == "vm-objects-cleanup") | .spec.activeDeadlineSeconds' -) + export OPERATOR_LIVE=0 MISSING_KINDS="" SLEEP_ADVANCES=1 + run_hook "$tmp" + unset SLEEP_ADVANCES + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + first_patch=$(grep -n ' patch ' "$tmp/calls" | head -1 | cut -d: -f1) + start=$(grep '^date ' "$tmp/calls" | head -1 | cut -d' ' -f2) + waited_until=$(head -n "${first_patch:-0}" "$tmp/calls" | grep '^date ' | tail -1 | cut -d' ' -f2) + if [ -z "$first_patch" ] || [ -z "$job_deadline" ] || [ $((waited_until - start)) -ge "$job_deadline" ]; then + echo "FAIL: first finalizer patch at $((waited_until - start))s, Job deadline ${job_deadline}s" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook leaves the finalizers to a live operator" { + tmp=$(mktemp -d) + write_fakes "$tmp" + write_state "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + export OPERATOR_LIVE=1 MISSING_KINDS="" + run_hook "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc" + cat "$tmp/out" "$tmp/err" + false + fi + if ! grep -q ' delete vmagents.operator.victoriametrics.com vmagent ' "$tmp/calls"; then + echo "FAIL: the VMAgent was not deleted" + cat "$tmp/calls" + false + fi + if grep -q ' patch ' "$tmp/calls"; then + echo "FAIL: the hook cleared finalizers the live operator was handling" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook exits 0 and cleans the other kinds when a CRD is gone" { + tmp=$(mktemp -d) + write_fakes "$tmp" + write_state "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + export OPERATOR_LIVE=0 MISSING_KINDS="vmrules vmpodscrapes vmnodescrapes" + run_hook "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc with a CRD gone, which would block the uninstall" + cat "$tmp/out" "$tmp/err" + false + fi + if grep -q "^vmagents|$MA_RELEASE|" "$tmp/state"; then + echo "FAIL: the VMAgent was left Terminating because another kind's CRD was gone" + cat "$tmp/state" "$tmp/calls" + false + fi + rm -rf "$tmp" +} + +@test "vm cleanup hook exits 0 without deleting anything when the release owns nothing" { + tmp=$(mktemp -d) + write_fakes "$tmp" + printf 'vmrules|other|%s|%s|foreign-rule\nvmrules|%s|other-namespace|%s|foreign-ns-rule\n' \ + "$MA_NAMESPACE" "$MA_NAMESPACE" "$MA_RELEASE" "$MA_NAMESPACE" > "$tmp/state" + render_hook vm-objects-cleanup "$tmp/hook.sh" + export OPERATOR_LIVE=0 MISSING_KINDS="" + run_hook "$tmp" + + if [ "$rc" -ne 0 ]; then + echo "FAIL: hook exited $rc with nothing to clean" + cat "$tmp/out" "$tmp/err" + false + fi + if grep -E -q ' (delete|patch) ' "$tmp/calls"; then + echo "FAIL: the hook deleted or patched with nothing of this release left" + cat "$tmp/calls" + false + fi + rm -rf "$tmp" +} diff --git a/hack/run-kubernetes-talos-spec_test.bats b/hack/run-kubernetes-talos-spec_test.bats index 9eabc119d..5901ee4c6 100644 --- a/hack/run-kubernetes-talos-spec_test.bats +++ b/hack/run-kubernetes-talos-spec_test.bats @@ -2,7 +2,7 @@ # ----------------------------------------------------------------------------- # Unit tests for talos_spec_block in hack/e2e-chainsaw/_lib/run-kubernetes.sh # -# It is the composition point for the tenant Kubernetes CR's `spec.talos`: the +# It is the composition point for the tenant KubernetesNodes CR's `spec.talos`: the # Talos OS image cache contributes imageFactoryURL when available. The result is # spliced into an unquoted heredoc directly under `spec:`, so an indentation slip # does not fail this function -- it produces a CR the API rejects, and every @@ -32,7 +32,7 @@ load test_helper # Splice a talos_spec_block result under `spec:` of a minimal CR and print the # document, so yq can be asked what the tenant apiserver would see. spec_doc() { - printf 'apiVersion: apps.cozystack.io/v1alpha1\nkind: Kubernetes\nmetadata:\n name: t\nspec:\n%s\n host: ""\n' "$1" + printf 'apiVersion: apps.cozystack.io/v1alpha1\nkind: KubernetesNodes\nmetadata:\n name: t\nspec:\n%s\n cluster: t\n' "$1" } @test "no Talos image cache renders nothing, so the chart default applies" { @@ -59,6 +59,6 @@ spec_doc() { # talos_spec_block writes imageFactoryURL by hand. If the chart ever renames # it, the CR would be rejected only at e2e time, and this file is the cheapest # place to notice. - yq -e '.talos | has("imageFactoryURL")' packages/apps/kubernetes/values.yaml >/dev/null \ - || { echo "packages/apps/kubernetes/values.yaml has no talos.imageFactoryURL; talos_spec_block would emit a key the chart rejects" >&2; exit 1; } + yq -e '.talos | has("imageFactoryURL")' packages/apps/kubernetes-nodes/values.yaml >/dev/null \ + || { echo "packages/apps/kubernetes-nodes/values.yaml has no talos.imageFactoryURL; talos_spec_block would emit a key the pool chart ignores" >&2; exit 1; } } diff --git a/hack/select-e2e.sh b/hack/select-e2e.sh index fcb7651e3..59db4fe25 100755 --- a/hack/select-e2e.sh +++ b/hack/select-e2e.sh @@ -8,8 +8,9 @@ # Output: # - empty no E2E impact, decided by one of the rules that select # nothing: step 1 for docs/, dashboards/ and *.md, step 2 for -# a *.disabled Chainsaw suite and for an -# examples/backups// with no suite, and +# a *.disabled Chainsaw suite, for an +# examples/backups// with no suite and for the CI-only +# tools cmd/backport-batch/ and cmd/api-gate/, and # inert_config_pattern for repo meta and agent config # - selected per the PackageSource dependency graph # - full list any path that affects all tests, OR an unrecognised @@ -497,6 +498,14 @@ while IFS= read -r file || [ -n "$file" ]; do echo "select-e2e: '$file' belongs to no single backup walkthrough — escalating to the full suite" >&2 trigger_full=1 continue ;; + cmd/backport-batch/*|cmd/api-gate/*) + # CI-only Go tools. No image builds them and nothing in the e2e sandbox + # runs them, so an edit cannot regress a suite. Matched here because + # full_suite_pattern takes all of cmd/ and is checked before the inert + # list. The plan job's `code` output and e2e-fork.yaml skip e2e for a + # diff made only of these and docs/; this arm keeps them from escalating + # a diff that also touches something else. + continue ;; esac # 3. Full-suite trigger diff --git a/hack/select-e2e_test.bats b/hack/select-e2e_test.bats index 6ce198830..f79d6b05c 100644 --- a/hack/select-e2e_test.bats +++ b/hack/select-e2e_test.bats @@ -202,6 +202,23 @@ assert_full_suite() { rm -rf "$tmp" } +@test "the CI-only tools select nothing and do not escalate the rest" { + tmp=$(mktemp -d) + cp -r packages/core/platform/sources "$tmp/sources" + printf '%s\n' cmd/backport-batch/report_test.go cmd/api-gate/main.go > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") + [ -z "$output" ] + # Beside a package, the package alone decides the selection. + printf '%s\n' cmd/backport-batch/main.go packages/apps/postgres/values.yaml > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources") + assert_selection "a CI-only tool must not widen a package selection" "$output" "postgres" + # Any other cmd/ directory is a shipped binary and still escalates. + echo "cmd/cozystack-api/main.go" > "$tmp/diff" + output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources" 2>/dev/null) + assert_full_suite "$output" + rm -rf "$tmp" +} + @test "kubernetes-application maps to the two kubernetes suites" { tmp=$(mktemp -d) cp -r packages/core/platform/sources "$tmp/sources" diff --git a/hack/testdata/linstor-error-report-alerts/unittest.yaml b/hack/testdata/linstor-error-report-alerts/unittest.yaml new file mode 100644 index 000000000..41e332169 --- /dev/null +++ b/hack/testdata/linstor-error-report-alerts/unittest.yaml @@ -0,0 +1,198 @@ +rule_files: + - rules.yaml + +evaluation_interval: 15s + +tests: + - name: controller pod moves to another node + interval: 30s + input_series: + - series: 'up{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a"}' + values: '1x38 0 stale' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a", hostname="node-a", module="SATELLITE"}' + values: '1x38 stale' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a", hostname="node-b", module="SATELLITE"}' + values: '2x38 stale' + - series: 'up{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b"}' + values: '_x41 1x79' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b", hostname="node-a", module="SATELLITE"}' + values: '_x41 1x58 2x20' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b", hostname="node-b", module="SATELLITE"}' + values: '_x41 2x79' + alert_rule_test: + - eval_time: 21m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 28m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 35m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 45m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 51m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + exp_alerts: + - exp_labels: + hostname: node-a + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-a" to inspect the reports. + + - name: an external label of the scrape changes + interval: 30s + input_series: + - series: 'up{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack"}' + values: '1x39 stale' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack", hostname="node-a", module="SATELLITE"}' + values: '1x39 stale' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack", hostname="linstor-controller-abc", module="CONTROLLER"}' + values: '1x39 stale' + - series: 'up{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a"}' + values: '_x41 1x79' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a", hostname="node-a", module="SATELLITE"}' + values: '_x41 1x79' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a", hostname="linstor-controller-abc", module="CONTROLLER"}' + values: '_x41 1x79' + alert_rule_test: + - eval_time: 21m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 30m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 21m + groupname: linstor.rules + alertname: linstorControllerErrorRate + - eval_time: 30m + groupname: linstor.rules + alertname: linstorControllerErrorRate + + - name: a satellite drops out of the metric while it is disconnected + interval: 30s + input_series: + - series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}' + values: '1x220' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}' + values: '1x38 stale _x80 1x100' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-b", module="SATELLITE"}' + values: '2x220' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-c", module="SATELLITE"}' + values: '1x38 stale _x80 2x100' + alert_rule_test: + - eval_time: 61m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + exp_alerts: + - exp_labels: + hostname: node-c + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-c" to inspect the reports. + - eval_time: 70m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + exp_alerts: + - exp_labels: + hostname: node-c + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-c" to inspect the reports. + - eval_time: 76m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + + - name: a new report fires for the 15 minute window + interval: 30s + input_series: + - series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}' + values: '1x120' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}' + values: '1x39 2x80' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-b", module="SATELLITE"}' + values: '2x120' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-c", module="SATELLITE"}' + values: '0x39 1x80' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="linstor-controller-abc", module="CONTROLLER"}' + values: '0x39 1x80' + alert_rule_test: + - eval_time: 21m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + exp_alerts: + - exp_labels: + hostname: node-a + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-a" to inspect the reports. + - exp_labels: + hostname: node-c + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-c" to inspect the reports. + - eval_time: 34m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + exp_alerts: + - exp_labels: + hostname: node-a + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-a" to inspect the reports. + - exp_labels: + hostname: node-c + severity: warning + exp_annotations: + description: | + LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --nodes node-c" to inspect the reports. + - eval_time: 36m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 21m + groupname: linstor.rules + alertname: linstorControllerErrorRate + exp_alerts: + - exp_labels: + hostname: linstor-controller-abc + severity: warning + exp_annotations: + description: | + LINSTOR Controller reports 1 errors in the last 15 minutes. + Use "linstor error-reports list --since 15minutes" to see them. + - eval_time: 36m + groupname: linstor.rules + alertname: linstorControllerErrorRate + + - name: a failed controller scrape still holds the alert back + interval: 30s + input_series: + - series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}' + values: '1x19 0x140 1x60' + - series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}' + values: '1x19 stale _x140 1x60' + alert_rule_test: + - eval_time: 85m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 93m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate + - eval_time: 100m + groupname: linstor.rules + alertname: linstorSatelliteErrorRate diff --git a/hack/testdata/workload-placement-alerts/unittest.yaml b/hack/testdata/workload-placement-alerts/unittest.yaml new file mode 100644 index 000000000..2565a8241 --- /dev/null +++ b/hack/testdata/workload-placement-alerts/unittest.yaml @@ -0,0 +1,572 @@ +rule_files: + - rules.yaml + +evaluation_interval: 15s + +tests: + - name: two of three StatefulSet pods on one node + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", node="node-b", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 59m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: StatefulSet + owner_name: db + node: node-a + severity: warning + exported_instance: tenant-a/db + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of StatefulSet tenant-a/db are on node node-a in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + + - name: StatefulSet pods spread over three nodes + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", node="node-b", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", node="node-c", created_by_kind="StatefulSet", created_by_name="db"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: five ReplicaSet pods on three nodes + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-bbbbb", node="node-a", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ddddd", node="node-b", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-eeeee", node="node-c", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-aaaaa", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-bbbbb", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ccccc", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ddddd", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-eeeee", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: DaemonSet surge pod next to the old one + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-x7k2p", node="node-a", created_by_kind="DaemonSet", created_by_name="node-agent"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-q9z4m", node="node-a", created_by_kind="DaemonSet", created_by_name="node-agent"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-b5n8w", node="node-b", created_by_kind="DaemonSet", created_by_name="node-agent"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-x7k2p", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-q9z4m", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-b5n8w", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: two pods of one Job on one node + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-h2k4m", node="node-a", created_by_kind="Job", created_by_name="backup-29345678"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-p8r6t", node="node-a", created_by_kind="Job", created_by_name="backup-29345678"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-h2k4m", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-p8r6t", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: etcd members of one EtcdCluster on one node + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-x7k2p", node="node-a", created_by_kind="EtcdMember", created_by_name="etcd-x7k2p"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-q9z4m", node="node-a", created_by_kind="EtcdMember", created_by_name="etcd-q9z4m"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-b5n8w", node="node-b", created_by_kind="EtcdMember", created_by_name="etcd-b5n8w"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-0", node="node-b", created_by_kind="EtcdMember", created_by_name="etcd-0"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-1", node="node-c", created_by_kind="EtcdMember", created_by_name="etcd-1"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-2", node="node-c", created_by_kind="EtcdMember", created_by_name="etcd-2"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-x7k2p", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-q9z4m", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-b5n8w", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-0", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-2", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: EtcdCluster + owner_name: etcd + node: node-a + severity: warning + exported_instance: tenant-a/etcd + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of EtcdCluster tenant-a/etcd are on node node-a in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + - exp_labels: + cluster: cozystack + namespace: tenant-b + owner_kind: EtcdCluster + owner_name: etcd + node: node-c + severity: warning + exported_instance: tenant-b/etcd + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of EtcdCluster tenant-b/etcd are on node node-c in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + + - name: replacement pod next to its terminating predecessor for twenty minutes + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '1x30 stale' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '_x10 1x140' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", phase="Running", node="node-c"}' + values: '1x30 stale' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", phase="Running", node="node-c"}' + values: '_x10 1x140' + alert_rule_test: + - eval_time: 29m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 75m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: the same overlap held past the hour + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '1x100 stale' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}' + values: '_x10 1x140' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", phase="Running", node="node-c"}' + values: '1x100 stale' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", phase="Running", node="node-c"}' + values: '_x10 1x140' + alert_rule_test: + - eval_time: 69m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 71m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: ReplicaSet + owner_name: web-6b8f4 + node: node-b + severity: warning + exported_instance: tenant-a/web-6b8f4 + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of ReplicaSet tenant-a/web-6b8f4 are on node node-b in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + - eval_time: 105m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: a cordoned node does not count until it is uncordoned + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '0x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '0x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x30 0x120' + - series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unschedulable", value="", effect="NoSchedule"}' + values: '1x30 stale' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", node="node-b", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", phase="Running", node="node-a"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", phase="Running", node="node-a"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", phase="Running", node="node-a"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 90m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 92m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: StatefulSet + owner_name: kv + node: node-a + severity: warning + exported_instance: tenant-a/kv + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of StatefulSet tenant-a/kv are on node node-a in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + + - name: a node that is down does not count + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '0x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '0x150' + - series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '0x150' + - series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unreachable", value="", effect="NoSchedule"}' + values: '1x150' + - series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unreachable", value="", effect="NoExecute"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", node="node-b", created_by_kind="StatefulSet", created_by_name="kv"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", phase="Running", node="node-a"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", phase="Running", node="node-a"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", phase="Running", node="node-a"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + - eval_time: 120m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: pods that are not running do not count + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", node="node-a", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-aaaaa", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", phase="Running", node="node-c"}' + values: '0x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", phase="Failed", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-ccccc", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: one pod each in two clusters that share workload and node names + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="kube-system", pod="coredns-5d78c9869d-x7k2p", node="node-a", created_by_kind="ReplicaSet", created_by_name="coredns-5d78c9869d"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="kubernetes-example", namespace="kube-system", pod="coredns-5d78c9869d-q9z4m", node="node-a", created_by_kind="ReplicaSet", created_by_name="coredns-5d78c9869d"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="kube-system", pod="coredns-5d78c9869d-x7k2p", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="kubernetes-example", namespace="kube-system", pod="coredns-5d78c9869d-q9z4m", phase="Running", node="node-b"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + + - name: ReplicaSet, CNPG Cluster and StrimziPodSet pods on one node + interval: 1m + input_series: + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}' + values: '1x150' + - series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-h2k4m", node="node-a", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-p8r6t", node="node-a", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-w3x5z", node="node-b", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-1", node="node-a", created_by_kind="Cluster", created_by_name="postgres-db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-2", node="node-b", created_by_kind="Cluster", created_by_name="postgres-db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-3", node="node-b", created_by_kind="Cluster", created_by_name="postgres-db"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-0", node="node-c", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-1", node="node-c", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}' + values: '1x150' + - series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-2", node="node-a", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-h2k4m", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-p8r6t", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-w3x5z", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-2", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-3", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-0", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-1", phase="Running", node="node-c"}' + values: '1x150' + - series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-2", phase="Running", node="node-c"}' + values: '1x150' + alert_rule_test: + - eval_time: 61m + groupname: workload-placement + alertname: PodReplicasColocated + exp_alerts: + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: ReplicaSet + owner_name: rfs-cache-6f7d8b9c5d + node: node-a + severity: warning + exported_instance: tenant-a/rfs-cache-6f7d8b9c5d + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of ReplicaSet tenant-a/rfs-cache-6f7d8b9c5d are on node node-a in cluster + cozystack, although the cluster has a schedulable node for each of its pods. Losing this + node stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: Cluster + owner_name: postgres-db + node: node-b + severity: warning + exported_instance: tenant-a/postgres-db + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of Cluster tenant-a/postgres-db are on node node-b in cluster cozystack, + although the cluster has a schedulable node for each of its pods. Losing this node + stops these pods together, which can cost a quorum-based workload its quorum. The + scheduler does not move running pods, so they stay together until they are recreated. + - exp_labels: + cluster: cozystack + namespace: tenant-a + owner_kind: StrimziPodSet + owner_name: kafka-kafka + node: node-c + severity: warning + exported_instance: tenant-a/kafka-kafka + service: workload-placement + exp_annotations: + summary: Pods of one workload share a node. + description: >- + 2 running pods of StrimziPodSet tenant-a/kafka-kafka are on node node-c in cluster + cozystack, although the cluster has a schedulable node for each of its pods. Losing + this node stops these pods together, which can cost a quorum-based workload its quorum. + The scheduler does not move running pods, so they stay together until they are + recreated. diff --git a/hack/workload-placement-alerts.bats b/hack/workload-placement-alerts.bats new file mode 100644 index 000000000..3f7b9b961 --- /dev/null +++ b/hack/workload-placement-alerts.bats @@ -0,0 +1,26 @@ +#!/usr/bin/env bats + +load test_helper + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +VMALERT_TOOL_IMAGE="victoriametrics/vmalert-tool:v1.136.0@sha256:417346d7b6fdb66d8e3984887c2ad114535ec7d983eeef53381b56c776f333f1" + +@test "workload placement alerts pass their vmalert-tool cases" { + docker info >/dev/null 2>&1 || { + echo "docker is required to run the cases in $VMALERT_TOOL_IMAGE" >&2 + return 1 + } + work=$(mktemp -d) + rc=0 + yq '.spec' "$REPO_ROOT/packages/system/monitoring-agents/alerts/workload-placement.yaml" > "$work/rules.yaml" || rc=$? + cp "$REPO_ROOT/hack/testdata/workload-placement-alerts/unittest.yaml" "$work/unittest.yaml" || rc=$? + if [ "$rc" -eq 0 ]; then + docker run --rm --network none -v "$work:/w:ro" "$VMALERT_TOOL_IMAGE" \ + unittest --files /w/unittest.yaml > "$work/out" 2>&1 || rc=$? + cat "$work/out" + fi + passed=$(grep -cx 'SUCCESS' "$work/out" 2>/dev/null || true) + rm -rf "$work" + [ "$rc" -eq 0 ] + [ "$passed" -eq 1 ] +} diff --git a/internal/backupcontroller/altinitystrategy_controller.go b/internal/backupcontroller/altinitystrategy_controller.go index dab780100..4314c3dbf 100644 --- a/internal/backupcontroller/altinitystrategy_controller.go +++ b/internal/backupcontroller/altinitystrategy_controller.go @@ -2,11 +2,11 @@ package backupcontroller import ( - "bytes" "context" "encoding/json" "fmt" "maps" + "regexp" "strings" "time" @@ -57,6 +57,12 @@ const ( // archive relies on. altinityArchiveTimeLayout = "20060102T150405Z" + // altinityArchiveRunTimeLayout and altinityArchiveUIDLength shape the + // archive name of a run; see altinityArchiveName for why they are this + // short. + altinityArchiveRunTimeLayout = "20060102150405" + altinityArchiveUIDLength = 12 + // Polling cadence for the Altinity Job lifecycle. Mirrors the CNPG / // Velero strategy cadence so behaviour is uniform across drivers. altinityPollInterval = 5 * time.Second @@ -187,11 +193,30 @@ func backupParameters(b *backupsv1alpha1.Backup) map[string]string { } // altinityArchiveName mints the archive name for one BackupJob from its -// persisted StartedAt, so every reconcile and every Pod retry of the Job -// writes the same archive. It keeps the clickhouse-- -// shape, because a Backup that records no archive is restored by the -// timestamp in that shape. +// persisted StartedAt and its UID, so every reconcile and every Pod retry of +// the Job writes the same archive and no other run writes it. StartedAt has +// one-second resolution, and two Plans on one schedule start their runs in +// the same second; the strategy takes a finished upload of its archive as an +// earlier attempt of its own run, so a shared name would let one run report +// the other's archive. +// +// The name is no longer than altinityArchiveNameBeforeUIDs, because it sits in +// every object key, a key is capped at 1024 bytes, and the legacy +// s3PathOverride prefix has no length limit of its own: the UID takes the room +// that dropping "clickhouse-", "T" and "Z" frees. It goes after the timestamp +// so names still sort chronologically, and the restore of a Backup that +// records no archive, which accepts only clickhouse--, +// never picks one of these. func altinityArchiveName(j *backupsv1alpha1.BackupJob) string { + uid := strings.ReplaceAll(string(j.UID), "-", "") + uid = uid[:min(len(uid), altinityArchiveUIDLength)] + return fmt.Sprintf("%s-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveRunTimeLayout), uid) +} + +// altinityArchiveNameBeforeUIDs is the name a controller that did not append +// the UID gave the same BackupJob, which a Job it created before an upgrade +// still writes. +func altinityArchiveNameBeforeUIDs(j *backupsv1alpha1.BackupJob) string { return fmt.Sprintf("clickhouse-%s-%s", j.Spec.ApplicationRef.Name, j.Status.StartedAt.UTC().Format(altinityArchiveTimeLayout)) } @@ -199,13 +224,16 @@ func altinityArchiveName(j *backupsv1alpha1.BackupJob) string { // the archive name. A Job created by a controller that predates archive // names, or rendered from a strategy that names its own archives, wrote an // archive under another name, and recording this one would point the Backup -// at an archive that does not exist. +// at an archive that does not exist. The name has to stand on its own: the +// name without the UID starts the name with it, and a strategy may append to +// either. func altinityJobCarriesArchive(job *batchv1.Job, archive string) bool { spec, err := json.Marshal(job.Spec.Template.Spec) if err != nil { return false } - return bytes.Contains(spec, []byte(archive)) + whole := regexp.MustCompile(`(^|[^A-Za-z0-9._-])` + regexp.QuoteMeta(archive) + `($|[^A-Za-z0-9._-])`) + return whole.Match(spec) } // renderAltinityTemplate runs the strategy's PodTemplateSpec through the @@ -365,7 +393,10 @@ func (r *BackupJobReconciler) reconcileAltinity(ctx context.Context, j *backupsv return ctrl.Result{}, nil } if !altinityJobCarriesArchive(batchJob, archive) { - archive = "" + archive = altinityArchiveNameBeforeUIDs(j) + if !altinityJobCarriesArchive(batchJob, archive) { + archive = "" + } } artifact, err := r.createAltinityBackupArtifact(ctx, j, resolved, archive) if err != nil { @@ -600,7 +631,7 @@ func (r *BackupJobReconciler) createAltinityBackupArtifact( // used name meets the earlier run's Backup here. Only one recording // this run's archive was created by an earlier reconcile of this run. if got := existing.Spec.DriverMetadata[altinityArchiveMetaKey]; got != archive { - return nil, fmt.Errorf("Backup %s/%s already exists and records archive %q, not %q that this run wrote", + return nil, fmt.Errorf("existing Backup %s/%s records archive %q, not %q that this run wrote", backup.Namespace, backup.Name, got, archive) } return existing, nil diff --git a/internal/backupcontroller/altinitystrategy_controller_test.go b/internal/backupcontroller/altinitystrategy_controller_test.go index 48592ac54..a9022ceb6 100644 --- a/internal/backupcontroller/altinitystrategy_controller_test.go +++ b/internal/backupcontroller/altinitystrategy_controller_test.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" @@ -733,14 +734,66 @@ func newAltinityArchiveStrategy(name string) *strategyv1alpha1.Altinity { func TestAltinityArchiveName_FromStartedAtInUTC(t *testing.T) { startedAt := metav1.NewTime(time.Date(2026, 3, 4, 8, 6, 7, 890000000, time.FixedZone("UTC+3", 3*3600))) j := &backupsv1alpha1.BackupJob{ - Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, - Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + ObjectMeta: metav1.ObjectMeta{UID: "0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, } - if got, want := altinityArchiveName(j), "clickhouse-ch-test-20260304T050607Z"; got != want { + if got, want := altinityArchiveName(j), "ch-test-20260304050607-0f3c9a1e5b7d"; got != want { t.Errorf("altinityArchiveName = %q, want %q", got, want) } } +// TestAltinityArchiveName_NoLongerThanTheNameWithoutUID pins the S3 key +// budget: the archive name is part of every object key, a key may not exceed +// 1024 bytes, and the legacy s3PathOverride prefix is unbounded, so a prefix +// that fit the name without the UID must still fit this one. +func TestAltinityArchiveName_NoLongerThanTheNameWithoutUID(t *testing.T) { + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + for n := 1; n <= 63; n++ { + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{UID: "0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef(strings.Repeat("a", n))}, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + } + name, legacy := altinityArchiveName(j), altinityArchiveNameBeforeUIDs(j) + if len(name) > len(legacy) { + t.Errorf("application name of %d characters: archive %q is longer than %q", n, name, legacy) + } + // The manifest key of a prefix that left the name without the UID + // exactly at the limit. + prefix := strings.Repeat("p", 1024-len("/"+legacy+"/metadata.json")) + if key := prefix + "/" + name + "/metadata.json"; len(key) > 1024 { + t.Errorf("application name of %d characters: manifest key is %d bytes", n, len(key)) + } + } +} + +// TestAltinityArchiveName_DistinctForRunsStartedInTheSameSecond pins that two +// BackupJobs of one application started within the same second, as two Plans +// on the same schedule are, write different archives. With a shared name, the +// strategy takes the other run's finished upload for an earlier attempt of its +// own and records an archive missing what changed since that upload. +func TestAltinityArchiveName_DistinctForRunsStartedInTheSameSecond(t *testing.T) { + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + run := func(name, uid string) *backupsv1alpha1.BackupJob { + return &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tenant-test", UID: types.UID(uid)}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: newAltinityClickHouseRef("ch-test")}, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt}, + } + } + daily := altinityArchiveName(run("daily-1", "11111111-1111-1111-1111-111111111111")) + weekly := altinityArchiveName(run("weekly-1", "22222222-2222-2222-2222-222222222222")) + if daily == weekly { + t.Fatalf("two runs started in the same second share archive %q", daily) + } + for _, name := range []string{daily, weekly} { + if !strings.HasPrefix(name, "ch-test-20260304050607-") { + t.Errorf("archive %q does not start with the application and start time, so names stop sorting chronologically", name) + } + } +} + // TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven drives one BackupJob // through Job creation and completion in separate reconciles. The archive // name has to come out of the persisted StartedAt both times: the Pod is @@ -751,7 +804,7 @@ func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) { strategy := newAltinityArchiveStrategy("clickhouse-strategy") startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) backupJob := &backupsv1alpha1.BackupJob{ - ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "test-bj-uid"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: newAltinityClickHouseRef("ch-test"), BackupClassName: "clickhouse-backup", @@ -768,7 +821,7 @@ func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) { } r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy)) ctx := context.Background() - const want = "clickhouse-ch-test-20260304T050607Z" + const want = "ch-test-20260304050607-testbjuid" if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { t.Fatalf("reconcileAltinity() creating the Job: %v", err) @@ -804,6 +857,90 @@ func TestReconcileAltinity_RecordsTheArchiveTheJobWasGiven(t *testing.T) { } } +// TestReconcileAltinity_RecordsTheArchiveOfAJobNamedBeforeUIDs covers a backup +// Job created by a controller that named archives without the BackupJob's UID +// and still running when the controller is upgraded: the Job is not +// re-rendered, so its archive carries the old name, and the Backup has to +// record that name rather than none. +func TestReconcileAltinity_RecordsTheArchiveOfAJobNamedBeforeUIDs(t *testing.T) { + const legacy = "clickhouse-ch-test-20260304T050607Z" + cases := []struct { + name string + archive string // archive the Job's Pod was rendered with + want string + }{ + {name: "the name without the UID", archive: legacy, want: legacy}, + {name: "a longer name that starts with it", archive: legacy + "-copy", want: ""}, + {name: "a longer name that ends with it", archive: "copy-" + legacy, want: ""}, + {name: "a longer name that continues it after a period", archive: legacy + ".copy", want: ""}, + {name: "a longer name that leads into it with a period", archive: "copy." + legacy, want: ""}, + {name: "a longer name that continues it after an underscore", archive: legacy + "_copy", want: ""}, + {name: "a longer name that leads into it with an underscore", archive: "copy_" + legacy, want: ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := recordAltinityArchiveOfJobRenderedWith(t, tc.archive); got != tc.want { + t.Errorf("Backup records archive %q, want %q", got, tc.want) + } + }) + } +} + +// recordAltinityArchiveOfJobRenderedWith completes a backup Job whose Pod +// carries the given archive in place of the one the controller rendered, and +// returns the archive the resulting Backup records. +func recordAltinityArchiveOfJobRenderedWith(t *testing.T, archive string) string { + t.Helper() + app := newAltinityClickHouseApp("ch-test", "tenant-test") + strategy := newAltinityArchiveStrategy("clickhouse-strategy") + startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) + backupJob := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "test-bj-uid"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: newAltinityClickHouseRef("ch-test"), + BackupClassName: "clickhouse-backup", + }, + Status: backupsv1alpha1.BackupJobStatus{StartedAt: &startedAt, Phase: backupsv1alpha1.BackupJobPhaseRunning}, + } + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: new(strategyv1alpha1.GroupVersion.Group), + Kind: strategyv1alpha1.AltinityStrategyKind, + Name: "clickhouse-strategy", + }, + } + r, _, _ := newAltinityTestEnv(t, app, clientfake.NewClientBuilder().WithObjects(backupJob, strategy)) + ctx := context.Background() + + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() creating the Job: %v", err) + } + job := &batchv1.Job{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: jobNameForBackupJob(backupJob)}, job); err != nil { + t.Fatalf("get backup Job: %v", err) + } + job.Spec.Template.Spec.Containers[0].Args[1] = "--archive=" + archive + if err := r.Update(ctx, job); err != nil { + t.Fatalf("render the backup Job with archive %q: %v", archive, err) + } + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := r.Status().Update(ctx, job); err != nil { + t.Fatalf("mark backup Job complete: %v", err) + } + if err := r.Get(ctx, client.ObjectKeyFromObject(backupJob), backupJob); err != nil { + t.Fatalf("refresh BackupJob: %v", err) + } + if _, err := r.reconcileAltinity(ctx, backupJob, resolved); err != nil { + t.Fatalf("reconcileAltinity() on the completed Job: %v", err) + } + + created := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: "tenant-test", Name: backupJob.Name}, created); err != nil { + t.Fatalf("get Backup: %v", err) + } + return created.Spec.DriverMetadata[altinityArchiveMetaKey] +} + // TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven covers a Job whose // Pod never carried the driver's archive name: one created before the // upgrade that introduced the names, or rendered from a strategy that names @@ -872,13 +1009,13 @@ func TestReconcileAltinity_RecordsNoArchiveTheJobWasNotGiven(t *testing.T) { // A Backup recording this run's own archive is the one an earlier reconcile // created before its status write was lost, and is adopted. func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) { - const thisRun = "clickhouse-ch-test-20260305T050607Z" + const thisRun = "ch-test-20260305050607-testbjuid" cases := []struct { name string existing string // archive the existing Backup records; empty for none wantSucceeded bool }{ - {name: "earlier run's Backup records another archive", existing: "clickhouse-ch-test-20260304T050607Z"}, + {name: "earlier run's Backup records another archive", existing: "ch-test-20260304050607-earlierbjuid"}, {name: "earlier run's Backup records no archive", existing: ""}, {name: "this run's Backup from a reconcile whose status write was lost", existing: thisRun, wantSucceeded: true}, } @@ -888,7 +1025,7 @@ func TestReconcileAltinity_BackupNameAlreadyTaken(t *testing.T) { strategy := newAltinityArchiveStrategy("clickhouse-strategy") startedAt := metav1.NewTime(time.Date(2026, 3, 5, 5, 6, 7, 0, time.UTC)) backupJob := &backupsv1alpha1.BackupJob{ - ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "test-bj-uid"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: newAltinityClickHouseRef("ch-test"), BackupClassName: "clickhouse-backup", @@ -1503,7 +1640,7 @@ func TestReconcileAltinityRestore_UnreadableHoldStateDoesNotStartTheJob(t *testi func TestReconcileAltinity_CredentialsWaitKeepsTheArchiveName(t *testing.T) { startedAt := metav1.NewTime(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)) backupJob := &backupsv1alpha1.BackupJob{ - ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test"}, + ObjectMeta: metav1.ObjectMeta{Name: "test-bj", Namespace: "tenant-test", UID: "0f3c9a1e-5b7d-4c2a-9e8f-1a2b3c4d5e6f"}, Spec: backupsv1alpha1.BackupJobSpec{ ApplicationRef: newAltinityClickHouseRef("ch-test"), BackupClassName: "clickhouse-backup", @@ -1522,7 +1659,7 @@ func TestReconcileAltinity_CredentialsWaitKeepsTheArchiveName(t *testing.T) { }, } ctx := context.Background() - const want = "clickhouse-ch-test-20260304T050607Z" + const want = "ch-test-20260304050607-0f3c9a1e5b7d" reconcile := func(step string) { t.Helper() j := &backupsv1alpha1.BackupJob{} diff --git a/internal/backupcontroller/artifact_size.go b/internal/backupcontroller/artifact_size.go index 6376c8e87..88ba71e2f 100644 --- a/internal/backupcontroller/artifact_size.go +++ b/internal/backupcontroller/artifact_size.go @@ -39,6 +39,13 @@ type s3ObjectLister interface { StatObject(ctx context.Context, bucket, key string) (int64, error) } +type s3ObjectRemover interface { + // RemoveObject deletes key. S3 answers 204 for a key that does not + // exist and minio-go takes 204 as success; a backend answering a + // missing key with any other status makes it an error. + RemoveObject(ctx context.Context, bucket, key string) error +} + // s3Connection is what a driver resolves from its rendered strategy to // read the bucket with the same identity the backup was written with. type s3Connection struct { @@ -67,8 +74,10 @@ type objectStore struct { // reader reads Secrets, which the manager's client fetches uncached, // and lists operator Backups, which it serves from the cache. reader client.Reader - // newLister overrides the S3 client; nil means a real minio client. - newLister func(conn s3Connection) (s3ObjectLister, error) + // newLister and newRemover override the S3 client; nil means a real + // minio client. + newLister func(conn s3Connection) (s3ObjectLister, error) + newRemover func(conn s3Connection) (s3ObjectRemover, error) } func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { @@ -78,6 +87,17 @@ func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) { return newMinioObjectLister(conn) } +func (s objectStore) remover(conn s3Connection) (s3ObjectRemover, error) { + if s.newRemover != nil { + return s.newRemover(conn) + } + c, err := newMinioObjectClient(conn) + if err != nil { + return nil, err + } + return c, nil +} + func (s objectStore) secretValue(ctx context.Context, namespace, name, key string) (string, error) { secret := &corev1.Secret{} if err := s.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { @@ -94,6 +114,14 @@ func (r *BackupJobReconciler) objectStore() objectStore { return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} } +func (r *RestoreJobReconciler) objectStore() objectStore { + return objectStore{reader: r.Client, newLister: r.newS3ObjectLister} +} + +func (r *BackupReconciler) objectStore() objectStore { + return objectStore{reader: r.Client, newLister: r.newS3ObjectLister, newRemover: r.newS3ObjectRemover} +} + func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) { return r.objectStore().lister(conn) } @@ -103,6 +131,14 @@ type minioObjectLister struct { } func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) { + c, err := newMinioObjectClient(conn) + if err != nil { + return nil, err + } + return c, nil +} + +func newMinioObjectClient(conn s3Connection) (*minioObjectLister, error) { opts := &minio.Options{ Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken), Region: conn.Region, @@ -168,6 +204,10 @@ func (l *minioObjectLister) StatObject(ctx context.Context, bucket, key string) return info.Size, nil } +func (l *minioObjectLister) RemoveObject(ctx context.Context, bucket, key string) error { + return l.client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{}) +} + // collectObjects drains a minio listing. minio-go ends a listing whose // context is cancelled between pages by closing the channel without an // error item, so a cancelled context is checked once the channel closes: diff --git a/internal/backupcontroller/backup_controller.go b/internal/backupcontroller/backup_controller.go index 1a8ec0b40..bc49baa6b 100644 --- a/internal/backupcontroller/backup_controller.go +++ b/internal/backupcontroller/backup_controller.go @@ -39,9 +39,22 @@ type BackupReconciler struct { client.Client Scheme *runtime.Scheme Recorder record.EventRecorder - // CredentialsConfig lets the Rabbitmq cleanup path project cozy-backups-creds - // into the Backup's namespace before spawning its best-effort delete Job. + // CredentialsConfig locates the platform backup credentials: the Job-based + // cleanups project them into the Backup's namespace, and the MariaDB + // cleanup reads them to reach the platform bucket. CredentialsConfig BackupCredentialsConfig + // APIReader is the manager's uncached reader. The MariaDB cleanup lists + // the RestoreJobs and operator Restores of a Backup, and the operator + // Backups that may own its dump, through it: a restore that started + // before the delete, or a run that has already uploaded its dump, may + // not be in the cache yet. Wired in SetupWithManager. + APIReader client.Reader + + // newS3ObjectLister and newS3ObjectRemover override the S3 client the + // MariaDB cleanup deletes a dump with; nil means a real minio client. + // Set by unit tests. + newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error) + newS3ObjectRemover func(conn s3Connection) (s3ObjectRemover, error) } func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -126,22 +139,20 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // orchestration" for tenant-facing guidance. return ctrl.Result{}, nil case strategyv1alpha1.MariaDBStrategyKind: - // Cozystack Backup deletion does NOT delete the operator-side - // k8s.mariadb.com/Backup CR or the backing S3/PVC archive. - // Lifecycle of that CR is owned by mariadb-operator's - // MaxRetention setting and by tenants / explicit teardown flows; - // the controller's RBAC has no delete verb on - // k8s.mariadb.com/backups for that reason. Surfacing this as an - // explicit branch (rather than falling through to the Velero - // default) keeps the contract honest: the MariaDB driver does - // not own the archive, so it does not delete it on CR removal. - // Mirrors the Altinity branch's "we do not own S3" stance. - return ctrl.Result{}, nil + // mariadb-operator's maxRetention prunes only from inside the next + // backup run of the same prefix and a Plan never deletes a Backup, so + // a dump on the platform bucket is deleted here or never. A dump in + // a tenant's own bucket is deleted too: the Backup proves it owns + // it, and tenants cannot delete Backups, so the delete comes from an + // administrator or from namespace teardown. Teardown can remove the + // operator Backup first, and the dump is then left behind with a + // Warning, since its storage is read from it (see cleanupMariaDBBackup). + return r.cleanupMariaDBBackup(ctx, backup) case strategyv1alpha1.MongoDBStrategyKind: // Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the // tenant's own bucket and its lifecycle stays the tenant's / psmdb // operator's — that branch is a no-op, the same "we do not own the - // archive" contract as Altinity / MariaDB / FDB. A useSystemBucket backup + // archive" contract as Altinity / FDB. A useSystemBucket backup // lands in the shared cozy-backups bucket with no psmdb task retention to // prune it, so the driver OWNS it: this deletes the operator // PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer @@ -154,7 +165,7 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // is governed by the next BackupJob's stopOtherFoundationDBBackups // call, by manual operator action, or by // examples/backups/foundationdb/cleanup.sh. Same "we do not own - // the archive" contract as Altinity / MariaDB — the explicit + // the archive" contract as Altinity — the explicit // branch guards the seam against a future driver refactor that // might incidentally stamp velero.io/backup-name onto FDB // driverMetadata via a shared helper. @@ -183,6 +194,9 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv // deletes the object and WAITS for that delete before the Backup is // removed, so no object is orphaned (see cleanupKafkaBackup). return r.cleanupKafkaBackup(ctx, backup) + case strategyv1alpha1.KubernetesStrategyKind: + // Owns two artifacts: the Velero backup and the etcd dump object. + return r.cleanupKubernetesBackup(ctx, backup) case strategyv1alpha1.VeleroStrategyKind: return ctrl.Result{}, r.cleanupVeleroBackup(ctx, backup) default: @@ -291,7 +305,18 @@ func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backup // SetupWithManager registers the BackupReconciler with the Manager. func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error { + r.APIReader = mgr.GetAPIReader() return ctrl.NewControllerManagedBy(mgr). For(&backupsv1alpha1.Backup{}). Complete(r) } + +// apiReader returns the uncached reader. Reconcilers built by hand in unit +// tests have none and fall back to their fake Client, which has no cache to +// bypass. +func (r *BackupReconciler) apiReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} diff --git a/internal/backupcontroller/backup_controller_test.go b/internal/backupcontroller/backup_controller_test.go index 579cb5fe6..d77f5d208 100644 --- a/internal/backupcontroller/backup_controller_test.go +++ b/internal/backupcontroller/backup_controller_test.go @@ -176,11 +176,8 @@ func TestBackupCleanup_Altinity_DoesNotFallThroughToVelero(t *testing.T) { // TestBackupCleanup_MariaDB_DoesNotFallThroughToVelero locks in the same // invariant as the Altinity test above for the MariaDB strategy. The -// dispatcher must explicitly route MariaDB-strategy Backups to a no-op -// cleanup rather than fall through to the Velero default - the MariaDB -// driver does not own the operator-side k8s.mariadb.com/Backup CR or the -// S3/PVC archive behind it (rbac.yaml grants no delete verb on -// k8s.mariadb.com/backups for that reason). The test seeds the Velero +// dispatcher must route MariaDB-strategy Backups to their own cleanup +// rather than fall through to the Velero default. The test seeds the Velero // metadata keys plus a matching velero.io/Backup; with the explicit branch // no DeleteBackupRequest is synthesised, and the Velero Backup survives. // Without the branch the default falls through to cleanupVeleroBackup, diff --git a/internal/backupcontroller/backupjob_controller.go b/internal/backupcontroller/backupjob_controller.go index b86278649..15ef65db4 100644 --- a/internal/backupcontroller/backupjob_controller.go +++ b/internal/backupcontroller/backupjob_controller.go @@ -214,6 +214,8 @@ func (r *BackupJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return r.reconcileRedis(ctx, j, resolved) case strategyv1alpha1.KafkaStrategyKind: return r.reconcileKafka(ctx, j, resolved) + case strategyv1alpha1.KubernetesStrategyKind: + return r.reconcileKubernetes(ctx, j, resolved) default: logger.V(1).Info("BackupJob resolved StrategyRef.Kind not supported, skipping", "backupjob", j.Name, @@ -240,6 +242,7 @@ func supportedBackupStrategyKinds() []string { strategyv1alpha1.RabbitmqStrategyKind, strategyv1alpha1.RedisStrategyKind, strategyv1alpha1.KafkaStrategyKind, + strategyv1alpha1.KubernetesStrategyKind, } } @@ -315,14 +318,6 @@ func (r *BackupJobReconciler) handleProjectionError(ctx context.Context, j *back } // markBackupJobFailed records a terminal Failed phase on the BackupJob. -// -// Coupling note: a failure that fires before reconcileCNPG's StartedAt block -// has set backupJob.Status.StartedAt leaves StartedAt nil. The CNPG path's -// cnpgDefaultBackupDeadline check only fires once StartedAt is set, so an -// early-failure retry that reaches the StartedAt block restarts the deadline -// budget from the retry's StartedAt - intentional, since retries are -// user-driven and a fresh budget is what users expect after correcting the -// cause of the previous failure. func (r *BackupJobReconciler) markBackupJobFailed(ctx context.Context, backupJob *backupsv1alpha1.BackupJob, message string) (ctrl.Result, error) { logger := getLogger(ctx) now := metav1.Now() diff --git a/internal/backupcontroller/backupjob_controller_test.go b/internal/backupcontroller/backupjob_controller_test.go index f802fe374..994654987 100644 --- a/internal/backupcontroller/backupjob_controller_test.go +++ b/internal/backupcontroller/backupjob_controller_test.go @@ -45,6 +45,7 @@ func TestSupportedBackupStrategyKindsMatchesDispatch(t *testing.T) { strategyv1alpha1.RabbitmqStrategyKind, strategyv1alpha1.RedisStrategyKind, strategyv1alpha1.KafkaStrategyKind, + strategyv1alpha1.KubernetesStrategyKind, } sort.Strings(got) sort.Strings(want) diff --git a/internal/backupcontroller/backupjob_restore_lock.go b/internal/backupcontroller/backupjob_restore_lock.go index 06f165285..6b662cad5 100644 --- a/internal/backupcontroller/backupjob_restore_lock.go +++ b/internal/backupcontroller/backupjob_restore_lock.go @@ -32,13 +32,13 @@ const ConditionReasonRestoreInProgress = "RestoreInProgress" // // A backup must not start against an application a restore is rewriting. It // would capture a half-restored state at best, and on the CNPG path it does -// worse: every BackupJob reconcile re-applies the Cluster's barman plugin, and -// one landing between the restore re-rendering the Cluster with its new -// serverName and the plugin being attached can put the restored cluster back -// on the source's WAL prefix, where its recovery fails with "Expected empty -// archive". Plan-created runs are the ones that hit this, since nothing else -// looks at restores before creating a BackupJob. -func activeRestoreTargeting(ctx context.Context, c client.Client, namespace string, app corev1.TypedLocalObjectReference) (string, error) { +// worse: a BackupJob applies the Cluster's barman plugin before it creates +// its Backup, and an apply landing between the restore re-rendering the +// Cluster with its new serverName and the plugin being attached can put the +// restored cluster back on the source's WAL prefix, where its recovery fails +// with "Expected empty archive". Plan-created runs are the ones that hit +// this, since nothing else looks at restores before creating a BackupJob. +func activeRestoreTargeting(ctx context.Context, c client.Reader, namespace string, app corev1.TypedLocalObjectReference) (string, error) { app = NormalizeApplicationRef(app) list := &backupsv1alpha1.RestoreJobList{} if err := c.List(ctx, list, client.InNamespace(namespace)); err != nil { @@ -80,7 +80,7 @@ func activeRestoreTargeting(ctx context.Context, c client.Client, namespace stri // reads options.targetNamespace, so the option is looked at only when the // Backup's strategy is Velero — the CNPG and Job-based drivers ignore it and // restore in place. -func restoreJobTarget(ctx context.Context, c client.Client, rj *backupsv1alpha1.RestoreJob) (corev1.TypedLocalObjectReference, bool, error) { +func restoreJobTarget(ctx context.Context, c client.Reader, rj *backupsv1alpha1.RestoreJob) (corev1.TypedLocalObjectReference, bool, error) { backup := &backupsv1alpha1.Backup{} if err := c.Get(ctx, client.ObjectKey{Namespace: rj.Namespace, Name: rj.Spec.BackupRef.Name}, backup); err != nil { if client.IgnoreNotFound(err) == nil { @@ -130,13 +130,14 @@ func ptrString(s *string) string { return *s } -// holdForRestore keeps a BackupJob that has not started yet waiting while a -// restore writes into its application. It stays non-terminal, so a scheduled -// run still happens once the restore ends, only later than its slot. +// holdForRestore keeps a BackupJob waiting while a restore writes into its +// application: one that has not started yet, or a CNPG one about to write +// its WAL archive destination. It stays non-terminal, so a scheduled run +// still happens once the restore ends, only later than its slot. // // A failed status write is returned so the manager retries it with backoff. -// The job is held either way, since nothing past the gate ran, but a hold -// nobody can see reads as a run stuck for no reason. +// The job is held either way, since it writes nothing past the hold, but a +// hold nobody can see reads as a run stuck for no reason. func (r *BackupJobReconciler) holdForRestore(ctx context.Context, j *backupsv1alpha1.BackupJob, restore string) (ctrl.Result, error) { if j.Status.Phase == backupsv1alpha1.BackupJobPhaseEmpty { j.Status.Phase = backupsv1alpha1.BackupJobPhasePending diff --git a/internal/backupcontroller/cnpg_artifact_size_test.go b/internal/backupcontroller/cnpg_artifact_size_test.go index 85e55e8c2..4afef7194 100644 --- a/internal/backupcontroller/cnpg_artifact_size_test.go +++ b/internal/backupcontroller/cnpg_artifact_size_test.go @@ -222,13 +222,13 @@ func cnpgListStub(prefix string, seen func(*http.Request)) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/xml") if _, ok := req.URL.Query()["location"]; ok { - fmt.Fprint(w, ``) + _, _ = fmt.Fprint(w, ``) return } if seen != nil { seen(req) } - fmt.Fprintf(w, ` + _, _ = fmt.Fprintf(w, ` bkt%[1]s21000false %[1]sbackup.info7"a"2026-10-21T10:01:00.000Z @@ -284,7 +284,7 @@ func TestCreateCNPGBackupArtifact_SizeBytes(t *testing.T) { r := reconcilerWithLister(t, l) r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", "creds")) - got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "srv", "postgresql", cnpgDestinationOf(rendered), rendered, newPostgresApp("pg", "tenant")) if err != nil { t.Fatalf("createCNPGBackupArtifact: %v", err) } @@ -311,7 +311,7 @@ func TestCreateCNPGBackupArtifact_ExistingBackupSkipsProbe(t *testing.T) { cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} - if _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")); err != nil { + if _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", cnpgDestinationOf(rendered), rendered, newPostgresApp("pg", "tenant")); err != nil { t.Fatalf("createCNPGBackupArtifact: %v", err) } if l.calls != 0 { @@ -331,7 +331,7 @@ func TestCreateCNPGBackupArtifact_AlreadyExistsRefetchReadsPastTheCache(t *testi store := newCNPGStrategyTestClient(t, backup) r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: notFoundOnce(t, store)} rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}} - got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", cnpgDestinationOf(rendered), rendered, newPostgresApp("pg", "tenant")) if err != nil || got == nil || got.Name != "bj-race" { t.Fatalf("createCNPGBackupArtifact = %v, %v; want the Backup that already exists", got, err) } @@ -349,7 +349,7 @@ func TestCreateCNPGBackupArtifact_UnreadableExistingBackupIsRetried(t *testing.T store := newCNPGStrategyTestClient(t, backup) r := &BackupJobReconciler{Client: staleCache(t, store), APIReader: failingReads(t, store)} rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}} - _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + _, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}}, "postgres-pg", "srv", "postgresql", cnpgDestinationOf(rendered), rendered, newPostgresApp("pg", "tenant")) if !errors.Is(err, errBackupNotReadBack) { t.Errorf("err = %v, want errBackupNotReadBack", err) } @@ -409,7 +409,7 @@ func TestCreateCNPGBackupArtifact_FailedExistenceCheckSkipsProbe(t *testing.T) { cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} - got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", rendered, newPostgresApp("pg", "tenant")) + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", cnpgDestinationOf(rendered), rendered, newPostgresApp("pg", "tenant")) if err != nil || got == nil { t.Fatalf("createCNPGBackupArtifact = %v, %v; a transient read must not fail the backup", got, err) } @@ -451,3 +451,128 @@ func TestReconcileCNPG_RejectedCreateFailsTheJob(t *testing.T) { t.Errorf("phase = %q, want Failed for a rejected Create", persisted.Status.Phase) } } + +// The probe measures the base backup where it was written: the destination +// recorded on the cnpg.io/Backup when it was created, not the strategy as +// it reads by the time the backup completes. +func TestCreateCNPGBackupArtifact_SizeProbeReadsTheRecordedDestination(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-moved"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} + oldCreds := "creds-before-edit" + written := cnpgDestination{DestinationPath: "s3://old-bkt/before/edit/", EndpointURL: "https://old.example:9000", S3SecretRef: &oldCreds} + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "before/edit/srv/base/20261021T100000/data.tar.gz", Size: 42}}} + r := reconcilerWithLister(t, l) + r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", oldCreds)) + + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", written, rendered, newPostgresApp("pg", "tenant")) + if err != nil { + t.Fatalf("createCNPGBackupArtifact: %v", err) + } + if l.bucket != "old-bkt" || l.prefix != "before/edit/srv/base/20261021T100000/" { + t.Errorf("listed %s/%s, want the recorded destination old-bkt/before/edit/srv/base/20261021T100000/", l.bucket, l.prefix) + } + if l.conn.Endpoint != "old.example:9000" { + t.Errorf("probed endpoint %q, want the recorded old.example:9000", l.conn.Endpoint) + } + if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 42 { + t.Errorf("Artifact = %#v, want SizeBytes 42", got.Status.Artifact) + } +} + +// A backup recorded without a credentials Secret is not measured with the +// Secret the strategy names now: that Secret belongs to another location. +func TestCreateCNPGBackupArtifact_SizeProbeSkipsARecordWithoutCredentials(t *testing.T) { + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-nocreds"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} + rendered := &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *newSizedBarmanObjectStore()} + written := cnpgDestination{DestinationPath: "s3://bkt/tenant/pg/", EndpointURL: "https://s3.example:8443"} + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "x", Size: 1}}} + r := reconcilerWithLister(t, l) + r.Client = newCNPGStrategyTestClient(t, s3CredsSecret("tenant", "creds")) + + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", written, rendered, newPostgresApp("pg", "tenant")) + if err != nil { + t.Fatalf("createCNPGBackupArtifact: %v", err) + } + if l.calls != 0 { + t.Errorf("S3 listed %d times with the strategy's Secret for a backup recorded without one", l.calls) + } + if got.Status.Artifact == nil || got.Status.Artifact.SizeBytes != 0 { + t.Errorf("Artifact = %#v, want no size", got.Status.Artifact) + } +} + +// probeRecordedBackup runs createCNPGBackupArtifact for a backup recorded as +// written with credentials Secret recordedSecret, against rendered, and +// returns the size it measured and the connection it measured through. +func probeRecordedBackup(t *testing.T, rendered *strategyv1alpha1.CNPGTemplate, recordedSecret *corev1.Secret) (int64, s3Connection) { + t.Helper() + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + j := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-probe"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: "pg"}, + }, + } + cnpgBk := &cnpgtypes.Backup{ObjectMeta: metav1.ObjectMeta{Name: "cnpg-bk"}, Status: cnpgtypes.BackupStatus{BackupID: "20261021T100000"}} + secretName := recordedSecret.Name + written := cnpgDestination{DestinationPath: "s3://bkt/tenant/pg/", EndpointURL: "https://s3.example:8443", S3SecretRef: &secretName} + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: "tenant/pg/srv/base/20261021T100000/data.tar.gz", Size: 42}}} + r := reconcilerWithLister(t, l) + r.Client = newCNPGStrategyTestClient(t, recordedSecret) + + got, err := r.createCNPGBackupArtifact(context.Background(), j, &ResolvedBackupConfig{}, cnpgBk, "postgres-pg", "srv", "postgresql", written, rendered, newPostgresApp("pg", "tenant")) + if err != nil { + t.Fatalf("createCNPGBackupArtifact: %v", err) + } + if got.Status.Artifact == nil { + t.Fatalf("no artifact") + } + return got.Status.Artifact.SizeBytes, l.conn +} + +// A strategy that has since dropped its credentials still measures a backup +// recorded with a Secret, through that Secret and its default keys. +func TestCreateCNPGBackupArtifact_SizeProbeOfARecordedSecretUnderAStrategyWithoutCredentials(t *testing.T) { + store := newSizedBarmanObjectStore() + store.S3Credentials = nil + size, conn := probeRecordedBackup(t, &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *store}, s3CredsSecret("tenant", "creds-recorded")) + if size != 42 { + t.Errorf("SizeBytes = %d, want 42 measured through the recorded Secret", size) + } + if conn.AccessKey != "AKID" { + t.Errorf("probed with access key %q, want the recorded Secret's AKID", conn.AccessKey) + } +} + +// The record names the Secret, not its keys: the strategy's key names stay, +// so a strategy with its own key names still reads the recorded Secret. +func TestCreateCNPGBackupArtifact_SizeProbeKeepsTheStrategysSecretKeys(t *testing.T) { + store := newSizedBarmanObjectStore() + store.S3Credentials.AccessKeyIDKey = "id" + store.S3Credentials.SecretAccessKeyKey = "key" + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "creds-recorded"}, + Data: map[string][]byte{"id": []byte("CUSTOMID"), "key": []byte("custom-secret")}, + } + size, conn := probeRecordedBackup(t, &strategyv1alpha1.CNPGTemplate{BarmanObjectStore: *store}, secret) + if size != 42 { + t.Errorf("SizeBytes = %d, want 42 measured with the strategy's key names", size) + } + if conn.AccessKey != "CUSTOMID" { + t.Errorf("probed with access key %q, want CUSTOMID read under the strategy's key name", conn.AccessKey) + } +} diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 19f255c6c..3fd620787 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -15,6 +15,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" @@ -175,11 +176,17 @@ const ( // against an unreachable target reproduces the same FATAL. cnpgRecoveryMaxInspectPods = 5 - // Cap on the wall-clock time a BackupJob waits for the barman-cloud - // plugin rollout before the cnpg.io/Backup exists. A Backup that exists - // is not capped: CloudNativePG keeps a healthy one pending while another - // backup of the same Cluster runs, or while it restarts one on another - // instance, so no fixed deadline tells those apart from a stuck one. + // Cap on the wall-clock time a BackupJob spends before its + // cnpg.io/Backup exists on each of two waits. The plugin rollout is + // counted from the attach, the later of the PluginRolledOut condition and + // the Cluster's cnpgPluginAttachedAtAnnotation, or from the release + // recorded in DestinationWriteHeld when that is later. Retrying the WAL + // archive destination write is counted from the first failure since the + // last successful write or the last wait for another backup or a + // restore. A Backup that exists is not capped: CloudNativePG keeps a + // healthy one pending while another backup of the same Cluster runs, or + // while it restarts one on another instance, so no fixed deadline tells + // those apart from a stuck one. cnpgDefaultBackupDeadline = 30 * time.Minute cnpgClusterHealthyPhase = "Cluster in healthy state" @@ -190,11 +197,69 @@ const ( // instance pod spec (the plugin injects its sidecar), so CNPG rolls every // instance, and a cnpg.io/Backup created meanwhile fails with "requested // plugin is not available" or "instance manager was restarted during - // backup" and never recovers. False holds the Backup back; its - // LastTransitionTime, written before the plugin is attached, is what the - // instance pods must postdate (see cnpgPluginRolloutPending). + // backup" and never recovers. False holds the Backup back; the instance + // pods must postdate its LastTransitionTime and the Cluster's + // cnpgPluginAttachedAtAnnotation (see cnpgReasonAttachingPlugin and + // cnpgPluginRolloutPending). backupCondPluginRolledOut = "PluginRolledOut" + // Reason of the PluginRolledOut latch recorded before the write that + // attaches the plugin. Every write of this job that attaches it replaces + // the condition with its own time, which retries can put long after the + // latch. + cnpgReasonAttachingPlugin = "AttachingPlugin" + + // Condition Type recorded while writing the WAL archive destination keeps + // failing. Its LastTransitionTime marks the first failure, so the retry + // budget is not spent by time the job waited for another backup of the + // app, or for a restore of it, before it tried. + backupCondDestinationApplied = "DestinationApplied" + + // Condition Type recorded while the job is held off writing the WAL + // archive destination, by another backup of the app in flight or by a + // restore of it, and set False when released. The release time restarts + // the plugin rollout deadline, which that wait must not spend; the + // instance pods still have to postdate the rollout's own start. + backupCondDestinationWriteHeld = "DestinationWriteHeld" + + // Annotation on a cnpg.io/Backup carrying, as JSON, the WAL archive + // destination written by the pass that created it: the destination + // path, the endpoint URL, the name of the credentials Secret and the + // serverName the Cluster archived under. + cnpgWrittenDestinationAnnotation = "backups.cozystack.io/cnpg-written-destination" + + // Annotation on a cnpg.io/Cluster carrying, as RFC 3339, when a backup + // job attached the barman-cloud plugin. It is written by the same apply + // that attaches the plugin, so unlike the PluginRolledOut condition it + // cannot be lost to a failed BackupJob status write after the attach. + cnpgPluginAttachedAtAnnotation = "backups.cozystack.io/barman-plugin-attached-at" + + // Annotation on the ObjectStore carrying, as RFC 3339, when a backup job + // last changed its configuration. It is written by the apply that makes + // the change, so it cannot be lost apart from it. + cnpgDestinationChangedAtAnnotation = "backups.cozystack.io/destination-changed-at" + + // Annotation on the ObjectStore naming the BackupJob whose apply last + // wrote it. Written by that apply, it holds the destination for that job + // while it runs and has no cnpg.io/Backup of its own yet; from then on + // the queue sees that Backup instead. + cnpgDestinationWrittenByAnnotation = "backups.cozystack.io/destination-written-by" + + // How long after an ObjectStore configuration change a Backup is held. + // The plugin-barman-cloud sidecar reads the ObjectStore through its own + // cache, kept for DefaultTTLSeconds = 10 (plugin-barman-cloud v0.13.0, + // internal/cnpgi/instance/internal/client/client.go) and invalidated + // only by the sidecar's own writes, so a Backup taken sooner can use the + // previous configuration and land in the previous bucket. The margin + // covers the cache's whole-second expiry and the record's whole-second + // timestamp. + cnpgObjectStoreCacheSettle = 15 * time.Second + + // How far ahead of this controller's clock a recorded time, of a plugin + // attach or of a destination change, may be before it is taken as + // corrupt rather than as written by a leader whose clock ran ahead. + cnpgAttachClockSkewTolerance = 5 * time.Minute + cnpgPodRoleLabel = "cnpg.io/podRole" cnpgPodRoleInstance = "instance" ) @@ -321,7 +386,53 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) awaitingRollout := existingBackup == nil && rollout != nil && rollout.Status == metav1.ConditionFalse - attachment, err := r.applyClusterPluginBackup(ctx, j.Namespace, clusterName, rendered, serverName, existingBackup != nil || awaitingRollout) + // The ObjectStore is the Cluster's WAL archive destination, shared by + // every backup of the app, so it is written only before this job's + // Backup exists, and only while no other job's Backup is in flight and + // no other running job without a Backup yet holds the destination it + // last wrote. + writeDestination := existingBackup == nil + if writeDestination { + // The gate in Reconcile sees only a job that has not started, and + // this one may have waited in the queue for as long as a restore + // takes to begin. Read from the apiserver: the cache can still miss + // a RestoreJob created a moment ago. + restore, err := activeRestoreTargeting(ctx, r.apiReader(), j.Namespace, j.Spec.ApplicationRef) + if err != nil { + return ctrl.Result{}, err + } + if restore != "" { + apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied) + apimeta.SetStatusCondition(&j.Status.Conditions, cnpgDestinationWriteHeld(ConditionReasonRestoreInProgress)) + return r.holdForRestore(ctx, j, restore) + } + holder, err := r.otherCNPGBackupInFlight(ctx, j, clusterName) + if err != nil { + return ctrl.Result{}, err + } + if holder == "" { + if holder, err = r.destinationClaimHolder(ctx, j, clusterName); err != nil { + return ctrl.Result{}, err + } + } + if holder != "" { + apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied) + apimeta.SetStatusCondition(&j.Status.Conditions, cnpgDestinationWriteHeld("WaitingForOtherBackup")) + return r.waitCNPGBackupJob(ctx, j, "WaitingForOtherBackup", fmt.Sprintf( + "waiting for BackupJob %s of the same application to finish its backup, which writes to the shared WAL archive destination", holder)) + } + if apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld) != nil { + apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: backupCondDestinationWriteHeld, + Status: metav1.ConditionFalse, + Reason: "Released", + Message: "nothing holds the WAL archive destination write any more", + }) + } + } + + attemptedAt := metav1.Now() + attachment, err := r.applyClusterPluginBackup(ctx, j.Namespace, clusterName, rendered, serverName, existingBackup != nil || awaitingRollout, writeDestination, j.Name) if err != nil { if apierrors.IsNotFound(err) { // HelmRelease has not yet rendered the Cluster (fresh app, or @@ -340,7 +451,29 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp } return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil } - return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to attach barman-cloud plugin to Cluster: %v", err)) + if !writeDestination { + // Only the live Cluster was read for a Backup already running; + // a failed read says nothing about that Backup. + return ctrl.Result{}, fmt.Errorf("read cnpg.io/Cluster %s/%s: %w", j.Namespace, clusterName, err) + } + message := fmt.Sprintf("failed to attach barman-cloud plugin to Cluster: %v", err) + if apierrors.IsInvalid(err) || apierrors.IsBadRequest(err) || apierrors.IsForbidden(err) { + return r.markBackupJobFailed(ctx, j, message) + } + apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: backupCondDestinationApplied, + Status: metav1.ConditionFalse, + Reason: "ClusterPluginApplyRetrying", + Message: message, + }) + failingSince := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied).LastTransitionTime + if cnpgBackupDeadlineExceeded(&failingSince) { + return r.markBackupJobFailed(ctx, j, message) + } + return r.waitCNPGBackupJob(ctx, j, "ClusterPluginApplyRetrying", message) + } + if writeDestination { + apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondDestinationApplied) } if attachment.serverName != serverName { logger.Info("preserving the live Cluster's barman serverName over the strategy template's", @@ -358,7 +491,7 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ Type: backupCondPluginRolledOut, Status: metav1.ConditionFalse, - Reason: "WaitingForPluginRollout", + Reason: cnpgReasonAttachingPlugin, Message: message, }) apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ @@ -372,12 +505,54 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp } return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil } - pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, rollout.LastTransitionTime) + if attachment.missing { + // This write attached the plugin, so its rollout starts here: + // instances recreated since the latch, or by a restore that + // re-rendered the Cluster without the plugin, run without it. A + // plugin someone else attached since the latch keeps it: the + // rollout that attach started may already be over. + message := fmt.Sprintf("attached the barman-cloud plugin to cnpg.io/Cluster %s/%s, waiting for its instances to roll", j.Namespace, clusterName) + apimeta.RemoveStatusCondition(&j.Status.Conditions, backupCondPluginRolledOut) + apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: backupCondPluginRolledOut, + Status: metav1.ConditionFalse, + Reason: "WaitingForPluginRollout", + Message: message, + LastTransitionTime: attemptedAt, + }) + apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "WaitingForPluginRollout", + Message: message, + }) + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil + } + // The Cluster's record covers an attach whose status write was lost. + attachedAt := rollout.LastTransitionTime + recorded, ok, err := cnpgRecordedTime(attachment.attachedAt) + if err != nil { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf( + "cannot tell when the barman-cloud plugin was attached: the %s annotation of cnpg.io/Cluster %s/%s: %v", + cnpgPluginAttachedAtAnnotation, j.Namespace, clusterName, err)) + } + if ok && recorded.After(attachedAt.Time) { + attachedAt = metav1.NewTime(recorded) + } + pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, attachedAt) if err != nil { return ctrl.Result{}, err } if pending != "" { - if cnpgBackupDeadlineExceeded(j.Status.StartedAt) { + deadlineFrom := attachedAt + if held := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); held != nil && + held.LastTransitionTime.After(deadlineFrom.Time) { + deadlineFrom = held.LastTransitionTime + } + if cnpgBackupDeadlineExceeded(&deadlineFrom) { return r.markBackupJobFailed(ctx, j, fmt.Sprintf( "barman-cloud plugin rollout did not finish within %s: %s", cnpgDefaultBackupDeadline, pending)) } @@ -401,6 +576,24 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp }) apimeta.RemoveStatusCondition(&j.Status.Conditions, "Ready") rolledOut = true + } else if existingBackup == nil && attachment.attachedAt != "" { + // Another backup job attached the plugin, and its rollout can still + // be running while the attach is recent; past the rollout deadline + // this job stops waiting. The record is that job's: one this job + // cannot use does not fail it. + recorded, _, err := cnpgRecordedTime(attachment.attachedAt) + if err != nil { + logger.Info("ignoring an unusable attach record another backup job left", + "cluster", clusterName, "annotation", cnpgPluginAttachedAtAnnotation, "error", err) + } else if time.Since(recorded) < cnpgDefaultBackupDeadline { + pending, err := r.cnpgPluginRolloutPending(ctx, j.Namespace, clusterName, attachment, metav1.NewTime(recorded)) + if err != nil { + return ctrl.Result{}, err + } + if pending != "" { + return r.waitCNPGBackupJob(ctx, j, "WaitingForPluginRollout", pending) + } + } } // The chart refuses a flavor change on a live Cluster, but the app keeps @@ -411,7 +604,15 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp "cluster", clusterName, "live", attachment.flavor, "application", specFlavor) } - cnpgBackup, err := r.ensureCNPGBackup(ctx, j, clusterName) + if changedAt := attachment.destinationChangedAt; !changedAt.IsZero() && time.Since(changedAt) < cnpgObjectStoreCacheSettle { + return r.waitCNPGBackupJob(ctx, j, "WaitingForDestinationToSettle", fmt.Sprintf( + "the WAL archive destination of cnpg.io/Cluster %s/%s changed at %s; the barman-cloud plugin can read the previous one for up to %s", + j.Namespace, clusterName, changedAt.UTC().Format(time.RFC3339), cnpgObjectStoreCacheSettle)) + } + + written := cnpgDestinationOf(rendered) + written.ServerName = serverName + cnpgBackup, err := r.ensureCNPGBackup(ctx, j, clusterName, existingBackup, written) if err != nil { return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure cnpg.io/Backup: %v", err)) } @@ -431,7 +632,16 @@ func (r *BackupJobReconciler) reconcileCNPG(ctx context.Context, j *backupsv1alp if j.Status.BackupRef != nil { return ctrl.Result{}, nil } - artifact, err := r.createCNPGBackupArtifact(ctx, j, resolved, cnpgBackup, clusterName, serverName, attachment.flavor, rendered, app) + destination, err := cnpgArtifactDestination(cnpgBackup, rendered) + if err != nil { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("refusing to create the Backup artifact: %v", err)) + } + // A restore can re-render the Cluster under another serverName + // between the backup and this pass. + if destination.ServerName != "" { + serverName = destination.ServerName + } + artifact, err := r.createCNPGBackupArtifact(ctx, j, resolved, cnpgBackup, clusterName, serverName, attachment.flavor, destination, rendered, app) if err != nil { return r.failBackupArtifact(ctx, j, err) } @@ -486,16 +696,15 @@ func cnpgBackupFailedMessage(b *cnpgtypes.Backup, errText string) string { return message } -// cnpgBackupDeadlineExceeded reports whether enough wall-clock time has -// elapsed since the BackupJob started that we should stop waiting for the -// barman-cloud plugin rollout and fail the run. Returns false when -// StartedAt is nil so the very first reconcile (which sets StartedAt) does -// not trip the gate. -func cnpgBackupDeadlineExceeded(startedAt *metav1.Time) bool { - if startedAt == nil { +// cnpgBackupDeadlineExceeded reports whether more than the backup deadline +// has passed since the BackupJob began the step it is waiting on: the plugin +// rollout or retrying the destination write (see cnpgDefaultBackupDeadline). +// Returns false for a nil time. +func cnpgBackupDeadlineExceeded(since *metav1.Time) bool { + if since == nil { return false } - return time.Since(startedAt.Time) > cnpgDefaultBackupDeadline + return time.Since(since.Time) > cnpgDefaultBackupDeadline } // cnpgPurgeNeeded decides whether the controller should delete the live @@ -601,7 +810,10 @@ var cnpgClusterGVR = schema.GroupVersionResource{ // With attach unset, a Cluster that has no barman-cloud plugin yet is left // untouched and only reported missing, so the caller can record that it is // about to roll the Cluster before it does. -func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, namespace, clusterName string, t *strategyv1alpha1.CNPGTemplate, serverName string, attach bool) (cnpgPluginAttachment, error) { +// +// With write unset nothing is written: only what the live Cluster carries is +// reported. +func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, namespace, clusterName string, t *strategyv1alpha1.CNPGTemplate, serverName string, attach, write bool, jobName string) (cnpgPluginAttachment, error) { // Live read: the cached Cluster can predate a restore re-render, and the // serverName, the flavor and the UID below must come from the current // object. @@ -618,6 +830,7 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name flavor: cnpgImageFlavor(existing.Spec.ImageName), clusterStatus: existing.Status, } + attachment.attachedAt = existing.Annotations[cnpgPluginAttachedAtAnnotation] if live := currentBarmanServerName(existing); live != "" { attachment.serverName = live } else { @@ -626,6 +839,9 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name return attachment, nil } } + if !write { + return attachment, nil + } serverName = attachment.serverName // The ObjectStore is named after the Cluster (distinct kind, same @@ -653,18 +869,116 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name RetentionPolicy: t.BarmanObjectStore.RetentionPolicy, InstanceSidecarConfiguration: barmanSidecarConfiguration(), } + changedAt, err := r.objectStoreChangeRecord(ctx, objStore) + if err != nil { + return cnpgPluginAttachment{}, err + } + // Only a job without a Backup of its own writes, and the queue let it + // through, so it holds the destination from this write while it runs + // and has not created its Backup. + objStore.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: jobName} + if !changedAt.IsZero() { + objStore.Annotations[cnpgDestinationChangedAtAnnotation] = changedAt.UTC().Format(time.RFC3339) + } if err := r.Patch(ctx, objStore, client.Apply, client.FieldOwner(cnpgFieldManager), client.ForceOwnership); err != nil { return cnpgPluginAttachment{}, fmt.Errorf("apply ObjectStore %s/%s: %w", namespace, objStoreName, err) } + attachment.destinationChangedAt = changedAt patch := newCNPGClusterPatch(namespace, clusterName) patch.Spec.Plugins = []cnpgtypes.PluginConfiguration{buildBarmanPlugin(objStoreName, serverName)} + // Re-sent unchanged on later applies: the field manager owns it, and an + // apply that left it out would delete it. + stamp := existing.Annotations[cnpgPluginAttachedAtAnnotation] + if attachment.missing { + stamp = metav1.Now().UTC().Format(time.RFC3339) + } + if stamp != "" { + patch.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: stamp} + } if err := r.Patch(ctx, patch, client.Apply, client.FieldOwner(cnpgFieldManager), client.ForceOwnership); err != nil { return cnpgPluginAttachment{}, err } return attachment, nil } +// objectStoreChangeRecord returns the cnpgDestinationChangedAtAnnotation the +// apply of desired should carry: the time of this write when it changes the +// configuration, otherwise the record the live ObjectStore already carries, +// re-sent so the apply does not delete it, and zero when there is none. +// "Changes" is judged on the configuration a server-side dry run of the same +// apply returns, not on desired, so values the apiserver defaults or another +// field manager owns do not read as a change on every pass. Creating the +// ObjectStore stamps no time: the plugin cannot hold a stale copy of an +// object that did not exist, short of one deleted and recreated by hand +// within the cache's lifetime. +func (r *BackupJobReconciler) objectStoreChangeRecord(ctx context.Context, desired *cnpgtypes.ObjectStore) (time.Time, error) { + live := &cnpgtypes.ObjectStore{} + if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(desired), live); err != nil { + if apierrors.IsNotFound(err) { + return time.Time{}, nil + } + return time.Time{}, fmt.Errorf("read ObjectStore %s/%s: %w", desired.Namespace, desired.Name, err) + } + dry := desired.DeepCopy() + if err := r.Patch(ctx, dry, client.Apply, client.FieldOwner(cnpgFieldManager), client.ForceOwnership, client.DryRunAll); err != nil { + return time.Time{}, fmt.Errorf("dry-run apply ObjectStore %s/%s: %w", desired.Namespace, desired.Name, err) + } + if !apiequality.Semantic.DeepEqual(live.Spec.Configuration, dry.Spec.Configuration) { + return time.Now(), nil + } + recorded, _, err := cnpgRecordedTime(live.Annotations[cnpgDestinationChangedAtAnnotation]) + if err != nil { + // Nothing says when the cache expires; this write's time is the + // latest it can be. + return time.Now(), nil + } + return recorded, nil +} + +// destinationClaimHolder returns the BackupJob other than j that the +// ObjectStore of clusterName names as the last to write it, while that job +// is still running and has no cnpg.io/Backup of its own, or "" when there is +// none. Such a job wrote the destination before otherCNPGBackupInFlight can +// see it: it can still be waiting for the plugin cache, or for the rollout of +// the plugin it attached. Once it has a Backup, that Backup holds the +// destination while CloudNativePG runs it, and the claim stops holding, so a +// job stuck after its Backup finished does not hold every later one. +func (r *BackupJobReconciler) destinationClaimHolder(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string) (string, error) { + store := &cnpgtypes.ObjectStore{} + if err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: clusterName}, store); err != nil { + if apierrors.IsNotFound(err) { + return "", nil + } + return "", fmt.Errorf("read ObjectStore %s/%s: %w", j.Namespace, clusterName, err) + } + holder := store.Annotations[cnpgDestinationWrittenByAnnotation] + if holder == "" || holder == j.Name { + return "", nil + } + holderJob := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: holder}, holderJob); err != nil { + if apierrors.IsNotFound(err) { + return "", nil + } + return "", err + } + if holderJob.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded || holderJob.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + return "", nil + } + holderBackups := &cnpgtypes.BackupList{} + if err := r.apiReader().List(ctx, holderBackups, client.InNamespace(j.Namespace), client.MatchingLabels{ + backupsv1alpha1.OwningJobNameLabel: holder, + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + }); err != nil { + return "", fmt.Errorf("list cnpg.io/Backups of BackupJob %s/%s: %w", j.Namespace, holder, err) + } + if len(holderBackups.Items) > 0 { + return "", nil + } + return holder, nil +} + // cnpgPluginAttachment is what applyClusterPluginBackup read off the live // Cluster before applying the plugin. type cnpgPluginAttachment struct { @@ -673,8 +987,15 @@ type cnpgPluginAttachment struct { // flavor is the image flavor the live Cluster runs. flavor string // missing reports a Cluster that carried no barman-cloud plugin. - missing bool - clusterStatus cnpgtypes.ClusterStatus + missing bool + // attachedAt is the cnpgPluginAttachedAtAnnotation the Cluster carried, + // "" when it carries no record: the plugin was attached by the chart, or + // before the record existed. + attachedAt string + // destinationChangedAt is the cnpgDestinationChangedAtAnnotation this + // pass applied to the ObjectStore, zero when it wrote none. + destinationChangedAt time.Time + clusterStatus cnpgtypes.ClusterStatus } // cnpgPluginRolloutPending returns why the rollout that attaching the @@ -720,6 +1041,26 @@ func (r *BackupJobReconciler) cnpgPluginRolloutPending(ctx context.Context, name return "", nil } +// cnpgRecordedTime parses a recorded time annotation, the plugin attach +// record on the Cluster or the destination change record on the ObjectStore; +// ok is false for "", an object that carries no record. A record further +// ahead of this controller's clock than cnpgAttachClockSkewTolerance is an +// error: every instance would predate it, and a deadline counted from it +// would not run out for as long as it lies ahead. +func cnpgRecordedTime(raw string) (recorded time.Time, ok bool, err error) { + if raw == "" { + return time.Time{}, false, nil + } + recorded, err = time.Parse(time.RFC3339, raw) + if err != nil { + return time.Time{}, false, err + } + if time.Until(recorded) > cnpgAttachClockSkewTolerance { + return time.Time{}, false, fmt.Errorf("%s is in the future", raw) + } + return recorded, true, nil +} + func podReady(pod *corev1.Pod) bool { for _, c := range pod.Status.Conditions { if c.Type == corev1.PodReady { @@ -747,16 +1088,23 @@ func currentBarmanServerName(c *cnpgtypes.Cluster) string { } // ensureCNPGBackup creates a one-shot postgresql.cnpg.io/Backup CR labelled -// with the BackupJob, or returns the existing one if a previous reconcile -// already created it. Idempotency relies on the OwningJob labels. -func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string) (*cnpgtypes.Backup, error) { - existing, err := r.findCNPGBackupForJob(ctx, j) - if err != nil { - return nil, err - } +// with the BackupJob, or returns existing, the job's Backup as looked up at +// the start of the pass. It does not look again: a reconcile of the same +// job cannot run in between, and a failed lookup here would come after the +// destination write and fail the job. +// +// written is the destination this pass wrote to the ObjectStore. A pass +// reaches the create only after that write: a job without a Backup writes +// the destination on every pass, and a pass that wrote nothing, or wrote it +// and waits for the plugin rollout, returns before this point. +func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string, existing *cnpgtypes.Backup, written cnpgDestination) (*cnpgtypes.Backup, error) { if existing != nil { return existing, nil } + record, err := json.Marshal(written) + if err != nil { + return nil, fmt.Errorf("encode the WAL archive destination: %w", err) + } obj := &cnpgtypes.Backup{ ObjectMeta: metav1.ObjectMeta{ @@ -767,6 +1115,7 @@ func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1 backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, cnpgClusterLabel: clusterName, }, + Annotations: map[string]string{cnpgWrittenDestinationAnnotation: string(record)}, }, Spec: cnpgtypes.BackupSpec{ Method: cnpgtypes.BackupMethodPlugin, @@ -784,23 +1133,167 @@ func (r *BackupJobReconciler) ensureCNPGBackup(ctx context.Context, j *backupsv1 // findCNPGBackupForJob returns the cnpg.io/Backup labelled with the // BackupJob's OwningJob{Name,Namespace}, if any. Returns (nil, nil) when // no match is found. +// +// A miss in the cache is checked against the apiserver: the cache can lag +// behind a Backup the previous pass created, and taking that miss as no +// Backup would write the destination under it and create a second one. func (r *BackupJobReconciler) findCNPGBackupForJob(ctx context.Context, j *backupsv1alpha1.BackupJob) (*cnpgtypes.Backup, error) { - list := &cnpgtypes.BackupList{} - if err := r.List(ctx, list, + opts := []client.ListOption{ client.InNamespace(j.Namespace), client.MatchingLabels{ backupsv1alpha1.OwningJobNameLabel: j.Name, backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, }, - ); err != nil { + } + list := &cnpgtypes.BackupList{} + if err := r.List(ctx, list, opts...); err != nil { return nil, err } + if len(list.Items) == 0 { + if err := r.apiReader().List(ctx, list, opts...); err != nil { + return nil, err + } + } if len(list.Items) == 0 { return nil, nil } return &list.Items[0], nil } +// otherCNPGBackupInFlight returns the name of another BackupJob that is still +// running and whose cnpg.io/Backup of clusterName has not finished, or "" when +// there is none. A Backup left behind by a finished or deleted job does not +// hold the queue. +// +// The Backups are read from the apiserver, not the cache, which can still +// miss a Backup another job's earlier pass created. Only the ones this +// controller created for the Cluster are asked for, by the labels +// ensureCNPGBackup has always set, so a waiting job's poll does not fetch +// the namespace's whole Backup history. +func (r *BackupJobReconciler) otherCNPGBackupInFlight(ctx context.Context, j *backupsv1alpha1.BackupJob, clusterName string) (string, error) { + list := &cnpgtypes.BackupList{} + if err := r.apiReader().List(ctx, list, client.InNamespace(j.Namespace), + client.MatchingLabels{cnpgClusterLabel: clusterName}, + client.HasLabels{backupsv1alpha1.OwningJobNameLabel}); err != nil { + return "", err + } + var holders []string + for i := range list.Items { + b := &list.Items[i] + owner := b.Labels[backupsv1alpha1.OwningJobNameLabel] + if b.Spec.Cluster.Name != clusterName || owner == "" || owner == j.Name { + continue + } + if b.Status.Phase == cnpgBackupPhaseComplete || b.Status.Phase == cnpgBackupPhaseFailed { + continue + } + ownerJob := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: owner}, ownerJob); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return "", err + } + if ownerJob.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded || ownerJob.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + continue + } + holders = append(holders, owner) + } + if len(holders) == 0 { + return "", nil + } + sort.Strings(holders) + return holders[0], nil +} + +func cnpgDestinationWriteHeld(reason string) metav1.Condition { + return metav1.Condition{ + Type: backupCondDestinationWriteHeld, + Status: metav1.ConditionTrue, + Reason: reason, + Message: "held off writing the WAL archive destination", + } +} + +// waitCNPGBackupJob requeues the BackupJob and records why it waits in its +// Ready condition, writing the status only when the conditions changed. +func (r *BackupJobReconciler) waitCNPGBackupJob(ctx context.Context, j *backupsv1alpha1.BackupJob, reason, message string) (ctrl.Result, error) { + if apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: reason, + Message: message, + }) { + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: cnpgPollInterval}, nil +} + +// cnpgDestination is the WAL archive destination recorded in +// cnpgWrittenDestinationAnnotation. S3SecretRef is nil when the strategy +// names no credentials Secret. ServerName is empty in a record written +// before it carried one. +type cnpgDestination struct { + DestinationPath string `json:"destinationPath"` + EndpointURL string `json:"endpointURL,omitempty"` + S3SecretRef *string `json:"s3SecretRef,omitempty"` + ServerName string `json:"serverName,omitempty"` +} + +func cnpgDestinationOf(t *strategyv1alpha1.CNPGTemplate) cnpgDestination { + d := cnpgDestination{ + DestinationPath: t.BarmanObjectStore.DestinationPath, + EndpointURL: t.BarmanObjectStore.EndpointURL, + } + if t.BarmanObjectStore.S3Credentials != nil { + name := t.BarmanObjectStore.S3Credentials.SecretRef.Name + d.S3SecretRef = &name + } + return d +} + +// cnpgArtifactDestination returns the destination the artifact of b names: +// the one recorded when b was created. A Backup without the record was +// created by a controller that kept none, and falls back to the strategy +// rendered now. A record that names no destination is an error: the +// strategy may have changed since the write, so it cannot stand in. +func cnpgArtifactDestination(b *cnpgtypes.Backup, rendered *strategyv1alpha1.CNPGTemplate) (cnpgDestination, error) { + raw, ok := b.Annotations[cnpgWrittenDestinationAnnotation] + if !ok { + return cnpgDestinationOf(rendered), nil + } + var d cnpgDestination + if err := json.Unmarshal([]byte(raw), &d); err != nil { + return cnpgDestination{}, fmt.Errorf("read the %s annotation of cnpg.io/Backup %s/%s: %w", cnpgWrittenDestinationAnnotation, b.Namespace, b.Name, err) + } + if d.DestinationPath == "" { + return cnpgDestination{}, fmt.Errorf("the %s annotation of cnpg.io/Backup %s/%s names no destination path", cnpgWrittenDestinationAnnotation, b.Namespace, b.Name) + } + return d, nil +} + +// cnpgProbeStore is the object store the size probe reads: the location +// and the credentials Secret recorded for the backup, with the Secret keys +// and the endpoint CA the record does not carry taken from the strategy. +// After a strategy edit those may not fit the recorded location; the probe +// then reports no size rather than measuring somewhere else. +func cnpgProbeStore(rendered *strategyv1alpha1.BarmanObjectStoreTemplate, destination cnpgDestination) *strategyv1alpha1.BarmanObjectStoreTemplate { + store := rendered.DeepCopy() + store.DestinationPath = destination.DestinationPath + store.EndpointURL = destination.EndpointURL + if destination.S3SecretRef == nil { + store.S3Credentials = nil + } else { + if store.S3Credentials == nil { + store.S3Credentials = &strategyv1alpha1.S3CredentialsTemplate{} + } + store.S3Credentials.SecretRef.Name = *destination.S3SecretRef + } + return store +} + // createCNPGBackupArtifact materialises a Cozystack Backup resource carrying // the metadata callers need to drive a future restore. The source app's // spec.databases / spec.users are snapshotted into Status.UnderlyingResources @@ -812,6 +1305,7 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( resolved *ResolvedBackupConfig, cnpgBackup *cnpgtypes.Backup, clusterName, serverName, flavor string, + destination cnpgDestination, rendered *strategyv1alpha1.CNPGTemplate, sourceApp *postgresapp.Postgres, ) (*backupsv1alpha1.Backup, error) { @@ -827,12 +1321,12 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( driverMD := map[string]string{ cnpgBackupNameKey: cnpgBackup.Name, cnpgServerNameKey: serverName, - cnpgDestinationPathKey: rendered.BarmanObjectStore.DestinationPath, - cnpgEndpointURLKey: rendered.BarmanObjectStore.EndpointURL, + cnpgDestinationPathKey: destination.DestinationPath, + cnpgEndpointURLKey: destination.EndpointURL, cnpgClusterNameKey: clusterName, } - if rendered.BarmanObjectStore.S3Credentials != nil { - driverMD[cnpgS3SecretRefKey] = rendered.BarmanObjectStore.S3Credentials.SecretRef.Name + if destination.S3SecretRef != nil { + driverMD[cnpgS3SecretRefKey] = *destination.S3SecretRef } // Copied rather than read back at restore time: barman retention deletes // the cnpg.io/Backup while the artifact can still be restored from. @@ -865,7 +1359,7 @@ func (r *BackupJobReconciler) createCNPGBackupArtifact( }, } if probe { - backup.Status.Artifact.SizeBytes = r.cnpgArtifactSizeBytes(ctx, j.Namespace, &rendered.BarmanObjectStore, serverName, cnpgBackup.Status.BackupID) + backup.Status.Artifact.SizeBytes = r.cnpgArtifactSizeBytes(ctx, j.Namespace, cnpgProbeStore(&rendered.BarmanObjectStore, destination), serverName, cnpgBackup.Status.BackupID) } if j.Spec.PlanRef != nil { backup.Spec.PlanRef = j.Spec.PlanRef @@ -2224,9 +2718,12 @@ func buildBarmanPlugin(objectStoreName, serverName string) cnpgtypes.PluginConfi // high-water mark on a 38 MB database is 254 MiB, so 128Mi cannot hold it. A // tenant that leaves resourceQuotas empty has no LimitRange, and there the // sidecar had no requests and no limit at all. 256Mi holds the measured working -// set and 1Gi is four times the measurement. No CPU limit is set, matching the -// entityOperator precedent in packages/apps/kafka: a throttled sidecar stalls -// WAL archiving instead of failing it. +// set and 1Gi is four times the measurement. The CPU limit is explicit for the +// same reason: left unset, that LimitRange gives the sidecar its 250m default, +// so gzip compression of full backups and WAL segments runs on a quarter of a +// core (a 1.6 GB database took 30 minutes to back up). One core keeps the +// sidecar from starving the instance while removing that ceiling; it counts +// against the tenant's limits.cpu budget, one core per instance pod. // // Checksum: since botocore ~1.36 the default (when_supported) attaches a // flexible checksum to every PutObject, which non-AWS S3-compatible backends @@ -2246,6 +2743,7 @@ func barmanSidecarConfiguration() *cnpgtypes.InstanceSidecarConfiguration { corev1.ResourceMemory: resource.MustParse("256Mi"), }, Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("1Gi"), }, }, diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 551209fac..780f7fb60 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -3,7 +3,9 @@ package backupcontroller import ( "context" + "errors" "fmt" + "slices" "strings" "testing" "time" @@ -13,13 +15,16 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/scheme" + clienttesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -997,10 +1002,9 @@ func TestUnmarshalCNPGBackupSnapshot_RejectsMissingOrWrongKind(t *testing.T) { } } -// TestCNPGBackupDeadlineExceeded locks in the wall-clock cap that prevents a -// perpetually-failed cnpg.io/Backup from pinning the BackupJob in Running -// forever. Without this guard the Plan-controller queue would back up -// behind a single broken backup. +// TestCNPGBackupDeadlineExceeded locks in the wall-clock cap on waiting for +// the barman-cloud plugin rollout, so a rollout that never finishes fails the +// BackupJob instead of pinning it in Running. func TestCNPGBackupDeadlineExceeded(t *testing.T) { cases := []struct { name string @@ -1073,7 +1077,7 @@ func TestCreateCNPGBackupArtifact_AlreadyExistsReturnsExisting(t *testing.T) { } sourceApp := newPostgresApp("pg", "tenant") - got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "postgres-pg", "postgresql", rendered, sourceApp) + got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "postgres-pg", "postgresql", cnpgDestinationOf(rendered), rendered, sourceApp) if err != nil { t.Fatalf("expected AlreadyExists to be swallowed, got error %v", err) } @@ -1099,7 +1103,7 @@ func TestApplyClusterPluginBackup_NotFoundOnMissingCluster(t *testing.T) { }, } - _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-missing", tmpl, "postgres-missing", true) + _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-missing", tmpl, "postgres-missing", true, true, "bj") if err == nil { t.Fatalf("expected NotFound error, got nil") } @@ -1739,7 +1743,7 @@ func TestApplyClusterPluginBackup_ReadsTheLiveClusterNotTheCache(t *testing.T) { BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/x/"}, } - got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true) + got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true, true, "bj") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -1949,7 +1953,7 @@ func TestApplyClusterPluginBackup_PreservesLiveServerName(t *testing.T) { BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/x/"}, } - got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true) + got, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "strategy-name", true, true, "bj") if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -2118,7 +2122,7 @@ func TestApplyClusterPluginBackup_PatchesExistingCluster(t *testing.T) { }, } - if _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "tenant-app", true); err != nil { + if _, err := r.applyClusterPluginBackup(context.Background(), "tenant", "postgres-app", tmpl, "tenant-app", true, true, "bj"); err != nil { t.Fatalf("unexpected error: %v", err) } @@ -2152,8 +2156,11 @@ func TestApplyClusterPluginBackup_PatchesExistingCluster(t *testing.T) { sc.Resources.Requests.Cpu().MilliValue() != 100 { t.Errorf("ObjectStore leaves Barman subject to undersized tenant resource defaults: %+v", sc.Resources) } - if _, ok := sc.Resources.Limits[corev1.ResourceCPU]; ok { - t.Errorf("ObjectStore caps sidecar CPU, which throttles WAL archiving instead of failing it: %+v", sc.Resources.Limits) + // Without an explicit CPU limit the tenant LimitRange caps the sidecar at + // its 250m default, and every backup and WAL upload compresses on a + // quarter of a core. + if sc.Resources.Limits.Cpu().MilliValue() != 1000 { + t.Errorf("ObjectStore leaves the sidecar CPU limit to the tenant LimitRange default: %+v", sc.Resources.Limits) } // The ObjectStore must be owner-referenced to the Cluster so Kubernetes GC // removes it when the Cluster is deleted (no orphan in the platform flow, @@ -2205,7 +2212,7 @@ func TestEnsureCNPGBackup_UsesPluginMethod(t *testing.T) { c := newCNPGStrategyTestClient(t, j) r := &BackupJobReconciler{Client: c} - bk, err := r.ensureCNPGBackup(context.Background(), j, "postgres-app") + bk, err := r.ensureCNPGBackup(context.Background(), j, "postgres-app", nil, cnpgDestination{DestinationPath: "s3://bucket/"}) if err != nil { t.Fatalf("ensureCNPGBackup: %v", err) } @@ -2418,6 +2425,13 @@ func TestReconcileCNPG_BackupPhaseOutcome(t *testing.T) { if j.Status.Phase != tc.wantPhase { t.Fatalf("BackupJob phase = %q (%s), want %q", j.Status.Phase, j.Status.Message, tc.wantPhase) } + stored := &backupsv1alpha1.BackupJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(job), stored); err != nil { + t.Fatalf("get BackupJob after reconcile: %v", err) + } + if stored.Status.Phase != tc.wantPhase { + t.Fatalf("stored BackupJob phase = %q, want %q", stored.Status.Phase, tc.wantPhase) + } message := j.Status.Message if tc.wantPhase == backupsv1alpha1.BackupJobPhaseRunning { @@ -2480,6 +2494,1700 @@ func TestReconcileCNPG_BackupPhaseOutcome(t *testing.T) { } } +// newAttachedObjectStoreFixture is a Cluster that already runs the barman-cloud +// plugin, whose ObjectStore currently points at liveDestination; the strategy +// of the fixture's BackupJob points at s3://bucket/. +func newAttachedObjectStoreFixture(t *testing.T, startedAt time.Time, liveDestination string) *cnpgRolloutFixture { + t.Helper() + f := newCNPGRolloutFixture(t, newRolloutBackupJob(startedAt), + newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(time.Now().Add(-time.Hour))) + store := &cnpgtypes.ObjectStore{ + ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: rolloutCluster}, + Spec: cnpgtypes.ObjectStoreSpec{Configuration: cnpgtypes.BarmanObjectStoreConfiguration{DestinationPath: liveDestination}}, + } + if err := f.c.Create(context.Background(), store); err != nil { + t.Fatalf("seed ObjectStore: %v", err) + } + return f +} + +func (f *cnpgRolloutFixture) liveDestination() string { + f.t.Helper() + store := &cnpgtypes.ObjectStore{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, store); err != nil { + f.t.Fatalf("get ObjectStore: %v", err) + } + return store.Spec.Configuration.DestinationPath +} + +// seedCNPGBackup creates a cnpg.io/Backup of cluster owned by the BackupJob +// named owner, labelled as ensureCNPGBackup labels it; an empty owner leaves +// it unlabelled, as a Backup created by hand. +func (f *cnpgRolloutFixture) seedCNPGBackup(name, owner, cluster, phase string) { + f.t.Helper() + var backupLabels map[string]string + if owner != "" { + backupLabels = map[string]string{ + backupsv1alpha1.OwningJobNameLabel: owner, + backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS, + cnpgClusterLabel: cluster, + } + } + b := &cnpgtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: name, Labels: backupLabels}, + Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: cluster}}, + Status: cnpgtypes.BackupStatus{Phase: phase}, + } + if err := f.c.Create(context.Background(), b); err != nil { + f.t.Fatalf("seed cnpg.io/Backup %s: %v", name, err) + } +} + +func (f *cnpgRolloutFixture) setJobCondition(cond metav1.Condition) { + f.t.Helper() + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + f.t.Fatalf("get BackupJob: %v", err) + } + j.Status.Conditions = append(j.Status.Conditions, cond) + if err := f.c.Status().Update(context.Background(), j); err != nil { + f.t.Fatalf("seed BackupJob condition: %v", err) + } +} + +func (f *cnpgRolloutFixture) seedOtherBackupJob(name string, phase backupsv1alpha1.BackupJobPhase) { + f.t.Helper() + other := newRolloutBackupJob(time.Now().Add(-time.Hour)) + other.Name = name + other.Status.Phase = phase + if err := f.c.Create(context.Background(), other); err != nil { + f.t.Fatalf("seed BackupJob %s: %v", name, err) + } +} + +// The ObjectStore is the Cluster's WAL archive destination, so a BackupJob +// whose cnpg.io/Backup is already running must not keep re-applying it: a +// second BackupJob of the same app with another destination would otherwise +// flip it back and forth under both backups. +func TestReconcileCNPG_RunningBackupDoesNotReapplyObjectStore(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") + // Two jobs can still pass the queue together, across a leader handover; + // once both have a Backup neither may wait on the other. + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + writes := 0 + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + writes++ + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + + j, _ := f.pass() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("BackupJob failed: %s", j.Status.Message) + } + if got := f.liveDestination(); got != "s3://other/" { + t.Fatalf("a pass over a running backup re-applied the ObjectStore: destination %q", got) + } + if writes != 0 { + t.Errorf("a pass over a running backup patched %d object(s); the ObjectStore and the Cluster plugin are only read", writes) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCNPGBackup" { + t.Fatalf("Ready = %+v, want the running backup followed rather than queued", ready) + } +} + +// The queue reads the apiserver on every pass of every waiting job, so it +// asks only for the Backups this controller created for the Cluster rather +// than the namespace's whole Backup history. +func TestReconcileCNPG_QueueListsOnlyTheClustersOwnedBackups(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") + var selectors []labels.Selector + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*cnpgtypes.BackupList); ok { + lo := &client.ListOptions{} + lo.ApplyOptions(opts) + selectors = append(selectors, lo.LabelSelector) + } + return cl.List(ctx, list, opts...) + }, + }) + f.pass() + owned := labels.Set{cnpgClusterLabel: rolloutCluster, backupsv1alpha1.OwningJobNameLabel: "bj-other"} + foreignCluster := labels.Set{cnpgClusterLabel: "postgres-other", backupsv1alpha1.OwningJobNameLabel: "bj-other"} + unowned := labels.Set{cnpgClusterLabel: rolloutCluster} + var queue labels.Selector + for _, s := range selectors { + if s == nil { + s = labels.Everything() + } + if s.Matches(owned) { + queue = s + } + } + if queue == nil { + t.Fatalf("no cnpg.io/Backup list would find another job's Backup; selectors = %v", selectors) + } + if queue.Matches(foreignCluster) || queue.Matches(unowned) { + t.Errorf("queue selector %q, want it to match only owned Backups of %s", queue, rolloutCluster) + } +} + +func TestReconcileCNPG_WaitsForAnotherRunningBackupOfTheApp(t *testing.T) { + cases := []struct { + name string + otherJob backupsv1alpha1.BackupJobPhase + seedJob bool + otherBackup string + owner string + cluster string + wantWait bool + }{ + {name: "another running job with a running Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", wantWait: true}, + {name: "another running job with a pending Backup holds it", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "pending", wantWait: true}, + {name: "a completed Backup of another job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: cnpgBackupPhaseComplete}, + {name: "a failed Backup of another job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: cnpgBackupPhaseFailed}, + {name: "a Backup of a failed job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseFailed, otherBackup: "pending"}, + {name: "a Backup of a succeeded job does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseSucceeded, otherBackup: "pending"}, + {name: "a Backup whose job is gone does not", otherBackup: "pending"}, + {name: "a Backup of another Cluster does not", seedJob: true, otherJob: backupsv1alpha1.BackupJobPhaseRunning, otherBackup: "running", cluster: "postgres-other"}, + {name: "a Backup created by hand does not", otherBackup: "running", owner: "-"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + if tc.seedJob { + f.seedOtherBackupJob("bj-other", tc.otherJob) + } + owner, cluster := "bj-other", rolloutCluster + if tc.owner == "-" { + owner = "" + } + if tc.cluster != "" { + cluster = tc.cluster + } + f.seedCNPGBackup("bj-other-cnpg", owner, cluster, tc.otherBackup) + // client-go refuses a Get with an empty name before it reaches + // the apiserver; the fake client answers NotFound instead. + statusWrites := 0 + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if key.Name == "" { + return errors.New("resource name may not be empty") + } + return cl.Get(ctx, key, obj, opts...) + }, + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + statusWrites++ + return cl.SubResource(sub).Update(ctx, obj, opts...) + }, + }) + + j, backups, res := f.passResult() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("BackupJob failed: %s", j.Status.Message) + } + ours := 0 + for _, b := range backups { + if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { + ours++ + } + } + ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready") + if tc.wantWait { + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("the waiting job applied its destination over a running backup: %q", got) + } + if ours != 0 { + t.Errorf("the waiting job created %d cnpg.io/Backup(s)", ours) + } + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "bj-other") { + t.Errorf("Ready = %+v, want False/WaitingForOtherBackup naming bj-other", ready) + } + if res.RequeueAfter == 0 { + t.Errorf("a queued job is not requeued, and nothing else wakes it when the holder finishes: %+v", res) + } + statusWrites = 0 + if _, _, res = f.passResult(); res.RequeueAfter == 0 { + t.Errorf("a queued job whose status did not change is not requeued: %+v", res) + } + if statusWrites != 0 { + t.Errorf("a second pass with the same holder wrote the BackupJob status %d time(s)", statusWrites) + } + + held := &cnpgtypes.Backup{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil { + t.Fatalf("get holder Backup: %v", err) + } + held.Status.Phase = cnpgBackupPhaseComplete + if err := f.c.Update(context.Background(), held); err != nil { + t.Fatalf("complete holder Backup: %v", err) + } + j, _ = f.pass() + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("once the holder finished, destination = %q, want the job's own s3://bucket/", got) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" { + t.Errorf("Ready still names the finished holder: %+v", ready) + } + return + } + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("destination = %q, want the job's own s3://bucket/", got) + } + if ready != nil && ready.Reason == "WaitingForOtherBackup" { + t.Errorf("Ready = %+v, the job was held", ready) + } + f.settleDestination() + _, backups = f.pass() + ours = 0 + for _, b := range backups { + if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { + ours++ + } + } + if ours != 1 { + t.Errorf("got %d cnpg.io/Backups for the job, want 1", ours) + } + }) + } +} + +func TestReconcileCNPG_ObjectStoreApplyErrors(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + webhookDown := apierrors.NewInternalError(errors.New(`failed calling webhook "mcluster.cnpg.io": connect: connection refused`)) + invalid := apierrors.NewInvalid(schema.GroupKind{Group: cnpgtypes.BarmanGroupVersion.Group, Kind: "ObjectStore"}, rolloutCluster, nil) + cases := []struct { + name string + err error + startedAt time.Time + // failingSince seeds an earlier failure of the same write. + failingSince time.Time + wantPhase backupsv1alpha1.BackupJobPhase + }{ + {name: "a transient error is retried", err: webhookDown, startedAt: time.Now()}, + {name: "a transient error failing since past the deadline fails", err: webhookDown, startedAt: expired, failingSince: expired, wantPhase: backupsv1alpha1.BackupJobPhaseFailed}, + {name: "a transient error after a long wait is retried", err: webhookDown, startedAt: expired}, + {name: "an invalid object fails at once", err: invalid, startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed}, + {name: "a bad request fails at once", err: apierrors.NewBadRequest("malformed"), startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed}, + {name: "a forbidden write fails at once", err: apierrors.NewForbidden(schema.GroupResource{Resource: "objectstores"}, rolloutCluster, errors.New("denied")), startedAt: time.Now(), wantPhase: backupsv1alpha1.BackupJobPhaseFailed}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, tc.startedAt, "s3://other/") + if !tc.failingSince.IsZero() { + f.setJobCondition(metav1.Condition{ + Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", + Message: "earlier failure", LastTransitionTime: metav1.NewTime(tc.failingSince), + }) + } + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok { + return tc.err + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + + j, backups, res := f.passResult() + if j.Status.Phase != tc.wantPhase { + t.Fatalf("BackupJob phase = %q (%s), want %q", j.Status.Phase, j.Status.Message, tc.wantPhase) + } + if len(backups) != 0 { + t.Errorf("a cnpg.io/Backup was created although the ObjectStore was not applied: %+v", backups) + } + if tc.wantPhase == "" { + ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready") + if ready == nil || ready.Reason != "ClusterPluginApplyRetrying" || !strings.Contains(ready.Message, "connection refused") { + t.Errorf("Ready = %+v, want ClusterPluginApplyRetrying carrying the error", ready) + } + if res.RequeueAfter == 0 { + t.Errorf("a retried write is not requeued: %+v", res) + } + } + }) + } +} + +// A write that recovers clears the retry state, so a later failure gets a +// fresh budget and Ready no longer reports the retry. +func TestReconcileCNPG_ObjectStoreApplyRecovers(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + failing := true + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok && failing { + return apierrors.NewInternalError(errors.New("webhook down")) + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + j, _ := f.pass() + if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond == nil || cond.Status != metav1.ConditionFalse { + t.Fatalf("%s = %+v after a failed write, want False", backupCondDestinationApplied, cond) + } + + failing = false + _, _ = f.pass() + f.settleDestination() + j, backups := f.pass() + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups after the write recovered, want 1", len(backups)) + } + if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil { + t.Errorf("%s = %+v left over after the write recovered", backupCondDestinationApplied, cond) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "ClusterPluginApplyRetrying" { + t.Errorf("Ready still reports the retry: %+v", ready) + } +} + +// An error reading the holder's BackupJob is not taken as "no holder". +func TestReconcileCNPG_HolderLookupErrorIsReturned(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if key.Name == "bj-other" { + return apierrors.NewServiceUnavailable("cache not synced") + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("reconcileCNPG swallowed the holder lookup error") + } + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination written although the holder could not be read: %q", got) + } +} + +// The rollout deadline counts from when the plugin was attached, not from +// the job's start, which may lie behind a long wait for another backup. +func TestReconcileCNPG_PluginRolloutDeadlineCountsFromAttach(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + f := newCNPGRolloutFixture(t, newRolloutBackupJob(expired), + newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) + + f.pass() // latch + j, _ := f.pass() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a rollout that just began failed on the job's old start: %s", j.Status.Message) + } + if cond := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); cond == nil || cond.Reason != "WaitingForPluginRollout" { + t.Fatalf("Ready = %+v, want WaitingForPluginRollout", cond) + } + attached := &cnpgtypes.Cluster{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil { + t.Fatalf("get Cluster: %v", err) + } + f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...)) + + rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + rollout.LastTransitionTime = metav1.NewTime(expired) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the rollout: %v", err) + } + j, _ = f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, "rollout did not finish") { + t.Fatalf("phase = %q (%s), want Failed on a rollout older than the deadline", j.Status.Phase, j.Status.Message) + } +} + +// A rollout dated by the Cluster's attach record runs out like one dated by +// the job's own condition: a record newer than the condition but older than +// the deadline fails the job too. +func TestReconcileCNPG_PluginRolloutDeadlineCountsFromTheAttachRecord(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second) + f := newCNPGRolloutFixture(t, newRolloutBackupJob(expired.Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(expired.Add(-time.Hour))) + + j, _ := f.pass() // latch + latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + if latch == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + latch.LastTransitionTime = metav1.NewTime(expired.Add(-10 * time.Minute)) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the latch: %v", err) + } + live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: expired.UTC().Format(time.RFC3339)} + f.observeCNPG(live) + + j, backups := f.pass() + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created against an instance older than the attach: %+v", backups) + } + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, "rollout did not finish") { + t.Fatalf("phase = %q (%s), want Failed on a rollout recorded longer ago than the deadline", j.Status.Phase, j.Status.Message) + } +} + +// The rollout is measured from the write that attached the plugin, not from +// the latch recorded before it: retries of a failing write can lie between +// the two, and an instance recreated in that time runs without the plugin. +func TestReconcileCNPG_PluginRolloutCountsFromTheSuccessfulAttach(t *testing.T) { + cases := []struct { + name string + // podBetween recreates the instance after the latch and before the + // attach; otherwise it predates both. + podBetween bool + }{ + {name: "an instance recreated before the attach still has to roll", podBetween: true}, + {name: "retries before the attach do not spend the rollout deadline"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second) + oldPod := newRolloutInstancePod(expired.Add(-time.Hour)) + f := newCNPGRolloutFixture(t, newRolloutBackupJob(expired.Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), oldPod) + + j, _ := f.pass() // latch + latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + if latch == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + latch.LastTransitionTime = metav1.NewTime(expired) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the latch: %v", err) + } + + failing := true + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.Cluster); ok && failing { + return apierrors.NewInternalError(errors.New(`failed calling webhook "mcluster.cnpg.io"`)) + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + if j, _ = f.pass(); j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a failed attach failed the job: %s", j.Status.Message) + } + if tc.podBetween { + f.replacePod(oldPod, newRolloutInstancePod(expired.Add(time.Minute))) + } + + failing = false + attachAt := time.Now().Truncate(time.Second) + j, backups := f.pass() + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created in the pass that attached the plugin: %+v", backups) + } + rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + if rollout == nil || rollout.Status != metav1.ConditionFalse || rollout.LastTransitionTime.Time.Before(attachAt) { + t.Fatalf("%s = %+v, want False since the attach at %s", backupCondPluginRolledOut, rollout, attachAt) + } + + attached := &cnpgtypes.Cluster{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil { + t.Fatalf("get Cluster: %v", err) + } + f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...)) + j, backups = f.pass() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a rollout that began with the attach failed: %s", j.Status.Message) + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created against an instance older than the attach: %+v", backups) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || !strings.Contains(ready.Message, "predates") { + t.Errorf("Ready = %+v, want the instance that predates the plugin", ready) + } + }) + } +} + +// The status write that records the attach can be lost after the attach +// itself went through, by a conflict, an apiserver that is briefly +// unavailable, or a restart, including on the retry of an attach that +// failed; an instance recreated between the latch and the attach still has +// to roll, and the rollout deadline still counts from the attach. +func TestReconcileCNPG_LostAttachRecordStillWaitsForTheRollout(t *testing.T) { + jobs := backupsv1alpha1.GroupVersion.WithResource("backupjobs").GroupResource() + for _, tc := range []struct { + name string + failFirstAttach bool + statusWriteError error + }{ + {name: "conflict", statusWriteError: apierrors.NewConflict(jobs, "bj", errors.New("the object has been modified"))}, + {name: "retried attach, apiserver unavailable", failFirstAttach: true, statusWriteError: apierrors.NewServiceUnavailable("the server is currently unable to handle the request")}, + } { + t.Run(tc.name, func(t *testing.T) { + latchedAt := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second) + oldPod := newRolloutInstancePod(latchedAt.Add(-time.Hour)) + f := newCNPGRolloutFixture(t, newRolloutBackupJob(latchedAt.Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), oldPod) + + j, _ := f.pass() // latch + latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + if latch == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + latch.LastTransitionTime = metav1.NewTime(latchedAt) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the latch: %v", err) + } + f.replacePod(oldPod, newRolloutInstancePod(latchedAt.Add(time.Minute))) + + base := f.r.Client + if tc.failFirstAttach { + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.Cluster); ok { + return apierrors.NewServiceUnavailable("the server is currently unable to handle the request") + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + if j, _ = f.pass(); apimeta.IsStatusConditionTrue(j.Status.Conditions, backupCondPluginRolledOut) { + t.Fatalf("a failed attach reported the rollout done: %+v", j.Status.Conditions) + } + if applied := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); applied == nil || applied.Reason != "ClusterPluginApplyRetrying" { + t.Fatalf("%s = %+v, want the attach retrying", backupCondDestinationApplied, applied) + } + } + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + return tc.statusWriteError + }, + }) + j = &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("the pass that attached the plugin reported no lost status write") + } + f.r.Client = base + + attached := &cnpgtypes.Cluster{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, attached); err != nil { + t.Fatalf("get Cluster: %v", err) + } + if len(attached.Spec.Plugins) == 0 { + t.Fatalf("the plugin was not attached") + } + live := newRolloutCluster(cnpgClusterHealthyPhase, attached.Spec.Plugins...) + live.Annotations = attached.Annotations + f.observeCNPG(live) + + var backups []cnpgtypes.Backup + for range 3 { + if j, backups = f.pass(); len(backups) != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + break + } + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created against an instance recreated before the attach: %+v", backups) + } + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed instead of waiting for the rollout: %s", j.Status.Message) + } + }) + } +} + +// A record older than the latch is left from an earlier attach of a plugin +// since removed; the plugin found now was attached by someone who keeps no +// record, after the latch, so an instance recreated between the record and +// the latch still has to roll. +func TestReconcileCNPG_AttachRecordOlderThanTheLatchKeepsTheLatch(t *testing.T) { + latchedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second) + oldPod := newRolloutInstancePod(latchedAt.Add(-2 * time.Hour)) + f := newCNPGRolloutFixture(t, newRolloutBackupJob(latchedAt.Add(-2*time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), oldPod) + + j, _ := f.pass() // latch + latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + if latch == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + latch.LastTransitionTime = metav1.NewTime(latchedAt) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the latch: %v", err) + } + f.replacePod(oldPod, newRolloutInstancePod(latchedAt.Add(-time.Minute))) + live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: latchedAt.Add(-time.Hour).UTC().Format(time.RFC3339)} + f.observeCNPG(live) + + var backups []cnpgtypes.Backup + for range 3 { + if j, backups = f.pass(); len(backups) != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + break + } + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created against an instance recreated before the latch: %+v", backups) + } + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed instead of waiting for the rollout: %s", j.Status.Message) + } +} + +func (f *cnpgRolloutFixture) liveObjectStore() *cnpgtypes.ObjectStore { + f.t.Helper() + store := &cnpgtypes.ObjectStore{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: rolloutCluster}, store); err != nil { + f.t.Fatalf("get ObjectStore: %v", err) + } + return store +} + +// setDestinationStamp rewrites the destination-change record on the live +// ObjectStore, as a write that long ago, or a corrupt one, would have left it. +func (f *cnpgRolloutFixture) setDestinationStamp(value string) { + f.t.Helper() + store := f.liveObjectStore() + if store.Annotations == nil { + store.Annotations = map[string]string{} + } + store.Annotations[cnpgDestinationChangedAtAnnotation] = value + if err := f.c.Update(context.Background(), store); err != nil { + f.t.Fatalf("stamp ObjectStore: %v", err) + } +} + +// settleDestination ages the destination-change record, when the ObjectStore +// carries one, as if the plugin's cache had expired since that write. +func (f *cnpgRolloutFixture) settleDestination() { + f.t.Helper() + if _, ok := f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]; ok { + f.setDestinationStamp(time.Now().Add(-time.Minute).UTC().Format(time.RFC3339)) + } +} + +// addJobWithDestination adds BackupJob name of the same application, resolved +// to a strategy of its own that writes destination, and returns a function +// running one pass of it. +func (f *cnpgRolloutFixture) addJobWithDestination(name, destination string) func() *backupsv1alpha1.BackupJob { + f.t.Helper() + ctx := context.Background() + strategy := &strategyv1alpha1.CNPG{ + ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy-" + name}, + Spec: strategyv1alpha1.CNPGSpec{Template: strategyv1alpha1.CNPGTemplate{ + BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: destination}, + }}, + } + if err := f.c.Create(ctx, strategy); err != nil { + f.t.Fatalf("seed strategy: %v", err) + } + job := newRolloutBackupJob(time.Now().Add(-time.Hour)) + job.Name = name + if err := f.c.Create(ctx, job); err != nil { + f.t.Fatalf("seed BackupJob %s: %v", name, err) + } + group := strategyv1alpha1.GroupVersion.Group + resolved := &ResolvedBackupConfig{StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &group, Kind: strategyv1alpha1.CNPGStrategyKind, Name: strategy.Name}} + return func() *backupsv1alpha1.BackupJob { + f.t.Helper() + key := client.ObjectKey{Namespace: rolloutNS, Name: name} + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(ctx, key, j); err != nil { + f.t.Fatalf("get BackupJob %s: %v", name, err) + } + if _, err := f.r.reconcileCNPG(ctx, j, resolved); err != nil { + f.t.Fatalf("reconcileCNPG %s: %v", name, err) + } + if err := f.c.Get(ctx, key, j); err != nil { + f.t.Fatalf("get BackupJob %s after reconcile: %v", name, err) + } + return j + } +} + +// Two jobs of the application with different destinations, neither with a +// Backup yet, would otherwise take turns rewriting the ObjectStore, each +// write restarting the other's wait for the plugin cache so that neither +// ever creates one. The job whose write came first holds the destination, +// by its claim until it has a Backup and by that Backup afterwards, and the +// other waits for it. +func TestReconcileCNPG_JobsWithDifferentDestinationsTakeTurns(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://third/") + pass2 := f.addJobWithDestination("bj2", "s3://other/") + + var j2 *backupsv1alpha1.BackupJob + for range 5 { + f.pass() + f.settleDestination() + j2 = pass2() + f.settleDestination() + } + backups := &cnpgtypes.BackupList{} + if err := f.c.List(context.Background(), backups, client.InNamespace(rolloutNS)); err != nil { + t.Fatalf("list cnpg.io/Backups: %v", err) + } + if len(backups.Items) != 1 || backups.Items[0].Labels[backupsv1alpha1.OwningJobNameLabel] != "bj" { + t.Fatalf("cnpg.io/Backups = %+v, want exactly one, of bj, whose write came first", backups.Items) + } + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("destination = %q, want bj's s3://bucket/ under its running Backup", got) + } + if ready := apimeta.FindStatusCondition(j2.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "BackupJob bj ") { + t.Errorf("bj2 Ready = %+v, want WaitingForOtherBackup naming bj", ready) + } +} + +// The claim names the job whose write last reached the ObjectStore, whether +// it changed the configuration, created the object or left it as it was; the +// change record is re-sent as it is by a write that changes nothing. +func TestReconcileCNPG_ClaimNamesTheJobThatWroteTheDestination(t *testing.T) { + t.Run("changed", func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.pass() + if got := f.liveObjectStore().Annotations[cnpgDestinationWrittenByAnnotation]; got != "bj" { + t.Errorf("%s = %q, want bj", cnpgDestinationWrittenByAnnotation, got) + } + }) + t.Run("created", func(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(time.Now().Add(-time.Hour))) + f.pass() + store := f.liveObjectStore() + if got := store.Annotations[cnpgDestinationWrittenByAnnotation]; got != "bj" { + t.Errorf("%s = %q, want bj", cnpgDestinationWrittenByAnnotation, got) + } + if _, ok := store.Annotations[cnpgDestinationChangedAtAnnotation]; ok { + t.Errorf("creating the ObjectStore stamped %s, which would hold the Backup for nothing", cnpgDestinationChangedAtAnnotation) + } + }) + for name, recorded := range map[string]map[string]string{ + "unchanged": {cnpgDestinationWrittenByAnnotation: "bj-old"}, + "unchanged with a stamp": {cnpgDestinationWrittenByAnnotation: "bj-old", cnpgDestinationChangedAtAnnotation: "2026-01-02T03:04:05Z"}, + } { + t.Run(name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") + store := f.liveObjectStore() + store.Annotations = recorded + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed claim: %v", err) + } + var sent []map[string]string + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + po := &client.PatchOptions{} + po.ApplyOptions(opts) + if _, ok := obj.(*cnpgtypes.ObjectStore); ok && !slices.Contains(po.DryRun, metav1.DryRunAll) { + sent = append(sent, obj.GetAnnotations()) + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + f.pass() + if len(sent) != 1 { + t.Fatalf("got %d ObjectStore writes, want 1", len(sent)) + } + if got := sent[0][cnpgDestinationWrittenByAnnotation]; got != "bj" { + t.Errorf("the write sent %s = %q, want bj, the job that wrote", cnpgDestinationWrittenByAnnotation, got) + } + if got, want := sent[0][cnpgDestinationChangedAtAnnotation], recorded[cnpgDestinationChangedAtAnnotation]; got != want { + t.Errorf("the write sent %s = %q, want the recorded %q", cnpgDestinationChangedAtAnnotation, got, want) + } + }) + } +} + +// A claim stops holding the destination once its job has finished or is +// gone, so the next job writes its own. +func TestReconcileCNPG_ClaimOfAFinishedJobDoesNotHold(t *testing.T) { + for name, phase := range map[string]backupsv1alpha1.BackupJobPhase{ + "succeeded": backupsv1alpha1.BackupJobPhaseSucceeded, + "failed": backupsv1alpha1.BackupJobPhaseFailed, + "gone": "", + } { + t.Run(name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + if name != "gone" { + f.seedOtherBackupJob("bj-other", phase) + } + store := f.liveObjectStore() + store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"} + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed claim: %v", err) + } + j, _ := f.pass() + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("destination = %q, want bj's s3://bucket/", got) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" { + t.Errorf("Ready = %+v, held by a %s job", ready, name) + } + }) + } +} + +// A job that attaches the plugin writes a configuration the ObjectStore may +// already carry, for instance after a restore re-rendered the Cluster +// without the plugin. Its write still claims the destination, so a job with +// another one waits through the rollout instead of switching it. +func TestReconcileCNPG_UnchangedWriteClaimsTheDestinationForTheRollout(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) + f.seedOtherBackupJob("bj-old", backupsv1alpha1.BackupJobPhaseSucceeded) + store := &cnpgtypes.ObjectStore{ + ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: rolloutCluster, + Annotations: map[string]string{cnpgDestinationWrittenByAnnotation: "bj-old"}}, + Spec: cnpgtypes.ObjectStoreSpec{Configuration: cnpgtypes.BarmanObjectStoreConfiguration{DestinationPath: "s3://bucket/"}}, + } + if err := f.c.Create(context.Background(), store); err != nil { + t.Fatalf("seed ObjectStore: %v", err) + } + pass2 := f.addJobWithDestination("bj2", "s3://other/") + + f.pass() // latch + if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, "Ready").Reason != "WaitingForPluginRollout" { + t.Fatalf("Ready = %+v, want bj waiting for the rollout of the plugin it attached", apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } + j2 := pass2() + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("destination = %q, switched during bj's rollout", got) + } + if ready := apimeta.FindStatusCondition(j2.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "BackupJob bj ") { + t.Errorf("bj2 Ready = %+v, want WaitingForOtherBackup naming bj", ready) + } +} + +// The claim covers only the window before its job has a Backup the queue +// can see. Once it has one, the Backup in flight holds the destination, for +// as long as CloudNativePG runs it, and a Backup that finished does not, even +// if its job is stuck before it finishes, so that job cannot hold every later +// backup of the application for as long as it lives. +func TestReconcileCNPG_ClaimOfAJobWithItsOwnBackupDoesNotHold(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, cnpgBackupPhaseComplete) + store := f.liveObjectStore() + store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"} + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed claim: %v", err) + } + j, _ := f.pass() + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("destination = %q, held by a job whose Backup already finished", got) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready != nil && ready.Reason == "WaitingForOtherBackup" { + t.Errorf("Ready = %+v, held by a job whose Backup already finished", ready) + } +} + +// A claim whose job cannot be read cannot be taken as released: writing over +// it would let two jobs flip the destination again. The error goes back to +// the controller for a retry. +func TestReconcileCNPG_ClaimHolderLookupErrorIsReturned(t *testing.T) { + unavailable := apierrors.NewServiceUnavailable("apiserver unavailable") + for name, setup := range map[string]func(f *cnpgRolloutFixture){ + "holder job read fails": func(f *cnpgRolloutFixture) { + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*backupsv1alpha1.BackupJob); ok && key.Name == "bj-other" { + return unavailable + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + }, + "holder Backups list fails": func(f *cnpgRolloutFixture) { + holders := labels.Set{backupsv1alpha1.OwningJobNameLabel: "bj-other", backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS} + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + lo := &client.ListOptions{} + lo.ApplyOptions(opts) + if _, ok := list.(*cnpgtypes.BackupList); ok && lo.LabelSelector != nil && lo.LabelSelector.Matches(holders) && !lo.LabelSelector.Matches(labels.Set{backupsv1alpha1.OwningJobNameLabel: "bj", backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS}) { + return unavailable + } + return cl.List(ctx, list, opts...) + }, + }) + }, + } { + t.Run(name, func(t *testing.T) { + assertClaimHolderLookupErrorReturned(t, setup) + }) + } +} + +func assertClaimHolderLookupErrorReturned(t *testing.T, setup func(f *cnpgRolloutFixture)) { + t.Helper() + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + store := f.liveObjectStore() + store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"} + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed claim: %v", err) + } + setup(f) + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("reconcileCNPG swallowed the claim holder lookup error") + } + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination written although the claim holder could not be read: %q", got) + } +} + +// A running job that holds the claim keeps the destination while it has no +// Backup yet, for instance while the plugin it attached rolls the instances. +func TestReconcileCNPG_ClaimOfARunningJobHolds(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + store := f.liveObjectStore() + store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-other"} + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed claim: %v", err) + } + j, backups, res := f.passResult() + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination = %q, rewritten under bj-other's claim", got) + } + if len(backups) != 0 { + t.Errorf("a cnpg.io/Backup was created under another job's claim: %+v", backups) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "bj-other") { + t.Errorf("Ready = %+v, want WaitingForOtherBackup naming bj-other", ready) + } + if res.RequeueAfter == 0 { + t.Errorf("a held job is not requeued: %+v", res) + } +} + +// The plugin sidecar reads the ObjectStore through a cache that nothing this +// controller writes invalidates, so a Backup created right after the +// destination changed can be taken against the previous one. The job holds +// the Backup until that cache has expired, and creates it afterwards. +func TestReconcileCNPG_BackupWaitsForThePluginCacheAfterADestinationChange(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + + j, backups, res := f.passResult() + if got := f.liveDestination(); got != "s3://bucket/" { + t.Fatalf("destination = %q, want the job's s3://bucket/", got) + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created in the pass that changed the destination: %+v", backups) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForDestinationToSettle" { + t.Errorf("Ready = %+v, want WaitingForDestinationToSettle", ready) + } + if res.RequeueAfter == 0 { + t.Errorf("a waiting job is not requeued: %+v", res) + } + stamp, err := time.Parse(time.RFC3339, f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]) + if err != nil || time.Since(stamp) > time.Minute { + t.Fatalf("%s = %v (%v), want the time of this write", cnpgDestinationChangedAtAnnotation, stamp, err) + } + + if _, backups = f.pass(); len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created while the plugin cache could still hold the old destination: %+v", backups) + } + + f.setDestinationStamp(time.Now().Add(-time.Minute).UTC().Format(time.RFC3339)) + if j, backups = f.pass(); len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups once the cache expired, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } +} + +// A write that leaves the configuration as it was cannot leave a stale +// cache behind: the Backup is created at once, and the record of the last +// change is re-sent as it is, since an apply that left it out would delete it. +func TestReconcileCNPG_UnchangedDestinationDoesNotWait(t *testing.T) { + const stamp = "2026-01-02T03:04:05Z" + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") + f.setDestinationStamp(stamp) + + if j, backups := f.pass(); len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups for an unchanged destination, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } + if got := f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]; got != stamp { + t.Errorf("%s = %q, want the recorded %q", cnpgDestinationChangedAtAnnotation, got, stamp) + } +} + +// The apiserver answers an apply with the merged object: fields it defaults, +// or that another field manager owns, are there although the job never sets +// them. Judged against what the job sends, every pass would read as a change +// and the Backup would never be created; judged against the dry run of the +// same apply, an unchanged configuration stays unchanged. +func TestReconcileCNPG_FieldsTheJobDoesNotSetAreNotAChange(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") + store := f.liveObjectStore() + store.Spec.Configuration.Wal = &cnpgtypes.WalBackupConfiguration{Compression: "gzip"} + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed ObjectStore: %v", err) + } + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + po := &client.PatchOptions{} + po.ApplyOptions(opts) + if os, ok := obj.(*cnpgtypes.ObjectStore); ok && slices.Contains(po.DryRun, metav1.DryRunAll) { + os.Spec.Configuration.Wal = &cnpgtypes.WalBackupConfiguration{Compression: "gzip"} + return nil + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + + if j, backups := f.pass(); len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups for a configuration only the server filled in, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } + if _, ok := f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]; ok { + t.Errorf("%s was set by a write that changed nothing", cnpgDestinationChangedAtAnnotation) + } +} + +// Without the live ObjectStore, or the dry run of the apply, the job cannot +// tell whether its write changes the destination, and taking the write as +// unchanged would create the Backup at once against a cache that may still +// hold the previous one. Either failure is retried like a failed write. +func TestReconcileCNPG_UnreadableObjectStoreIsRetried(t *testing.T) { + unavailable := apierrors.NewServiceUnavailable("apiserver unavailable") + for _, tc := range []struct { + name string + setup func(f *cnpgRolloutFixture) + }{ + {name: "live read fails", setup: func(f *cnpgRolloutFixture) { + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok { + return unavailable + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + }}, + {name: "claim read fails", setup: func(f *cnpgRolloutFixture) { + reads := 0 + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok { + if reads++; reads == 1 { + return unavailable + } + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + }}, + {name: "live read at the write fails", setup: func(f *cnpgRolloutFixture) { + reads := 0 + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok { + if reads++; reads > 1 { + return unavailable + } + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + }}, + {name: "dry run fails", setup: func(f *cnpgRolloutFixture) { + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + po := &client.PatchOptions{} + po.ApplyOptions(opts) + if _, ok := obj.(*cnpgtypes.ObjectStore); ok && slices.Contains(po.DryRun, metav1.DryRunAll) { + return unavailable + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + tc.setup(f) + assertObjectStoreReadRetried(t, f) + }) + } +} + +// assertObjectStoreReadRetried runs one pass and checks it is retried, either +// by returning the error to the controller or by waiting with +// ClusterPluginApplyRetrying, without a Backup, a destination write or a +// failed job. +func assertObjectStoreReadRetried(t *testing.T, f *cnpgRolloutFixture) { + t.Helper() + ctx := context.Background() + key := client.ObjectKey{Namespace: rolloutNS, Name: "bj"} + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(ctx, key, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + res, reconcileErr := f.r.reconcileCNPG(ctx, j, f.resolved) + if err := f.c.Get(ctx, key, j); err != nil { + t.Fatalf("get BackupJob after reconcile: %v", err) + } + backups := &cnpgtypes.BackupList{} + if err := f.c.List(ctx, backups, client.InNamespace(rolloutNS)); err != nil { + t.Fatalf("list cnpg.io/Backups: %v", err) + } + if len(backups.Items) != 0 { + t.Fatalf("a cnpg.io/Backup was created although the ObjectStore could not be read: %+v", backups.Items) + } + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination = %q, written without knowing whether it changed", got) + } + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed on a transient read: %s", j.Status.Message) + } + if reconcileErr != nil { + return + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "ClusterPluginApplyRetrying" { + t.Errorf("Ready = %+v, want ClusterPluginApplyRetrying", ready) + } + if res.RequeueAfter == 0 { + t.Errorf("a retried read is not requeued: %+v", res) + } +} + +// A change record that cannot be read, or that lies further ahead than +// controller clocks drift, says nothing about when the cache expires, so the +// write replaces it with its own time and the job waits once. It claims the +// destination for that wait, or another job would write its own meanwhile. +func TestReconcileCNPG_UnusableDestinationRecordIsReplaced(t *testing.T) { + for name, value := range map[string]string{ + "does not parse": "yesterday", + "far ahead": time.Now().Add(time.Hour).UTC().Format(time.RFC3339), + } { + t.Run(name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://bucket/") + f.seedOtherBackupJob("bj-old", backupsv1alpha1.BackupJobPhaseSucceeded) + store := f.liveObjectStore() + store.Annotations = map[string]string{cnpgDestinationWrittenByAnnotation: "bj-old"} + if err := f.c.Update(context.Background(), store); err != nil { + t.Fatalf("seed claim: %v", err) + } + f.setDestinationStamp(value) + + j, backups := f.pass() + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created with an unusable change record: %+v", backups) + } + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed: %s", j.Status.Message) + } + stamp, err := time.Parse(time.RFC3339, f.liveObjectStore().Annotations[cnpgDestinationChangedAtAnnotation]) + if err != nil || time.Since(stamp) > time.Minute || time.Until(stamp) > time.Minute { + t.Fatalf("%s = %v (%v), want it replaced with the time of this write", cnpgDestinationChangedAtAnnotation, stamp, err) + } + if got := f.liveObjectStore().Annotations[cnpgDestinationWrittenByAnnotation]; got != "bj" { + t.Errorf("%s = %q, want bj, which waits on the replaced record", cnpgDestinationWrittenByAnnotation, got) + } + }) + } +} + +// A later write to a Cluster that already carries the plugin re-sends the +// attach record as it is: moving it would make the instances that rolled +// with the plugin look older than the attach. +func TestReconcileCNPG_LaterWriteKeepsTheAttachRecord(t *testing.T) { + const stamp = "2026-01-02T03:04:05Z" + cluster := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + cluster.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: stamp} + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), cluster, + newRolloutInstancePod(time.Now().Add(-time.Hour))) + + var sent []map[string]string + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.Cluster); ok { + sent = append(sent, obj.GetAnnotations()) + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + if _, backups := f.pass(); len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups, want the one this pass creates", len(backups)) + } + if len(sent) != 1 { + t.Fatalf("got %d Cluster writes, want 1", len(sent)) + } + if got := sent[0][cnpgPluginAttachedAtAnnotation]; got != stamp { + t.Errorf("the write sent %s = %q, want the recorded %q", cnpgPluginAttachedAtAnnotation, got, stamp) + } +} + +// An attach record that does not parse cannot tell which instances rolled +// with the plugin, so the job fails naming it instead of guessing. One further +// ahead than controller clocks drift apart would hold every instance as older +// than the plugin and push the rollout deadline out by as far as it lies +// ahead, so it fails the same way. +func TestReconcileCNPG_UnreadableAttachRecordFailsTheJob(t *testing.T) { + for _, tc := range []struct{ name, record string }{ + {name: "does not parse", record: "yesterday"}, + {name: "in the future", record: time.Now().Add(time.Hour).UTC().Format(time.RFC3339)}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) + if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: tc.record} + f.observeCNPG(live) + + j, backups := f.pass() + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created with an unreadable attach record: %+v", backups) + } + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, cnpgPluginAttachedAtAnnotation) { + t.Errorf("phase %q, message %q; want Failed naming %s", j.Status.Phase, j.Status.Message, cnpgPluginAttachedAtAnnotation) + } + }) + } +} + +// A record a little ahead of this controller's clock is what a leadership +// change between controllers whose clocks differ leaves behind, so the job +// waits for the clocks to agree rather than failing an attach that worked. +func TestReconcileCNPG_AttachRecordSlightlyAheadWaits(t *testing.T) { + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), newRolloutInstancePod(time.Now().Add(-time.Hour))) + if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + live := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + live.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: time.Now().Add(time.Minute).UTC().Format(time.RFC3339)} + f.observeCNPG(live) + + j, backups, res := f.passResult() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed on a record a minute ahead: %s", j.Status.Message) + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created before the rollout could be judged: %+v", backups) + } + if res.RequeueAfter == 0 { + t.Errorf("a waiting job is not requeued: %+v", res) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" { + t.Errorf("Ready = %+v, want WaitingForPluginRollout", ready) + } +} + +// A job that attached nothing itself still finds the plugin another backup +// job attached a moment ago, while CNPG rolls the instances for it; a Backup +// created then fails for good. It waits for that rollout as long as the +// attach is recent enough for it to be running, and an attach record it +// cannot read is not its own to fail on. +func TestReconcileCNPG_JobWithoutLatchWaitsForAnotherJobsRollout(t *testing.T) { + cases := []struct { + name string + record string + podCreated time.Time + wantBackup bool + }{ + {name: "recent attach, instance not rolled yet", record: time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-10 * time.Minute), wantBackup: false}, + {name: "recent attach, instance rolled", record: time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-time.Minute), wantBackup: true}, + {name: "attach older than the rollout deadline", record: time.Now().Add(-2 * cnpgDefaultBackupDeadline).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-3 * cnpgDefaultBackupDeadline), wantBackup: true}, + {name: "unreadable record", record: "yesterday", podCreated: time.Now().Add(-time.Hour), wantBackup: true}, + {name: "record further ahead than clocks drift", record: time.Now().Add(time.Hour).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-time.Hour), wantBackup: true}, + {name: "record ahead within clock drift", record: time.Now().Add(time.Minute).UTC().Format(time.RFC3339), podCreated: time.Now().Add(-time.Hour), wantBackup: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cluster := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + cluster.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: tc.record} + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), cluster, + newRolloutInstancePod(tc.podCreated.Truncate(time.Second))) + + j, backups, res := f.passResult() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the job failed: %s", j.Status.Message) + } + if got := len(backups) == 1; got != tc.wantBackup { + t.Fatalf("cnpg.io/Backup created = %t, want %t; Ready = %+v", got, tc.wantBackup, apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } + if !tc.wantBackup { + if res.RequeueAfter == 0 { + t.Errorf("a waiting job is not requeued: %+v", res) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" { + t.Errorf("Ready = %+v, want WaitingForPluginRollout", ready) + } + } + }) + } +} + +// A job whose own Backup already exists is past every wait before it: it +// tracks that Backup even while another job's recent attach is still rolling +// the instances, since waiting would only hold the result of a Backup that +// runs regardless. +func TestReconcileCNPG_JobWithItsOwnBackupDoesNotWaitForAnotherJobsRollout(t *testing.T) { + cluster := newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)) + cluster.Annotations = map[string]string{cnpgPluginAttachedAtAnnotation: time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339)} + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), cluster, + newRolloutInstancePod(time.Now().Add(-10*time.Minute).Truncate(time.Second))) + f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") + + j, _ := f.pass() + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCNPGBackup" { + t.Errorf("Ready = %+v, want WaitingForCNPGBackup tracking the job's own Backup", ready) + } +} + +// A plugin attached by someone else after the latch, another backup job or +// the chart re-rendering a restored Cluster outside the platform flow, +// started the rollout before this job's write, which only finds it there: +// instances that rolled since the latch carry the plugin. +func TestReconcileCNPG_PluginAttachedByAnotherWriterKeepsTheLatch(t *testing.T) { + oldPod := newRolloutInstancePod(time.Now().Add(-time.Hour).Truncate(time.Second)) + f := newCNPGRolloutFixture(t, newRolloutBackupJob(time.Now().Add(-time.Hour)), + newRolloutCluster(cnpgClusterHealthyPhase), oldPod) + + j, _ := f.pass() // latch + latch := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut) + if latch == nil { + t.Fatalf("no %s latch: %+v", backupCondPluginRolledOut, j.Status.Conditions) + } + latchedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second) + latch.LastTransitionTime = metav1.NewTime(latchedAt) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the latch: %v", err) + } + f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster))) + f.replacePod(oldPod, newRolloutInstancePod(latchedAt.Add(5*time.Minute))) + + var backups []cnpgtypes.Backup + for range 3 { + if j, backups = f.pass(); len(backups) != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + break + } + } + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups against instances that rolled with the plugin; Ready = %+v", + len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } +} + +// Waiting for another backup resets the retry budget of the destination +// write, so the queue does not spend it. +func TestReconcileCNPG_QueueResetsDestinationRetryBudget(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + f := newAttachedObjectStoreFixture(t, expired, "s3://other/") + f.setJobCondition(metav1.Condition{ + Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", + Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired), + }) + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + + j, _ := f.pass() + if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil { + t.Fatalf("%s = %+v kept while waiting for another backup", backupCondDestinationApplied, cond) + } + + held := &cnpgtypes.Backup{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil { + t.Fatalf("get holder Backup: %v", err) + } + held.Status.Phase = cnpgBackupPhaseComplete + if err := f.c.Update(context.Background(), held); err != nil { + t.Fatalf("complete holder Backup: %v", err) + } + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok { + return apierrors.NewInternalError(errors.New("webhook down")) + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + j, _ = f.pass() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the first failure after the queue failed the job: %s", j.Status.Message) + } +} + +// seedRestoreOfApp starts a RestoreJob that writes into the fixture's app. +func (f *cnpgRolloutFixture) seedRestoreOfApp() *backupsv1alpha1.RestoreJob { + f.t.Helper() + appGroup := backupsv1alpha1.DefaultApplicationAPIGroup + strategyGroup := strategyv1alpha1.GroupVersion.Group + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "app-backup"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &appGroup, Kind: postgresAppKind, Name: rolloutApp}, + StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, + }, + } + restore := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: rolloutNS, Name: "app-restore"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + } + for _, obj := range []client.Object{backup, restore} { + if err := f.c.Create(context.Background(), obj); err != nil { + f.t.Fatalf("seed %T: %v", obj, err) + } + } + return restore +} + +// The restore check before the destination write reads the apiserver: the +// cache can still miss a RestoreJob created a moment ago, and a write landing +// in that gap can put the restored Cluster back on the source's WAL prefix. +func TestReconcileCNPG_RestoreCheckReadsPastTheCache(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.seedRestoreOfApp() + f.r.APIReader = f.c + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok { + return nil + } + return cl.List(ctx, list, opts...) + }, + }) + + j, backups := f.pass() + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination written during a restore the cache had not caught: %q", got) + } + if len(backups) != 0 { + t.Errorf("a cnpg.io/Backup was created during a restore the cache had not caught: %+v", backups) + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != ConditionReasonRestoreInProgress { + t.Errorf("Ready = %+v, want %s", ready, ConditionReasonRestoreInProgress) + } +} + +// A job past its start writes the WAL archive destination only while no +// restore rewrites its application. The restore gate in Reconcile looks only +// at a job that has not started, and a job released from the queue, or one +// still before its first write, has started long ago: a write landing during +// the restore can put the restored Cluster back on the source's WAL prefix. +func TestReconcileCNPG_HoldsTheDestinationWriteWhileTheAppIsRestored(t *testing.T) { + for _, queued := range []bool{true, false} { + t.Run(fmt.Sprintf("released from the queue=%t", queued), func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + if queued { + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + if j, _ := f.pass(); apimeta.FindStatusCondition(j.Status.Conditions, "Ready").Reason != "WaitingForOtherBackup" { + t.Fatalf("Ready = %+v, want the job queued", apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } + } + restore := f.seedRestoreOfApp() + if queued { + held := &cnpgtypes.Backup{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil { + t.Fatalf("get holder Backup: %v", err) + } + held.Status.Phase = cnpgBackupPhaseComplete + if err := f.c.Update(context.Background(), held); err != nil { + t.Fatalf("complete holder Backup: %v", err) + } + holder := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other"}, holder); err != nil { + t.Fatalf("get holder BackupJob: %v", err) + } + holder.Status.Phase = backupsv1alpha1.BackupJobPhaseSucceeded + if err := f.c.Status().Update(context.Background(), holder); err != nil { + t.Fatalf("finish holder BackupJob: %v", err) + } + } + + j, backups, res := f.passResult() + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination written during a restore of the app: %q", got) + } + for _, b := range backups { + if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { + t.Errorf("a cnpg.io/Backup was created during a restore of the app: %s", b.Name) + } + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != ConditionReasonRestoreInProgress { + t.Errorf("Ready = %+v, want %s", ready, ConditionReasonRestoreInProgress) + } + if res.RequeueAfter == 0 { + t.Errorf("a held job is not requeued: %+v", res) + } + + restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded + if err := f.c.Status().Update(context.Background(), restore); err != nil { + t.Fatalf("finish the restore: %v", err) + } + f.pass() + if got := f.liveDestination(); got != "s3://bucket/" { + t.Errorf("once the restore finished, destination = %q, want the job's own s3://bucket/", got) + } + }) + } +} + +// A job whose Backup already runs writes nothing, so a restore does not hold +// it: holding would strand a backup that is already moving data. +func TestReconcileCNPG_RestoreDoesNotHoldARunningBackup(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") + f.seedRestoreOfApp() + + j, _ := f.pass() + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForCNPGBackup" { + t.Errorf("Ready = %+v, want the running backup still followed", ready) + } +} + +// An error looking for a restore is not taken as "no restore". +func TestReconcileCNPG_RestoreLookupErrorIsReturned(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now().Add(-time.Hour), "s3://other/") + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok { + return apierrors.NewServiceUnavailable("cache not synced") + } + return cl.List(ctx, list, opts...) + }, + }) + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("reconcileCNPG swallowed the restore lookup error") + } + if got := f.liveDestination(); got != "s3://other/" { + t.Errorf("destination written although restores could not be listed: %q", got) + } +} + +// Waiting for a restore resets the retry budget of the destination write, +// as waiting for another backup does, so the restore does not spend it. +func TestReconcileCNPG_RestoreResetsDestinationRetryBudget(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + f := newAttachedObjectStoreFixture(t, expired, "s3://other/") + f.setJobCondition(metav1.Condition{ + Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", + Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired), + }) + restore := f.seedRestoreOfApp() + + j, _ := f.pass() + if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationApplied); cond != nil { + t.Fatalf("%s = %+v kept while waiting for a restore", backupCondDestinationApplied, cond) + } + + restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded + if err := f.c.Status().Update(context.Background(), restore); err != nil { + t.Fatalf("finish the restore: %v", err) + } + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*cnpgtypes.ObjectStore); ok { + return apierrors.NewInternalError(errors.New("webhook down")) + } + return cl.Patch(ctx, obj, patch, opts...) + }, + }) + if j, _ = f.pass(); j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("the first failure after the restore failed the job: %s", j.Status.Message) + } +} + +// With two jobs holding the queue the wait names the same one on every pass, +// whatever order the cache lists their Backups in, so a waiting job does not +// rewrite its status each poll. +func TestReconcileCNPG_QueueNamesOneHolderAcrossPasses(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedOtherBackupJob("holder-z", backupsv1alpha1.BackupJobPhaseRunning) + f.seedOtherBackupJob("holder-a", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("a-cnpg", "holder-z", rolloutCluster, "running") + f.seedCNPGBackup("z-cnpg", "holder-a", rolloutCluster, "running") + statusWrites := 0 + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, cl client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + statusWrites++ + return cl.SubResource(sub).Update(ctx, obj, opts...) + }, + }) + + j, _ := f.pass() + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" || !strings.Contains(ready.Message, "holder-a ") { + t.Fatalf("Ready = %+v, want WaitingForOtherBackup naming holder-a", ready) + } + statusWrites = 0 + f.pass() + if statusWrites != 0 { + t.Errorf("a second pass with the same two holders wrote the BackupJob status %d time(s)", statusWrites) + } +} + +func TestOtherCNPGBackupInFlight_IgnoresTheJobsOwnBackup(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + holder, err := f.r.otherCNPGBackupInFlight(context.Background(), j, rolloutCluster) + if err != nil { + t.Fatalf("otherCNPGBackupInFlight: %v", err) + } + if holder != "" { + t.Fatalf("the job waits on its own Backup: holder %q", holder) + } +} + +// Once the job's Backup exists the Cluster is only read; a failed read is +// returned for a retry and neither fails the running backup nor starts the +// destination retry budget. +func TestReconcileCNPG_ClusterReadErrorDoesNotFailRunningBackup(t *testing.T) { + for name, readErr := range map[string]error{ + "internal error": apierrors.NewInternalError(errors.New("blip")), + // A write answered Forbidden fails the job; a read must not. + "forbidden": apierrors.NewForbidden(schema.GroupResource{Group: cnpgtypes.GroupName, Resource: "clusters"}, rolloutCluster, errors.New("denied")), + } { + t.Run(name, func(t *testing.T) { testClusterReadErrorKeepsBackup(t, readErr) }) + } +} + +func testClusterReadErrorKeepsBackup(t *testing.T, readErr error) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + f := newAttachedObjectStoreFixture(t, expired, "s3://other/") + f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") + f.setJobCondition(metav1.Condition{ + Type: backupCondDestinationApplied, Status: metav1.ConditionFalse, Reason: "ClusterPluginApplyRetrying", + Message: "earlier failure", LastTransitionTime: metav1.NewTime(expired), + }) + dyn := f.r.Interface.(*dynamicfake.FakeDynamicClient) + dyn.PrependReactor("get", "clusters", func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, readErr + }) + + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("a failed Cluster read was not returned for a retry") + } + persisted := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, persisted); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if persisted.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a failed Cluster read failed the running backup: %s", persisted.Status.Message) + } +} + // cnpgRolloutFixture drives reconcileCNPG against one Postgres app whose // Cluster and instance pods a test swaps between passes, standing in for // CNPG rolling the instances. @@ -2557,6 +4265,15 @@ func newRolloutInstancePod(created time.Time) *corev1.Pod { // pass runs one reconcile and returns the persisted BackupJob and the // cnpg.io/Backups that exist afterwards. func (f *cnpgRolloutFixture) pass() (*backupsv1alpha1.BackupJob, []cnpgtypes.Backup) { + f.t.Helper() + j, backups, _ := f.passResult() + return j, backups +} + +// passResult is pass that also returns the reconcile result: the controller +// watches only BackupJobs, so a pass that waits on anything else is woken +// only by its own requeue. +func (f *cnpgRolloutFixture) passResult() (*backupsv1alpha1.BackupJob, []cnpgtypes.Backup, ctrl.Result) { f.t.Helper() ctx := context.Background() key := client.ObjectKey{Namespace: rolloutNS, Name: "bj"} @@ -2564,7 +4281,8 @@ func (f *cnpgRolloutFixture) pass() (*backupsv1alpha1.BackupJob, []cnpgtypes.Bac if err := f.c.Get(ctx, key, j); err != nil { f.t.Fatalf("get BackupJob: %v", err) } - if _, err := f.r.reconcileCNPG(ctx, j, f.resolved); err != nil { + res, err := f.r.reconcileCNPG(ctx, j, f.resolved) + if err != nil { f.t.Fatalf("reconcileCNPG: %v", err) } got := &backupsv1alpha1.BackupJob{} @@ -2575,7 +4293,7 @@ func (f *cnpgRolloutFixture) pass() (*backupsv1alpha1.BackupJob, []cnpgtypes.Bac if err := f.c.List(ctx, backups, client.InNamespace(rolloutNS)); err != nil { f.t.Fatalf("list cnpg.io/Backups: %v", err) } - return got, backups.Items + return got, backups.Items, res } // observeCNPG makes the next live read see cluster, as CNPG has written it. @@ -2684,6 +4402,76 @@ func TestReconcileCNPG_AttachedPluginDoesNotDelayBackup(t *testing.T) { if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut); cond != nil { t.Errorf("%s condition = %+v on a Cluster the driver did not roll", backupCondPluginRolledOut, cond) } + if cond := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld); cond != nil { + t.Errorf("%s condition = %+v on a job nothing held", backupCondDestinationWriteHeld, cond) + } +} + +// An in-place restore of the platform flow recreates the Cluster without the +// barman-cloud plugin, so a job held through it attaches the plugin again +// once released. That attach starts a rollout of its own, which the +// instances restored after the job's first rollout began have not done. +func TestReconcileCNPG_ReattachAfterRestoreRestartsTheRollout(t *testing.T) { + stampedAt := time.Now().Add(-10 * time.Minute).Truncate(time.Second) + j := newRolloutBackupJob(stampedAt.Add(-time.Hour)) + j.Status.Conditions = []metav1.Condition{{ + Type: backupCondPluginRolledOut, + Status: metav1.ConditionFalse, + Reason: "WaitingForPluginRollout", + LastTransitionTime: metav1.NewTime(stampedAt), + }} + f := newCNPGRolloutFixture(t, j, newRolloutCluster(cnpgClusterHealthyPhase), + newRolloutInstancePod(time.Now().Add(-2*time.Minute))) + restore := f.seedRestoreOfApp() + if got, _ := f.pass(); apimeta.FindStatusCondition(got.Status.Conditions, "Ready").Reason != ConditionReasonRestoreInProgress { + t.Fatalf("Ready = %+v, want the job held by the restore", apimeta.FindStatusCondition(got.Status.Conditions, "Ready")) + } + restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded + if err := f.c.Status().Update(context.Background(), restore); err != nil { + t.Fatalf("finish the restore: %v", err) + } + + attachAt := time.Now().Truncate(time.Second) + got, backups := f.pass() + if len(backups) != 0 { + t.Fatalf("cnpg.io/Backup created in the pass that attached the plugin to the restored Cluster: %+v", backups) + } + rollout := apimeta.FindStatusCondition(got.Status.Conditions, backupCondPluginRolledOut) + if rollout == nil || rollout.Status != metav1.ConditionFalse || rollout.LastTransitionTime.Time.Before(attachAt) { + t.Fatalf("%s = %+v, want False since the attach at %s", backupCondPluginRolledOut, rollout, attachAt) + } +} + +// A release older than the rollout, from a wait before the plugin was +// attached, does not shorten the rollout that began after it. +func TestReconcileCNPG_EarlierReleaseDoesNotShortenTheRollout(t *testing.T) { + expired := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)) + j := newRolloutBackupJob(expired.Add(-time.Hour)) + j.Status.Conditions = []metav1.Condition{ + { + Type: backupCondDestinationWriteHeld, + Status: metav1.ConditionFalse, + Reason: "Released", + LastTransitionTime: metav1.NewTime(expired), + }, + { + Type: backupCondPluginRolledOut, + Status: metav1.ConditionFalse, + Reason: "WaitingForPluginRollout", + LastTransitionTime: metav1.NewTime(time.Now().Add(-time.Minute)), + }, + } + f := newCNPGRolloutFixture(t, j, + newRolloutCluster("Waiting for user action", buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(expired.Add(-time.Hour))) + + got, backups := f.pass() + if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a rollout that began a minute ago failed on an older release: %s", got.Status.Message) + } + if len(backups) != 0 { + t.Fatalf("a cnpg.io/Backup was created against an unfinished rollout: %+v", backups) + } } // TestReconcileCNPG_PluginRolloutWaitIsBounded: a rollout that never ends @@ -2713,6 +4501,142 @@ func TestReconcileCNPG_PluginRolloutWaitIsBounded(t *testing.T) { } } +// A job whose plugin rollout started, and which then waited for another +// backup of the app or for a restore of it, gets the rollout deadline back +// from the moment it was released: that wait is not a rollout of its own. +// The instance pods still count as rolled from the rollout's own start. +func TestReconcileCNPG_HeldJobRolloutDeadlineCountsFromRelease(t *testing.T) { + type hold struct { + name string + seed func(f *cnpgRolloutFixture) (release func()) + waiting string + } + holds := []hold{ + { + name: "another backup", + waiting: "WaitingForOtherBackup", + seed: func(f *cnpgRolloutFixture) func() { + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + return func() { + held := &cnpgtypes.Backup{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj-other-cnpg"}, held); err != nil { + f.t.Fatalf("get holder Backup: %v", err) + } + held.Status.Phase = cnpgBackupPhaseComplete + if err := f.c.Update(context.Background(), held); err != nil { + f.t.Fatalf("complete holder Backup: %v", err) + } + } + }, + }, + { + name: "a restore", + waiting: ConditionReasonRestoreInProgress, + seed: func(f *cnpgRolloutFixture) func() { + restore := f.seedRestoreOfApp() + return func() { + restore.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded + if err := f.c.Status().Update(context.Background(), restore); err != nil { + f.t.Fatalf("finish the restore: %v", err) + } + } + }, + }, + } + stampedAt := time.Now().Add(-(cnpgDefaultBackupDeadline + time.Minute)).Truncate(time.Second) + newHeldRollout := func(t *testing.T, h hold, cluster *cnpgtypes.Cluster, pod *corev1.Pod) *cnpgRolloutFixture { + t.Helper() + j := newRolloutBackupJob(stampedAt.Add(-time.Hour)) + j.Status.Conditions = []metav1.Condition{{ + Type: backupCondPluginRolledOut, + Status: metav1.ConditionFalse, + Reason: "WaitingForPluginRollout", + LastTransitionTime: metav1.NewTime(stampedAt), + }} + f := newCNPGRolloutFixture(t, j, cluster, pod) + release := h.seed(f) + got, backups := f.pass() + for _, b := range backups { + if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { + t.Fatalf("a held job created cnpg.io/Backup %s", b.Name) + } + } + if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a held job did not wait: %s", got.Status.Message) + } + if ready := apimeta.FindStatusCondition(got.Status.Conditions, "Ready"); ready == nil || ready.Reason != h.waiting { + t.Fatalf("Ready = %+v, want %s", ready, h.waiting) + } + release() + return f + } + + for _, h := range holds { + t.Run(h.name+": the rollout is still waited for after the release", func(t *testing.T) { + f := newHeldRollout(t, h, + newRolloutCluster("Waiting for user action", buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(stampedAt.Add(-time.Hour))) + + releasedAt := time.Now().Truncate(time.Second) + j, backups := f.pass() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("time held by %s was charged to the rollout: %s", h.name, j.Status.Message) + } + for _, b := range backups { + if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { + t.Fatalf("cnpg.io/Backup %s was created against an unfinished rollout", b.Name) + } + } + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForPluginRollout" { + t.Fatalf("Ready = %+v, want WaitingForPluginRollout", ready) + } + released := apimeta.FindStatusCondition(j.Status.Conditions, backupCondDestinationWriteHeld) + if released == nil || released.Status != metav1.ConditionFalse || released.LastTransitionTime.Time.Before(releasedAt) { + t.Fatalf("%s = %+v, want False since the release at %s", backupCondDestinationWriteHeld, released, releasedAt) + } + if rollout := apimeta.FindStatusCondition(j.Status.Conditions, backupCondPluginRolledOut); rollout == nil || !rollout.LastTransitionTime.Time.Equal(stampedAt) { + t.Errorf("%s = %+v, want its stamp kept at %s", backupCondPluginRolledOut, rollout, stampedAt) + } + + released.LastTransitionTime = metav1.NewTime(stampedAt) + if err := f.c.Status().Update(context.Background(), j); err != nil { + t.Fatalf("age the release: %v", err) + } + j, _ = f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, "rollout did not finish") { + t.Fatalf("phase = %q (%s), want Failed once the rollout outlived the deadline after the release", j.Status.Phase, j.Status.Message) + } + }) + + t.Run(h.name+": an instance that rolled while the job was held counts", func(t *testing.T) { + f := newHeldRollout(t, h, + newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, rolloutCluster)), + newRolloutInstancePod(stampedAt.Add(time.Minute))) + + var j *backupsv1alpha1.BackupJob + own := 0 + for range 3 { + var backups []cnpgtypes.Backup + j, backups = f.pass() + own = 0 + for _, b := range backups { + if b.Labels[backupsv1alpha1.OwningJobNameLabel] == "bj" { + own++ + } + } + if own != 0 || j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + break + } + } + if own != 1 { + t.Fatalf("got %d cnpg.io/Backups of the job against an instance that rolled after the rollout began; Ready = %+v, phase %q (%s)", + own, apimeta.FindStatusCondition(j.Status.Conditions, "Ready"), j.Status.Phase, j.Status.Message) + } + }) + } +} + // TestReconcileCNPGRestore_MissingStrategyFailsClosedAfterDeadline covers the // restore-path blocker: a RestoreJob whose referenced CNPG Strategy CR never // appears must NOT requeue forever. requeueRestoreStrategyNotReady gates @@ -4336,7 +6260,7 @@ func TestCreateCNPGBackupArtifact_RecordsMajorVersion(t *testing.T) { rendered := &strategyv1alpha1.CNPGTemplate{ BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://b/"}, } - got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "postgres-pg", "postgresql", rendered, newPostgresApp("pg", "tenant")) + got, err := r.createCNPGBackupArtifact(context.Background(), j, resolved, cnpgBk, "postgres-pg", "postgres-pg", "postgresql", cnpgDestinationOf(rendered), rendered, newPostgresApp("pg", "tenant")) if err != nil { t.Fatalf("createCNPGBackupArtifact: %v", err) } @@ -4517,3 +6441,339 @@ func TestReconcileCNPGRestore_RefusesCrossMajorRestore(t *testing.T) { }) } } + +func (f *cnpgRolloutFixture) setStrategyDestination(destination, endpoint, secret string) { + f.t.Helper() + strategy := &strategyv1alpha1.CNPG{} + if err := f.c.Get(context.Background(), client.ObjectKey{Name: "cnpg-strategy"}, strategy); err != nil { + f.t.Fatalf("get strategy: %v", err) + } + strategy.Spec.Template.BarmanObjectStore.DestinationPath = destination + strategy.Spec.Template.BarmanObjectStore.EndpointURL = endpoint + strategy.Spec.Template.BarmanObjectStore.S3Credentials = &strategyv1alpha1.S3CredentialsTemplate{SecretRef: corev1.LocalObjectReference{Name: secret}} + if err := f.c.Update(context.Background(), strategy); err != nil { + f.t.Fatalf("update strategy: %v", err) + } +} + +// completeOwnBackup applies edit, when given, to the job's cnpg.io/Backup and +// marks it completed. +func (f *cnpgRolloutFixture) completeOwnBackup(edit func(b *cnpgtypes.Backup)) { + f.t.Helper() + backups := &cnpgtypes.BackupList{} + if err := f.c.List(context.Background(), backups, client.InNamespace(rolloutNS)); err != nil { + f.t.Fatalf("list cnpg.io/Backups: %v", err) + } + for i := range backups.Items { + b := &backups.Items[i] + if b.Labels[backupsv1alpha1.OwningJobNameLabel] != "bj" { + continue + } + if edit != nil { + edit(b) + } + b.Status.Phase = cnpgBackupPhaseComplete + if err := f.c.Update(context.Background(), b); err != nil { + f.t.Fatalf("complete cnpg.io/Backup: %v", err) + } + return + } + f.t.Fatalf("the job has no cnpg.io/Backup") +} + +func (f *cnpgRolloutFixture) artifact() *backupsv1alpha1.Backup { + f.t.Helper() + artifact := &backupsv1alpha1.Backup{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, artifact); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + f.t.Fatalf("get Backup artifact: %v", err) + } + return artifact +} + +// The destination is written by the pass that creates the cnpg.io/Backup and +// not again while it runs, so the artifact has to name what that pass wrote: +// a strategy edited while the backup runs does not reach the backup, and must +// not reach its artifact either. The BackupJob, which a tenant can edit, +// carries no part of it. +func TestReconcileCNPG_ArtifactNamesTheDestinationItsBackupWasCreatedWith(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.setStrategyDestination("s3://bucket/", "https://s3.one", "creds-one") + + f.pass() + f.settleDestination() + j, backups := f.pass() + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups once the destination settled, want 1; Ready = %+v", len(backups), apimeta.FindStatusCondition(j.Status.Conditions, "Ready")) + } + if got := f.liveDestination(); got != "s3://bucket/" { + t.Fatalf("destination = %q, want s3://bucket/", got) + } + want := `{"destinationPath":"s3://bucket/","endpointURL":"https://s3.one","s3SecretRef":"creds-one","serverName":"` + rolloutCluster + `"}` + if got := backups[0].Annotations[cnpgWrittenDestinationAnnotation]; got != want { + t.Fatalf("%s on the created cnpg.io/Backup = %q, want %q", cnpgWrittenDestinationAnnotation, got, want) + } + + f.setStrategyDestination("s3://edited/", "https://s3.two", "creds-two") + j.Annotations = map[string]string{cnpgWrittenDestinationAnnotation: `{"destinationPath":"s3://tenant/","endpointURL":"https://tenant","s3SecretRef":"creds-one"}`} + if err := f.c.Update(context.Background(), j); err != nil { + t.Fatalf("annotate BackupJob: %v", err) + } + f.completeOwnBackup(nil) + j, _ = f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) + } + artifact := f.artifact() + if artifact == nil { + t.Fatalf("no Backup artifact") + } + md := artifact.Spec.DriverMetadata + if md[cnpgDestinationPathKey] != "s3://bucket/" || md[cnpgEndpointURLKey] != "https://s3.one" || md[cnpgS3SecretRefKey] != "creds-one" { + t.Fatalf("artifact driver metadata = %v, want the destination written with the Backup (s3://bucket/, https://s3.one, creds-one)", md) + } +} + +// The serverName is the archive prefix under the destination, so the record +// carries it too: a restore that re-renders the Cluster between the backup +// and its artifact gives the live Cluster another one. A record written +// before it carried a serverName falls back to the live Cluster. +func TestReconcileCNPG_ArtifactNamesTheServerNameItsBackupWasCreatedWith(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") + _, backups := f.pass() + want := `{"destinationPath":"s3://bucket/","serverName":"` + rolloutCluster + `"}` + if len(backups) != 1 || backups[0].Annotations[cnpgWrittenDestinationAnnotation] != want { + t.Fatalf("cnpg.io/Backups after the first pass = %+v, want one recording %s", backups, want) + } + f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, "restored-server"))) + f.completeOwnBackup(nil) + j, _ := f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) + } + artifact := f.artifact() + if got := artifact.Spec.DriverMetadata[cnpgServerNameKey]; got != rolloutCluster { + t.Errorf("artifact serverName = %q, want %q recorded with the Backup", got, rolloutCluster) + } + if !strings.HasPrefix(artifact.Status.Artifact.URI, "cnpg://"+rolloutCluster+"/") { + t.Errorf("artifact URI = %q, want it under %s", artifact.Status.Artifact.URI, rolloutCluster) + } +} + +func TestReconcileCNPG_ArtifactOfARecordWithoutServerName(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") + f.pass() + f.observeCNPG(newRolloutCluster(cnpgClusterHealthyPhase, buildBarmanPlugin(rolloutCluster, "live-server"))) + f.completeOwnBackup(func(b *cnpgtypes.Backup) { + b.Annotations[cnpgWrittenDestinationAnnotation] = `{"destinationPath":"s3://bucket/"}` + }) + j, _ := f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) + } + if got := f.artifact().Spec.DriverMetadata[cnpgServerNameKey]; got != "live-server" { + t.Errorf("artifact serverName = %q, want the live Cluster's live-server", got) + } +} + +// A strategy without credentials records none, and the artifact names none. +func TestReconcileCNPG_ArtifactOfARecordWithoutCredentials(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") + if _, backups := f.pass(); len(backups) != 1 || backups[0].Annotations[cnpgWrittenDestinationAnnotation] != `{"destinationPath":"s3://bucket/","serverName":"`+rolloutCluster+`"}` { + t.Fatalf("cnpg.io/Backups after the first pass = %+v, want one recording only s3://bucket/", backups) + } + f.setStrategyDestination("s3://edited/", "https://s3.two", "creds-two") + f.completeOwnBackup(nil) + j, _ := f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) + } + md := f.artifact().Spec.DriverMetadata + if _, ok := md[cnpgS3SecretRefKey]; ok || md[cnpgDestinationPathKey] != "s3://bucket/" || md[cnpgEndpointURLKey] != "" { + t.Fatalf("artifact driver metadata = %v, want s3://bucket/ with no endpoint and no credentials", md) + } +} + +// A record that names no usable destination cannot say where the backup is, +// so no artifact is created rather than one pointing at the strategy as it +// reads now. +func TestReconcileCNPG_ArtifactRefusedForAnUnusableDestinationRecord(t *testing.T) { + for name, value := range map[string]string{ + "empty": "", + "unreadable": "{not json", + "not an object": `"s3://bucket/"`, + "null": "null", + "no fields": "{}", + "empty destination": `{"destinationPath":"","endpointURL":"https://s3.one"}`, + // Decoding sets the destination path and skips the mistyped + // endpoint, which would leave the artifact on the default one. + "mistyped endpoint": `{"destinationPath":"s3://bucket/","endpointURL":7}`, + } { + t.Run(name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") + if _, backups := f.pass(); len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups after the first pass, want 1", len(backups)) + } + f.completeOwnBackup(func(b *cnpgtypes.Backup) { + if b.Annotations == nil { + b.Annotations = map[string]string{} + } + b.Annotations[cnpgWrittenDestinationAnnotation] = value + }) + j, _ := f.pass() + if f.artifact() != nil { + t.Fatalf("an artifact was created from the record %q", value) + } + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(j.Status.Message, cnpgWrittenDestinationAnnotation) { + t.Fatalf("phase = %q (%s), want Failed naming %s", j.Status.Phase, j.Status.Message, cnpgWrittenDestinationAnnotation) + } + }) + } +} + +// A cnpg.io/Backup created by a controller that recorded nothing keeps +// getting its artifact from the strategy. +func TestReconcileCNPG_ArtifactOfABackupWithoutARecord(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") + f.seedCNPGBackup("bj-legacy", "bj", rolloutCluster, cnpgBackupPhaseComplete) + + j, _ := f.pass() + if j.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded { + t.Fatalf("phase = %q (%s), want Succeeded", j.Status.Phase, j.Status.Message) + } + artifact := f.artifact() + if artifact == nil || artifact.Spec.DriverMetadata[cnpgDestinationPathKey] != "s3://bucket/" { + t.Fatalf("artifact = %+v, want one built from the strategy", artifact) + } +} + +// cacheLaggingBehind makes the reconciler's cached client miss the named +// cnpg.io/Backups, as an informer does right after they are created, while +// its uncached reader sees them. +func (f *cnpgRolloutFixture) cacheLaggingBehind(hidden ...string) { + f.r.APIReader = f.c + f.r.Client = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := cl.List(ctx, list, opts...); err != nil { + return err + } + if backups, ok := list.(*cnpgtypes.BackupList); ok { + backups.Items = slices.DeleteFunc(backups.Items, func(b cnpgtypes.Backup) bool { + return slices.Contains(hidden, b.Name) + }) + } + return nil + }, + }) +} + +// Reconciles run one at a time, so a Backup created by one pass exists by +// the next; the queue has to see it even when the cache does not yet, or +// the next job rewrites the destination under it and the record on that +// Backup no longer names where its WAL goes. +func TestReconcileCNPG_QueueSeesABackupTheCacheHasNotCaught(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedOtherBackupJob("bj-other", backupsv1alpha1.BackupJobPhaseRunning) + f.seedCNPGBackup("bj-other-cnpg", "bj-other", rolloutCluster, "running") + f.cacheLaggingBehind("bj-other-cnpg") + + j, backups := f.pass() + if ready := apimeta.FindStatusCondition(j.Status.Conditions, "Ready"); ready == nil || ready.Reason != "WaitingForOtherBackup" { + t.Fatalf("Ready = %+v, want WaitingForOtherBackup", ready) + } + if got := f.liveDestination(); got != "s3://other/" { + t.Fatalf("destination = %q, rewritten under the other job's Backup", got) + } + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups, want only the other job's", len(backups)) + } +} + +// The job's own Backup, missing from the cache right after it was created, +// is neither created again nor written under. +func TestReconcileCNPG_OwnBackupTheCacheHasNotCaught(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.seedCNPGBackup("bj-cnpg", "bj", rolloutCluster, "running") + f.cacheLaggingBehind("bj-cnpg") + + _, backups := f.pass() + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups, want the job's one", len(backups)) + } + if got := f.liveDestination(); got != "s3://other/" { + t.Fatalf("destination = %q, rewritten under the job's own Backup", got) + } +} + +// failingBackupLists makes the uncached reader fail the cnpg.io/Backup lists +// for which fail, given whether the list selects only the job's own Backup +// (it would not find another job's) and how many such lists came before, +// returns true. +func (f *cnpgRolloutFixture) failingBackupLists(fail func(own bool, n int) bool) { + counts := map[bool]int{} + anotherJobs := labels.Set{ + backupsv1alpha1.OwningJobNameLabel: "bj-other", + backupsv1alpha1.OwningJobNamespaceLabel: rolloutNS, + cnpgClusterLabel: rolloutCluster, + } + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*cnpgtypes.BackupList); ok { + o := &client.ListOptions{} + o.ApplyOptions(opts) + own := o.LabelSelector != nil && !o.LabelSelector.Matches(anotherJobs) + counts[own]++ + if fail(own, counts[own]) { + return apierrors.NewServiceUnavailable("apiserver unavailable") + } + } + return cl.List(ctx, list, opts...) + }, + }) +} + +// A failed apiserver read of the Backups is retried, not taken as the job +// failing: nothing has been written yet that the retry would not redo. +func TestReconcileCNPG_BackupListErrorIsReturned(t *testing.T) { + for name, own := range map[string]bool{"own Backup": true, "queue": false} { + t.Run(name, func(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://other/") + f.failingBackupLists(func(o bool, _ int) bool { return o == own }) + j := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, j); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if _, err := f.r.reconcileCNPG(context.Background(), j, f.resolved); err == nil { + t.Fatalf("reconcileCNPG swallowed the Backup list error") + } + got := &backupsv1alpha1.BackupJob{} + if err := f.c.Get(context.Background(), client.ObjectKey{Namespace: rolloutNS, Name: "bj"}, got); err != nil { + t.Fatalf("get BackupJob: %v", err) + } + if got.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("BackupJob failed on a Backup list error: %s", got.Status.Message) + } + if dest := f.liveDestination(); dest != "s3://other/" { + t.Fatalf("destination = %q, written although the Backups could not be read", dest) + } + }) + } +} + +// The pass that writes the destination looks its own Backup up once, at the +// start: a second lookup before the create could only fail, after the write, +// and fail the job with it. +func TestReconcileCNPG_LooksUpItsOwnBackupOncePerPass(t *testing.T) { + f := newAttachedObjectStoreFixture(t, time.Now(), "s3://bucket/") + f.failingBackupLists(func(own bool, n int) bool { return own && n > 1 }) + + j, backups := f.pass() + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("BackupJob failed: %s", j.Status.Message) + } + if len(backups) != 1 { + t.Fatalf("got %d cnpg.io/Backups, want 1", len(backups)) + } +} diff --git a/internal/backupcontroller/default_objects_gate.go b/internal/backupcontroller/default_objects_gate.go index 9aea96f31..94075651e 100644 --- a/internal/backupcontroller/default_objects_gate.go +++ b/internal/backupcontroller/default_objects_gate.go @@ -593,12 +593,13 @@ func (g *DefaultObjectsGate) inspectObjects(ctx context.Context, backupClass *ba group = *strategy.StrategyRef.APIGroup } // When the Velero BSL is disabled (velero.bslEnabled=false, surfaced - // here as an empty VeleroNamespace) the chart gates the Velero - // Strategy CRs off the SAME flag, so they are never rendered. The - // BackupClass still routes to them unconditionally, so counting them - // as missing would force a Helm upgrade every MinForceInterval - // forever against a render that can never produce them. - if g.VeleroNamespace == "" && group == strategyAPIGroup && kind == "Velero" { + // here as an empty VeleroNamespace) the chart gates the Strategy CRs + // that back up through Velero (Velero, Kubernetes) off the SAME flag, + // so they are never rendered. The BackupClass still routes to them + // unconditionally, so counting them as missing would force a Helm + // upgrade every MinForceInterval forever against a render that can + // never produce them. + if g.VeleroNamespace == "" && group == strategyAPIGroup && (kind == "Velero" || kind == "Kubernetes") { continue } key := fmt.Sprintf("%s/%s", kind, name) diff --git a/internal/backupcontroller/default_objects_gate_test.go b/internal/backupcontroller/default_objects_gate_test.go index e287c9872..a07beea7f 100644 --- a/internal/backupcontroller/default_objects_gate_test.go +++ b/internal/backupcontroller/default_objects_gate_test.go @@ -37,6 +37,7 @@ var gateGVKs = []schema.GroupVersionKind{ {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Etcd"}, {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Altinity"}, {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Velero"}, + {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Kubernetes"}, {Group: "velero.io", Version: "v1", Kind: "BackupStorageLocation"}, {Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}, } @@ -109,6 +110,7 @@ func cozyDefaultBackupClass() *backupsv1alpha1.BackupClass { ref("Altinity", "cozy-default-altinity"), ref("Velero", "cozy-default-velero-vminstance"), ref("Velero", "cozy-default-velero-vmdisk"), + ref("Kubernetes", "cozy-default-kubernetes"), }, }, } @@ -149,6 +151,7 @@ func allDefaultObjects() []runtime.Object { strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), + strategyObject("Kubernetes", "cozy-default-kubernetes"), bslObject(), } } @@ -288,8 +291,8 @@ func TestCheckForcesWhenObjectsMissing(t *testing.T) { t.Fatal("expected a forced Helm upgrade when default objects are missing") } // 6 strategyRefs (Velero twice under different names) + the BSL. - if len(missing) != 7 { - t.Fatalf("missing = %v, want 7 entries", missing) + if len(missing) != 8 { + t.Fatalf("missing = %v, want 8 entries", missing) } ann := forceAnnotations(t, dyn) forceAt, ok := ann["reconcile.fluxcd.io/forceAt"] @@ -319,6 +322,7 @@ func TestCheckNoopWhenAllPresent(t *testing.T) { strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), + strategyObject("Kubernetes", "cozy-default-kubernetes"), bslObject(), ) @@ -454,8 +458,8 @@ func TestCheckThrottlesTheTwoReleasesIndependently(t *testing.T) { if !forced { t.Fatal("objects release throttled by the earlier credentials force") } - if len(missing) != 7 { - t.Fatalf("missing = %v, want the 6 strategies + the BSL", missing) + if len(missing) != 8 { + t.Fatalf("missing = %v, want the 7 strategies + the BSL", missing) } if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" { t.Errorf("own HelmRelease not forced: %v", ann) @@ -482,7 +486,7 @@ func TestForceSkipsSuspendedHelmRelease(t *testing.T) { if forced { t.Error("forced reported true for a suspended HelmRelease that ignores the annotations") } - if len(missing) != 7 { + if len(missing) != 8 { t.Fatalf("missing = %v, want the objects still reported", missing) } if ann := forceAnnotations(t, dyn); len(ann) != 0 { @@ -599,6 +603,7 @@ func TestMissingObjectsIgnoresUnmappedKinds(t *testing.T) { strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), + strategyObject("Kubernetes", "cozy-default-kubernetes"), bslObject(), ) @@ -658,6 +663,7 @@ func TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent(t *testing.T) { strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), + strategyObject("Kubernetes", "cozy-default-kubernetes"), ) // Rebuild the RESTMapper without BackupStorageLocation: the velero.io API // is not served on this cluster. @@ -1053,8 +1059,8 @@ func TestCheckKeepsForcingForUnannotatedObjects(t *testing.T) { g.lastForce = base.Add(-time.Minute) g.now = func() time.Time { return base } - if missing, forced, err := g.Check(context.Background()); err != nil || forced || len(missing) != 7 { - t.Fatalf("throttled Check: missing = %v, forced = %v, err = %v; want 7 reported, none forced", missing, forced, err) + if missing, forced, err := g.Check(context.Background()); err != nil || forced || len(missing) != len(allDefaultObjects()) { + t.Fatalf("throttled Check: missing = %v, forced = %v, err = %v; want %d reported, none forced", missing, forced, err, len(allDefaultObjects())) } g.now = func() time.Time { return base.Add(5 * time.Minute) } @@ -1065,8 +1071,8 @@ func TestCheckKeepsForcingForUnannotatedObjects(t *testing.T) { g.now = func() time.Time { return base.Add(time.Hour) } missing, forced, err := g.Check(context.Background()) - if err != nil || !forced || len(missing) != 7 { - t.Fatalf("after a render that left them unannotated: missing = %v, forced = %v, err = %v; want 7 reported and forced again", missing, forced, err) + if err != nil || !forced || len(missing) != len(allDefaultObjects()) { + t.Fatalf("after a render that left them unannotated: missing = %v, forced = %v, err = %v; want %d reported and forced again", missing, forced, err, len(allDefaultObjects())) } if again := forceAnnotations(t, dyn)["reconcile.fluxcd.io/forceAt"]; again == first { t.Errorf("not forced again for unannotated objects: forceAt still %q", first) diff --git a/internal/backupcontroller/etcdstrategy_controller.go b/internal/backupcontroller/etcdstrategy_controller.go index 2f2f2732c..4f530ca2f 100644 --- a/internal/backupcontroller/etcdstrategy_controller.go +++ b/internal/backupcontroller/etcdstrategy_controller.go @@ -134,14 +134,14 @@ var etcdClusterGVR = schema.GroupVersionResource{ // the wrong CRD via the typed client. func validateEtcdApplicationRef(ref corev1.TypedLocalObjectReference) error { if ref.Kind != etcdAppKind { - return fmt.Errorf("Etcd strategy supports applicationRef.kind=%q, got %q", etcdAppKind, ref.Kind) + return fmt.Errorf("the Etcd strategy supports applicationRef.kind=%q, got %q", etcdAppKind, ref.Kind) } apiGroup := "" if ref.APIGroup != nil { apiGroup = *ref.APIGroup } if apiGroup != "" && apiGroup != etcdapp.GroupName { - return fmt.Errorf("Etcd strategy supports applicationRef.apiGroup=%q, got %q", etcdapp.GroupName, apiGroup) + return fmt.Errorf("the Etcd strategy supports applicationRef.apiGroup=%q, got %q", etcdapp.GroupName, apiGroup) } return nil } diff --git a/internal/backupcontroller/kafkastrategy_controller.go b/internal/backupcontroller/kafkastrategy_controller.go index 70f594d65..f7c34ce53 100644 --- a/internal/backupcontroller/kafkastrategy_controller.go +++ b/internal/backupcontroller/kafkastrategy_controller.go @@ -104,10 +104,10 @@ func kafkaClusterName(appName string) string { // apps.cozystack.io/Kafka application. func validateKafkaApplicationRef(ref corev1.TypedLocalObjectReference) error { if ref.Kind != kafkaApplicationKind { - return fmt.Errorf("Kafka strategy only supports applicationRef.kind=%q, got %q", kafkaApplicationKind, ref.Kind) + return fmt.Errorf("the Kafka strategy only supports applicationRef.kind=%q, got %q", kafkaApplicationKind, ref.Kind) } if ref.APIGroup != nil && *ref.APIGroup != "" && *ref.APIGroup != backupsv1alpha1.DefaultApplicationAPIGroup { - return fmt.Errorf("Kafka strategy only supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup) + return fmt.Errorf("the Kafka strategy only supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup) } return nil } @@ -528,7 +528,7 @@ func (r *BackupJobReconciler) createKafkaBackupArtifact( // RabbitMQ driver's guard). if existing.Spec.ApplicationRef.Kind != backup.Spec.ApplicationRef.Kind || existing.Spec.ApplicationRef.Name != backup.Spec.ApplicationRef.Name { - return nil, fmt.Errorf("Backup %s/%s already exists for a different application (%s/%s, not %s/%s)", + return nil, fmt.Errorf("existing Backup %s/%s belongs to a different application (%s/%s, not %s/%s)", backup.Namespace, backup.Name, existing.Spec.ApplicationRef.Kind, existing.Spec.ApplicationRef.Name, backup.Spec.ApplicationRef.Kind, backup.Spec.ApplicationRef.Name) diff --git a/internal/backupcontroller/kubernetesstrategy_controller.go b/internal/backupcontroller/kubernetesstrategy_controller.go new file mode 100644 index 000000000..1fd47f195 --- /dev/null +++ b/internal/backupcontroller/kubernetesstrategy_controller.go @@ -0,0 +1,862 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "maps" + "regexp" + "slices" + "sort" + "strconv" + "strings" + "time" + + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/template" +) + +const ( + kubernetesAppKind = "Kubernetes" + kubernetesReleasePrefix = "kubernetes-" + + kubernetesLabelMode = "kubernetes.strategy.backups.cozystack.io/mode" + kubernetesModeBackup = "backup" + kubernetesModeRestore = "restore" + kubernetesModeCleanup = "cleanup" + // kubernetesModeVerify checks that the dump object exists before a restore + // changes anything. + kubernetesModeVerify = "verify" + + // kubernetesCleanupAttemptsAnnotation counts failed cleanup Jobs on the + // Backup, so a persistently failing delete (revoked credentials, a removed + // bucket) ends in a released Backup and a Warning rather than a Job storm. + kubernetesCleanupAttemptsAnnotation = "kubernetes.strategy.backups.cozystack.io/cleanup-attempts" + kubernetesCleanupMaxAttempts = 5 + + kubernetesMetaPrefix = "kubernetes.strategy.backups.cozystack.io/" + kubernetesObjectMetaKey = kubernetesMetaPrefix + "object" + kubernetesDatastorePrefixMetaKey = kubernetesMetaPrefix + "datastore-prefix" + kubernetesDatastoreEndpointsMetaKey = kubernetesMetaPrefix + "datastore-endpoints" + kubernetesHelmReleasesMetaKey = kubernetesMetaPrefix + "helmreleases" + kubernetesDataVolumesMetaKey = kubernetesMetaPrefix + "datavolumes" + kubernetesCertWarningsMetaKey = kubernetesMetaPrefix + "certificate-warnings" + kubernetesControlPlaneUIDMetaKey = kubernetesMetaPrefix + "tenantcontrolplane-uid" + kubernetesParamPrefix = kubernetesMetaPrefix + "parameter/" + + // kubernetesCondCertificates is set on the BackupJob once the control-plane + // certificates have been checked; Status=True means at least one expires + // inside the warning window. + kubernetesCondCertificates = "CertificatesExpiring" + + kubernetesPollInterval = 5 * time.Second + // Leaf certificates are rotated by Kamaji and cert-manager, so a near + // expiry is only worth a warning when it is this close. + kubernetesDefaultCertThreshold = 720 * time.Hour + // A CA must outlive every backup that may still be restored, so its + // window is the Velero backup TTL; this is the fallback when none is set. + kubernetesDefaultBackupTTL = 720 * time.Hour + + helmReleaseNameAnnotation = "meta.helm.sh/release-name" + clusterNameLabel = "cluster.x-k8s.io/cluster-name" + helmInstanceLabel = "app.kubernetes.io/instance" + kamajiNameLabel = "kamaji.clastix.io/name" +) + +var ( + tenantControlPlaneGVR = schema.GroupVersionResource{Group: "kamaji.clastix.io", Version: "v1alpha1", Resource: "tenantcontrolplanes"} + kamajiDataStoreGVR = schema.GroupVersionResource{Group: "kamaji.clastix.io", Version: "v1alpha1", Resource: "datastores"} + kamajiControlPlaneGVR = schema.GroupVersionResource{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Resource: "kamajicontrolplanes"} + machineDeploymentGVR = schema.GroupVersionResource{Group: "cluster.x-k8s.io", Version: "v1beta1", Resource: "machinedeployments"} + machineGVR = schema.GroupVersionResource{Group: "cluster.x-k8s.io", Version: "v1beta1", Resource: "machines"} + deploymentGVR = schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} +) + +// kubernetesCertificateSecrets lists, by suffix after the release name, the +// Secrets whose certificates a backup checks. Kamaji and cert-manager rotate +// the leaves; the CAs are irreplaceable, so a restore needs them valid. +var kubernetesCertificateSecrets = []string{ + "ca", + "front-proxy-ca-certificate", + "api-server-certificate", + "api-server-kubelet-client-certificate", + "front-proxy-client-certificate", + "talos-ca", + "talos-tls-cert", +} + +// kubernetesUnlabelledSecrets are issued by cert-manager without the Helm +// instance label the Velero selector relies on; the driver adds it before the +// backup so the Talos PKI is not silently left out. +var kubernetesUnlabelledSecrets = []string{"talos-ca", "talos-tls-cert"} + +func kubernetesReleaseName(appName string) string { + return kubernetesReleasePrefix + appName +} + +func validateKubernetesApplicationRef(ref corev1.TypedLocalObjectReference) error { + if ref.Kind != kubernetesAppKind { + return fmt.Errorf("kubernetes strategy supports applicationRef.kind %s, got %q", kubernetesAppKind, ref.Kind) + } + if ref.APIGroup != nil && *ref.APIGroup != "" && *ref.APIGroup != backupsv1alpha1.DefaultApplicationAPIGroup { + return fmt.Errorf("kubernetes strategy supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup) + } + return nil +} + +func kubernetesObjectKey(namespace, appName, backupName string) string { + return fmt.Sprintf("%s/%s/%s.etcd.json", namespace, appName, backupName) +} + +// kubernetesDatastore is where the tenant cluster keeps its state: a key +// prefix in the etcd DataStore shared by every cluster of the tenant. +type kubernetesDatastore struct { + // ControlPlaneUID identifies the cluster across a delete and re-create + // under the same name, which Prefix does not on a Kamaji that keys the + // schema on the namespace and name. + ControlPlaneUID string + Prefix string + Endpoints []string + CertificateSecret string + // SourcePrefix is the prefix the restored dump was taken under. It + // differs from Prefix when a deleted cluster is rebuilt: Kamaji keys a new + // TenantControlPlane's schema on its UID. + SourcePrefix string +} + +func (d kubernetesDatastore) templateContext() map[string]any { + source := d.SourcePrefix + if source == "" { + source = d.Prefix + } + return map[string]any{ + "Prefix": d.Prefix, + "SourcePrefix": source, + "Endpoints": strings.Join(d.Endpoints, ","), + "CertificateSecret": d.CertificateSecret, + } +} + +// etcdEndpointURL gives a scheme to a DataStore endpoint: Kamaji lists them as +// host:port and always talks TLS to an etcd DataStore. +func etcdEndpointURL(endpoint string) string { + if strings.Contains(endpoint, "://") { + return endpoint + } + return "https://" + endpoint +} + +// resolveKubernetesDatastore reads the cluster's key prefix and client +// certificate from its TenantControlPlane and the client URLs from the +// DataStore it points at. +func resolveKubernetesDatastore(ctx context.Context, dyn dynamic.Interface, namespace, appName string) (*kubernetesDatastore, error) { + release := kubernetesReleaseName(appName) + tcp, err := dyn.Resource(tenantControlPlaneGVR).Namespace(namespace).Get(ctx, release, metav1.GetOptions{}) + if err != nil { + return nil, err + } + if driver, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "driver"); driver != "" && driver != "etcd" { + return nil, fmt.Errorf("TenantControlPlane %s/%s uses the %q datastore driver; only etcd is supported", namespace, release, driver) + } + prefix, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "setup", "schema") + if prefix == "" { + prefix, _, _ = unstructured.NestedString(tcp.Object, "spec", "dataStoreSchema") + } + if prefix == "" { + return nil, fmt.Errorf("TenantControlPlane %s/%s reports no datastore schema yet", namespace, release) + } + dsName, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "dataStoreName") + if dsName == "" { + dsName, _, _ = unstructured.NestedString(tcp.Object, "spec", "dataStore") + } + if dsName == "" { + return nil, fmt.Errorf("TenantControlPlane %s/%s names no DataStore", namespace, release) + } + certSecret, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "certificate", "secretName") + if certSecret == "" { + certSecret = release + "-datastore-certificate" + } + ds, err := dyn.Resource(kamajiDataStoreGVR).Get(ctx, dsName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get DataStore %s: %w", dsName, err) + } + raw, _, _ := unstructured.NestedStringSlice(ds.Object, "spec", "endpoints") + if len(raw) == 0 { + return nil, fmt.Errorf("DataStore %s lists no endpoints", dsName) + } + endpoints := make([]string, 0, len(raw)) + for _, e := range raw { + endpoints = append(endpoints, etcdEndpointURL(e)) + } + return &kubernetesDatastore{ControlPlaneUID: string(tcp.GetUID()), Prefix: prefix, Endpoints: endpoints, CertificateSecret: certSecret}, nil +} + +// kubernetesCertificateWarnings reports every certificate in the given Secrets +// that expires inside its window: caWindow for CAs, leafWindow for the rest. +// Output is sorted so a condition message is stable across reconciles. +func kubernetesCertificateWarnings(secrets []corev1.Secret, now time.Time, caWindow, leafWindow time.Duration) []string { + seen := map[string]bool{} + var out []string + for _, s := range secrets { + for key, data := range s.Data { + if !strings.HasSuffix(key, ".crt") { + continue + } + for rest := data; ; { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + if block.Type != "CERTIFICATE" { + continue + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + continue + } + window := leafWindow + kind := "certificate" + if cert.IsCA { + window = caWindow + kind = "CA" + } + if !cert.NotAfter.Before(now.Add(window)) { + continue + } + msg := fmt.Sprintf("%s %s/%s (CN=%s) expires %s", kind, s.Name, key, cert.Subject.CommonName, cert.NotAfter.UTC().Format(time.RFC3339)) + if !seen[msg] { + seen[msg] = true + out = append(out, msg) + } + } + } + } + sort.Strings(out) + return out +} + +// kubernetesHelmReleaseNames returns the HelmRelease of the cluster followed +// by those of its worker pools. Pools live in separate kubernetes-nodes +// releases; the MachineDeployments of the cluster name them, which also +// covers the older layout where the cluster release owned its pools. +func kubernetesHelmReleaseNames(mds []unstructured.Unstructured, release string) []string { + names := []string{release} + for _, md := range mds { + if cluster, _, _ := unstructured.NestedString(md.Object, "spec", "clusterName"); cluster != release { + continue + } + if owner := md.GetAnnotations()[helmReleaseNameAnnotation]; owner != "" && !slices.Contains(names, owner) { + names = append(names, owner) + } + } + return names +} + +// sanitizeHelmRelease keeps what is needed to recreate a HelmRelease and +// drops server-set metadata and status. +func sanitizeHelmRelease(hr *unstructured.Unstructured) map[string]any { + labels := map[string]string{} + for k, v := range hr.GetLabels() { + if !strings.HasPrefix(k, "velero.io/") { + labels[k] = v + } + } + meta := map[string]any{ + "name": hr.GetName(), + "namespace": hr.GetNamespace(), + } + if len(labels) > 0 { + meta["labels"] = labels + } + spec, _, _ := unstructured.NestedMap(hr.Object, "spec") + delete(spec, "suspend") + return map[string]any{ + "apiVersion": hr.GetAPIVersion(), + "kind": hr.GetKind(), + "metadata": meta, + "spec": spec, + } +} + +// kubernetesSchemaPattern is every datastore schema Kamaji assigns: a +// TenantControlPlane UID, or _ on an older Kamaji. The Job +// turns a schema into the prefix it runs `etcdctl del --prefix` on in the +// etcd every cluster of the tenant shares, so a value that is empty, holds a +// slash or a dot, or comes from an edited Backup must never reach it. +var kubernetesSchemaPattern = regexp.MustCompile(`^[a-z0-9_-]+$`) + +func validateKubernetesDatastore(ds kubernetesDatastore) error { + if !kubernetesSchemaPattern.MatchString(ds.Prefix) { + return fmt.Errorf("refusing datastore schema %q: want %s", ds.Prefix, kubernetesSchemaPattern) + } + if ds.SourcePrefix != "" && !kubernetesSchemaPattern.MatchString(ds.SourcePrefix) { + return fmt.Errorf("refusing source datastore schema %q: want %s", ds.SourcePrefix, kubernetesSchemaPattern) + } + return nil +} + +func renderKubernetesTemplate( + tmpl corev1.PodTemplateSpec, + appName, namespace, mode, objectKey string, + ds kubernetesDatastore, + parameters map[string]string, +) (*corev1.PodTemplateSpec, error) { + if err := validateKubernetesDatastore(ds); err != nil { + return nil, err + } + return template.Template(&tmpl, map[string]any{ + "Release": map[string]string{ + "Name": appName, + "Namespace": namespace, + "Prefix": kubernetesReleasePrefix, + }, + "Mode": mode, + "ObjectKey": objectKey, + "Datastore": ds.templateContext(), + "Parameters": parameters, + }) +} + +func kubernetesStrategyParameters(b *backupsv1alpha1.Backup) map[string]string { + out := map[string]string{} + for k, v := range b.Spec.DriverMetadata { + if p := strings.TrimPrefix(k, kubernetesParamPrefix); p != k && p != "" { + out[p] = v + } + } + return out +} + +func kubernetesBackupDatastore(b *backupsv1alpha1.Backup) kubernetesDatastore { + ds := kubernetesDatastore{ + Prefix: b.Spec.DriverMetadata[kubernetesDatastorePrefixMetaKey], + CertificateSecret: kubernetesReleaseName(b.Spec.ApplicationRef.Name) + "-datastore-certificate", + } + if e := b.Spec.DriverMetadata[kubernetesDatastoreEndpointsMetaKey]; e != "" { + ds.Endpoints = strings.Split(e, ",") + } + return ds +} + +func kubernetesBackupTTL(s *strategyv1alpha1.Kubernetes) time.Duration { + if s.Spec.Velero.Spec.TTL.Duration > 0 { + return s.Spec.Velero.Spec.TTL.Duration + } + return kubernetesDefaultBackupTTL +} + +func kubernetesCertThreshold(s *strategyv1alpha1.Kubernetes) time.Duration { + if s.Spec.CertificateWarningThreshold != nil && s.Spec.CertificateWarningThreshold.Duration > 0 { + return s.Spec.CertificateWarningThreshold.Duration + } + return kubernetesDefaultCertThreshold +} + +// --------------------------------------------------------------------------- +// BackupJob path +// --------------------------------------------------------------------------- + +// reconcileKubernetes drives a BackupJob for a tenant Kubernetes application in +// three steps, each observed on later reconciles: check the control-plane +// certificates, dump the cluster's etcd prefix with a Job, then back up the +// host-side Secrets and volumes with Velero. The Backup records what a restore +// needs to put both halves back. +func (r *BackupJobReconciler) reconcileKubernetes(ctx context.Context, j *backupsv1alpha1.BackupJob, resolved *ResolvedBackupConfig) (ctrl.Result, error) { + logger := getLogger(ctx) + logger.Debug("reconciling Kubernetes strategy", "backupjob", j.Name, "phase", j.Status.Phase) + + if j.Status.Phase == backupsv1alpha1.BackupJobPhaseSucceeded || + j.Status.Phase == backupsv1alpha1.BackupJobPhaseFailed { + return ctrl.Result{}, nil + } + if err := validateKubernetesApplicationRef(j.Spec.ApplicationRef); err != nil { + return r.markBackupJobFailed(ctx, j, err.Error()) + } + + if j.Status.StartedAt == nil { + fresh := &backupsv1alpha1.BackupJob{} + if err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: j.Name}, fresh); err != nil { + return ctrl.Result{}, err + } + if fresh.Status.StartedAt == nil { + base := fresh.DeepCopy() + now := metav1.Now() + fresh.Status.StartedAt = &now + fresh.Status.Phase = backupsv1alpha1.BackupJobPhaseRunning + if err := r.Status().Patch(ctx, fresh, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + j.Status.StartedAt = fresh.Status.StartedAt + j.Status.Phase = fresh.Status.Phase + } + + strategy := &strategyv1alpha1.Kubernetes{} + if err := r.Get(ctx, client.ObjectKey{Name: resolved.StrategyRef.Name}, strategy); err != nil { + if apierrors.IsNotFound(err) { + return r.requeueStrategyNotReady(ctx, j, resolved.StrategyRef.Name) + } + return ctrl.Result{}, err + } + + app, err := r.getApplicationUnstructured(ctx, j.Namespace, j.Spec.ApplicationRef) + if err != nil { + if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application not found: %s/%s", j.Namespace, j.Spec.ApplicationRef.Name)) + } + return ctrl.Result{}, err + } + + appName := j.Spec.ApplicationRef.Name + release := kubernetesReleaseName(appName) + ds, err := resolveKubernetesDatastore(ctx, r.Interface, j.Namespace, appName) + if err != nil { + if apierrors.IsNotFound(err) { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application %s/%s has no TenantControlPlane: %v", j.Namespace, appName, err)) + } + return r.markBackupJobFailed(ctx, j, err.Error()) + } + + // A restore that failed partway leaves the control plane stopped on a + // possibly partial prefix, marked with the replica count it had. Backing + // that up would turn the damage into a recovery point, so a run that has + // not dumped anything yet refuses. + if err := r.Get(ctx, client.ObjectKey{Namespace: j.Namespace, Name: jobNameForBackupJob(j)}, &batchv1.Job{}); apierrors.IsNotFound(err) { + kcp, kerr := r.Interface.Resource(kamajiControlPlaneGVR).Namespace(j.Namespace).Get(ctx, release, metav1.GetOptions{}) + if kerr != nil && !apierrors.IsNotFound(kerr) && !apimeta.IsNoMatchError(kerr) { + return ctrl.Result{}, kerr + } + if kerr == nil { + if _, stopped := kcp.GetAnnotations()[kubernetesReplicasBeforeAnnotation]; stopped { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("Kubernetes application %s/%s is stopped by an unfinished restore; finish or repair the restore before backing it up", j.Namespace, appName)) + } + } + } else if err != nil { + return ctrl.Result{}, err + } + + if apimeta.FindStatusCondition(j.Status.Conditions, kubernetesCondCertificates) == nil { + return r.checkKubernetesCertificates(ctx, j, strategy, release) + } + + objectKey := kubernetesObjectKey(j.Namespace, appName, j.Name) + rendered, err := renderKubernetesTemplate(strategy.Spec.Template, appName, j.Namespace, kubernetesModeBackup, objectKey, *ds, resolved.Parameters) + if err != nil { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to template Kubernetes strategy: %v", err)) + } + dump, err := ensureBackupBatchJob(ctx, r.Client, r.Scheme, j, j.Namespace, jobNameForBackupJob(j), + map[string]string{ + kubernetesLabelMode: kubernetesModeBackup, + backupsv1alpha1.OwningJobNameLabel: j.Name, + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + }, rendered) + if err != nil { + return r.markBackupJobFailed(ctx, j, fmt.Sprintf("failed to ensure etcd dump Job: %v", err)) + } + switch jobConditionState(dump) { + case batchv1.JobFailed: + msg := jobFailureMessage(dump) + if msg == "" { + msg = "etcd dump Job reported Failed" + } + return r.markBackupJobFailed(ctx, j, msg) + case batchv1.JobComplete: + default: + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + veleroBackups := &velerov1.BackupList{} + if err := r.List(ctx, veleroBackups, client.InNamespace(veleroNamespace), client.MatchingLabels{ + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + backupsv1alpha1.OwningJobNameLabel: j.Name, + }); err != nil { + return ctrl.Result{}, err + } + if len(veleroBackups.Items) == 0 { + if err := r.labelKubernetesSecrets(ctx, j.Namespace, release); err != nil { + return ctrl.Result{}, err + } + if err := r.createKubernetesVeleroBackup(ctx, j, strategy, app, resolved); err != nil { + return r.failKubernetesBackupAfterDump(ctx, j, strategy, *ds, objectKey, resolved, fmt.Sprintf("failed to create Velero Backup: %v", err)) + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + if len(veleroBackups.Items) > 1 { + return r.markBackupJobFailed(ctx, j, "more than one Velero Backup references this BackupJob") + } + vb := &veleroBackups.Items[0] + switch vb.Status.Phase { + case velerov1.BackupPhaseCompleted: + case velerov1.BackupPhaseFailed, velerov1.BackupPhasePartiallyFailed, velerov1.BackupPhaseFailedValidation: + return r.failKubernetesBackupAfterDump(ctx, j, strategy, *ds, objectKey, resolved, formatVeleroBackupFailureMessageForBackupJob(ctx, r.Client, vb)) + default: + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + if j.Status.BackupRef != nil { + return ctrl.Result{}, nil + } + // A failed create is retried rather than failing the run: the dump and + // the Velero backup already exist, and no Backup would own them. + artifact, err := r.createKubernetesBackupArtifact(ctx, j, resolved, vb, *ds, objectKey) + if err != nil { + return ctrl.Result{}, fmt.Errorf("create Backup artifact: %w", err) + } + now := metav1.Now() + j.Status.BackupRef = &corev1.LocalObjectReference{Name: artifact.Name} + j.Status.CompletedAt = &now + j.Status.Phase = backupsv1alpha1.BackupJobPhaseSucceeded + apimeta.SetStatusCondition(&j.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "BackupCompleted", + Message: "etcd prefix and Velero backup completed", + }) + return ctrl.Result{}, r.Status().Update(ctx, j) +} + +// failKubernetesBackupAfterDump fails a run whose etcd dump is already in the +// bucket. No Backup will own the dump, so a cleanup-mode Job owned by the +// BackupJob removes it; the dump holds every Secret of the cluster. +func (r *BackupJobReconciler) failKubernetesBackupAfterDump(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, ds kubernetesDatastore, objectKey string, resolved *ResolvedBackupConfig, msg string) (ctrl.Result, error) { + rendered, err := renderKubernetesTemplate(strategy.Spec.Template, j.Spec.ApplicationRef.Name, j.Namespace, kubernetesModeCleanup, objectKey, ds, resolved.Parameters) + if err == nil { + _, err = ensureBackupBatchJob(ctx, r.Client, r.Scheme, j, j.Namespace, j.Name+"-discard", + map[string]string{ + kubernetesLabelMode: kubernetesModeCleanup, + backupsv1alpha1.OwningJobNameLabel: j.Name, + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + }, rendered) + } + if err != nil { + msg = fmt.Sprintf("%s; the etcd dump %s could not be scheduled for deletion: %v", msg, objectKey, err) + } + return r.markBackupJobFailed(ctx, j, msg) +} + +// checkKubernetesCertificates records on the BackupJob whether any +// control-plane certificate expires inside its window. It never fails the +// run: an expiring certificate is a reason to act, not to skip the backup. +func (r *BackupJobReconciler) checkKubernetesCertificates(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, release string) (ctrl.Result, error) { + var secrets []corev1.Secret + for _, suffix := range kubernetesCertificateSecrets { + s := corev1.Secret{} + err := r.Get(ctx, types.NamespacedName{Namespace: j.Namespace, Name: release + "-" + suffix}, &s) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return ctrl.Result{}, err + } + secrets = append(secrets, s) + } + warnings := kubernetesCertificateWarnings(secrets, time.Now(), kubernetesBackupTTL(strategy), kubernetesCertThreshold(strategy)) + cond := metav1.Condition{ + Type: kubernetesCondCertificates, + Status: metav1.ConditionFalse, + Reason: "CertificatesValid", + Message: "no control-plane certificate expires within the warning window", + } + if len(warnings) > 0 { + cond.Status = metav1.ConditionTrue + cond.Reason = "CertificatesExpiring" + cond.Message = strings.Join(warnings, "; ") + if r.Recorder != nil { + r.Recorder.Event(j, corev1.EventTypeWarning, "CertificatesExpiring", cond.Message) + } + } + apimeta.SetStatusCondition(&j.Status.Conditions, cond) + if err := r.Status().Update(ctx, j); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil +} + +func (r *BackupJobReconciler) labelKubernetesSecrets(ctx context.Context, namespace, release string) error { + for _, suffix := range kubernetesUnlabelledSecrets { + s := &corev1.Secret{} + err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: release + "-" + suffix}, s) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return err + } + if s.Labels[helmInstanceLabel] == release { + continue + } + base := s.DeepCopy() + if s.Labels == nil { + s.Labels = map[string]string{} + } + s.Labels[helmInstanceLabel] = release + if err := r.Patch(ctx, s, client.MergeFrom(base)); err != nil { + return fmt.Errorf("label Secret %s/%s: %w", namespace, s.Name, err) + } + } + return nil +} + +func (r *BackupJobReconciler) createKubernetesVeleroBackup(ctx context.Context, j *backupsv1alpha1.BackupJob, strategy *strategyv1alpha1.Kubernetes, app map[string]any, resolved *ResolvedBackupConfig) error { + spec, err := template.Template(&strategy.Spec.Velero.Spec, map[string]any{ + "Application": app, + "Parameters": resolved.Parameters, + }) + if err != nil { + return err + } + // The volumes an in-place restore deletes and replaces are listed before + // Velero starts: each of them is then in the Velero backup unless it was + // deleted first, and a volume created while the backup runs, which Velero + // may have missed, is never deleted by a restore of it. + volumes, err := r.kubernetesDataVolumeNames(ctx, j.Namespace, kubernetesReleaseName(j.Spec.ApplicationRef.Name)) + if err != nil { + return err + } + vb := &velerov1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s.%s-", j.Namespace, j.Name), + Namespace: veleroNamespace, + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: j.Name, + backupsv1alpha1.OwningJobNamespaceLabel: j.Namespace, + }, + Annotations: map[string]string{kubernetesDataVolumesMetaKey: strings.Join(volumes, ",")}, + }, + Spec: *spec, + } + if err := r.Create(ctx, vb); err != nil { + return err + } + if r.Recorder != nil { + r.Recorder.Event(j, corev1.EventTypeNormal, "VeleroBackupCreated", fmt.Sprintf("Created Velero Backup %s/%s", veleroNamespace, vb.Name)) + } + return nil +} + +// kubernetesDataVolumeNames lists the claims backing the cluster's +// PersistentVolumes: the DataVolumes and PVCs the CSI driver labels with the +// cluster name. A PVC is listed even without a DataVolume, so an in-place +// restore clears it too rather than leaving Velero to skip it. +func (r *BackupJobReconciler) kubernetesDataVolumeNames(ctx context.Context, ns, release string) ([]string, error) { + selector := clusterNameLabel + "=" + release + seen := map[string]bool{} + dvs, err := r.Interface.Resource(dataVolumeGVR).Namespace(ns).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, err + } + for _, dv := range dvs.Items { + seen[dv.GetName()] = true + } + pvcs := &corev1.PersistentVolumeClaimList{} + if err := r.List(ctx, pvcs, client.InNamespace(ns), client.MatchingLabels{clusterNameLabel: release}); err != nil { + return nil, err + } + for _, pvc := range pvcs.Items { + seen[pvc.Name] = true + } + names := make([]string, 0, len(seen)) + for name := range seen { + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + +// createKubernetesBackupArtifact records, besides the Velero backup and the +// etcd object, the cluster's HelmReleases and data DataVolumes. The +// HelmReleases are kept here rather than in Velero because selecting them by +// application label would also select the worker system disks, which the +// lineage webhook labels after the same application. +func (r *BackupJobReconciler) createKubernetesBackupArtifact( + ctx context.Context, + j *backupsv1alpha1.BackupJob, + resolved *ResolvedBackupConfig, + vb *velerov1.Backup, + ds kubernetesDatastore, + objectKey string, +) (*backupsv1alpha1.Backup, error) { + release := kubernetesReleaseName(j.Spec.ApplicationRef.Name) + mds, err := r.Interface.Resource(machineDeploymentGVR).Namespace(j.Namespace).List(ctx, metav1.ListOptions{}) + if err != nil && !apierrors.IsNotFound(err) && !apimeta.IsNoMatchError(err) { + return nil, err + } + var mdItems []unstructured.Unstructured + if mds != nil { + mdItems = mds.Items + } + var releases []map[string]any + for _, name := range kubernetesHelmReleaseNames(mdItems, release) { + hr, err := r.Interface.Resource(helmReleaseGVR).Namespace(j.Namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + continue + } + return nil, err + } + releases = append(releases, sanitizeHelmRelease(hr)) + } + releasesJSON, err := json.Marshal(releases) + if err != nil { + return nil, err + } + + driverMD := map[string]string{ + veleroBackupNameMetadataKey: vb.Name, + veleroBackupNamespaceMetadataKey: vb.Namespace, + kubernetesObjectMetaKey: objectKey, + kubernetesDatastorePrefixMetaKey: ds.Prefix, + kubernetesDatastoreEndpointsMetaKey: strings.Join(ds.Endpoints, ","), + kubernetesHelmReleasesMetaKey: string(releasesJSON), + kubernetesDataVolumesMetaKey: vb.Annotations[kubernetesDataVolumesMetaKey], + kubernetesControlPlaneUIDMetaKey: ds.ControlPlaneUID, + } + if c := apimeta.FindStatusCondition(j.Status.Conditions, kubernetesCondCertificates); c != nil && c.Status == metav1.ConditionTrue { + driverMD[kubernetesCertWarningsMetaKey] = c.Message + } + for k, v := range resolved.Parameters { + driverMD[kubernetesParamPrefix+k] = v + } + + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: j.Name, Namespace: j.Namespace}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: j.Spec.ApplicationRef, + StrategyRef: resolved.StrategyRef, + TakenAt: metav1.Now(), + DriverMetadata: driverMD, + PlanRef: j.Spec.PlanRef, + }, + Status: backupsv1alpha1.BackupStatus{ + Phase: backupsv1alpha1.BackupPhaseReady, + Artifact: &backupsv1alpha1.BackupArtifact{URI: fmt.Sprintf("velero://%s/%s", vb.Namespace, vb.Name)}, + }, + } + if err := r.Create(ctx, backup); err != nil { + if !apierrors.IsAlreadyExists(err) { + return nil, err + } + existing := &backupsv1alpha1.Backup{} + if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name}, existing); err != nil { + return nil, err + } + return existing, nil + } + return backup, nil +} + +// --------------------------------------------------------------------------- +// Backup deletion cleanup +// --------------------------------------------------------------------------- + +// cleanupKubernetesBackup removes both halves of a deleted Backup: the Velero +// backup through a DeleteBackupRequest, and the etcd dump object through a +// cleanup-mode Job, waiting for the Job before the Backup is released. The +// escape hatches match the Redis driver's. +func (r *BackupReconciler) cleanupKubernetesBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + if err := r.cleanupVeleroBackup(ctx, backup); err != nil { + return ctrl.Result{}, err + } + + jobName := backup.Name + "-cleanup" + if backup.Annotations[redisSkipArtifactCleanupAnnotation] == "true" { + existing := &batchv1.Job{} + if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: jobName}, existing); err == nil { + _ = r.deleteRedisCleanupJob(ctx, existing) + } + return ctrl.Result{}, nil + } + objectKey := backup.Spec.DriverMetadata[kubernetesObjectMetaKey] + if objectKey == "" { + return ctrl.Result{}, nil + } + + strategy := &strategyv1alpha1.Kubernetes{} + if err := r.Get(ctx, client.ObjectKey{Name: backup.Spec.StrategyRef.Name}, strategy); err != nil { + if apierrors.IsNotFound(err) { + return r.releaseRedisCleanup(ctx, backup, objectKey, "strategy CR is gone"), nil + } + return ctrl.Result{}, err + } + + job := &batchv1.Job{} + err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: jobName}, job) + switch { + case apierrors.IsNotFound(err): + terminating, nsErr := r.namespaceTerminating(ctx, backup.Namespace) + if nsErr != nil { + return ctrl.Result{}, nsErr + } + if terminating { + return r.releaseRedisCleanup(ctx, backup, objectKey, "namespace is terminating"), nil + } + if perr := ProjectBackupCredentials(ctx, r.Client, r.CredentialsConfig, backup.Namespace); perr != nil { + return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cannot project credentials: %v", perr)), nil + } + rendered, rerr := renderKubernetesTemplate(strategy.Spec.Template, backup.Spec.ApplicationRef.Name, backup.Namespace, + kubernetesModeCleanup, objectKey, kubernetesBackupDatastore(backup), kubernetesStrategyParameters(backup)) + if rerr != nil { + return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cleanup template render failed: %v", rerr)), nil + } + cleanup := buildRedisCleanupJob(backup.Namespace, jobName, rendered) + cleanup.Labels = map[string]string{kubernetesLabelMode: kubernetesModeCleanup} + delete(cleanup.Spec.Template.Labels, redisLabelMode) + maps.Copy(cleanup.Spec.Template.Labels, cleanup.Labels) + if cerr := r.Create(ctx, cleanup); cerr != nil && !apierrors.IsAlreadyExists(cerr) { + if apierrors.IsForbidden(cerr) || apierrors.IsInvalid(cerr) { + return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("cannot create cleanup Job: %v", cerr)), nil + } + return ctrl.Result{}, cerr + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + case err != nil: + return ctrl.Result{}, err + } + + if !job.DeletionTimestamp.IsZero() { + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + switch jobConditionState(job) { + case batchv1.JobComplete: + _ = r.deleteRedisCleanupJob(ctx, job) + return ctrl.Result{}, nil + case batchv1.JobFailed: + attempts, _ := strconv.Atoi(backup.Annotations[kubernetesCleanupAttemptsAnnotation]) + attempts++ + if attempts >= kubernetesCleanupMaxAttempts { + _ = r.deleteRedisCleanupJob(ctx, job) + return r.releaseRedisCleanup(ctx, backup, objectKey, fmt.Sprintf("the cleanup Job failed %d times: %s", attempts, jobFailureMessage(job))), nil + } + base := backup.DeepCopy() + if backup.Annotations == nil { + backup.Annotations = map[string]string{} + } + backup.Annotations[kubernetesCleanupAttemptsAnnotation] = strconv.Itoa(attempts) + if err := r.Patch(ctx, backup, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, err + } + _ = r.deleteRedisCleanupJob(ctx, job) + return ctrl.Result{RequeueAfter: time.Duration(attempts) * 30 * time.Second}, nil + default: + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } +} diff --git a/internal/backupcontroller/kubernetesstrategy_controller_test.go b/internal/backupcontroller/kubernetesstrategy_controller_test.go new file mode 100644 index 000000000..590f82ae9 --- /dev/null +++ b/internal/backupcontroller/kubernetesstrategy_controller_test.go @@ -0,0 +1,1483 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "math/big" + "strconv" + "strings" + "testing" + "time" + + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/scheme" + clienttesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" +) + +const ( + k8sTestNS = "tenant-a" + k8sTestApp = "c1" + k8sTestRelease = "kubernetes-c1" +) + +var kubernetesAppGVR = schema.GroupVersionResource{Group: backupsv1alpha1.DefaultApplicationAPIGroup, Version: "v1alpha1", Resource: "kubernetes"} + +func testCertPEM(t *testing.T, isCA bool, notAfter time.Time) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test"}, + NotBefore: notAfter.Add(-time.Hour), + NotAfter: notAfter, + IsCA: isCA, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +func unstr(gvk schema.GroupVersionKind, ns, name string, fields map[string]any) *unstructured.Unstructured { + u := &unstructured.Unstructured{Object: map[string]any{}} + for k, v := range fields { + u.Object[k] = v + } + u.SetGroupVersionKind(gvk) + u.SetNamespace(ns) + u.SetName(name) + return u +} + +func TestKubernetesCertificateWarnings(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + secrets := []corev1.Secret{ + {ObjectMeta: metav1.ObjectMeta{Name: "ca"}, Data: map[string][]byte{ + "ca.crt": testCertPEM(t, true, now.Add(10*24*time.Hour)), + "ca.key": []byte("not a certificate"), + }}, + {ObjectMeta: metav1.ObjectMeta{Name: "leaf-soon"}, Data: map[string][]byte{ + "tls.crt": testCertPEM(t, false, now.Add(24*time.Hour)), + }}, + {ObjectMeta: metav1.ObjectMeta{Name: "leaf-later"}, Data: map[string][]byte{ + "tls.crt": testCertPEM(t, false, now.Add(10*24*time.Hour)), + }}, + } + + got := kubernetesCertificateWarnings(secrets, now, 30*24*time.Hour, 2*24*time.Hour) + if len(got) != 2 { + t.Fatalf("got %d warnings %v, want 2", len(got), got) + } + if !strings.HasPrefix(got[0], "CA ca/ca.crt") || !strings.HasPrefix(got[1], "certificate leaf-soon/tls.crt") { + t.Fatalf("unexpected warnings %v", got) + } + + if got := kubernetesCertificateWarnings(secrets, now, time.Hour, time.Hour); len(got) != 0 { + t.Fatalf("want no warnings inside a short window, got %v", got) + } +} + +func TestKubernetesHelmReleaseNames(t *testing.T) { + mdGVK := schema.GroupVersionKind{Group: "cluster.x-k8s.io", Version: "v1beta1", Kind: "MachineDeployment"} + md := func(name, cluster, owner string) unstructured.Unstructured { + u := unstr(mdGVK, k8sTestNS, name, map[string]any{"spec": map[string]any{"clusterName": cluster}}) + u.SetAnnotations(map[string]string{helmReleaseNameAnnotation: owner}) + return *u + } + got := kubernetesHelmReleaseNames([]unstructured.Unstructured{ + md("a", k8sTestRelease, "kubernetes-nodes-c1-a"), + md("b", k8sTestRelease, "kubernetes-nodes-c1-b"), + md("legacy", k8sTestRelease, k8sTestRelease), + md("other", "kubernetes-c2", "kubernetes-nodes-c2-a"), + }, k8sTestRelease) + want := []string{k8sTestRelease, "kubernetes-nodes-c1-a", "kubernetes-nodes-c1-b"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("got %v, want %v", got, want) + } +} + +func TestParseKubernetesRestoreOptions(t *testing.T) { + o, err := parseKubernetesRestoreOptions(nil) + if err != nil || !o.restoreEtcd() || !o.restoreVolumes() { + t.Fatalf("defaults: %+v %v", o, err) + } + o, err = parseKubernetesRestoreOptions(&runtime.RawExtension{Raw: []byte(`{"volumes":false}`)}) + if err != nil || !o.restoreEtcd() || o.restoreVolumes() { + t.Fatalf("etcd only: %+v %v", o, err) + } + if _, err := parseKubernetesRestoreOptions(&runtime.RawExtension{Raw: []byte(`{"etcd":false,"volumes":false}`)}); err == nil { + t.Fatal("want an error when both halves are disabled") + } +} + +func TestKubernetesVeleroRestoreSpecPhases(t *testing.T) { + strategy := &strategyv1alpha1.Kubernetes{} + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS}, + Spec: backupsv1alpha1.BackupSpec{ApplicationRef: corev1.TypedLocalObjectReference{Kind: kubernetesAppKind, Name: k8sTestApp}}, + } + secrets, err := kubernetesVeleroRestoreSpec(strategy, backup, kubernetesPhaseSecrets, k8sTestRelease) + if err != nil { + t.Fatal(err) + } + if strings.Join(secrets.IncludedResources, ",") != "secrets" || len(secrets.OrLabelSelectors) != 2 { + t.Fatalf("secrets phase: %+v", secrets) + } + volumes, err := kubernetesVeleroRestoreSpec(strategy, backup, kubernetesPhaseVolumes, k8sTestRelease) + if err != nil { + t.Fatal(err) + } + if volumes.OrLabelSelectors[0].MatchLabels[clusterNameLabel] != k8sTestRelease || volumes.RestorePVs == nil || !*volumes.RestorePVs { + t.Fatalf("volumes phase: %+v", volumes) + } + if _, err := kubernetesVeleroRestoreSpec(strategy, backup, "bogus", k8sTestRelease); err == nil { + t.Fatal("want an error for an unknown phase") + } +} + +type kubernetesTestEnv struct { + backup *BackupJobReconciler + restore *RestoreJobReconciler + client client.Client +} + +// newKubernetesTestEnv seeds a live tenant cluster: the application, its +// HelmReleases, TenantControlPlane, DataStore, KamajiControlPlane, one worker +// pool and one data DataVolume. +const k8sTestControlPlaneUID = "6b0c4f1e-2d3a-4e5f-8a9b-0c1d2e3f4a5b" + +func newKubernetesTestEnv(t *testing.T, objs ...client.Object) *kubernetesTestEnv { + t.Helper() + s := runtime.NewScheme() + _ = scheme.AddToScheme(s) + _ = backupsv1alpha1.AddToScheme(s) + _ = strategyv1alpha1.AddToScheme(s) + _ = velerov1.AddToScheme(s) + + listKinds := map[schema.GroupVersionResource]string{ + kubernetesAppGVR: "KubernetesList", + helmReleaseGVR: "HelmReleaseList", + tenantControlPlaneGVR: "TenantControlPlaneList", + kamajiDataStoreGVR: "DataStoreList", + kamajiControlPlaneGVR: "KamajiControlPlaneList", + machineDeploymentGVR: "MachineDeploymentList", + machineGVR: "MachineList", + dataVolumeGVR: "DataVolumeList", + deploymentGVR: "DeploymentList", + } + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(s, listKinds) + seed := func(gvr schema.GroupVersionResource, u *unstructured.Unstructured) { + if err := dyn.Tracker().Create(gvr, u, u.GetNamespace()); err != nil { + t.Fatalf("seed %s: %v", u.GetName(), err) + } + } + appGVK := schema.GroupVersionKind{Group: backupsv1alpha1.DefaultApplicationAPIGroup, Version: "v1alpha1", Kind: kubernetesAppKind} + seed(kubernetesAppGVR, unstr(appGVK, k8sTestNS, k8sTestApp, map[string]any{"spec": map[string]any{}})) + hrGVK := schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"} + for _, name := range []string{k8sTestRelease, "kubernetes-nodes-c1-md0"} { + hr := unstr(hrGVK, k8sTestNS, name, map[string]any{"spec": map[string]any{"interval": "5m"}}) + hr.SetLabels(map[string]string{"apps.cozystack.io/application.name": name, "velero.io/backup-name": "old"}) + seed(helmReleaseGVR, hr) + } + tcp := unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, k8sTestRelease, map[string]any{ + "status": map[string]any{"storage": map[string]any{ + "driver": "etcd", + "dataStoreName": "tenant-a", + "setup": map[string]any{"schema": "tenant_a_kubernetes_c1"}, + "certificate": map[string]any{"secretName": k8sTestRelease + "-datastore-certificate"}, + }}, + }) + tcp.SetUID(k8sTestControlPlaneUID) + seed(tenantControlPlaneGVR, tcp) + seed(kamajiDataStoreGVR, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "DataStore"}, "", "tenant-a", map[string]any{ + "spec": map[string]any{"endpoints": []any{"etcd.tenant-a.svc:2379"}}, + })) + seed(kamajiControlPlaneGVR, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, k8sTestRelease, map[string]any{ + "spec": map[string]any{"replicas": int64(2)}, + })) + md := unstr(schema.GroupVersionKind{Group: "cluster.x-k8s.io", Version: "v1beta1", Kind: "MachineDeployment"}, k8sTestNS, k8sTestRelease+"-md0", map[string]any{ + "spec": map[string]any{"clusterName": k8sTestRelease, "replicas": int64(3)}, + }) + md.SetAnnotations(map[string]string{helmReleaseNameAnnotation: "kubernetes-nodes-c1-md0"}) + seed(machineDeploymentGVR, md) + dv := unstr(schema.GroupVersionKind{Group: "cdi.kubevirt.io", Version: "v1beta1", Kind: "DataVolume"}, k8sTestNS, "pvc-1234", nil) + dv.SetLabels(map[string]string{clusterNameLabel: k8sTestRelease}) + seed(dataVolumeGVR, dv) + seed(deploymentGVR, unstr(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, k8sTestNS, k8sTestRelease+"-cluster-autoscaler", map[string]any{ + "spec": map[string]any{"replicas": int64(1)}, + })) + + base := []client.Object{ + &velerov1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: veleroNamespace, Name: "tenant-a.bj1-xyz"}, + Status: velerov1.BackupStatus{Phase: velerov1.BackupPhaseCompleted}, + }, + &strategyv1alpha1.Kubernetes{ + ObjectMeta: metav1.ObjectMeta{Name: "cozy-default-kubernetes"}, + Spec: strategyv1alpha1.KubernetesSpec{ + Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "etcd", + Image: "img", + Args: []string{"{{ .Mode }} {{ .ObjectKey }} {{ .Datastore.Prefix }} {{ .Datastore.Endpoints }} {{ .Datastore.CertificateSecret }} {{ .Datastore.SourcePrefix }}"}, + }}}}, + Velero: strategyv1alpha1.VeleroTemplate{Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"{{ .Application.metadata.namespace }}"}, + TTL: metav1.Duration{Duration: 720 * time.Hour}, + }}, + }, + }, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: k8sTestRelease + "-talos-ca"}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: k8sTestRelease + "-ca"}, Data: map[string][]byte{ + "ca.crt": testCertPEM(t, true, time.Now().Add(24*time.Hour)), + }}, + } + c := clientfake.NewClientBuilder().WithScheme(s). + WithObjects(append(base, objs...)...). + WithStatusSubresource(&backupsv1alpha1.BackupJob{}, &backupsv1alpha1.RestoreJob{}, &backupsv1alpha1.Backup{}). + Build() + mapper := &mockRESTMapper{mapping: &meta.RESTMapping{Resource: kubernetesAppGVR, GroupVersionKind: appGVK, Scope: meta.RESTScopeNamespace}} + return &kubernetesTestEnv{ + backup: &BackupJobReconciler{Client: c, Interface: dyn, RESTMapper: mapper, Scheme: s, Recorder: record.NewFakeRecorder(20)}, + restore: &RestoreJobReconciler{Client: c, Interface: dyn, RESTMapper: mapper, Scheme: s, Recorder: record.NewFakeRecorder(20)}, + client: c, + } +} + +func k8sResolved() *ResolvedBackupConfig { + return &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{APIGroup: new("strategy.backups.cozystack.io"), Kind: strategyv1alpha1.KubernetesStrategyKind, Name: "cozy-default-kubernetes"}, + Parameters: map[string]string{"p": "v"}, + } +} + +func TestReconcileKubernetesBackup(t *testing.T) { + ctx := context.Background() + bj := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj1"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{ + APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp, + }}, + } + env := newKubernetesTestEnv(t, bj) + get := func() *backupsv1alpha1.BackupJob { + out := &backupsv1alpha1.BackupJob{} + if err := env.client.Get(ctx, client.ObjectKeyFromObject(bj), out); err != nil { + t.Fatal(err) + } + return out + } + step := func() { + t.Helper() + if _, err := env.backup.reconcileKubernetes(ctx, get(), k8sResolved()); err != nil { + t.Fatal(err) + } + } + + step() // StartedAt + step() // certificates + cond := meta.FindStatusCondition(get().Status.Conditions, kubernetesCondCertificates) + if cond == nil || cond.Status != metav1.ConditionTrue || !strings.Contains(cond.Message, k8sTestRelease+"-ca/ca.crt") { + t.Fatalf("certificate condition: %+v", cond) + } + + step() // dump Job + job := &batchv1.Job{} + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj1-backup"}, job); err != nil { + t.Fatalf("dump Job: %v", err) + } + args := job.Spec.Template.Spec.Containers[0].Args[0] + want := "backup tenant-a/c1/bj1.etcd.json tenant_a_kubernetes_c1 https://etcd.tenant-a.svc:2379 kubernetes-c1-datastore-certificate tenant_a_kubernetes_c1" + if args != want { + t.Fatalf("rendered args %q, want %q", args, want) + } + step() + vbs := &velerov1.BackupList{} + _ = env.client.List(ctx, vbs, client.MatchingLabels{backupsv1alpha1.OwningJobNameLabel: "bj1"}) + if len(vbs.Items) != 0 { + t.Fatal("Velero Backup created before the dump Job completed") + } + + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := env.client.Status().Update(ctx, job); err != nil { + t.Fatal(err) + } + // A claim the CSI driver created without a DataVolume is still one an + // in-place restore has to clear before Velero writes it back. + if err := env.client.Create(ctx, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Namespace: k8sTestNS, Name: "pvc-plain", Labels: map[string]string{clusterNameLabel: k8sTestRelease}, + }}); err != nil { + t.Fatal(err) + } + step() + if err := env.client.List(ctx, vbs, client.MatchingLabels{backupsv1alpha1.OwningJobNameLabel: "bj1"}); err != nil || len(vbs.Items) != 1 { + t.Fatalf("Velero Backups: %d %v", len(vbs.Items), err) + } + // A volume created while Velero runs may be missing from its backup, so a + // restore of this backup must never delete it. + late := unstr(schema.GroupVersionKind{Group: "cdi.kubevirt.io", Version: "v1beta1", Kind: "DataVolume"}, k8sTestNS, "pvc-late", nil) + late.SetLabels(map[string]string{clusterNameLabel: k8sTestRelease}) + if _, err := env.backup.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Create(ctx, late, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if ns := vbs.Items[0].Spec.IncludedNamespaces; len(ns) != 1 || ns[0] != k8sTestNS { + t.Fatalf("Velero spec not rendered: %v", ns) + } + talosCA := &corev1.Secret{} + _ = env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: k8sTestRelease + "-talos-ca"}, talosCA) + if talosCA.Labels[helmInstanceLabel] != k8sTestRelease { + t.Fatalf("talos CA not labelled for the Velero selector: %v", talosCA.Labels) + } + + vb := vbs.Items[0] + vb.Status.Phase = velerov1.BackupPhaseCompleted + if err := env.client.Update(ctx, &vb); err != nil { + t.Fatal(err) + } + // A pool that is suspended while the backup runs must not come back + // suspended when the cluster is rebuilt from it. + if _, err := env.backup.Interface.Resource(helmReleaseGVR).Namespace(k8sTestNS).Patch(ctx, "kubernetes-nodes-c1-md0", types.MergePatchType, []byte(`{"spec":{"suspend":true}}`), metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + step() + if got := get(); got.Status.Phase != backupsv1alpha1.BackupJobPhaseSucceeded || got.Status.BackupRef == nil { + t.Fatalf("BackupJob not succeeded: %+v", got.Status) + } + backup := &backupsv1alpha1.Backup{} + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj1"}, backup); err != nil { + t.Fatal(err) + } + md := backup.Spec.DriverMetadata + if md[veleroBackupNameMetadataKey] != vb.Name || md[kubernetesDataVolumesMetaKey] != "pvc-1234,pvc-plain" || + md[kubernetesDatastorePrefixMetaKey] != "tenant_a_kubernetes_c1" || md[kubernetesParamPrefix+"p"] != "v" || + md[kubernetesCertWarningsMetaKey] == "" || md[kubernetesControlPlaneUIDMetaKey] != k8sTestControlPlaneUID { + t.Fatalf("driver metadata: %v", md) + } + releases, err := kubernetesBackupHelmReleases(backup) + if err != nil || len(releases) != 2 { + t.Fatalf("recorded HelmReleases: %v", releases) + } + raw, _ := json.Marshal(releases) + if strings.Contains(string(raw), "velero.io/") || strings.Contains(string(raw), "suspend") { + t.Fatalf("Velero labels or the suspend flag leaked into the recorded HelmReleases: %s", raw) + } +} + +func k8sBackupForRestore() *backupsv1alpha1.Backup { + return &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "b1"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp}, + StrategyRef: k8sResolved().StrategyRef, + DriverMetadata: map[string]string{ + veleroBackupNameMetadataKey: "tenant-a.bj1-xyz", + kubernetesObjectMetaKey: "tenant-a/c1/bj1.etcd.json", + kubernetesDatastorePrefixMetaKey: "tenant_a_kubernetes_c1", + kubernetesControlPlaneUIDMetaKey: k8sTestControlPlaneUID, + kubernetesDatastoreEndpointsMetaKey: "https://etcd.tenant-a.svc:2379", + kubernetesDataVolumesMetaKey: "pvc-1234", + kubernetesHelmReleasesMetaKey: `[{"apiVersion":"helm.toolkit.fluxcd.io/v2","kind":"HelmRelease","metadata":{"name":"kubernetes-c1"},"spec":{"interval":"5m"}}]`, + }, + }, + } +} + +// k8sRestoreRun drives one RestoreJob through the test environment. +type k8sRestoreRun struct { + t *testing.T + env *kubernetesTestEnv + rj *backupsv1alpha1.RestoreJob + backup *backupsv1alpha1.Backup +} + +func newK8sRestoreRun(t *testing.T, options string, backup *backupsv1alpha1.Backup) *k8sRestoreRun { + t.Helper() + rj := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "rj"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backup.Name}}, + } + if options != "" { + rj.Spec.Options = &runtime.RawExtension{Raw: []byte(options)} + } + return &k8sRestoreRun{t: t, env: newKubernetesTestEnv(t, backup, rj), rj: rj, backup: backup} +} + +func (r *k8sRestoreRun) get() *backupsv1alpha1.RestoreJob { + r.t.Helper() + out := &backupsv1alpha1.RestoreJob{} + if err := r.env.client.Get(context.Background(), client.ObjectKeyFromObject(r.rj), out); err != nil { + r.t.Fatal(err) + } + return out +} + +func (r *k8sRestoreRun) step() { + r.t.Helper() + if _, err := r.env.restore.reconcileKubernetesRestore(context.Background(), r.get(), r.backup); err != nil { + r.t.Fatal(err) + } +} + +func (r *k8sRestoreRun) finishJob(name string, condType batchv1.JobConditionType) { + r.t.Helper() + job := &batchv1.Job{} + if err := r.env.client.Get(context.Background(), client.ObjectKey{Namespace: k8sTestNS, Name: name}, job); err != nil { + r.t.Fatalf("Job %s: %v", name, err) + } + job.Status.Conditions = []batchv1.JobCondition{{Type: condType, Status: corev1.ConditionTrue, Message: "boom"}} + if err := r.env.client.Status().Update(context.Background(), job); err != nil { + r.t.Fatal(err) + } +} + +func (r *k8sRestoreRun) finishVeleroRestore(phase velerov1.RestorePhase) { + r.t.Helper() + list := &velerov1.RestoreList{} + if err := r.env.client.List(context.Background(), list); err != nil || len(list.Items) == 0 { + r.t.Fatalf("Velero Restores: %v %v", len(list.Items), err) + } + vr := list.Items[len(list.Items)-1] + vr.Status.Phase = phase + if err := r.env.client.Update(context.Background(), &vr); err != nil { + r.t.Fatal(err) + } +} + +func (r *k8sRestoreRun) obj(gvr schema.GroupVersionResource, name string) *unstructured.Unstructured { + r.t.Helper() + u, err := r.env.restore.Interface.Resource(gvr).Namespace(k8sTestNS).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + r.t.Fatalf("get %s %s: %v", gvr.Resource, name, err) + } + return u +} + +func (r *k8sRestoreRun) replicas(gvr schema.GroupVersionResource, name string) (int64, bool) { + n, found, _ := unstructured.NestedInt64(r.obj(gvr, name).Object, "spec", "replicas") + return n, found +} + +func (r *k8sRestoreRun) suspended(name string) bool { + s, _, _ := unstructured.NestedBool(r.obj(helmReleaseGVR, name).Object, "spec", "suspend") + return s +} + +func (r *k8sRestoreRun) addMachine(name string) { + r.t.Helper() + m := unstr(schema.GroupVersionKind{Group: "cluster.x-k8s.io", Version: "v1beta1", Kind: "Machine"}, k8sTestNS, name, nil) + m.SetLabels(map[string]string{clusterNameLabel: k8sTestRelease}) + if _, err := r.env.restore.Interface.Resource(machineGVR).Namespace(k8sTestNS).Create(context.Background(), m, metav1.CreateOptions{}); err != nil { + r.t.Fatal(err) + } +} + +// verifyAndCapture runs the preflight and the mode decision. +func (r *k8sRestoreRun) verifyAndCapture(etcd bool) { + r.t.Helper() + r.step() // StartedAt + r.step() // preflight + if etcd { + r.finishJob("rj-verify", batchv1.JobComplete) + r.step() + } + if !meta.IsStatusConditionTrue(r.get().Status.Conditions, kubernetesCondVerified) { + r.t.Fatalf("backup not verified: %+v", r.get().Status) + } + r.step() // capture +} + +func TestReconcileKubernetesRestoreInPlaceDefault(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.addMachine("w-1") + run.verifyAndCapture(true) + + run.step() // stop: a worker is still there + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatal("control plane scaled down while a worker Machine still needs it to delete its Node") + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 0 { + t.Fatalf("worker pool not scaled down: %d", n) + } + if n, _ := run.replicas(deploymentGVR, k8sTestRelease+"-cluster-autoscaler"); n != 0 { + t.Fatalf("autoscaler not scaled down: %d", n) + } + if _, ok := run.obj(machineGVR, "w-1").GetAnnotations()[machineExcludeDrainAnnotation]; !ok { + t.Fatal("Machine not released from draining") + } + if c := meta.FindStatusCondition(run.get().Status.Conditions, kubernetesCondStopped); c == nil || c.Status != metav1.ConditionFalse || !strings.Contains(c.Message, "Machine") { + t.Fatalf("stop progress not reported: %+v", c) + } + if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err != nil { + t.Fatal("data DataVolume deleted while a worker may still mount it") + } + + if err := run.env.restore.Interface.Resource(machineGVR).Namespace(k8sTestNS).Delete(context.Background(), "w-1", metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + run.step() // stop settles + if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err == nil { + t.Fatal("backed-up data DataVolume not deleted once the workers were gone") + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("control plane not scaled down after the workers were gone") + } + if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) { + t.Fatal("cluster not reported stopped") + } + + run.step() // etcd Job + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() // etcd done + run.step() // volumes Velero Restore + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() // volumes done + run.step() // resume + + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("RestoreJob not succeeded: %+v", got.Status) + } + for _, c := range []struct { + gvr schema.GroupVersionResource + name string + want int64 + }{ + {kamajiControlPlaneGVR, k8sTestRelease, 2}, + {machineDeploymentGVR, k8sTestRelease + "-md0", 3}, + {deploymentGVR, k8sTestRelease + "-cluster-autoscaler", 1}, + } { + if n, _ := run.replicas(c.gvr, c.name); n != c.want { + t.Errorf("%s %s replicas = %d, want %d", c.gvr.Resource, c.name, n, c.want) + } + if _, left := run.obj(c.gvr, c.name).GetAnnotations()[kubernetesReplicasBeforeAnnotation]; left { + t.Errorf("%s %s keeps the restore's replica record", c.gvr.Resource, c.name) + } + } + if run.suspended(k8sTestRelease) || run.suspended("kubernetes-nodes-c1-md0") { + t.Fatal("HelmRelease left suspended") + } +} + +func TestReconcileKubernetesRestoreInPlaceEtcdOnly(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatalf("control plane not scaled down: %d", n) + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("workers touched by an etcd-only restore: %d", n) + } + run.step() + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("RestoreJob not succeeded: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("control plane replicas not restored: %d", n) + } + list := &velerov1.RestoreList{} + _ = run.env.client.List(context.Background(), list) + if len(list.Items) != 0 { + t.Fatal("an etcd-only restore created a Velero Restore") + } +} + +func TestReconcileKubernetesRestoreKeepsPreSuspendedRelease(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + patch := []byte(`{"spec":{"suspend":true}}`) + if _, err := run.env.restore.Interface.Resource(helmReleaseGVR).Namespace(k8sTestNS).Patch(context.Background(), "kubernetes-nodes-c1-md0", types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + run.verifyAndCapture(true) + run.step() + run.step() + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() + run.step() + if run.get().Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("RestoreJob not succeeded: %+v", run.get().Status) + } + if !run.suspended("kubernetes-nodes-c1-md0") { + t.Fatal("a HelmRelease suspended before the restore was resumed by it") + } + if run.suspended(k8sTestRelease) { + t.Fatal("the cluster HelmRelease the restore suspended was left suspended") + } +} + +func TestReconcileKubernetesRestoreEtcdFailureLeavesControlPlaneStopped(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() + run.step() + run.addJobPod("rj-restore", kubernetesJobWriting) + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !meta.IsStatusConditionTrue(got.Status.Conditions, kubernetesCondEtcdWriteFailed) { + t.Fatalf("want a failed restore with the etcd write marked failed: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("control plane brought back up on a possibly partial etcd prefix") + } + if !strings.Contains(got.Status.Message+fmtConditions(got), ".pre-restore") { + t.Fatalf("failure does not point at the pre-restore copy: %+v", got.Status) + } + + run.env.restore.cleanupKubernetesRestore(context.Background(), got) + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("deleting the RestoreJob brought the control plane up on a possibly partial prefix") + } +} + +func fmtConditions(rj *backupsv1alpha1.RestoreJob) string { + var b strings.Builder + for _, c := range rj.Status.Conditions { + b.WriteString(c.Message) + } + return b.String() +} + +func TestCleanupKubernetesRestoreResumesStoppedCluster(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop: workers gone, volumes deleted, control plane stopped + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatalf("control plane not stopped: %d", n) + } + run.env.restore.cleanupKubernetesRestore(context.Background(), run.get()) + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("deleting the RestoreJob left the control plane at %d replicas", n) + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("deleting the RestoreJob left the worker pool at %d replicas", n) + } + if run.suspended(k8sTestRelease) { + t.Fatal("deleting the RestoreJob left the HelmRelease suspended") + } +} + +func TestReconcileKubernetesRestoreStopDeadline(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.addMachine("w-1") + run.verifyAndCapture(true) + run.step() // stop starts, a worker remains + rj := run.get() + cond := meta.FindStatusCondition(rj.Status.Conditions, kubernetesCondStopped) + cond.LastTransitionTime = metav1.NewTime(time.Now().Add(-restorePreparationDeadline - time.Minute)) + if err := run.env.client.Status().Update(context.Background(), rj); err != nil { + t.Fatal(err) + } + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a stop that never settles must fail the restore: %+v", got.Status) + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("worker pool not resumed after the stop timed out: %d", n) + } + if run.suspended(k8sTestRelease) { + t.Fatal("HelmRelease left suspended after the stop timed out") + } +} + +func TestReconcileKubernetesRestoreVerifiesBeforeChangingAnything(t *testing.T) { + backup := k8sBackupForRestore() + backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] = "expired" + run := newK8sRestoreRun(t, "", backup) + run.step() + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a missing Velero backup must fail the restore: %+v", got.Status) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster touched before the backup was verified") + } + if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err != nil { + t.Fatal("data DataVolume deleted before the backup was verified") + } + + backup = k8sBackupForRestore() + backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] = "partial" + run = newK8sRestoreRun(t, "", backup) + if err := run.env.client.Create(context.Background(), &velerov1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: veleroNamespace, Name: "partial"}, + Status: velerov1.BackupStatus{Phase: velerov1.BackupPhasePartiallyFailed}, + }); err != nil { + t.Fatal(err) + } + run.step() + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a Velero backup that did not complete must fail the restore: %+v", got.Status) + } + + run = newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.step() + run.step() + run.finishJob("rj-verify", batchv1.JobFailed) + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a missing etcd dump must fail the restore: %+v", got.Status) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster touched before the dump was verified") + } +} + +func k8sBackupOfDeletedCluster() *backupsv1alpha1.Backup { + backup := k8sBackupForRestore() + backup.Spec.ApplicationRef.Name = "gone" + backup.Spec.DriverMetadata[kubernetesHelmReleasesMetaKey] = `[` + + `{"apiVersion":"helm.toolkit.fluxcd.io/v2","kind":"HelmRelease","metadata":{"name":"kubernetes-gone"},"spec":{"interval":"5m"}},` + + `{"apiVersion":"helm.toolkit.fluxcd.io/v2","kind":"HelmRelease","metadata":{"name":"kubernetes-nodes-gone-md0"},"spec":{"interval":"5m"}}]` + return backup +} + +// TestReconcileKubernetesRestoreRecreate rebuilds a deleted cluster whose new +// TenantControlPlane gets a fresh, UID-keyed schema: the dump must be written +// under that schema, and the worker pools come back only after it is. +func TestReconcileKubernetesRestoreRecreate(t *testing.T) { + ctx := context.Background() + run := newK8sRestoreRun(t, "", k8sBackupOfDeletedCluster()) + run.verifyAndCapture(true) + if state := kubernetesStateFromRestoreJob(run.get()); state == nil || state.Mode != kubernetesRestoreModeRecreate || state.Release != "kubernetes-gone" { + t.Fatalf("want recreate mode, got %+v", state) + } + run.step() // secrets + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + run.step() // volumes + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + + run.step() // cluster HelmRelease, waiting for the TenantControlPlane + dyn := run.env.restore.Interface + if _, err := dyn.Resource(helmReleaseGVR).Namespace(k8sTestNS).Get(ctx, "kubernetes-gone", metav1.GetOptions{}); err != nil { + t.Fatalf("cluster HelmRelease not recreated: %v", err) + } + if _, err := dyn.Resource(helmReleaseGVR).Namespace(k8sTestNS).Get(ctx, "kubernetes-nodes-gone-md0", metav1.GetOptions{}); err == nil { + t.Fatal("worker pool recreated before the etcd state was written back") + } + if c := meta.FindStatusCondition(run.get().Status.Conditions, kubernetesCondRecreated); c == nil || c.Status != metav1.ConditionFalse { + t.Fatalf("wait for the new control plane not reported: %+v", c) + } + run.step() // a later poll, the control plane still coming up + if ann := run.obj(helmReleaseGVR, "kubernetes-gone").GetAnnotations(); ann["reconcile.fluxcd.io/forceAt"] != "" { + t.Fatalf("the cluster HelmRelease the rebuild created is forced through a Helm upgrade on a poll: %v", ann) + } + + const newSchema = "0f2c1e4a-77b1-4c39-9a3e-6d1f2b9c8e10" + if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "status": map[string]any{"storage": map[string]any{"driver": "etcd", "dataStoreName": "tenant-a", "setup": map[string]any{"schema": newSchema}}}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := dyn.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "spec": map[string]any{"replicas": int64(2)}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + run.step() // control plane is up + run.step() // stop it for the etcd write + if n, _ := run.replicas(kamajiControlPlaneGVR, "kubernetes-gone"); n != 0 { + t.Fatalf("new control plane not stopped before the etcd write: %d", n) + } + run.step() // etcd Job + job := &batchv1.Job{} + if err := run.env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "rj-restore"}, job); err != nil { + t.Fatal(err) + } + args := job.Spec.Template.Spec.Containers[0].Args[0] + if !strings.Contains(args, " "+newSchema+" ") || !strings.HasSuffix(args, " tenant_a_kubernetes_c1") || !strings.Contains(args, "kubernetes-gone-datastore-certificate") { + t.Fatalf("restore Job does not write the dump under the new cluster's schema: %q", args) + } + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() + run.step() // resume, worker pools + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("rebuild not succeeded: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, "kubernetes-gone"); n != 2 { + t.Fatalf("new control plane not started again: %d", n) + } + if _, err := dyn.Resource(helmReleaseGVR).Namespace(k8sTestNS).Get(ctx, "kubernetes-nodes-gone-md0", metav1.GetOptions{}); err != nil { + t.Fatalf("worker pool not recreated: %v", err) + } +} + +func TestReconcileKubernetesRestoreRecreateSecretsFailure(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupOfDeletedCluster()) + run.verifyAndCapture(true) + run.step() // secrets restore + list := &velerov1.RestoreList{} + if err := run.env.client.List(context.Background(), list); err != nil || len(list.Items) != 1 { + t.Fatalf("Velero Restores: %d %v", len(list.Items), err) + } + vr := list.Items[0] + if vr.Labels[kubernetesRestorePhaseLabel] != kubernetesPhaseSecrets || vr.Spec.BackupName != "tenant-a.bj1-xyz" || vr.Spec.ResourceModifier == nil { + t.Fatalf("secrets Restore: %+v", vr) + } + cm := &corev1.ConfigMap{} + if err := run.env.client.Get(context.Background(), client.ObjectKey{Namespace: veleroNamespace, Name: vr.Spec.ResourceModifier.Name}, cm); err != nil { + t.Fatal(err) + } + if !strings.Contains(cm.Data["resource-modifier-rules.yaml"], "ownerReferences: null") { + t.Fatalf("modifiers do not strip ownerReferences: %s", cm.Data["resource-modifier-rules.yaml"]) + } + run.finishVeleroRestore(velerov1.RestorePhaseFailed) + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("a failed Velero Restore must fail the RestoreJob: %+v", got.Status) + } +} + +func TestReconcileKubernetesRestoreRecreateRejectsBadReleaseRecord(t *testing.T) { + backup := k8sBackupForRestore() + backup.Spec.ApplicationRef.Name = "gone" + backup.Spec.DriverMetadata[kubernetesHelmReleasesMetaKey] = "not json" + run := newK8sRestoreRun(t, "", backup) + run.verifyAndCapture(true) + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("an unreadable HelmRelease record must fail the rebuild: %+v", got.Status) + } +} + +func TestReconcileKubernetesRestoreRejectsRename(t *testing.T) { + backup := k8sBackupForRestore() + run := newK8sRestoreRun(t, "", backup) + rj := run.get() + rj.Spec.TargetApplicationRef = &corev1.TypedLocalObjectReference{Kind: kubernetesAppKind, Name: "other"} + if _, err := run.env.restore.reconcileKubernetesRestore(context.Background(), rj, backup); err != nil { + t.Fatal(err) + } + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("restore into another application must fail: %+v", got.Status) + } +} + +func TestResolveKubernetesDatastoreRejectsNonEtcd(t *testing.T) { + env := newKubernetesTestEnv(t) + dyn := env.backup.Interface + tcp, _ := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Get(context.Background(), k8sTestRelease, metav1.GetOptions{}) + _ = unstructured.SetNestedField(tcp.Object, "PostgreSQL", "status", "storage", "driver") + if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Update(context.Background(), tcp, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := resolveKubernetesDatastore(context.Background(), dyn, k8sTestNS, k8sTestApp); err == nil { + t.Fatal("want an error for a non-etcd datastore") + } +} + +func TestCleanupKubernetesBackupGivesUpAfterRepeatedFailures(t *testing.T) { + ctx := context.Background() + backup := k8sBackupForRestore() + backup.Annotations = map[string]string{kubernetesCleanupAttemptsAnnotation: strconv.Itoa(kubernetesCleanupMaxAttempts - 1)} + failed := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "b1-cleanup"}, + Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue}}}, + } + env := newKubernetesTestEnv(t, backup, failed) + br := &BackupReconciler{Client: env.client, Scheme: env.backup.Scheme, Recorder: record.NewFakeRecorder(10)} + res, err := br.cleanupKubernetesBackup(ctx, backup) + if err != nil { + t.Fatal(err) + } + if res.RequeueAfter != 0 { + t.Fatalf("cleanup keeps retrying after %d failures: %+v", kubernetesCleanupMaxAttempts, res) + } +} + +func (r *k8sRestoreRun) addJobPod(job, message string) { + r.t.Helper() + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: job + "-pod", Labels: map[string]string{"job-name": job}}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: "etcd", + State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 1, Message: message + "\n"}}, + }}}, + } + if err := r.env.client.Create(context.Background(), pod); err != nil { + r.t.Fatal(err) + } +} + +func TestReconcileKubernetesRestoreFailureBeforeWriteResumes(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + run.step() // etcd Job + run.addJobPod("rj-restore", "") + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || meta.IsStatusConditionTrue(got.Status.Conditions, kubernetesCondEtcdWriteFailed) { + t.Fatalf("a Job that never touched the prefix must fail without marking it partial: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("control plane left stopped although the prefix was never changed: %d", n) + } +} + +func TestCleanupKubernetesRestoreKeepsClusterStoppedWhileRestoreJobRuns(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + run.step() // etcd Job, still running + run.env.restore.cleanupKubernetesRestore(context.Background(), run.get()) + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("deleting the RestoreJob brought the control plane up while the restore Job may be rewriting the prefix") + } + job := &batchv1.Job{} + if err := run.env.client.Get(context.Background(), client.ObjectKey{Namespace: k8sTestNS, Name: "rj-restore"}, job); err == nil { + t.Fatal("the running restore Job was left to keep writing") + } +} + +func TestReconcileKubernetesRestoreIgnoresModeAnnotation(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + rj := run.get() + rj.Annotations = map[string]string{"kubernetes.strategy.backups.cozystack.io/restore-state": `{"mode":"recreate","release":"kubernetes-c1"}`} + if err := run.env.client.Update(context.Background(), rj); err != nil { + t.Fatal(err) + } + run.verifyAndCapture(true) + if state := kubernetesStateFromRestoreJob(run.get()); state == nil || state.Mode != kubernetesRestoreModeInPlace { + t.Fatalf("a live cluster must be restored in place whatever the RestoreJob claims: %+v", state) + } +} + +func TestReconcileKubernetesBackupDiscardsDumpWhenVeleroFails(t *testing.T) { + ctx := context.Background() + bj := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj2"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{ + APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp, + }}, + } + env := newKubernetesTestEnv(t, bj) + step := func() { + t.Helper() + got := &backupsv1alpha1.BackupJob{} + if err := env.client.Get(ctx, client.ObjectKeyFromObject(bj), got); err != nil { + t.Fatal(err) + } + if _, err := env.backup.reconcileKubernetes(ctx, got, k8sResolved()); err != nil { + t.Fatal(err) + } + } + step() + step() + step() + dump := &batchv1.Job{} + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj2-backup"}, dump); err != nil { + t.Fatal(err) + } + dump.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}} + if err := env.client.Status().Update(ctx, dump); err != nil { + t.Fatal(err) + } + step() + vbs := &velerov1.BackupList{} + if err := env.client.List(ctx, vbs, client.MatchingLabels{backupsv1alpha1.OwningJobNameLabel: "bj2"}); err != nil || len(vbs.Items) != 1 { + t.Fatalf("Velero Backups: %d %v", len(vbs.Items), err) + } + vb := vbs.Items[0] + vb.Status.Phase = velerov1.BackupPhaseFailed + if err := env.client.Update(ctx, &vb); err != nil { + t.Fatal(err) + } + step() + got := &backupsv1alpha1.BackupJob{} + _ = env.client.Get(ctx, client.ObjectKeyFromObject(bj), got) + if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("BackupJob not failed: %+v", got.Status) + } + discard := &batchv1.Job{} + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj2-discard"}, discard); err != nil { + t.Fatalf("the orphaned etcd dump is not scheduled for deletion: %v", err) + } + if args := discard.Spec.Template.Spec.Containers[0].Args[0]; !strings.HasPrefix(args, "cleanup tenant-a/c1/bj2.etcd.json") { + t.Fatalf("discard Job does not delete this run's dump: %q", args) + } +} + +func TestReconcileKubernetesRestoreRecreateFailureBeforeWriteStartsControlPlane(t *testing.T) { + ctx := context.Background() + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupOfDeletedCluster()) + run.verifyAndCapture(true) + run.step() // secrets + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + run.step() // cluster HelmRelease + dyn := run.env.restore.Interface + if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "status": map[string]any{"storage": map[string]any{"driver": "etcd", "dataStoreName": "tenant-a", "setup": map[string]any{"schema": "uid-1"}}}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := dyn.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "spec": map[string]any{"replicas": int64(2)}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + run.step() // control plane is up + run.step() // stopped + run.step() // etcd Job + run.addJobPod("rj-restore", "") + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("rebuild not failed: %+v", got.Status) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, "kubernetes-gone"); n != 2 { + t.Fatalf("the rebuilt control plane is left stopped although its prefix was never changed: %d", n) + } +} + +func TestReconcileKubernetesRestoreJobThatNeverRanResumes(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + run.step() // etcd Job + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "rj-restore-pod", Labels: map[string]string{"job-name": "rj-restore"}}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: "etcd", + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"}}, + }}}, + } + if err := run.env.client.Create(context.Background(), pod); err != nil { + t.Fatal(err) + } + run.step() // still waiting on the Job + if meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondEtcdJobStarted) { + t.Fatal("a container that never ran was taken for a started Job") + } + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("control plane left stopped after a restore Job that never ran: %d", n) + } +} + +func TestReconcileKubernetesRestoreJobPodsGoneAfterStart(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + run.step() // stop + run.step() // etcd Job + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "rj-restore-pod", Labels: map[string]string{"job-name": "rj-restore"}}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: "etcd", + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }}}, + } + if err := run.env.client.Create(context.Background(), pod); err != nil { + t.Fatal(err) + } + run.step() + if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondEtcdJobStarted) { + t.Fatal("a running restore Job was not recorded as started") + } + // The Job deadline deletes its Pods and fails it. + if err := run.env.client.Delete(context.Background(), pod); err != nil { + t.Fatal(err) + } + run.finishJob("rj-restore", batchv1.JobFailed) + run.step() + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 0 { + t.Fatal("control plane brought up after a restore Job that ran and left no trace of how far it got") + } +} + +func TestReconcileKubernetesRestoreRetriesStopAfterAPIError(t *testing.T) { + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + run.verifyAndCapture(true) + failed := false + run.env.restore.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("patch", "kamajicontrolplanes", func(clienttesting.Action) (bool, runtime.Object, error) { + if failed { + return false, nil, nil + } + failed = true + return true, nil, fmt.Errorf("etcdserver: request timed out") + }) + run.step() + got := run.get() + if got.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("one API error aborted the restore: %+v", got.Status) + } + if c := meta.FindStatusCondition(got.Status.Conditions, kubernetesCondStopped); c == nil || !strings.Contains(c.Message, "API error") { + t.Fatalf("the retried API error is not reported: %+v", c) + } + run.step() + if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) { + t.Fatal("the stop did not go through once the API answered again") + } +} + +// driveRecreateToEtcdJob runs a rebuild up to the etcd restore Job, the new +// TenantControlPlane reporting schema. +func driveRecreateToEtcdJob(t *testing.T, run *k8sRestoreRun, schemaName string) { + t.Helper() + ctx := context.Background() + run.verifyAndCapture(true) + run.step() // secrets + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + run.step() // volumes + run.finishVeleroRestore(velerov1.RestorePhaseCompleted) + run.step() + run.step() // cluster HelmRelease + dyn := run.env.restore.Interface + if _, err := dyn.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "kamaji.clastix.io", Version: "v1alpha1", Kind: "TenantControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "status": map[string]any{"storage": map[string]any{"driver": "etcd", "dataStoreName": "tenant-a", "setup": map[string]any{"schema": schemaName}}}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := dyn.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Create(ctx, unstr(schema.GroupVersionKind{Group: "controlplane.cluster.x-k8s.io", Version: "v1alpha1", Kind: "KamajiControlPlane"}, k8sTestNS, "kubernetes-gone", map[string]any{ + "spec": map[string]any{"replicas": int64(2)}, + }), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + run.step() // control plane is up + run.step() // stopped + run.step() // etcd Job +} + +func TestReconcileKubernetesRestoreRecreateForcesSurvivingPool(t *testing.T) { + ctx := context.Background() + run := newK8sRestoreRun(t, "", k8sBackupOfDeletedCluster()) + // Deleting only the Kubernetes application leaves its KubernetesNodes + // pool's HelmRelease behind, while the pool's MachineDeployment went + // with the CAPI Cluster. + pool := unstr(schema.GroupVersionKind{Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}, k8sTestNS, "kubernetes-nodes-gone-md0", map[string]any{"spec": map[string]any{"interval": "5m"}}) + if _, err := run.env.restore.Interface.Resource(helmReleaseGVR).Namespace(k8sTestNS).Create(ctx, pool, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + driveRecreateToEtcdJob(t, run, "uid-1") + run.finishJob("rj-restore", batchv1.JobComplete) + run.step() + run.step() + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded { + t.Fatalf("rebuild not succeeded: %+v", got.Status) + } + ann := run.obj(helmReleaseGVR, "kubernetes-nodes-gone-md0").GetAnnotations() + if ann["reconcile.fluxcd.io/forceAt"] == "" || ann["reconcile.fluxcd.io/forceAt"] != ann["reconcile.fluxcd.io/requestedAt"] { + t.Fatalf("the surviving pool is not forced to recreate its MachineDeployment: %v", ann) + } +} + +func TestReconcileKubernetesBackupRefusesClusterStoppedByRestore(t *testing.T) { + ctx := context.Background() + bj := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj3"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{ + APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp, + }}, + } + env := newKubernetesTestEnv(t, bj) + patch := []byte(`{"metadata":{"annotations":{"` + kubernetesReplicasBeforeAnnotation + `":"2"}},"spec":{"replicas":0}}`) + if _, err := env.backup.Interface.Resource(kamajiControlPlaneGVR).Namespace(k8sTestNS).Patch(ctx, k8sTestRelease, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + for range 2 { + got := &backupsv1alpha1.BackupJob{} + _ = env.client.Get(ctx, client.ObjectKeyFromObject(bj), got) + if _, err := env.backup.reconcileKubernetes(ctx, got, k8sResolved()); err != nil { + t.Fatal(err) + } + } + got := &backupsv1alpha1.BackupJob{} + _ = env.client.Get(ctx, client.ObjectKeyFromObject(bj), got) + if got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed { + t.Fatalf("a cluster stopped by an unfinished restore was backed up: %+v", got.Status) + } + if err := env.client.Get(ctx, client.ObjectKey{Namespace: k8sTestNS, Name: "bj3-backup"}, &batchv1.Job{}); err == nil { + t.Fatal("the etcd prefix of a stopped cluster was dumped") + } +} + +func TestStopKubernetesClusterWaitsForTerminatingControlPlane(t *testing.T) { + ctx := context.Background() + run := newK8sRestoreRun(t, `{"volumes":false}`, k8sBackupForRestore()) + cp := unstr(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, k8sTestNS, k8sTestRelease, map[string]any{ + "spec": map[string]any{"selector": map[string]any{"matchLabels": map[string]any{"kamaji.clastix.io/name": k8sTestRelease}}}, + "status": map[string]any{"replicas": int64(0)}, + }) + if _, err := run.env.restore.Interface.Resource(deploymentGVR).Namespace(k8sTestNS).Create(ctx, cp, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "apiserver-0", Labels: map[string]string{"kamaji.clastix.io/name": k8sTestRelease}}} + if err := run.env.client.Create(ctx, pod); err != nil { + t.Fatal(err) + } + run.verifyAndCapture(true) + run.step() + if meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) { + t.Fatal("cluster reported stopped while a control-plane Pod is still terminating") + } + if err := run.env.client.Delete(ctx, pod); err != nil { + t.Fatal(err) + } + run.step() + if !meta.IsStatusConditionTrue(run.get().Status.Conditions, kubernetesCondStopped) { + t.Fatal("cluster not reported stopped once its control plane was gone") + } +} + +func TestKubernetesJobReachedWriteConservativeBranches(t *testing.T) { + for _, tc := range []struct { + name string + status corev1.ContainerStatus + want bool + }{ + {"running", corev1.ContainerStatus{Name: "etcd", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}, true}, + {"restarted after writing", corev1.ContainerStatus{Name: "etcd", + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"}}, + LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Message: kubernetesJobWriting}}}, true}, + {"never ran", corev1.ContainerStatus{Name: "etcd", State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"}}}, false}, + {"failed before writing", corev1.ContainerStatus{Name: "etcd", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 1}}}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "p", Labels: map[string]string{"job-name": "rj-restore"}}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{tc.status}}, + } + if err := run.env.client.Create(context.Background(), pod); err != nil { + t.Fatal(err) + } + got, err := run.env.restore.kubernetesJobReachedWrite(context.Background(), run.get(), "rj-restore") + if err != nil || got != tc.want { + t.Fatalf("reached write = %v (%v), want %v", got, err, tc.want) + } + }) + } +} + +// TestReconcileKubernetesRestoreFailsWhenClusterAPIIsNotServed covers a Cluster +// API that no longer serves the version the driver lists. Reading that as a +// cluster without workers would let the volumes path delete DataVolumes that +// running workers still mount. +func TestReconcileKubernetesRestoreFailsWhenClusterAPIIsNotServed(t *testing.T) { + for _, tc := range []struct{ resource, kind string }{ + {"machinedeployments", "MachineDeployment"}, + {"machines", "Machine"}, + } { + t.Run(tc.resource, func(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.verifyAndCapture(true) + run.env.restore.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("list", tc.resource, func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "cluster.x-k8s.io", Kind: tc.kind}, SearchedVersions: []string{"v1beta1"}} + }) + run.step() + + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(got.Status.Message, "Cluster API is not served") { + t.Fatalf("restore not failed on an unserved Cluster API: %+v", got.Status) + } + if _, err := run.env.restore.Interface.Resource(dataVolumeGVR).Namespace(k8sTestNS).Get(context.Background(), "pvc-1234", metav1.GetOptions{}); err != nil { + t.Fatalf("backed-up DataVolume deleted: %v", err) + } + if n, _ := run.replicas(kamajiControlPlaneGVR, k8sTestRelease); n != 2 { + t.Fatalf("control plane left at %d replicas", n) + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("worker pool left at %d replicas", n) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster HelmRelease left suspended") + } + }) + } +} + +// TestReconcileKubernetesRestoreRefusesRecreatedCluster covers an application +// deleted after the backup and created again under the same name. Its new CA +// and service-account key come back only through a rebuild, so an in-place +// restore must refuse before it changes anything. Backups without a recorded +// UID are told apart by the datastore schema. +func TestReconcileKubernetesRestoreRefusesRecreatedCluster(t *testing.T) { + for _, tc := range []struct { + name string + edit func(map[string]string) + }{ + {"recorded UID", func(md map[string]string) { md[kubernetesControlPlaneUIDMetaKey] = "uid-of-the-deleted-cluster" }}, + {"schema of a backup without the UID", func(md map[string]string) { + delete(md, kubernetesControlPlaneUIDMetaKey) + md[kubernetesDatastorePrefixMetaKey] = "uid-of-the-deleted-cluster" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + backup := k8sBackupForRestore() + tc.edit(backup.Spec.DriverMetadata) + run := newK8sRestoreRun(t, "", backup) + run.verifyAndCapture(true) + + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(got.Status.Message, "deleted and created again") { + t.Fatalf("restore into a re-created cluster not refused: %+v", got.Status) + } + if kubernetesStateFromRestoreJob(got) != nil { + t.Fatal("restore mode recorded for a refused restore") + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster HelmRelease suspended by a refused restore") + } + if n, _ := run.replicas(machineDeploymentGVR, k8sTestRelease+"-md0"); n != 3 { + t.Fatalf("worker pool scaled to %d by a refused restore", n) + } + }) + } +} + +// TestFailedKubernetesRestoreResumesWithoutClusterAPI covers Cluster API going +// unserved after the workers were stopped: the pools cannot be read back, but +// the autoscaler and the suspended HelmReleases must still be put back. +func TestFailedKubernetesRestoreResumesWithoutClusterAPI(t *testing.T) { + run := newK8sRestoreRun(t, "", k8sBackupForRestore()) + run.addMachine("w-1") + run.verifyAndCapture(true) + run.step() // workers stopping, the Machine still there + if n, _ := run.replicas(deploymentGVR, k8sTestRelease+"-cluster-autoscaler"); n != 0 { + t.Fatalf("autoscaler not stopped: %d", n) + } + for _, res := range []string{"machinedeployments", "machines"} { + run.env.restore.Interface.(*dynamicfake.FakeDynamicClient).PrependReactor("list", res, func(clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, &meta.NoKindMatchError{GroupKind: schema.GroupKind{Group: "cluster.x-k8s.io", Kind: "MachineDeployment"}, SearchedVersions: []string{"v1beta1"}} + }) + } + run.step() + + if got := run.get(); got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("restore not failed: %+v", got.Status) + } + if n, _ := run.replicas(deploymentGVR, k8sTestRelease+"-cluster-autoscaler"); n != 1 { + t.Fatalf("autoscaler left at %d replicas", n) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster HelmRelease left suspended") + } +} + +// TestRenderKubernetesTemplateRefusesUnexpectedSchema pins the allowlist every +// strategy Job is rendered through: the schema becomes the prefix the restore +// deletes in the etcd the whole tenant shares. +func TestRenderKubernetesTemplateRefusesUnexpectedSchema(t *testing.T) { + tmpl := corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "etcd", Args: []string{"{{ .Datastore.Prefix }} {{ .Datastore.SourcePrefix }}"}}}}} + render := func(prefix, source string) error { + _, err := renderKubernetesTemplate(tmpl, k8sTestApp, k8sTestNS, kubernetesModeRestore, "key", kubernetesDatastore{Prefix: prefix, SourcePrefix: source}, nil) + return err + } + for _, ok := range []string{"0f2c1e4a-77b1-4c39-9a3e-6d1f2b9c8e10", "tenant_root_kubernetes_bk1"} { + if err := render(ok, ok); err != nil { + t.Errorf("schema %q refused: %v", ok, err) + } + } + if err := render("tenant_a_kubernetes_c1", ""); err != nil { + t.Errorf("a Job without a source schema refused: %v", err) + } + for _, bad := range []string{"", "a/b", "/", "..", "a..b", "a.b", "Tenant_a", "a b", "a\nb", "a*"} { + if render(bad, "tenant_a_kubernetes_c1") == nil { + t.Errorf("schema %q accepted", bad) + } + if bad != "" && render("tenant_a_kubernetes_c1", bad) == nil { + t.Errorf("source schema %q accepted", bad) + } + } +} + +// TestReconcileKubernetesRestoreRefusesEditedPrefix covers a Backup whose +// recorded prefix was edited: the restore fails in its preflight, before any +// Job runs against the shared etcd or the cluster is stopped. +func TestReconcileKubernetesRestoreRefusesEditedPrefix(t *testing.T) { + backup := k8sBackupForRestore() + backup.Spec.DriverMetadata[kubernetesDatastorePrefixMetaKey] = "tenant_a_kubernetes_c1/.." + run := newK8sRestoreRun(t, "", backup) + run.step() // StartedAt + run.step() // preflight + + got := run.get() + if got.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed || !strings.Contains(got.Status.Message, "refusing") { + t.Fatalf("restore from an edited prefix not refused: %+v", got.Status) + } + jobs := &batchv1.JobList{} + if err := run.env.client.List(context.Background(), jobs); err != nil || len(jobs.Items) != 0 { + t.Fatalf("Jobs created for an edited prefix: %d %v", len(jobs.Items), err) + } + if run.suspended(k8sTestRelease) { + t.Fatal("cluster HelmRelease suspended") + } +} + +// TestReconcileKubernetesBackupRefusesUnexpectedSchema covers a live +// TenantControlPlane reporting a schema outside the allowlist. +func TestReconcileKubernetesBackupRefusesUnexpectedSchema(t *testing.T) { + ctx := context.Background() + bj := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: k8sTestNS, Name: "bj1"}, + Spec: backupsv1alpha1.BackupJobSpec{ApplicationRef: corev1.TypedLocalObjectReference{ + APIGroup: new(backupsv1alpha1.DefaultApplicationAPIGroup), Kind: kubernetesAppKind, Name: k8sTestApp, + }}, + } + env := newKubernetesTestEnv(t, bj) + tcp, err := env.backup.Interface.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Get(ctx, k8sTestRelease, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(tcp.Object, "tenant_a/kubernetes_c1", "status", "storage", "setup", "schema"); err != nil { + t.Fatal(err) + } + if _, err := env.backup.Interface.Resource(tenantControlPlaneGVR).Namespace(k8sTestNS).Update(ctx, tcp, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + get := func() *backupsv1alpha1.BackupJob { + out := &backupsv1alpha1.BackupJob{} + if err := env.client.Get(ctx, client.ObjectKeyFromObject(bj), out); err != nil { + t.Fatal(err) + } + return out + } + for i := 0; i < 4 && get().Status.Phase != backupsv1alpha1.BackupJobPhaseFailed; i++ { + if _, err := env.backup.reconcileKubernetes(ctx, get(), k8sResolved()); err != nil { + t.Fatal(err) + } + } + if got := get(); got.Status.Phase != backupsv1alpha1.BackupJobPhaseFailed || !strings.Contains(got.Status.Message, "refusing") { + t.Fatalf("backup of an unexpected schema not refused: %+v", got.Status) + } + jobs := &batchv1.JobList{} + if err := env.client.List(ctx, jobs); err != nil || len(jobs.Items) != 0 { + t.Fatalf("dump Job created for an unexpected schema: %d %v", len(jobs.Items), err) + } +} diff --git a/internal/backupcontroller/kubernetesstrategy_restore.go b/internal/backupcontroller/kubernetesstrategy_restore.go new file mode 100644 index 000000000..830a25927 --- /dev/null +++ b/internal/backupcontroller/kubernetesstrategy_restore.go @@ -0,0 +1,1129 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/template" +) + +const ( + // kubernetesCondMode records on the RestoreJob whether it rebuilds a + // deleted cluster or rolls a live one back (Reason), and which cluster + // (Message). It lives in status, which the creator of a RestoreJob cannot + // set, so a RestoreJob cannot claim a live cluster is gone. + kubernetesCondMode = "KubernetesRestoreMode" + kubernetesRestorePhaseLabel = "kubernetes.strategy.backups.cozystack.io/restore-phase" + + // What a restore changes on a live cluster is recorded on the objects it + // changes rather than on the RestoreJob, so the cluster can be put back + // after the RestoreJob is deleted, and a retry after a failed restore + // keeps the counts from before the first one rather than the zeros. + kubernetesReplicasBeforeAnnotation = "kubernetes.strategy.backups.cozystack.io/replicas-before-restore" + kubernetesSuspendedByAnnotation = "kubernetes.strategy.backups.cozystack.io/suspended-by-restore" + kubernetesReplicasUnset = "unset" + + kubernetesRestoreModeRecreate = "recreate" + kubernetesRestoreModeInPlace = "in-place" + + kubernetesPhaseSecrets = "secrets" + kubernetesPhaseVolumes = "volumes" + + machineExcludeDrainAnnotation = "machine.cluster.x-k8s.io/exclude-node-draining" + machineExcludeVolumeDetachAnnotation = "machine.cluster.x-k8s.io/exclude-wait-for-node-volume-detach" + + kubernetesCondVerified = "KubernetesBackupVerified" + kubernetesCondRecreated = "KubernetesClusterRecreated" + kubernetesCondStopped = "KubernetesClusterStopped" + kubernetesCondSecretsRestored = "KubernetesSecretsRestored" + kubernetesCondEtcdRestored = "KubernetesEtcdRestored" + kubernetesCondVolumesRestored = "KubernetesVolumesRestored" + // kubernetesCondEtcdWriteFailed marks a restore whose etcd Job failed + // after it started replacing the prefix. The control plane is then left + // stopped, by a failure and by the RestoreJob's deletion alike. + kubernetesCondEtcdWriteFailed = "KubernetesEtcdWriteFailed" + // kubernetesCondEtcdJobStarted records that a container of the restore + // Job was seen running, which is the only trace left once the Job's + // deadline has deleted its Pods. + kubernetesCondEtcdJobStarted = "KubernetesEtcdJobStarted" + + // The restore Job writes this to its termination message just before it + // deletes the prefix, so a failed Job tells whether the prefix is still + // intact. + kubernetesJobWriting = "writing" +) + +// KubernetesRestoreOptions is the typed shape of RestoreJob.Spec.Options for +// the Kubernetes driver. Both halves of the backup are restored unless one is +// switched off. +type KubernetesRestoreOptions struct { + // Etcd restores the cluster's etcd prefix (its API objects). + Etcd *bool `json:"etcd,omitempty"` + // Volumes restores the DataVolumes backing the cluster's PersistentVolumes. + Volumes *bool `json:"volumes,omitempty"` +} + +func (o KubernetesRestoreOptions) restoreEtcd() bool { return boolDefault(o.Etcd, true) } +func (o KubernetesRestoreOptions) restoreVolumes() bool { return boolDefault(o.Volumes, true) } + +func parseKubernetesRestoreOptions(opts *runtime.RawExtension) (KubernetesRestoreOptions, error) { + var out KubernetesRestoreOptions + if opts == nil || len(opts.Raw) == 0 { + return out, nil + } + if err := json.Unmarshal(opts.Raw, &out); err != nil { + return KubernetesRestoreOptions{}, fmt.Errorf("decode restoreJob.spec.options: %w", err) + } + if !out.restoreEtcd() && !out.restoreVolumes() { + return KubernetesRestoreOptions{}, fmt.Errorf("restoreJob.spec.options disables both etcd and volumes; nothing to restore") + } + return out, nil +} + +type kubernetesRestoreState struct { + Mode string `json:"mode"` + Release string `json:"release"` +} + +func kubernetesAutoscalerName(release string) string { + return release + "-cluster-autoscaler" +} + +func kubernetesBackupDataVolumes(b *backupsv1alpha1.Backup) []string { + raw := b.Spec.DriverMetadata[kubernetesDataVolumesMetaKey] + if raw == "" { + return nil + } + return strings.Split(raw, ",") +} + +func kubernetesBackupHelmReleases(b *backupsv1alpha1.Backup) ([]map[string]any, error) { + var out []map[string]any + if err := json.Unmarshal([]byte(b.Spec.DriverMetadata[kubernetesHelmReleasesMetaKey]), &out); err != nil { + return nil, fmt.Errorf("decode recorded HelmReleases: %w", err) + } + if len(out) == 0 { + return nil, fmt.Errorf("the Backup records no HelmReleases") + } + return out, nil +} + +// kubernetesPreRestoreKey is where the restore Job keeps the prefix it is +// about to replace. The Job writes it only when none is there and deletes it +// once the restore succeeds, so after a failed restore it holds the prefix +// as it was before the first failed attempt, and a retry does not overwrite +// it with a partial one. It is fixed per backup, so the Backup's cleanup +// removes it. +func kubernetesPreRestoreKey(objectKey string) string { + return objectKey + ".pre-restore" +} + +// reconcileKubernetesRestore restores a tenant Kubernetes application in +// place, under its own name and namespace. +// +// When the application is gone, the restore rebuilds it: control-plane PKI +// Secrets first (the etcd Job authenticates with one of them), then the etcd +// prefix, the volumes, and finally the HelmReleases, so the control plane +// comes up on restored state and the existing CA. +// +// When the application is live, the restore stops what the selected halves +// need stopped — the workers for volumes, so the volumes are detached, then +// the control plane for etcd, so nothing writes while the prefix is replaced +// — writes the state back, and returns the cluster to the replica counts it +// had. +// +// Nothing is changed before the backup's artifacts are confirmed to exist. +func (r *RestoreJobReconciler) reconcileKubernetesRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + if rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded || + rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + return ctrl.Result{}, nil + } + if err := validateKubernetesApplicationRef(backup.Spec.ApplicationRef); err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + appName := backup.Spec.ApplicationRef.Name + if t := rj.Spec.TargetApplicationRef; t != nil && ((t.Name != "" && t.Name != appName) || (t.Kind != "" && t.Kind != kubernetesAppKind)) { + return r.markRestoreJobFailed(ctx, rj, "the Kubernetes strategy restores only into the application the backup was taken from") + } + if rj.Namespace != backup.Namespace { + return r.markRestoreJobFailed(ctx, rj, "the Kubernetes strategy restores only into the namespace the backup was taken from") + } + opts, err := parseKubernetesRestoreOptions(rj.Spec.Options) + if err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + + if rj.Status.StartedAt == nil { + base := rj.DeepCopy() + now := metav1.Now() + rj.Status.StartedAt = &now + rj.Status.Phase = backupsv1alpha1.RestoreJobPhaseRunning + if err := r.Status().Patch(ctx, rj, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + strategy := &strategyv1alpha1.Kubernetes{} + if err := r.Get(ctx, client.ObjectKey{Name: backup.Spec.StrategyRef.Name}, strategy); err != nil { + if apierrors.IsNotFound(err) { + return r.requeueRestoreStrategyNotReady(ctx, rj, backup.Spec.StrategyRef.Name) + } + return ctrl.Result{}, err + } + + if !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondVerified) { + done, msg, err := r.verifyKubernetesBackup(ctx, rj, backup, strategy, opts) + if err != nil || msg != "" || !done { + return r.kubernetesStepResult(ctx, rj, nil, err, msg, done) + } + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondVerified, "the backup's artifacts are present") + } + + state := kubernetesStateFromRestoreJob(rj) + if state == nil { + return r.captureKubernetesRestoreState(ctx, rj, backup) + } + + if state.Mode == kubernetesRestoreModeInPlace { + return r.reconcileKubernetesInPlace(ctx, rj, backup, strategy, opts, state) + } + return r.reconcileKubernetesRecreate(ctx, rj, backup, strategy, opts, state) +} + +func kubernetesStateFromRestoreJob(rj *backupsv1alpha1.RestoreJob) *kubernetesRestoreState { + c := apimeta.FindStatusCondition(rj.Status.Conditions, kubernetesCondMode) + if c == nil { + return nil + } + mode := kubernetesRestoreModeInPlace + if c.Reason == "Recreate" { + mode = kubernetesRestoreModeRecreate + } + return &kubernetesRestoreState{Mode: mode, Release: c.Message} +} + +// verifyKubernetesBackup confirms, before anything is stopped or deleted, +// that the Velero backup completed and still exists, and that the etcd dump +// object is in the bucket. The dump is checked by a verify-mode run of the +// strategy Job, the only component holding the bucket credentials. +func (r *RestoreJobReconciler) verifyKubernetesBackup(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, opts KubernetesRestoreOptions) (bool, string, error) { + name := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] + if name == "" { + return false, "the Backup records no Velero backup", nil + } + vb := &velerov1.Backup{} + if err := r.Get(ctx, client.ObjectKey{Namespace: veleroNamespace, Name: name}, vb); err != nil { + if apierrors.IsNotFound(err) { + return false, fmt.Sprintf("Velero backup %s/%s no longer exists", veleroNamespace, name), nil + } + return false, "", err + } + if vb.Status.Phase != velerov1.BackupPhaseCompleted { + return false, fmt.Sprintf("Velero backup %s/%s is %q, not Completed", veleroNamespace, name, vb.Status.Phase), nil + } + if !opts.restoreEtcd() { + return true, "", nil + } + return r.runKubernetesJob(ctx, rj, backup, strategy, kubernetesModeVerify, rj.Name+"-verify", kubernetesBackupDatastore(backup)) +} + +// captureKubernetesRestoreState decides between rebuilding a deleted cluster +// and restoring a live one. +func (r *RestoreJobReconciler) captureKubernetesRestoreState(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + release := kubernetesReleaseName(backup.Spec.ApplicationRef.Name) + reason := "Recreate" + _, err := r.Resource(helmReleaseGVR).Namespace(rj.Namespace).Get(ctx, release, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + case err != nil: + return ctrl.Result{}, err + default: + reason = "InPlace" + mismatch, err := r.kubernetesClusterMismatch(ctx, rj.Namespace, release, backup) + if err != nil { + return ctrl.Result{}, err + } + if mismatch != "" { + return r.markRestoreJobFailed(ctx, rj, mismatch) + } + } + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: kubernetesCondMode, + Status: metav1.ConditionTrue, + Reason: reason, + Message: release, + }) + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil +} + +// capiNotServed reports a list of a Cluster API resource that failed because +// the apiserver does not serve that resource at the version the driver asks +// for: a NotFound on a collection, or no RESTMapper match. +func capiNotServed(err error) bool { + return err != nil && (apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err)) +} + +// kubernetesClusterMismatch reports why the live cluster is not the one the +// backup was taken from, or "" when it is or cannot be told apart. A cluster +// deleted and created again under the same name has a new CA, service-account +// key and Talos secrets, which only a rebuild restores: writing the old prefix +// into it would leave every token and node signed by keys it no longer has. +// Backups that predate the recorded UID fall back to the datastore schema, +// which tells the two apart only on a Kamaji that keys it on the UID. +func (r *RestoreJobReconciler) kubernetesClusterMismatch(ctx context.Context, ns, release string, backup *backupsv1alpha1.Backup) (string, error) { + tcp, err := r.Resource(tenantControlPlaneGVR).Namespace(ns).Get(ctx, release, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return "", nil + } + if err != nil { + return "", err + } + md := backup.Spec.DriverMetadata + recorded, live := md[kubernetesControlPlaneUIDMetaKey], string(tcp.GetUID()) + if recorded == "" { + recorded = md[kubernetesDatastorePrefixMetaKey] + live, _, _ = unstructured.NestedString(tcp.Object, "status", "storage", "setup", "schema") + } + if recorded == "" || live == "" || recorded == live { + return "", nil + } + return fmt.Sprintf("TenantControlPlane %s/%s is not the cluster this backup was taken from (%s, backup has %s): it was deleted and created again under the same name, so an in-place restore would write the old state over a new CA and service-account key; delete the application and restore again to rebuild it with the backed-up PKI", ns, release, live, recorded), nil +} + +func (r *RestoreJobReconciler) listClusterMachineDeployments(ctx context.Context, ns, release string) ([]unstructured.Unstructured, error) { + list, err := r.Resource(machineDeploymentGVR).Namespace(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + var out []unstructured.Unstructured + for _, md := range list.Items { + if cluster, _, _ := unstructured.NestedString(md.Object, "spec", "clusterName"); cluster == release { + out = append(out, md) + } + } + return out, nil +} + +// --------------------------------------------------------------------------- +// Rebuild a deleted cluster +// --------------------------------------------------------------------------- + +// reconcileKubernetesRecreate rebuilds a deleted cluster. The control-plane +// PKI and the volumes come back first, then the cluster's own HelmRelease. +// Kamaji gives the new TenantControlPlane a fresh datastore schema (keyed on +// its UID), so the etcd half can only be written once that schema exists: +// the restore then stops the new control plane like a live one, writes the +// dump into the new prefix, and starts it again. The worker pools are +// recreated last, so their workers join the restored cluster rather than an +// empty one. +func (r *RestoreJobReconciler) reconcileKubernetesRecreate(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, opts KubernetesRestoreOptions, state *kubernetesRestoreState) (ctrl.Result, error) { + releases, err := kubernetesBackupHelmReleases(backup) + if err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + + if !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondSecretsRestored) { + done, msg, err := r.runKubernetesVeleroRestore(ctx, rj, backup, strategy, kubernetesPhaseSecrets, state.Release) + if err != nil || msg != "" || !done { + return r.kubernetesStepResult(ctx, rj, nil, err, msg, done) + } + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondSecretsRestored, "control-plane Secrets restored") + } + if opts.restoreVolumes() && !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondVolumesRestored) { + done, msg, err := r.runKubernetesVeleroRestore(ctx, rj, backup, strategy, kubernetesPhaseVolumes, state.Release) + if err != nil || msg != "" || !done { + return r.kubernetesStepResult(ctx, rj, nil, err, msg, done) + } + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondVolumesRestored, "volumes restored") + } + + if !opts.restoreEtcd() { + if err := r.createKubernetesHelmReleases(ctx, rj.Namespace, releases, true); err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + return r.markKubernetesRestoreSucceeded(ctx, rj, "cluster rebuilt from backup without its etcd state") + } + + if !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondRecreated) { + if err := r.createKubernetesHelmReleases(ctx, rj.Namespace, releases[:1], false); err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + ready, err := r.tenantControlPlaneSchemaReady(ctx, rj.Namespace, state.Release) + if err != nil { + return ctrl.Result{}, err + } + if ready { + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondRecreated, "the control plane is up with a datastore schema") + } + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: kubernetesCondRecreated, + Status: metav1.ConditionFalse, + Reason: "Waiting", + Message: "waiting for TenantControlPlane " + state.Release + " to report its datastore schema", + }) + cond := apimeta.FindStatusCondition(rj.Status.Conditions, kubernetesCondRecreated) + if time.Since(cond.LastTransitionTime.Time) > restorePreparationDeadline { + return r.markRestoreJobFailed(ctx, rj, fmt.Sprintf("TenantControlPlane %s/%s reported no datastore schema within %s", rj.Namespace, state.Release, restorePreparationDeadline)) + } + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + return r.reconcileKubernetesInPlace(ctx, rj, backup, strategy, KubernetesRestoreOptions{Etcd: new(true), Volumes: new(false)}, state) +} + +func (r *RestoreJobReconciler) tenantControlPlaneSchemaReady(ctx context.Context, ns, release string) (bool, error) { + tcp, err := r.Resource(tenantControlPlaneGVR).Namespace(ns).Get(ctx, release, metav1.GetOptions{}) + if err != nil { + return false, client.IgnoreNotFound(err) + } + schema, _, _ := unstructured.NestedString(tcp.Object, "status", "storage", "setup", "schema") + return schema != "", nil +} + +// createKubernetesHelmReleases recreates the recorded HelmReleases. A worker +// pool's HelmRelease outlives the deletion of its cluster (KubernetesNodes is +// an application of its own), but its MachineDeployment was garbage-collected +// with the CAPI Cluster, and nothing re-applies an unchanged release. One +// that exists is therefore forced through a Helm upgrade, which recreates +// the objects its release is missing. The cluster's own release is created +// while the rebuild polls for its schema, so it passes force=false: forcing +// it on every poll would re-run its chart while Kamaji brings it up. +func (r *RestoreJobReconciler) createKubernetesHelmReleases(ctx context.Context, ns string, releases []map[string]any, force bool) error { + for _, hr := range releases { + obj := &unstructured.Unstructured{Object: hr} + obj.SetNamespace(ns) + _, err := r.Resource(helmReleaseGVR).Namespace(ns).Create(ctx, obj, metav1.CreateOptions{}) + if err == nil { + continue + } + if !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("recreate HelmRelease %s/%s: %w", ns, obj.GetName(), err) + } + if !force { + continue + } + now := time.Now().UTC().Format(time.RFC3339Nano) + patch, merr := json.Marshal(map[string]any{"metadata": map[string]any{"annotations": map[string]string{ + "reconcile.fluxcd.io/requestedAt": now, + "reconcile.fluxcd.io/forceAt": now, + }}}) + if merr != nil { + return merr + } + if _, err := r.Resource(helmReleaseGVR).Namespace(ns).Patch(ctx, obj.GetName(), types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("force HelmRelease %s/%s to reconcile: %w", ns, obj.GetName(), err) + } + } + return nil +} + +// --------------------------------------------------------------------------- +// Restore a live cluster +// --------------------------------------------------------------------------- + +func (r *RestoreJobReconciler) reconcileKubernetesInPlace(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, opts KubernetesRestoreOptions, state *kubernetesRestoreState) (ctrl.Result, error) { + if !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondStopped) { + stopped, waiting, err := r.stopKubernetesCluster(ctx, rj, backup, opts, state.Release) + if capiNotServed(err) { + // Reading this as "no workers" would delete volumes that running + // workers still mount. Workers are listed before anything is + // stopped, so the resume below has only what it already stopped + // to put back. + return r.failKubernetesInPlace(ctx, rj, state, fmt.Sprintf("Cluster API is not served at %s: %v", machineDeploymentGVR.GroupVersion(), err)) + } + if err != nil { + // Retried like any other wait, under the same deadline: a single + // apiserver blip must not abort a restore. + stopped, waiting = false, fmt.Sprintf("a retry after an API error: %v", err) + } + if stopped { + return r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondStopped, "cluster stopped for restore") + } + // The condition's transition time marks when stopping began: it is + // kept while the status stays False, whatever the message says. + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: kubernetesCondStopped, + Status: metav1.ConditionFalse, + Reason: "Stopping", + Message: "waiting for " + waiting, + }) + cond := apimeta.FindStatusCondition(rj.Status.Conditions, kubernetesCondStopped) + if time.Since(cond.LastTransitionTime.Time) > restorePreparationDeadline { + return r.failKubernetesInPlace(ctx, rj, state, fmt.Sprintf("the cluster did not stop within %s: still waiting for %s", restorePreparationDeadline, waiting)) + } + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + + if res, next, err := r.kubernetesRestoreData(ctx, rj, backup, strategy, opts, state); !next { + return res, err + } + + if err := r.resumeKubernetesCluster(ctx, rj.Namespace, state.Release); err != nil { + return ctrl.Result{}, err + } + if state.Mode == kubernetesRestoreModeRecreate { + releases, err := kubernetesBackupHelmReleases(backup) + if err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + if err := r.createKubernetesHelmReleases(ctx, rj.Namespace, releases[1:], true); err != nil { + return r.markRestoreJobFailed(ctx, rj, err.Error()) + } + return r.markKubernetesRestoreSucceeded(ctx, rj, "cluster rebuilt from backup") + } + return r.markKubernetesRestoreSucceeded(ctx, rj, "cluster restored in place") +} + +// stopKubernetesCluster suspends the cluster's HelmReleases so Flux does not +// undo the scale-down, then stops the workers when volumes are restored — +// deleting the data DataVolumes and PVCs the backup holds once nothing mounts +// them — and the control plane when etcd is. Workers go first: CAPI removes a +// Machine's Node through the tenant API. It reports whether all of that has +// settled and, if not, what it is waiting for. +func (r *RestoreJobReconciler) stopKubernetesCluster(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, opts KubernetesRestoreOptions, release string) (bool, string, error) { + ns := rj.Namespace + mds, err := r.listClusterMachineDeployments(ctx, ns, release) + if err != nil { + return false, "", err + } + for _, hr := range kubernetesHelmReleaseNames(mds, release) { + if err := r.suspendHelmReleaseForRestore(ctx, ns, hr); err != nil { + return false, "", err + } + } + + if opts.restoreVolumes() { + if err := r.stopReplicas(ctx, deploymentGVR, ns, kubernetesAutoscalerName(release)); err != nil { + return false, "", err + } + for _, md := range mds { + if err := r.stopReplicas(ctx, machineDeploymentGVR, ns, md.GetName()); err != nil { + return false, "", err + } + } + remaining, err := r.releaseClusterMachines(ctx, ns, release) + if err != nil { + return false, "", err + } + if remaining > 0 { + return false, fmt.Sprintf("%d worker Machine(s) to be deleted", remaining), nil + } + var pending []string + for _, name := range kubernetesBackupDataVolumes(backup) { + dvGone, err := r.deleteDataVolume(ctx, ns, name) + if err != nil { + return false, "", err + } + pvcGone, err := r.deletePVC(ctx, ns, name) + if err != nil { + return false, "", err + } + if !dvGone || !pvcGone { + pending = append(pending, name) + } + } + if len(pending) > 0 { + return false, "data volumes to be deleted: " + strings.Join(pending, ", "), nil + } + } + + if opts.restoreEtcd() { + if err := r.stopReplicas(ctx, kamajiControlPlaneGVR, ns, release); err != nil { + return false, "", err + } + cp, err := r.Resource(deploymentGVR).Namespace(ns).Get(ctx, release, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + case err != nil: + return false, "", err + default: + if n, _, _ := unstructured.NestedInt64(cp.Object, "status", "replicas"); n > 0 { + return false, fmt.Sprintf("%d control-plane Pod(s) to stop", n), nil + } + // status.replicas leaves out Pods that are still terminating, and + // a terminating kube-apiserver can still write to etcd. + selector, _, _ := unstructured.NestedStringMap(cp.Object, "spec", "selector", "matchLabels") + if len(selector) > 0 { + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(ns), client.MatchingLabels(selector)); err != nil { + return false, "", err + } + if len(pods.Items) > 0 { + return false, fmt.Sprintf("%d control-plane Pod(s) to terminate", len(pods.Items)), nil + } + } + } + } + return true, "", nil +} + +// releaseClusterMachines lets the cluster's Machines finish deleting without +// draining their Nodes or waiting for volumes to detach: the workers are +// recreated after the restore, and a drain can block on a PodDisruptionBudget. +// It returns how many Machines remain. +func (r *RestoreJobReconciler) releaseClusterMachines(ctx context.Context, ns, release string) (int, error) { + machines, err := r.Resource(machineGVR).Namespace(ns).List(ctx, metav1.ListOptions{LabelSelector: clusterNameLabel + "=" + release}) + if err != nil { + return 0, err + } + patch := []byte(`{"metadata":{"annotations":{"` + machineExcludeDrainAnnotation + `":"","` + machineExcludeVolumeDetachAnnotation + `":""}}}`) + for _, m := range machines.Items { + ann := m.GetAnnotations() + if _, ok := ann[machineExcludeDrainAnnotation]; ok { + if _, ok := ann[machineExcludeVolumeDetachAnnotation]; ok { + continue + } + } + if _, err := r.Resource(machineGVR).Namespace(ns).Patch(ctx, m.GetName(), types.MergePatchType, patch, metav1.PatchOptions{}); err != nil && !apierrors.IsNotFound(err) { + return 0, err + } + } + return len(machines.Items), nil +} + +// suspendHelmReleaseForRestore suspends a HelmRelease and marks it as +// suspended by the restore. One already suspended is left alone and unmarked, +// so resuming the cluster does not resume it. +func (r *RestoreJobReconciler) suspendHelmReleaseForRestore(ctx context.Context, ns, name string) error { + hr, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return client.IgnoreNotFound(err) + } + if suspended, _, _ := unstructured.NestedBool(hr.Object, "spec", "suspend"); suspended { + return nil + } + patch := []byte(`{"metadata":{"annotations":{"` + kubernetesSuspendedByAnnotation + `":"true"}},"spec":{"suspend":true}}`) + _, err = r.Resource(helmReleaseGVR).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + return client.IgnoreNotFound(err) +} + +// stopReplicas scales an object to zero, first recording its replica count +// on it. A count already recorded is kept: it was taken before an earlier +// restore stopped the object. +func (r *RestoreJobReconciler) stopReplicas(ctx context.Context, gvr schema.GroupVersionResource, ns, name string) error { + obj, err := r.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return client.IgnoreNotFound(err) + } + annotations := map[string]any{} + if _, recorded := obj.GetAnnotations()[kubernetesReplicasBeforeAnnotation]; !recorded { + before := kubernetesReplicasUnset + if n, found, _ := unstructured.NestedInt64(obj.Object, "spec", "replicas"); found { + before = strconv.FormatInt(n, 10) + } + annotations[kubernetesReplicasBeforeAnnotation] = before + } + patch, err := json.Marshal(map[string]any{ + "metadata": map[string]any{"annotations": annotations}, + "spec": map[string]any{"replicas": 0}, + }) + if err != nil { + return err + } + _, err = r.Resource(gvr).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + return client.IgnoreNotFound(err) +} + +// resumeReplicas puts back the replica count stopReplicas recorded — removing +// the field when it was unset, so its owner (the autoscaler, for a +// MachineDeployment) takes it back — and drops the record. +func (r *RestoreJobReconciler) resumeReplicas(ctx context.Context, gvr schema.GroupVersionResource, ns, name string) error { + obj, err := r.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return client.IgnoreNotFound(err) + } + before, recorded := obj.GetAnnotations()[kubernetesReplicasBeforeAnnotation] + if !recorded { + return nil + } + var replicas any + if before != kubernetesReplicasUnset { + n, err := strconv.ParseInt(before, 10, 64) + if err != nil { + return fmt.Errorf("%s %s/%s: bad %s %q", gvr.Resource, ns, name, kubernetesReplicasBeforeAnnotation, before) + } + replicas = n + } + patch, err := json.Marshal(map[string]any{ + "metadata": map[string]any{"annotations": map[string]any{kubernetesReplicasBeforeAnnotation: nil}}, + "spec": map[string]any{"replicas": replicas}, + }) + if err != nil { + return err + } + _, err = r.Resource(gvr).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + return client.IgnoreNotFound(err) +} + +// resumeKubernetesCluster undoes stopKubernetesCluster from the records on +// the objects themselves, so it needs nothing from the RestoreJob and is safe +// to call more than once. +func (r *RestoreJobReconciler) resumeKubernetesCluster(ctx context.Context, ns, release string) error { + var errs []string + mds, err := r.listClusterMachineDeployments(ctx, ns, release) + if err != nil && !capiNotServed(err) { + return err + } + if err := r.resumeReplicas(ctx, kamajiControlPlaneGVR, ns, release); err != nil { + errs = append(errs, err.Error()) + } + for _, md := range mds { + if err := r.resumeReplicas(ctx, machineDeploymentGVR, ns, md.GetName()); err != nil { + errs = append(errs, err.Error()) + } + } + if err := r.resumeReplicas(ctx, deploymentGVR, ns, kubernetesAutoscalerName(release)); err != nil { + errs = append(errs, err.Error()) + } + for _, name := range kubernetesHelmReleaseNames(mds, release) { + hr, err := r.Resource(helmReleaseGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + errs = append(errs, err.Error()) + } + continue + } + if _, marked := hr.GetAnnotations()[kubernetesSuspendedByAnnotation]; !marked { + continue + } + patch := []byte(`{"metadata":{"annotations":{"` + kubernetesSuspendedByAnnotation + `":null}},"spec":{"suspend":false}}`) + if _, err := r.Resource(helmReleaseGVR).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, err.Error()) + } + } + if len(errs) > 0 { + return fmt.Errorf("resume cluster: %s", strings.Join(errs, "; ")) + } + return nil +} + +// failKubernetesInPlace fails a live-cluster restore. The cluster is resumed +// unless the etcd prefix may have been partly replaced: bringing a control +// plane up on that would serve a mix of old and restored objects, so it is +// left stopped for a retry or a manual repair. +func (r *RestoreJobReconciler) failKubernetesInPlace(ctx context.Context, rj *backupsv1alpha1.RestoreJob, state *kubernetesRestoreState, msg string) (ctrl.Result, error) { + if apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdWriteFailed) { + return r.markRestoreJobFailed(ctx, rj, msg) + } + if err := r.resumeKubernetesCluster(ctx, rj.Namespace, state.Release); err != nil { + msg = fmt.Sprintf("%s; the cluster could not be fully resumed: %v", msg, err) + } + return r.markRestoreJobFailed(ctx, rj, msg) +} + +// cleanupKubernetesRestore runs when a RestoreJob is deleted. A live cluster +// it stopped is resumed, unless its etcd prefix may be partly replaced, which +// is left for the next restore to repair and reported as a Warning event. +func (r *RestoreJobReconciler) cleanupKubernetesRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob) { + state := kubernetesStateFromRestoreJob(rj) + if state == nil || state.Release == "" || rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseSucceeded { + return + } + partial := apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdWriteFailed) + // A restore Job still running may be rewriting the prefix right now; + // one that failed after it started writing left it partial. Either way + // the control plane must not come up on it. The Job is deleted so it + // stops writing; the garbage collector would only get to it later. + job := &batchv1.Job{} + switch err := r.Get(ctx, client.ObjectKey{Namespace: rj.Namespace, Name: jobNameForRestoreJob(rj)}, job); { + case err == nil && jobConditionState(job) != batchv1.JobComplete: + wrote, werr := r.kubernetesJobReachedWrite(ctx, rj, job.Name) + if werr != nil || wrote || jobConditionState(job) != batchv1.JobFailed { + partial = true + } + policy := metav1.DeletePropagationBackground + _ = r.Delete(ctx, job, &client.DeleteOptions{PropagationPolicy: &policy}) + case err != nil && !apierrors.IsNotFound(err): + partial = true + } + if partial { + if r.Recorder != nil { + r.Recorder.Eventf(rj, corev1.EventTypeWarning, "ClusterLeftStopped", + "%s/%s stays stopped: its etcd prefix may be partly restored; run a new restore from a Backup", rj.Namespace, state.Release) + } + return + } + if err := r.resumeKubernetesCluster(ctx, rj.Namespace, state.Release); err != nil && r.Recorder != nil { + r.Recorder.Eventf(rj, corev1.EventTypeWarning, "CleanupFailed", "resume %s/%s: %v", rj.Namespace, state.Release, err) + } +} + +// --------------------------------------------------------------------------- +// Steps shared by both modes +// --------------------------------------------------------------------------- + +// kubernetesRestoreData writes back the selected halves, etcd before volumes. +// next is true when both are done and the caller may move on; otherwise the +// returned result is the reconcile's. +func (r *RestoreJobReconciler) kubernetesRestoreData(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, opts KubernetesRestoreOptions, state *kubernetesRestoreState) (ctrl.Result, bool, error) { + if opts.restoreEtcd() && !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdRestored) { + ds, err := resolveKubernetesDatastore(ctx, r.Interface, rj.Namespace, backup.Spec.ApplicationRef.Name) + if err != nil { + res, rerr := r.kubernetesStepResult(ctx, rj, state, nil, fmt.Sprintf("resolve the cluster's datastore: %v", err), false) + return res, false, rerr + } + ds.SourcePrefix = backup.Spec.DriverMetadata[kubernetesDatastorePrefixMetaKey] + done, msg, err := r.runKubernetesJob(ctx, rj, backup, strategy, kubernetesModeRestore, jobNameForRestoreJob(rj), *ds) + if msg != "" && err == nil { + wrote, werr := r.kubernetesJobReachedWrite(ctx, rj, jobNameForRestoreJob(rj)) + if werr != nil { + return ctrl.Result{}, false, werr + } + if !wrote { + res, rerr := r.kubernetesStepResult(ctx, rj, state, nil, "etcd restore failed before it changed the prefix: "+msg, false) + return res, false, rerr + } + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: kubernetesCondEtcdWriteFailed, + Status: metav1.ConditionTrue, + Reason: "RestoreJobFailed", + Message: msg, + }) + msg = fmt.Sprintf("etcd restore failed: %s. The prefix may be partly restored, so the control plane is left stopped. "+ + "A copy of the prefix from before the first restore of this backup that did not finish is kept at %s in the backup bucket; run a new RestoreJob to retry", + msg, kubernetesPreRestoreKey(backup.Spec.DriverMetadata[kubernetesObjectMetaKey])) + } + if err == nil && msg == "" && !done && !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdJobStarted) { + started, serr := r.kubernetesJobStarted(ctx, rj.Namespace, jobNameForRestoreJob(rj)) + if serr != nil { + return ctrl.Result{}, false, serr + } + if started { + res, cerr := r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondEtcdJobStarted, "the etcd restore Job is running") + return res, false, cerr + } + } + if err != nil || msg != "" || !done { + res, rerr := r.kubernetesStepResult(ctx, rj, state, err, msg, done) + return res, false, rerr + } + res, err := r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondEtcdRestored, "etcd prefix restored") + return res, false, err + } + if opts.restoreVolumes() && !apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondVolumesRestored) { + done, msg, err := r.runKubernetesVeleroRestore(ctx, rj, backup, strategy, kubernetesPhaseVolumes, kubernetesReleaseName(backup.Spec.ApplicationRef.Name)) + if err != nil || msg != "" || !done { + res, rerr := r.kubernetesStepResult(ctx, rj, state, err, msg, done) + return res, false, rerr + } + res, err := r.setKubernetesRestoreCondition(ctx, rj, kubernetesCondVolumesRestored, "volumes restored") + return res, false, err + } + return ctrl.Result{}, true, nil +} + +// kubernetesStepResult turns a step's outcome into the reconcile result: an +// API error is retried, a failure message fails the restore (resuming a live +// cluster first where that is safe), and an unfinished step is polled. +func (r *RestoreJobReconciler) kubernetesStepResult(ctx context.Context, rj *backupsv1alpha1.RestoreJob, state *kubernetesRestoreState, err error, failure string, done bool) (ctrl.Result, error) { + switch { + case err != nil: + return ctrl.Result{}, err + case failure != "": + // A rebuild stops its new control plane for the etcd half the same + // way a live restore does, so both put back whatever they stopped. + if state != nil { + return r.failKubernetesInPlace(ctx, rj, state, failure) + } + return r.markRestoreJobFailed(ctx, rj, failure) + case !done: + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil + } + return ctrl.Result{}, nil +} + +// kubernetesJobReachedWrite reports whether any Pod of the restore Job may +// have changed the prefix: one running now, or one that terminated with the +// marker the script writes just before it deletes the prefix. A container +// that never ran (an image it could not pull) wrote nothing. When the Job's +// Pods are gone (its deadline deletes them), the answer is whether any of +// them was ever seen running. +func (r *RestoreJobReconciler) kubernetesJobReachedWrite(ctx context.Context, rj *backupsv1alpha1.RestoreJob, jobName string) (bool, error) { + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(rj.Namespace), client.MatchingLabels{"job-name": jobName}); err != nil { + return false, err + } + if len(pods.Items) == 0 { + return apimeta.IsStatusConditionTrue(rj.Status.Conditions, kubernetesCondEtcdJobStarted), nil + } + for _, p := range pods.Items { + for _, cs := range p.Status.ContainerStatuses { + switch { + case cs.State.Running != nil: + return true, nil + case cs.State.Terminated != nil: + if strings.TrimSpace(cs.State.Terminated.Message) == kubernetesJobWriting { + return true, nil + } + case cs.LastTerminationState.Terminated != nil: + if strings.TrimSpace(cs.LastTerminationState.Terminated.Message) == kubernetesJobWriting { + return true, nil + } + } + } + } + return false, nil +} + +// kubernetesJobStarted reports whether a container of the restore Job has +// run, which is what kubernetesJobReachedWrite falls back on once the Job's +// Pods are gone. +func (r *RestoreJobReconciler) kubernetesJobStarted(ctx context.Context, ns, jobName string) (bool, error) { + pods := &corev1.PodList{} + if err := r.List(ctx, pods, client.InNamespace(ns), client.MatchingLabels{"job-name": jobName}); err != nil { + return false, err + } + for _, p := range pods.Items { + for _, cs := range p.Status.ContainerStatuses { + if cs.State.Running != nil || cs.State.Terminated != nil || cs.LastTerminationState.Terminated != nil { + return true, nil + } + } + } + return false, nil +} + +// runKubernetesJob runs the strategy Job in the given mode under the given +// name against ds. It returns done once the Job completed, or a failure +// message once it failed. +func (r *RestoreJobReconciler) runKubernetesJob(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, mode, name string, ds kubernetesDatastore) (bool, string, error) { + if ds.Prefix == "" || len(ds.Endpoints) == 0 { + return false, "the Backup records no etcd datastore prefix or endpoints", nil + } + rendered, err := renderKubernetesTemplate(strategy.Spec.Template, backup.Spec.ApplicationRef.Name, rj.Namespace, + mode, backup.Spec.DriverMetadata[kubernetesObjectMetaKey], ds, kubernetesStrategyParameters(backup)) + if err != nil { + return false, fmt.Sprintf("failed to template Kubernetes strategy: %v", err), nil + } + job, err := ensureBackupBatchJob(ctx, r.Client, r.Scheme, rj, rj.Namespace, name, + map[string]string{ + kubernetesLabelMode: mode, + backupsv1alpha1.OwningJobNameLabel: rj.Name, + backupsv1alpha1.OwningJobNamespaceLabel: rj.Namespace, + }, rendered) + if err != nil { + return false, "", err + } + switch jobConditionState(job) { + case batchv1.JobComplete: + return true, "", nil + case batchv1.JobFailed: + msg := jobFailureMessage(job) + if msg == "" { + msg = fmt.Sprintf("%s Job reported Failed", mode) + } + return false, msg, nil + } + return false, "", nil +} + +// runKubernetesVeleroRestore restores one phase of the Velero backup: +// control-plane Secrets, or the cluster's data DataVolumes and PVCs. It +// returns done once the Velero Restore completed. +func (r *RestoreJobReconciler) runKubernetesVeleroRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, strategy *strategyv1alpha1.Kubernetes, phase, release string) (bool, string, error) { + veleroBackup := backup.Spec.DriverMetadata[veleroBackupNameMetadataKey] + if veleroBackup == "" { + return false, "the Backup records no Velero backup", nil + } + labels := map[string]string{ + backupsv1alpha1.OwningJobNameLabel: rj.Name, + backupsv1alpha1.OwningJobNamespaceLabel: rj.Namespace, + kubernetesRestorePhaseLabel: phase, + } + restores := &velerov1.RestoreList{} + if err := r.List(ctx, restores, client.InNamespace(veleroNamespace), client.MatchingLabels(labels)); err != nil { + return false, "", err + } + if len(restores.Items) == 0 { + spec, err := kubernetesVeleroRestoreSpec(strategy, backup, phase, release) + if err != nil { + return false, fmt.Sprintf("failed to build Velero Restore: %v", err), nil + } + spec.BackupName = veleroBackup + cm, err := r.kubernetesResourceModifiers(ctx, rj, phase, labels) + if err != nil { + return false, "", err + } + spec.ResourceModifier = &corev1.TypedLocalObjectReference{APIGroup: new(""), Kind: "ConfigMap", Name: cm.Name} + restore := &velerov1.Restore{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s.%s-%s-", rj.Namespace, rj.Name, phase), + Namespace: veleroNamespace, + Labels: labels, + }, + Spec: *spec, + } + if err := r.Create(ctx, restore); err != nil { + return false, "", err + } + if r.Recorder != nil { + r.Recorder.Event(rj, corev1.EventTypeNormal, "VeleroRestoreCreated", fmt.Sprintf("Created Velero Restore %s/%s", veleroNamespace, restore.Name)) + } + return false, "", nil + } + restore := &restores.Items[0] + switch restore.Status.Phase { + case velerov1.RestorePhaseCompleted: + return true, "", nil + case velerov1.RestorePhaseFailed, velerov1.RestorePhasePartiallyFailed, velerov1.RestorePhaseFailedValidation: + msg := fmt.Sprintf("Velero Restore %s/%s finished %s", restore.Namespace, restore.Name, restore.Status.Phase) + if restore.Status.FailureReason != "" { + msg += ": " + restore.Status.FailureReason + } + return false, msg, nil + } + return false, "", nil +} + +// kubernetesVeleroRestoreSpec starts from the strategy's restore spec and +// narrows it to one phase. The selectors mirror the ones the shipped backup +// spec uses, so a phase restores exactly its share of the backup. +func kubernetesVeleroRestoreSpec(strategy *strategyv1alpha1.Kubernetes, backup *backupsv1alpha1.Backup, phase, release string) (*velerov1.RestoreSpec, error) { + spec := &velerov1.RestoreSpec{} + if strategy.Spec.Velero.RestoreSpec != nil { + rendered, err := template.Template(strategy.Spec.Velero.RestoreSpec, map[string]any{ + "Application": map[string]any{ + "metadata": map[string]any{"name": backup.Spec.ApplicationRef.Name, "namespace": backup.Namespace}, + "kind": backup.Spec.ApplicationRef.Kind, + }, + "Parameters": kubernetesStrategyParameters(backup), + }) + if err != nil { + return nil, err + } + spec = rendered + } + spec.IncludedNamespaces = []string{backup.Namespace} + spec.LabelSelector = nil + switch phase { + case kubernetesPhaseSecrets: + spec.IncludedResources = []string{"secrets"} + spec.OrLabelSelectors = []*metav1.LabelSelector{ + {MatchLabels: map[string]string{kamajiNameLabel: release}}, + {MatchLabels: map[string]string{helmInstanceLabel: release}}, + } + case kubernetesPhaseVolumes: + spec.IncludedResources = []string{"persistentvolumeclaims", "datavolumes.cdi.kubevirt.io"} + spec.OrLabelSelectors = []*metav1.LabelSelector{ + {MatchLabels: map[string]string{clusterNameLabel: release}}, + } + spec.RestorePVs = new(true) + default: + return nil, fmt.Errorf("unknown restore phase %q", phase) + } + return spec, nil +} + +// kubernetesResourceModifiers creates the ConfigMap of resource modifiers for +// a phase. Restored Secrets drop their ownerReferences, which point at +// objects of the deleted cluster and would let the garbage collector delete +// them again; restored PVCs and DataVolumes let CDI adopt the restored claim. +func (r *RestoreJobReconciler) kubernetesResourceModifiers(ctx context.Context, rj *backupsv1alpha1.RestoreJob, phase string, labels map[string]string) (*corev1.ConfigMap, error) { + var rules []resourceModifierRule + switch phase { + case kubernetesPhaseSecrets: + patch, err := marshalPatchData(map[string]any{"metadata": map[string]any{"ownerReferences": nil}}) + if err != nil { + return nil, err + } + rules = append(rules, resourceModifierRule{ + Conditions: resourceModifierConditions{GroupResource: "secrets", ResourceNameRegex: ".*", Namespaces: []string{rj.Namespace}}, + MergePatches: []mergePatch{{PatchData: patch}}, + }) + case kubernetesPhaseVolumes: + patch, err := marshalPatchData(map[string]any{"metadata": map[string]any{"annotations": map[string]string{cdiAllowClaimAdoption: "true"}}}) + if err != nil { + return nil, err + } + for _, gr := range []string{"persistentvolumeclaims", "datavolumes.cdi.kubevirt.io"} { + rules = append(rules, resourceModifierRule{ + Conditions: resourceModifierConditions{GroupResource: gr, ResourceNameRegex: ".*", Namespaces: []string{rj.Namespace}}, + MergePatches: []mergePatch{{PatchData: patch}}, + }) + } + } + data, err := yaml.Marshal(resourceModifiers{Version: "v1", ResourceModifierRules: rules}) + if err != nil { + return nil, err + } + name := fmt.Sprintf("restore-modifiers-%s-%s-%s", rj.Namespace, rj.Name, phase) + if len(name) > 253 { + name = name[:253] + } + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: veleroNamespace, Labels: labels}, + Data: map[string]string{"resource-modifier-rules.yaml": string(data)}, + } + if err := r.Create(ctx, cm); err != nil { + if !apierrors.IsAlreadyExists(err) { + return nil, err + } + existing := &corev1.ConfigMap{} + if err := r.Get(ctx, types.NamespacedName{Namespace: veleroNamespace, Name: name}, existing); err != nil { + return nil, err + } + existing.Data = cm.Data + if err := r.Update(ctx, existing); err != nil { + return nil, err + } + return existing, nil + } + return cm, nil +} + +func (r *RestoreJobReconciler) setKubernetesRestoreCondition(ctx context.Context, rj *backupsv1alpha1.RestoreJob, condType, message string) (ctrl.Result, error) { + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: condType, + Status: metav1.ConditionTrue, + Reason: "Done", + Message: message, + }) + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: kubernetesPollInterval}, nil +} + +func (r *RestoreJobReconciler) markKubernetesRestoreSucceeded(ctx context.Context, rj *backupsv1alpha1.RestoreJob, message string) (ctrl.Result, error) { + now := metav1.Now() + rj.Status.CompletedAt = &now + rj.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "RestoreCompleted", + Message: message, + }) + return ctrl.Result{}, r.Status().Update(ctx, rj) +} diff --git a/internal/backupcontroller/mariadb_artifact_size.go b/internal/backupcontroller/mariadb_artifact_size.go index 6397325ca..4e0525517 100644 --- a/internal/backupcontroller/mariadb_artifact_size.go +++ b/internal/backupcontroller/mariadb_artifact_size.go @@ -39,19 +39,19 @@ type mariadbDump struct { LastModified time.Time } -// mariadbArtifactSizeBytes measures the dump mariadb-operator uploaded for -// mdbBackup. Returns 0 - leaving Artifact.SizeBytes absent - when the dump -// cannot be found or told apart: an unknown size must not be reported as a -// measured one. -func (r *BackupJobReconciler) mariadbArtifactSizeBytes(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) int64 { +// attributedMariaDBDump returns the dump mariadb-operator uploaded for +// mdbBackup. ok is false when the dump cannot be found or told apart, and +// the Backup then reports neither a size nor a dump: an unknown dump must +// not be reported as a measured one. +func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (dump mariadbDump, ok bool) { ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout) defer cancel() dump, err := findMariaDBDump(ctx, r.objectStore(), mdbBackup, completedAt) if err != nil { - getLogger(ctx).Info("MariaDB backup size not measured; leaving it unset", "backup", mdbBackup.Name, "error", err) - return 0 + getLogger(ctx).Info("MariaDB dump not attributed; leaving its size and name unset", "backup", mdbBackup.Name, "error", err) + return mariadbDump{}, false } - return dump.Size + return dump, true } // findMariaDBDump lists the bucket mdbBackup wrote to and returns the one @@ -60,22 +60,26 @@ func (r *BackupJobReconciler) mariadbArtifactSizeBytes(ctx context.Context, mdbB // from can have been edited since, and pointing at another bucket now says // nothing about where this dump went. func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) { + lister, _, err := mariadbStorage(ctx, store, mdbBackup) + if err != nil { + return mariadbDump{}, err + } + return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt, false) +} + +// findMariaDBDumpWith is findMariaDBDump reading the bucket through lister, +// for a caller that reaches the bucket by another endpoint than the one +// mdbBackup names. withUnfinished lets runs that have not completed contest +// the dump too, as neighbourMariaDBRuns describes. +func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time, withUnfinished bool) (mariadbDump, error) { s3 := mdbBackup.Spec.Storage.S3 if s3 == nil { return mariadbDump{}, errors.New("the operator Backup does not store to S3") } - others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup) + others, err := neighbourMariaDBRuns(ctx, reader, mdbBackup, withUnfinished) if err != nil { return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err) } - conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3) - if err != nil { - return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err) - } - lister, err := store.lister(conn) - if err != nil { - return mariadbDump{}, fmt.Errorf("build S3 client: %w", err) - } // Non-recursive, as mariadb-operator lists it: the default prefix is also // the default CNPG destinationPath, so an app of each kind with the same // name shares it and a recursive listing would walk the WAL archive. @@ -88,6 +92,44 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt return matchMariaDBDump(objects, prefix, own, others) } +// mariadbDumpExists reports whether the dump file name is still in the +// storage mdbBackup wrote to. +func mariadbDumpExists(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, name string) (bool, error) { + lister, s3, err := mariadbStorage(ctx, store, mdbBackup) + if err != nil { + return false, err + } + key := mariadbOperatorS3Prefix(s3.Prefix) + name + objects, err := lister.ListObjects(ctx, s3.Bucket, key, false) + if err != nil { + return false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) + } + for _, obj := range objects { + if obj.Key == key { + return true, nil + } + } + return false, nil +} + +// mariadbStorage opens the bucket mdbBackup wrote to, with the credentials +// its own spec names. +func mariadbStorage(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup) (s3ObjectLister, *mariadbtypes.S3Storage, error) { + s3 := mdbBackup.Spec.Storage.S3 + if s3 == nil { + return nil, nil, errors.New("the operator Backup does not store to S3") + } + conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3) + if err != nil { + return nil, nil, fmt.Errorf("resolve S3 credentials: %w", err) + } + lister, err := store.lister(conn) + if err != nil { + return nil, nil, fmt.Errorf("build S3 client: %w", err) + } + return lister, s3, nil +} + // mariadbRun is the span in which an operator Backup can have written its // dump: its lifetime, widened by mariadbSizeProbeClockSkew on each side. type mariadbRun struct { @@ -95,6 +137,10 @@ type mariadbRun struct { lower, upper time.Time } +// mariadbUnfinishedRunEnd closes the window of a run that has not completed, +// which can still write its dump at any time. +var mariadbUnfinishedRunEnd = time.Date(9999, time.December, 31, 0, 0, 0, 0, time.UTC) + func newMariaDBRun(b *mariadbtypes.Backup, created, completed time.Time) mariadbRun { return mariadbRun{ name: b.Namespace + "/" + b.Name, @@ -120,8 +166,8 @@ func (r mariadbRun) placesOnEitherClock(d mariadbDump) bool { // dump in the window on one clock only may be this run's own, misplaced by // a skewed clock, which leaves any other candidate unproven. A neighbour's // dump is still taken if this run's own is missing from the listing, or -// misplaced on both clocks, and the neighbour's operator Backup is gone or -// has not completed. +// misplaced on both clocks, and the neighbour's operator Backup is gone or, +// unless others includes unfinished runs, has not completed. func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, others []mariadbRun) (mariadbDump, error) { var near []mariadbDump for _, obj := range objects { @@ -153,14 +199,17 @@ func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, oth return d, nil } -// finishedMariaDBRuns returns the runs of the other operator Backups that +// neighbourMariaDBRuns returns the runs of the other operator Backups that // write under the same bucket and prefix as b and have completed, whether -// their Job succeeded or failed. One still running is left out: its window -// has no end yet, and an operator Backup stuck short of Complete, as one -// whose Job pod never starts is, would otherwise contest every later run of -// the application for good. The endpoint is not compared, since one has -// several spellings, and a false neighbour only leaves a size unset. -func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup) ([]mariadbRun, error) { +// their Job succeeded or failed. One still running is left out unless +// withUnfinished is set: its window has no end yet, and an operator Backup +// stuck short of Complete, as one whose Job pod never starts is, would +// otherwise contest every later run of the application for good. That is +// the price of deleting a dump, since a run can upload its dump and prune +// older ones through maxRetention before it completes. The endpoint is not +// compared, since one has several spellings, and a false neighbour only +// keeps a dump unattributed. +func neighbourMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup, withUnfinished bool) ([]mariadbRun, error) { list := &mariadbtypes.BackupList{} if err := reader.List(ctx, list); err != nil { return nil, err @@ -177,6 +226,9 @@ func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbty } cond := apimeta.FindStatusCondition(other.Status.Conditions, mariadbtypes.ConditionTypeComplete) if cond == nil || cond.Status != metav1.ConditionTrue { + if withUnfinished { + runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, mariadbUnfinishedRunEnd)) + } continue } runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, cond.LastTransitionTime.Time)) diff --git a/internal/backupcontroller/mariadb_artifact_size_test.go b/internal/backupcontroller/mariadb_artifact_size_test.go index 9d649c15c..239ec296c 100644 --- a/internal/backupcontroller/mariadb_artifact_size_test.go +++ b/internal/backupcontroller/mariadb_artifact_size_test.go @@ -29,6 +29,14 @@ func dumpKey(prefix string, ts time.Time, compression string) string { return fmt.Sprintf("%sbackup.%s.%s.sql", prefix, ts.UTC().Format(time.RFC3339), compression) } +// probedSize is the size a Backup reports for a probe result. +func probedSize(dump mariadbDump, attributed bool) int64 { + if !attributed { + return 0 + } + return dump.Size +} + // mariadbBackupSucceeded is the Complete condition mariadb-operator sets on a // Backup whose Job succeeded. A failed Job also gets Status=True, with // Reason=JobFailed, so a fixture without the reason is not a success. @@ -105,7 +113,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)}, {Key: prefix + "0-backup-target.txt", Size: 7}, }} - got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed) + got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)) if got != 4242 { t.Errorf("size = %d, want 4242", got) } @@ -130,7 +138,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { } l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 3, LastModified: created.Add(time.Minute)}}} r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), ca) - if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed); got != 3 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed)); got != 3 { t.Errorf("size = %d, want 3", got) } if !l.conn.Secure || string(l.conn.CAPEM) != "PEM" { @@ -148,7 +156,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { } l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4, LastModified: created.Add(time.Minute)}}} r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), sts) - if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed); got != 4 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed)); got != 4 { t.Errorf("size = %d, want 4", got) } if l.conn.SessionToken != "TOKEN" { @@ -160,7 +168,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9, LastModified: created.Add(time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 9 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 9 { t.Errorf("size = %d, want 9", got) } }) @@ -173,7 +181,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(-4*time.Minute), "gzip"), Size: 7, LastModified: created.Add(time.Minute)}, {Key: dumpKey(prefix, created.Add(30*time.Second), "gzip"), Size: 666, LastModified: created.Add(-4 * time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0: no dump is inside the window on both clocks", got) } }) @@ -182,7 +190,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, completed.Add(30*time.Second), "gzip"), Size: 5, LastModified: completed.Add(30 * time.Second)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 5 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 5 { t.Errorf("size = %d, want 5", got) } }) @@ -191,7 +199,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds")) r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } withShortProbeTimeout(t, func() { - if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -202,7 +210,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { {Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1, LastModified: created.Add(time.Minute)}, {Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2, LastModified: created.Add(2 * time.Minute)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0 (ambiguous)", got) } }) @@ -211,14 +219,14 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { l := &fakeS3Lister{objects: []s3ObjectInfo{ {Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1, LastModified: completed.Add(time.Hour)}, }} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) t.Run("list error leaves size unset", func(t *testing.T) { l := &fakeS3Lister{err: errors.New("AccessDenied")} - if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -226,7 +234,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) { t.Run("missing credentials Secret leaves size unset", func(t *testing.T) { c := newMariaDBStrategyTestClient(t) r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()} - if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 { t.Errorf("size = %d, want 0", got) } }) @@ -464,7 +472,7 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization") gotDelimiter = req.URL.Query().Get("delimiter") w.Header().Set("Content-Type", "application/xml") - fmt.Fprintf(w, ` + _, _ = fmt.Fprintf(w, ` bkttenant/db/11000false %s31337"x"2026-10-21T10:01:00.000Z @@ -481,7 +489,7 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) { s3.Prefix = "tenant/db/" s3.Region = "us-east-1" - if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute)); got != 31337 { + if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute))); got != 31337 { t.Fatalf("size = %d, want 31337", got) } if gotPath != "/bkt/" || gotPrefix != "tenant/db/" || gotDelimiter != "/" { @@ -523,6 +531,69 @@ func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) { } } +// TestCreateMariaDBBackupArtifact_RecordsDump pins that the Backup names +// its dump only when the probe attributed one to the operator Backup. +func TestCreateMariaDBBackupArtifact_RecordsDump(t *testing.T) { + apps := mariadbapp.GroupName + created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + completed := metav1.NewTime(created.Add(3 * time.Minute)) + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "src/" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + own := s3ObjectInfo{Key: "src/backup.2026-10-21T10:01:00Z.gzip.sql", Size: 2048, LastModified: created.Add(2 * time.Minute)} + second := s3ObjectInfo{Key: "src/backup.2026-10-21T10:02:00Z.gzip.sql", Size: 1, LastModified: created.Add(2 * time.Minute)} + offset := s3ObjectInfo{Key: "src/backup.2026-10-21T14:01:00+04:00.sql", Size: 7, LastModified: created.Add(2 * time.Minute)} + + cases := []struct { + name string + lister *fakeS3Lister + wantFile, wantTime string + }{ + {name: "an attributed dump is recorded", lister: &fakeS3Lister{objects: []s3ObjectInfo{own}}, + wantFile: "backup.2026-10-21T10:01:00Z.gzip.sql", wantTime: "2026-10-21T10:01:00Z"}, + {name: "a dump named with an offset is timed in UTC", lister: &fakeS3Lister{objects: []s3ObjectInfo{offset}}, + wantFile: "backup.2026-10-21T14:01:00+04:00.sql", wantTime: "2026-10-21T10:01:00Z"}, + {name: "two dumps in the lifetime record none", lister: &fakeS3Lister{objects: []s3ObjectInfo{own, second}}}, + {name: "no dump in the lifetime records none", lister: &fakeS3Lister{}}, + {name: "a failed listing records none", lister: &fakeS3Lister{err: errors.New("AccessDenied")}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mdbBackup := operatorBackupStoringTo(t, rendered.Storage.S3, created) + mdbBackup.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(completed)} + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-dump"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + }, + } + r := reconcilerWithLister(t, tc.lister, job, s3CredsSecret("tenant", "creds")) + if _, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, rendered); err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + persisted := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "bj-dump"}, persisted); err != nil { + t.Fatalf("get Backup: %v", err) + } + md := persisted.Spec.DriverMetadata + file, hasFile := md[mariadbDumpFileKey] + at, hasTime := md[mariadbDumpTimeKey] + if tc.wantFile == "" { + if hasFile || hasTime { + t.Errorf("driverMetadata = %v; want no dump recorded", md) + } + return + } + if file != tc.wantFile || at != tc.wantTime { + t.Errorf("recorded dump %q at %q; want %q at %q", file, at, tc.wantFile, tc.wantTime) + } + }) + } +} + // TestCreateMariaDBBackupArtifact_StrategyEditedMidBackup pins that the // probe reads where the operator Backup wrote its dump, not where the // strategy, edited while the backup ran, now points. diff --git a/internal/backupcontroller/mariadb_restore_dump_test.go b/internal/backupcontroller/mariadb_restore_dump_test.go new file mode 100644 index 000000000..5ec96e663 --- /dev/null +++ b/internal/backupcontroller/mariadb_restore_dump_test.go @@ -0,0 +1,531 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +// The source run of the restore fixture: operator Backup op-bk of +// application src, alive from mdbRestoreT0 to mdbRestoreT0+40s, writing +// under tenant/src/ with mdbRestoreDump as its dump. +var mdbRestoreT0 = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + +const ( + mdbRestorePrefix = "tenant/src/" + mdbRestoreDump = "backup.2026-10-21T10:00:05Z.gzip.sql" + mdbRestoreTime = "2026-10-21T10:00:05Z" + mdbNewerDump = "backup.2026-10-21T11:00:05Z.gzip.sql" +) + +type mariadbRestoreFixture struct { + c client.Client + r *RestoreJobReconciler + rj *backupsv1alpha1.RestoreJob + backup *backupsv1alpha1.Backup + src *mariadbtypes.Backup + lister *fakeS3Lister + events *record.FakeRecorder +} + +// newMariaDBRestoreFixture seeds a RestoreJob of Backup src-bk, whose +// driverMetadata is driverMD plus the operator Backup's name, into the +// existing application target. extra objects are seeded too. +func newMariaDBRestoreFixture(t *testing.T, driverMD map[string]string, lister *fakeS3Lister, extra ...client.Object) *mariadbRestoreFixture { + t.Helper() + apps := mariadbapp.GroupName + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/src" + src := operatorBackupStoringTo(t, s3, mdbRestoreT0) + src.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)))} + + md := map[string]string{mariadbBackupNameKey: src.Name} + for k, v := range driverMD { + md[k] = v + } + backup := &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-bk"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps}, + TakenAt: metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)), + DriverMetadata: md, + }, + } + started := metav1.Now() + rj := &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"}, + Spec: backupsv1alpha1.RestoreJobSpec{ + BackupRef: corev1.LocalObjectReference{Name: backup.Name}, + TargetApplicationRef: &corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "target", APIGroup: &apps}, + }, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &started, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + target := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("target")}} + objs := append([]client.Object{src, backup, rj, target}, extra...) + c := newMariaDBStrategyTestClient(t, objs...) + fetched := &backupsv1alpha1.RestoreJob{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), fetched); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + events := record.NewFakeRecorder(10) + return &mariadbRestoreFixture{ + c: c, + rj: fetched, + backup: backup, + src: src, + lister: lister, + events: events, + r: &RestoreJobReconciler{ + Client: c, + Recorder: events, + newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) { + lister.conn = conn + return lister, nil + }, + }, + } +} + +func (f *mariadbRestoreFixture) reconcile(t *testing.T) ctrl.Result { + t.Helper() + res, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + return res +} + +func (f *mariadbRestoreFixture) restores(t *testing.T) []mariadbtypes.Restore { + t.Helper() + list := &mariadbtypes.RestoreList{} + if err := f.c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { + t.Fatalf("list operator Restores: %v", err) + } + return list.Items +} + +// wantFailed asserts the RestoreJob failed with a message holding every +// fragment, and that no operator Restore was created. +func (f *mariadbRestoreFixture) wantFailed(t *testing.T, fragments ...string) { + t.Helper() + if f.rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("phase = %q, want Failed", f.rj.Status.Phase) + } + for _, s := range fragments { + if !strings.Contains(f.rj.Status.Message, s) { + t.Errorf("message %q does not contain %q", f.rj.Status.Message, s) + } + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want none", len(got)) + } +} + +// wantWaiting asserts the RestoreJob is kept waiting on the dump check +// with Ready=False saying why, and that no operator Restore was created. +func (f *mariadbRestoreFixture) wantWaiting(t *testing.T, res ctrl.Result, fragments ...string) { + t.Helper() + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed: %s", f.rj.Status.Message) + } + if res.RequeueAfter != mariadbDumpCheckRetryInterval { + t.Errorf("RequeueAfter = %s, want %s", res.RequeueAfter, mariadbDumpCheckRetryInterval) + } + persisted := &backupsv1alpha1.RestoreJob{} + if err := f.c.Get(context.Background(), client.ObjectKeyFromObject(f.rj), persisted); err != nil { + t.Fatalf("get RestoreJob: %v", err) + } + cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready") + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "DumpCheckPending" { + t.Fatalf("Ready = %+v, want False/DumpCheckPending", cond) + } + for _, s := range fragments { + if !strings.Contains(cond.Message, s) { + t.Errorf("Ready message %q does not contain %q", cond.Message, s) + } + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want none", len(got)) + } +} + +// wantRestore asserts exactly one operator Restore of op-bk into target, +// asking for targetTime. +func (f *mariadbRestoreFixture) wantRestore(t *testing.T, targetTime string) { + t.Helper() + got := f.restores(t) + if len(got) != 1 { + t.Fatalf("created %d operator Restores; want one", len(got)) + } + spec := got[0].Spec + if spec.BackupRef == nil || spec.BackupRef.Name != f.src.Name || spec.MariaDBRef.Name != mariadbNameForApp("target") { + t.Errorf("Restore spec = %+v, want backupRef %s into %s", spec, f.src.Name, mariadbNameForApp("target")) + } + if spec.TargetRecoveryTime != targetTime { + t.Errorf("targetRecoveryTime = %q, want %q", spec.TargetRecoveryTime, targetTime) + } +} + +func recordedDump(name string) map[string]string { + return map[string]string{mariadbDumpFileKey: name, mariadbDumpTimeKey: mdbRestoreTime} +} + +// A Backup being deleted is deleting its dump, so a restore that starts +// now can find it gone once the operator reads it, and the operator then +// restores the nearest other dump. The Backup is read from the API server, +// since the copy the reconcile was handed can predate the delete. +func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + present := func() *fakeS3Lister { + return &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}}} + } + cases := map[string]map[string]string{"recorded dump": recordedDump(mdbRestoreDump), "legacy Backup": nil} + for name, md := range cases { + t.Run(name+", being deleted", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, md, present(), creds) + ctx := context.Background() + live := &backupsv1alpha1.Backup{} + if err := f.c.Get(ctx, client.ObjectKeyFromObject(f.backup), live); err != nil { + t.Fatal(err) + } + live.Finalizers = []string{backupFinalizer} + if err := f.c.Update(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Delete(ctx, live); err != nil { + t.Fatal(err) + } + f.reconcile(t) + f.wantFailed(t, "src-bk", "being deleted") + }) + t.Run(name+", already gone", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, md, present(), creds) + if err := f.c.Delete(context.Background(), f.backup.DeepCopy()); err != nil { + t.Fatal(err) + } + f.reconcile(t) + f.wantFailed(t, "src-bk", "is gone") + }) + } + // The refusal applies only before the operator Restore exists. One + // created by an earlier reconcile but not yet in the cache must be + // found and followed, or the RestoreJob fails under a running restore + // and the cleanup deletes the dump that restore reads. + t.Run("an operator Restore the cache has not seen yet is followed", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + ctx := context.Background() + live := &backupsv1alpha1.Backup{} + if err := f.c.Get(ctx, client.ObjectKeyFromObject(f.backup), live); err != nil { + t.Fatal(err) + } + live.Finalizers = []string{backupFinalizer} + if err := f.c.Update(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Delete(ctx, live); err != nil { + t.Fatal(err) + } + if err := f.c.Get(ctx, client.ObjectKeyFromObject(live), live); err != nil || live.DeletionTimestamp.IsZero() { + t.Fatalf("Backup not left being deleted: %v", err) + } + live.ResourceVersion = "" + apiServer := newMariaDBStrategyTestClient(t, live, &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }}) + f.r.APIReader = apiServer + f.reconcile(t) + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed under a running restore: %s", f.rj.Status.Message) + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores; want the existing one followed", len(got)) + } + }) + // A failed lookup says nothing about whether the operator Restore + // exists, so the RestoreJob must not be failed under one that runs. + t.Run("an operator Restore lookup that fails is retried", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup); err == nil { + t.Error("err = nil, want the failed lookup returned for a retry") + } + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Fatalf("RestoreJob failed on a lookup error: %s", f.rj.Status.Message) + } + if got := f.restores(t); len(got) != 0 { + t.Errorf("created %d operator Restores without knowing whether one exists", len(got)) + } + }) + t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds) + f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + f.wantWaiting(t, f.reconcile(t), "src-bk", "apiserver unavailable") + }) +} + +func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + + t.Run("restores the recorded dump by its own timestamp", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{ + {Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}, + {Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}, + }} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds) + f.reconcile(t) + f.wantRestore(t, mdbRestoreTime) + if l.bucket != "bkt" || l.prefix != mdbRestorePrefix+mdbRestoreDump || l.conn.AccessKey != "AKID" { + t.Errorf("checked %s/%s as %q, want bkt/%s%s with the operator Backup's credentials", l.bucket, l.prefix, l.conn.AccessKey, mdbRestorePrefix, mdbRestoreDump) + } + }) + + t.Run("a recorded dump gone from storage fails the RestoreJob", func(t *testing.T) { + l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}}} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds) + f.reconcile(t) + f.wantFailed(t, mdbRestoreDump, "no longer exists in storage", "s3://bkt/"+mdbRestorePrefix) + }) + + t.Run("a storage that cannot be read keeps the RestoreJob waiting", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds) + f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "Access Denied.") + }) + + t.Run("missing credentials keep the RestoreJob waiting", func(t *testing.T) { + l := &fakeS3Lister{} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l) + f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "resolve S3 credentials") + if l.calls != 0 { + t.Errorf("S3 listed %d times without credentials", l.calls) + } + }) + + t.Run("a hanging endpoint is given up at the check timeout", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{}, creds) + f.r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + var res ctrl.Result + var err error + withShortDumpCheckTimeout(t, func() { res, err = f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) }) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + f.wantWaiting(t, res, mdbRestoreDump, context.DeadlineExceeded.Error()) + }) + + t.Run("a storage still unreadable at the deadline fails the RestoreJob with the cause", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds) + long := metav1.NewTime(time.Now().Add(-mariadbDefaultRestoreDeadline - time.Minute)) + f.rj.Status.StartedAt = &long + f.reconcile(t) + f.wantFailed(t, mdbRestoreDump, "Access Denied.", "gave up after") + }) + + t.Run("a recorded name that is not a dump fails the RestoreJob unchecked", func(t *testing.T) { + l := &fakeS3Lister{} + f := newMariaDBRestoreFixture(t, recordedDump("dump.sql"), l, creds) + f.reconcile(t) + f.wantFailed(t, `"dump.sql"`) + if l.calls != 0 { + t.Errorf("S3 listed %d times for a name that is not a dump", l.calls) + } + }) + + t.Run("an operator Restore created earlier is followed without a check", func(t *testing.T) { + l := &fakeS3Lister{err: errors.New("Access Denied.")} + existing := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }} + f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds, existing) + res := f.reconcile(t) + if l.calls != 0 { + t.Errorf("S3 listed %d times for a RestoreJob whose operator Restore exists", l.calls) + } + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed || res.RequeueAfter != mariadbPollInterval { + t.Errorf("phase %q, requeue %s; want the existing Restore polled", f.rj.Status.Phase, res.RequeueAfter) + } + if got := f.restores(t); len(got) != 1 { + t.Errorf("%d operator Restores; want only the existing one", len(got)) + } + }) +} + +// TestReconcileMariaDBRestore_LegacyBackup drives Backups that record no +// dump: one proven to belong to the source run is restored by its +// timestamp, and anything else restores the newest dump with a Warning. +func TestReconcileMariaDBRestore_LegacyBackup(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + own := s3ObjectInfo{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)} + newer := s3ObjectInfo{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)} + second := s3ObjectInfo{Key: mdbRestorePrefix + "backup.2026-10-21T10:00:30Z.gzip.sql", LastModified: mdbRestoreT0.Add(35 * time.Second)} + + t.Run("a dump proven to be the run's own is restored by its timestamp", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{objects: []s3ObjectInfo{own, newer}}, creds) + f.reconcile(t) + f.wantRestore(t, mdbRestoreTime) + if ev := drainEvents(f.events); len(ev) != 0 { + t.Errorf("events %q; want none for a proven dump", ev) + } + }) + + cases := []struct { + name string + lister *fakeS3Lister + cause string + }{ + {name: "two dumps in the run's lifetime", lister: &fakeS3Lister{objects: []s3ObjectInfo{own, second, newer}}, cause: "cannot be told apart"}, + {name: "no dump in the run's lifetime", lister: &fakeS3Lister{objects: []s3ObjectInfo{newer}}, cause: "no dump in the bucket belongs"}, + {name: "a storage that cannot be read", lister: &fakeS3Lister{err: errors.New("Access Denied.")}, cause: "Access Denied."}, + } + wantBestEffort := func(t *testing.T, f *mariadbRestoreFixture, cause string) { + t.Helper() + f.wantRestore(t, "") + ev := drainEvents(f.events) + if len(ev) != 1 || !strings.HasPrefix(ev[0], "Warning BestEffortDump ") || !strings.Contains(ev[0], cause) || !strings.Contains(ev[0], "src-bk records no dump") { + t.Errorf("events %q; want one BestEffortDump warning naming the Backup and %q", ev, cause) + } + } + for _, tc := range cases { + t.Run(tc.name+" restores the newest dump with a warning", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, nil, tc.lister, creds) + f.reconcile(t) + wantBestEffort(t, f, tc.cause) + }) + } + + t.Run("a hanging endpoint is given up at the check timeout", func(t *testing.T) { + f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{}, creds) + f.r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil } + var err error + withShortDumpCheckTimeout(t, func() { _, err = f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) }) + if err != nil { + t.Fatalf("reconcileMariaDBRestore: %v", err) + } + wantBestEffort(t, f, context.DeadlineExceeded.Error()) + }) +} + +// TestReconcileMariaDBRestore_FailedBackupRun drives a Backup made of an +// operator Backup whose Job failed: there is no dump of its own to +// restore, whatever the Backup records. +func TestReconcileMariaDBRestore_FailedBackupRun(t *testing.T) { + creds := s3CredsSecret("tenant", "creds") + failed := func(status metav1.ConditionStatus) []metav1.Condition { + return []metav1.Condition{{ + Type: mariadbtypes.ConditionTypeComplete, Status: status, Reason: mariadbtypes.ConditionReasonJobFailed, + Message: "Failed", LastTransitionTime: metav1.NewTime(mdbRestoreT0.Add(3 * time.Minute)), + }} + } + listing := []s3ObjectInfo{{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}} + + cases := []struct { + name string + driverMD map[string]string + status metav1.ConditionStatus + }{ + {name: "Complete=True/JobFailed with no recorded dump", status: metav1.ConditionTrue}, + {name: "Complete=True/JobFailed with a recorded dump", driverMD: recordedDump(mdbRestoreDump), status: metav1.ConditionTrue}, + {name: "Complete=False/JobFailed", status: metav1.ConditionFalse}, + } + for _, tc := range cases { + t.Run(tc.name+" fails the RestoreJob", func(t *testing.T) { + l := &fakeS3Lister{objects: listing} + f := newMariaDBRestoreFixture(t, tc.driverMD, l, creds) + f.src.Status.Conditions = failed(tc.status) + if err := f.c.Update(context.Background(), f.src); err != nil { + t.Fatalf("update operator Backup: %v", err) + } + f.reconcile(t) + f.wantFailed(t, "k8s.mariadb.com/Backup tenant/op-bk failed", "left no dump to restore") + if l.calls != 0 { + t.Errorf("S3 listed %d times for a failed backup run", l.calls) + } + }) + } + + t.Run("an operator Restore created earlier is followed", func(t *testing.T) { + existing := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", Name: "rj-abcde", + Labels: map[string]string{ + backupsv1alpha1.OwningJobNameLabel: "rj", + backupsv1alpha1.OwningJobNamespaceLabel: "tenant", + }, + }} + f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{objects: listing}, creds, existing) + f.src.Status.Conditions = failed(metav1.ConditionTrue) + if err := f.c.Update(context.Background(), f.src); err != nil { + t.Fatalf("update operator Backup: %v", err) + } + f.reconcile(t) + if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed { + t.Errorf("RestoreJob failed although its operator Restore was already created: %s", f.rj.Status.Message) + } + }) +} + +func drainEvents(r *record.FakeRecorder) []string { + var out []string + for { + select { + case e := <-r.Events: + out = append(out, e) + default: + return out + } + } +} + +// withShortDumpCheckTimeout shortens the restore's dump check timeout and +// fails the test if fn is not back well before the default would expire. +func withShortDumpCheckTimeout(t *testing.T, fn func()) { + t.Helper() + saved := mariadbRestoreDumpCheckTimeout + mariadbRestoreDumpCheckTimeout = 50 * time.Millisecond + defer func() { mariadbRestoreDumpCheckTimeout = saved }() + done := make(chan struct{}) + go func() { + fn() + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("the dump check did not give up at its timeout") + } +} diff --git a/internal/backupcontroller/mariadbstrategy_cleanup.go b/internal/backupcontroller/mariadbstrategy_cleanup.go new file mode 100644 index 000000000..912bb0038 --- /dev/null +++ b/internal/backupcontroller/mariadbstrategy_cleanup.go @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +const ( + mariadbSkipArtifactCleanupAnnotation = "backups.cozystack.io/skip-artifact-cleanup" + + // Retry cadence while a dump cannot be deleted or a restore from the + // Backup is still running. + mariadbCleanupRetryInterval = 30 * time.Second +) + +// mariadbDumpCleanupTimeout bounds the object-store round trips of one +// cleanup attempt. The Backup controller has a single worker, so an +// unreachable endpoint stalls every other Backup for this long. A variable +// so tests can shorten it. +var mariadbDumpCleanupTimeout = 10 * time.Second + +// cleanupMariaDBBackup deletes the dump a MariaDB Backup was taken as, then +// the operator Backup that wrote it. Every run of an application writes +// under one prefix, so the only key deleted is one proven to be this +// Backup's: the dump it records, unless contestedMariaDBDump finds another +// run that could have written it, or, for a Backup taken before dumps were +// recorded, the one findMariaDBDump attributes to its operator Backup. When +// no dump is proven the Backup is released with a Warning and nothing is +// deleted, the operator Backup included, since it is what any later +// attribution would start from. A failed delete holds the Backup and +// retries, as the Redis and Rabbitmq cleanups do. +func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) { + logger := getLogger(ctx) + if backup.Annotations[mariadbSkipArtifactCleanupAnnotation] == "true" { + logger.Debug("skipping MariaDB artifact cleanup per annotation", "backup", backup.Name, "annotation", mariadbSkipArtifactCleanupAnnotation) + return ctrl.Result{}, nil + } + + opName := backup.Spec.DriverMetadata[mariadbBackupNameKey] + if opName == "" { + return r.releaseMariaDBCleanup(ctx, backup, "the dump", "the Backup records no k8s.mariadb.com/Backup"), nil + } + + // The operator Restore reads the dump through the operator Backup, so + // neither may go while a restore from this Backup runs. + restore, err := r.activeRestoreOf(ctx, backup, opName) + if err != nil { + return ctrl.Result{}, err + } + if restore != "" { + logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restore", restore) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactCleanupWaiting", + "%s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump", + restore, mariadbSkipArtifactCleanupAnnotation) + } + return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil + } + + op := &mariadbtypes.Backup{} + if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: opName}, op); err != nil { + if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) { + return r.releaseMariaDBCleanup(ctx, backup, "the dump", + fmt.Sprintf("k8s.mariadb.com/Backup %s is gone, so the storage and credentials its dump was written with cannot be read", opName)), nil + } + return ctrl.Result{}, err + } + + s3 := op.Spec.Storage.S3 + if s3 == nil { + return r.releaseMariaDBCleanup(ctx, backup, fmt.Sprintf("the dump on the storage volume of k8s.mariadb.com/Backup %s", opName), + "PVC and Volume storage is not reachable from the controller"), nil + } + prefix := mariadbOperatorS3Prefix(s3.Prefix) + location := fmt.Sprintf("s3://%s/%s", s3.Bucket, prefix) + + // A failed Job is Complete=True too, with reason JobFailed, and wrote no + // dump: whatever its window holds is another run's. + if cond := apimeta.FindStatusCondition(op.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond == nil || + cond.Status != metav1.ConditionTrue || cond.Reason != mariadbtypes.ConditionReasonJobComplete { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("k8s.mariadb.com/Backup %s did not complete successfully, so no dump is its own", opName)), nil + } + + name := backup.Spec.DriverMetadata[mariadbDumpFileKey] + if name != "" { + if _, ok := parseMariaDBLogicalBackupFile(name); !ok { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)), nil + } + } + + conn, err := r.mariadbCleanupConnection(ctx, op) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("resolve S3 credentials: %w", err)) + } + + s3ctx, cancel := context.WithTimeout(ctx, mariadbDumpCleanupTimeout) + defer cancel() + store := r.objectStore() + lister, err := store.lister(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + absent := false + if name != "" { + other, found, err := contestedMariaDBDump(s3ctx, lister, r.apiReader(), op, name) + absent = !found + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location+name, err) + } + if other != "" { + return r.releaseMariaDBCleanup(ctx, backup, location+name, + fmt.Sprintf("the recorded dump also falls in the lifetime of k8s.mariadb.com/Backup %s, which may have written it; the operator Backup is kept", other)), nil + } + } else { + dump, err := findMariaDBDumpWith(s3ctx, lister, r.apiReader(), op, backup.Spec.TakenAt.Time, true) + if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) { + return r.releaseMariaDBCleanup(ctx, backup, location, + fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil + } + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, err) + } + name = dump.Name + // Recorded first: once the dump is gone a retry could not match it + // again, and could match a neighbour's instead. + base := backup.DeepCopy() + backup.Spec.DriverMetadata[mariadbDumpFileKey] = name + backup.Spec.DriverMetadata[mariadbDumpTimeKey] = dump.Timestamp.UTC().Format(time.RFC3339) + if err := r.Patch(ctx, backup, client.MergeFrom(base)); err != nil { + return ctrl.Result{}, fmt.Errorf("record dump %s on Backup %s: %w", name, backup.Name, err) + } + } + + remover, err := store.remover(conn) + if err != nil { + return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err)) + } + if err := remover.RemoveObject(s3ctx, s3.Bucket, prefix+name); err != nil { + return r.retryMariaDBCleanup(ctx, backup, location+name, err) + } + policy := metav1.DeletePropagationBackground + if err := r.Delete(ctx, op, &client.DeleteOptions{PropagationPolicy: &policy}); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete k8s.mariadb.com/Backup %s/%s: %w", op.Namespace, op.Name, err) + } + if absent { + logger.Info("MariaDB dump already absent; deleted its operator Backup", "backup", backup.Name, "dump", location+name, "endpoint", conn.Endpoint, "operatorBackup", opName) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactAlreadyAbsent", + "%s was not found when listed through %s, so it was deleted earlier or never reached that storage; k8s.mariadb.com/Backup %s is deleted", + location+name, conn.Endpoint, opName) + } + return ctrl.Result{}, nil + } + logger.Info("deleted MariaDB dump and its operator Backup", "backup", backup.Name, "dump", location+name, "operatorBackup", opName) + return ctrl.Result{}, nil +} + +// contestedMariaDBDump returns the name of another operator Backup on op's +// bucket and prefix whose run, finished or not, could have written the dump +// name, or "" when none could, and whether the listing found the dump. The +// size probe that recorded name let only runs finished by then contest it, +// and a run can upload its dump and prune op's through maxRetention before +// it completes. A dump no longer in the bucket is contested by nobody: +// deleting it again removes nothing. +func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, bool, error) { + ts, ok := parseMariaDBLogicalBackupFile(name) + if !ok { + return "", false, fmt.Errorf("%q is not the name of a mariadb-operator dump", name) + } + s3 := op.Spec.Storage.S3 + key := mariadbOperatorS3Prefix(s3.Prefix) + name + objects, err := lister.ListObjects(ctx, s3.Bucket, key, false) + if err != nil { + return "", false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err) + } + i := slices.IndexFunc(objects, func(obj s3ObjectInfo) bool { return obj.Key == key }) + if i < 0 { + return "", false, nil + } + others, err := neighbourMariaDBRuns(ctx, reader, op, true) + if err != nil { + return "", true, fmt.Errorf("list operator Backups: %w", err) + } + d := mariadbDump{Key: key, Name: name, Timestamp: ts, LastModified: objects[i].LastModified} + for _, other := range others { + if other.placesOnEitherClock(d) { + return other.name, true, nil + } + } + return "", true, nil +} + +// activeRestoreOf names a restore from backup that has not finished: a +// RestoreJob of it, or an operator Restore reading the dump of opName, the +// operator Backup it was made of; "" when there is none. The operator +// Restore is checked on its own because a RestoreJob can fail, on its +// deadline, while the Restore it created keeps reading the dump. +// mariadb-operator marks a Restore Complete=True once its Job has +// completed or failed. +func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup, opName string) (string, error) { + list := &backupsv1alpha1.RestoreJobList{} + if err := r.apiReader().List(ctx, list, client.InNamespace(backup.Namespace)); err != nil { + return "", fmt.Errorf("list RestoreJobs: %w", err) + } + for i := range list.Items { + rj := &list.Items[i] + if rj.Spec.BackupRef.Name != backup.Name { + continue + } + if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded && rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed { + return "RestoreJob " + rj.Name, nil + } + } + restores := &mariadbtypes.RestoreList{} + if err := r.apiReader().List(ctx, restores, client.InNamespace(backup.Namespace)); err != nil { + // Without the operator's CRDs no Restore can run; the operator + // Backup read that follows releases the Backup. + if apimeta.IsNoMatchError(err) { + return "", nil + } + return "", fmt.Errorf("list k8s.mariadb.com/Restores: %w", err) + } + for i := range restores.Items { + restore := &restores.Items[i] + if restore.Spec.BackupRef == nil || restore.Spec.BackupRef.Name != opName { + continue + } + if !apimeta.IsStatusConditionTrue(restore.Status.Conditions, mariadbtypes.ConditionTypeComplete) { + return "k8s.mariadb.com/Restore " + restore.Name, nil + } + } + return "", nil +} + +// mariadbCleanupConnection returns how to reach the bucket op wrote to. A +// dump on the platform bucket, which op names by its bucket and by the +// Secret the platform credentials are projected to, is reached as the +// platform reaches it now, so a Backup taken before the endpoint moved is +// still deleted. Any other bucket, or the platform bucket while the +// platform credentials are missing or malformed, is reached as op recorded +// it; a delete that fails there is handled by retryMariaDBCleanup like any +// other. A failed read of the platform credentials is retried instead: the +// recorded endpoint may be the one the platform moved off, where the dump +// would be found missing and reported deleted. +func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) { + if r.CredentialsConfig.IsEnabled() { + conn, bucket, err := r.platformS3Connection(ctx) + var projErr *ProjectionError + switch { + case errors.As(err, &projErr) && projErr.Reason == ReasonAPIError: + return s3Connection{}, err + case err != nil: + getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err) + case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName: + // A strategy can pin a private CA for the platform bucket; the + // platform credentials carry none. + if tls := op.Spec.Storage.S3.TLS; conn.Secure && tls != nil && tls.CASecretKeyRef != nil { + ca, err := r.objectStore().secretValue(ctx, op.Namespace, tls.CASecretKeyRef.Name, tls.CASecretKeyRef.Key) + if err != nil { + return s3Connection{}, err + } + conn.CAPEM = []byte(ca) + } + return conn, nil + } + } + return mariadbS3Connection(ctx, r.objectStore(), op.Namespace, op.Spec.Storage.S3) +} + +// platformS3Connection reads the platform's backup bucket from the Secret +// cozy-backups-creds is projected from, and its TLS from the scheme of +// BACKUP_STORAGE_ENDPOINT, which is how the default strategy sets +// tls.enabled. It reads the source rather than projecting it: projecting +// writes a Secret, which a terminating namespace refuses. +func (r *BackupReconciler) platformS3Connection(ctx context.Context) (s3Connection, string, error) { + cfg := r.CredentialsConfig + src := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Namespace: cfg.SourceNamespace, Name: cfg.SourceSecretName}, src); err != nil { + if apierrors.IsNotFound(err) { + return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMissing, + Message: fmt.Sprintf("source credentials Secret %s/%s not found", cfg.SourceNamespace, cfg.SourceSecretName)} + } + return s3Connection{}, "", &ProjectionError{Reason: ReasonAPIError, Message: fmt.Sprintf("read source credentials Secret: %v", err)} + } + creds, err := parseSourceSecret(src) + if err != nil { + return s3Connection{}, "", err + } + if creds.endpoint == "" { + creds.endpoint = stripScheme(cfg.Endpoint) + } + if creds.endpoint == "" { + return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMalformed, + Message: fmt.Sprintf("endpoint is empty in both source Secret %s/%s and BACKUP_STORAGE_ENDPOINT", cfg.SourceNamespace, cfg.SourceSecretName)} + } + if creds.region == "" { + creds.region = cfg.Region + } + return s3Connection{ + Endpoint: creds.endpoint, + Secure: strings.HasPrefix(cfg.Endpoint, "https://"), + Region: creds.region, + AccessKey: creds.accessKey, + SecretKey: creds.secretKey, + }, creds.bucket, nil +} + +// retryMariaDBCleanup holds the Backup after a failed attempt and says so, +// unless its namespace is going away, where nothing it needs can be read +// any longer and holding it would hold the namespace. +func (r *BackupReconciler) retryMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location string, cause error) (ctrl.Result, error) { + terminating, err := r.namespaceTerminating(ctx, backup.Namespace) + if err != nil { + return ctrl.Result{}, err + } + if terminating { + return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("namespace is terminating and the dump cannot be deleted: %v", cause)), nil + } + getLogger(ctx).Info("cannot delete MariaDB dump; retrying", "backup", backup.Name, "location", location, "error", cause) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactCleanupFailed", + "cannot delete the dump under %s: %v; retrying. Annotate the Backup %s=true to release it and leave the dump", + location, cause, mariadbSkipArtifactCleanupAnnotation) + } + return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil +} + +// releaseMariaDBCleanup lets the Backup go without deleting anything further +// and records a Warning naming what was left, so the give-up is visible. +func (r *BackupReconciler) releaseMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location, reason string) ctrl.Result { + getLogger(ctx).Info("releasing Backup without deleting its dump", "backup", backup.Name, "location", location, "reason", reason) + if r.Recorder != nil { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactNotDeleted", "left %s behind: %s", location, reason) + } + return ctrl.Result{} +} diff --git a/internal/backupcontroller/mariadbstrategy_cleanup_test.go b/internal/backupcontroller/mariadbstrategy_cleanup_test.go new file mode 100644 index 000000000..7efb3046f --- /dev/null +++ b/internal/backupcontroller/mariadbstrategy_cleanup_test.go @@ -0,0 +1,1379 @@ +// SPDX-License-Identifier: Apache-2.0 +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + strategyv1alpha1 "github.com/cozystack/cozystack/api/backups/strategy/v1alpha1" + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp" + "github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes" +) + +// fakeS3Bucket stands in for the object store on both the listing and the +// delete path, and records what was asked of it. +type fakeS3Bucket struct { + objects []s3ObjectInfo + listErr error + removeErr error + + conns []s3Connection + removed []string +} + +func (f *fakeS3Bucket) ListObjects(_ context.Context, _, _ string, _ bool) ([]s3ObjectInfo, error) { + return f.objects, f.listErr +} + +func (f *fakeS3Bucket) RemoveObject(_ context.Context, bucket, key string) error { + if f.removeErr != nil { + return f.removeErr + } + f.removed = append(f.removed, bucket+"/"+key) + return nil +} + +var ( + cleanupCreated = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC) + cleanupCompleted = cleanupCreated.Add(3 * time.Minute) + cleanupDumpName = "backup." + cleanupCreated.Add(time.Minute).Format(time.RFC3339) + ".gzip.sql" +) + +const cleanupPrefix = "tenant/db/" + +// cleanupOperatorBackup is the operator Backup a MariaDB Backup was made of: +// it stored to bkt under tenant/db with the tenant's creds Secret, and its +// Job succeeded. +func cleanupOperatorBackup(t *testing.T) *mariadbtypes.Backup { + t.Helper() + s3 := newRenderedMariaDBTemplate().Storage.S3 + s3.Prefix = "tenant/db" + op := operatorBackupStoringTo(t, s3, cleanupCreated) + op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(cleanupCompleted))} + return op +} + +// cleanupBackup is a MariaDB Backup being deleted, made of operator Backup +// op-bk, recording dumpFile when it is not empty. +func cleanupBackup(dumpFile string) *backupsv1alpha1.Backup { + apiGroup := strategyv1alpha1.GroupVersion.Group + md := map[string]string{ + mariadbBackupNameKey: "op-bk", + mariadbBackupNamespaceKey: "tenant", + } + if dumpFile != "" { + md[mariadbDumpFileKey] = dumpFile + } + now := metav1.Now() + return &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "tenant", + Name: "bk", + DeletionTimestamp: &now, + Finalizers: []string{backupFinalizer}, + }, + Spec: backupsv1alpha1.BackupSpec{ + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: &apiGroup, Kind: strategyv1alpha1.MariaDBStrategyKind, Name: "cozy-default-mariadb", + }, + TakenAt: metav1.NewTime(cleanupCompleted), + DriverMetadata: md, + }, + } +} + +func tenantNamespace() *corev1.Namespace { + return &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "tenant"}} +} + +func cleanupReconciler(t *testing.T, bucket *fakeS3Bucket, objs ...client.Object) (*BackupReconciler, *record.FakeRecorder) { + t.Helper() + return cleanupReconcilerWithClient(newMariaDBStrategyTestClient(t, objs...), bucket) +} + +func cleanupReconcilerWithClient(c client.Client, bucket *fakeS3Bucket) (*BackupReconciler, *record.FakeRecorder) { + rec := record.NewFakeRecorder(20) + return &BackupReconciler{ + Client: c, + Scheme: c.Scheme(), + Recorder: rec, + newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) { + bucket.conns = append(bucket.conns, conn) + return bucket, nil + }, + newS3ObjectRemover: func(conn s3Connection) (s3ObjectRemover, error) { + bucket.conns = append(bucket.conns, conn) + return bucket, nil + }, + }, rec +} + +func operatorBackupExists(t *testing.T, c client.Client) bool { + t.Helper() + err := c.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "op-bk"}, &mariadbtypes.Backup{}) + if err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("get operator Backup: %v", err) + } + return err == nil +} + +func requeues(res ctrl.Result) bool { + return res.Requeue || res.RequeueAfter > 0 +} + +func hasEvent(events []string, prefix string) bool { + for _, e := range events { + if strings.HasPrefix(e, prefix) { + return true + } + } + return false +} + +// assertReleasedUntouched checks a Backup that is let go with nothing +// deleted: no S3 delete, the operator Backup kept, and a Warning saying so. +func assertReleasedUntouched(t *testing.T, r *BackupReconciler, rec *record.FakeRecorder, bucket *fakeS3Bucket, res ctrl.Result, err error) { + t.Helper() + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if len(bucket.removed) != 0 { + t.Errorf("removed %v, want nothing deleted from the bucket", bucket.removed) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted, want it kept") + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } +} + +func TestCleanupMariaDBBackup_DeletesRecordedDumpThenOperatorBackup(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + cleanupPrefix + cleanupDumpName}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } + if len(bucket.conns) == 0 || bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connections = %+v, want the operator Backup's endpoint and Secret", bucket.conns) + } + if events := drainEvents(rec); len(events) != 0 { + t.Errorf("events = %v, want none", events) + } +} + +func TestCleanupMariaDBBackup_FailedDeleteKeepsOperatorBackupAndRetries(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("503 SlowDown")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil { + t.Fatalf("err = %v", err) + } + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted although its dump was not") + } + events := drainEvents(rec) + if !hasEvent(events, "Warning ArtifactCleanupFailed") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed naming the skip annotation", events) + } +} + +func TestCleanupMariaDBBackup_FailedOperatorBackupDeleteRetries(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil { + t.Fatal("err = nil, want the failed operator Backup delete returned so the Backup is held") + } +} + +// A legacy Backup records the dump it proved before deleting it, so a retry +// after a failed operator Backup delete finds the key by name instead of +// matching again against a bucket that no longer holds it. +func TestBackupReconcile_LegacyBackupRetryAfterAFailedOperatorBackupDelete(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + backup := cleanupBackup("") + failDelete := true + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok && failDelete { + return errors.New("apiserver unavailable") + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)} + + if _, err := r.Reconcile(context.Background(), req); err == nil { + t.Fatal("first pass: err = nil, want the failed operator Backup delete returned") + } + got := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), req.NamespacedName, got); err != nil { + t.Fatalf("Backup gone after a failed operator Backup delete: %v", err) + } + if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName { + t.Errorf("driverMetadata = %v, want the proven dump recorded before its delete", got.Spec.DriverMetadata) + } + + // The dump is gone now, as the first pass deleted it. + bucket.objects = nil + failDelete = false + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("second pass: %v", err) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted on the retry") + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) { + t.Errorf("Backup still present (err=%v), want its finalizer removed", err) + } + if events := drainEvents(rec); hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want no claim that the deleted dump was left behind", events) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupThatCannotRecordItsDumpDeletesNothing(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + backup := cleanupBackup("") + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*backupsv1alpha1.Backup); ok { + return errors.New("apiserver unavailable") + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + + if _, err := r.cleanupMariaDBBackup(context.Background(), backup); err == nil { + t.Fatal("err = nil, want the failed record returned so the Backup is held") + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted before the dump is recorded", bucket.removed, operatorBackupExists(t, r.Client)) + } +} + +func TestCleanupMariaDBBackup_RejectsARecordedNameThatIsNotADump(t *testing.T) { + for _, name := range []string{"../other/backup.2026-10-21T10:01:00Z.sql", "x.sql", "sub/" + cleanupDumpName} { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(name)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupDeletesTheProvenDump(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)}, + {Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}, + }} + backup := cleanupBackup("") + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } + if events := drainEvents(rec); len(events) != 0 { + t.Errorf("events = %v, want none", events) + } + got := &backupsv1alpha1.Backup{} + if err := r.Get(context.Background(), client.ObjectKeyFromObject(backup), got); err != nil { + t.Fatal(err) + } + if got.Spec.DriverMetadata[mariadbDumpFileKey] != cleanupDumpName || got.Spec.DriverMetadata[mariadbDumpTimeKey] != cleanupCreated.Add(time.Minute).Format(time.RFC3339) { + t.Errorf("driverMetadata = %v, want the proven dump and its time recorded", got.Spec.DriverMetadata) + } +} + +// The window is the operator Backup's lifetime, closed by the Backup's +// takenAt: a dump stamped after takenAt plus the skew is not this run's. +func TestCleanupMariaDBBackup_LegacyWindowEndsAtTakenAt(t *testing.T) { + late := cleanupCompleted.Add(mariadbSizeProbeClockSkew + time.Minute) + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, late, "gzip"), LastModified: late}, + }} + op := cleanupOperatorBackup(t) + // The operator's own Complete transition is not what bounds the window. + op.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(late.Add(time.Hour)))} + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_LegacyBackupWithoutAProvenDumpIsReleased(t *testing.T) { + inWindow := cleanupCreated.Add(time.Minute) + cases := map[string][]s3ObjectInfo{ + "no dump in its lifetime": { + {Key: dumpKey(cleanupPrefix, cleanupCreated.Add(-24*time.Hour), "gzip"), LastModified: cleanupCreated.Add(-24 * time.Hour)}, + }, + "two dumps in its lifetime": { + {Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow}, + {Key: dumpKey(cleanupPrefix, inWindow.Add(30*time.Second), "gzip"), LastModified: inWindow.Add(30 * time.Second)}, + }, + "a dump placed by one clock only": { + {Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow.Add(time.Hour)}, + }, + } + for name, objects := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: objects} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpANeighbourCouldOwnIsReleased(t *testing.T) { + inWindow := cleanupCreated.Add(time.Minute) + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: dumpKey(cleanupPrefix, inWindow, "gzip"), LastModified: inWindow}}} + neighbour := cleanupOperatorBackup(t) + neighbour.Name = "op-neighbour" + neighbour.CreationTimestamp = metav1.NewTime(cleanupCreated.Add(30 * time.Second)) + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), neighbour, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +// A run started before this one completed can upload its dump and prune +// this run's through maxRetention before its own operator Backup completes. +// Its dump is then the only one in this run's window, and only its open +// window says it may be its own. +func TestCleanupMariaDBBackup_LegacyBackupWhoseDumpAnUnfinishedRunCouldOwnIsReleased(t *testing.T) { + started := cleanupCreated.Add(2 * time.Minute) + cases := map[string][]metav1.Condition{ + "running": {{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(started), + }}, + "not yet reconciled": nil, + } + for name, conditions := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{ + {Key: dumpKey(cleanupPrefix, started.Add(90*time.Second), "gzip"), LastModified: started.Add(100 * time.Second)}, + }} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(started) + running.Status.Conditions = conditions + backup := cleanupBackup("") + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +// An unfinished run contests only what it could have written: a dump +// that predates it, on both clocks, is still this run's. +func TestCleanupMariaDBBackup_LegacyBackupDeletesItsDumpDespiteALaterUnfinishedRun(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + backup := cleanupBackup("") + r, _ := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } +} + +// The size probe that records a dump lets only finished runs contest it, so +// a run still unfinished then, which uploaded its dump and pruned this +// run's through maxRetention, can have its dump recorded here. Deletion +// contests the recorded dump again, with every run as it stands now. +func TestCleanupMariaDBBackup_RecordedDumpARunUnfinishedAtTheProbeCouldOwnIsReleased(t *testing.T) { + started := cleanupCreated.Add(time.Minute) + theirs := dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip") + cases := map[string][]metav1.Condition{ + "running": {{ + Type: mariadbtypes.ConditionTypeComplete, Status: metav1.ConditionFalse, + Reason: mariadbtypes.ConditionReasonJobRunning, Message: "Running", LastTransitionTime: metav1.NewTime(started), + }}, + "not yet reconciled": nil, + "completed since": {mariadbBackupSucceeded(metav1.NewTime(cleanupCompleted.Add(5 * time.Minute)))}, + } + for name, conditions := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: started.Add(time.Minute)}}} + other := cleanupOperatorBackup(t) + other.Name = "op-other" + other.CreationTimestamp = metav1.NewTime(started) + other.Status.Conditions = conditions + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), other, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(strings.TrimPrefix(theirs, cleanupPrefix))) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +// Either clock placing the recorded dump in another run's lifetime is +// enough: the backup pod's clock can be skewed while the store's is not. +func TestCleanupMariaDBBackup_RecordedDumpAnotherRunPlacesOnOneClockIsReleased(t *testing.T) { + stamped := cleanupCreated.Add(20 * time.Second) + uploaded := cleanupCreated.Add(2 * time.Minute) + theirs := dumpKey(cleanupPrefix, stamped, "gzip") + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: uploaded}}} + other := cleanupOperatorBackup(t) + other.Name = "op-other" + other.CreationTimestamp = metav1.NewTime(uploaded.Add(30 * time.Second)) + other.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), other, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(strings.TrimPrefix(theirs, cleanupPrefix))) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +// The listing is by prefix, so a longer key sharing it is not the dump and +// contests nothing. +func TestCleanupMariaDBBackup_RecordedDumpIgnoresAKeyItIsOnlyAPrefixOf(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key + ".tmp", LastModified: cleanupCompleted.Add(2 * time.Hour)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } +} + +// A name the dump time cannot be read from leaves one clock out of the +// contest, so it is an error rather than a contest on half the evidence. +func TestContestedMariaDBDump_RejectsANameWithoutADumpTime(t *testing.T) { + op := cleanupOperatorBackup(t) + name := "backup.not-a-time.sql" + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + name, LastModified: cleanupCreated}}} + c := newMariaDBStrategyTestClient(t, op) + if _, _, err := contestedMariaDBDump(context.Background(), bucket, c, op, name); err == nil { + t.Fatal("err = nil, want the unreadable dump time reported") + } +} + +// A retry after the dump was deleted but the operator Backup was not finds +// nothing to contest, whichever run's lifetime the dump's name falls in. +func TestCleanupMariaDBBackup_RecordedDumpAlreadyGoneIsNotContested(t *testing.T) { + bucket := &fakeS3Bucket{} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCreated) + running.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted") + } + events := drainEvents(rec) + if hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want no ArtifactNotDeleted", events) + } + // The listing may have reached other storage than the dump went to, as + // after an endpoint move, so the absence is reported with where it was + // looked for rather than logged as a delete. + if joined := strings.Join(events, "\n"); !hasEvent(events, "Normal ArtifactAlreadyAbsent") || + !strings.Contains(joined, "s3://bkt/"+cleanupPrefix+cleanupDumpName) || !strings.Contains(joined, "through s3.example:9000") { + t.Errorf("events = %v, want a Normal ArtifactAlreadyAbsent naming the key and the endpoint listed", events) + } +} + +// The runs that may own a dump are read from the API server: a run whose +// operator Backup has not reached the cache yet can already have uploaded +// its dump, and missing it would delete that dump as this Backup's. +func TestCleanupMariaDBBackup_ContestsWithRunsTheCacheHasNotSeen(t *testing.T) { + started := cleanupCreated.Add(time.Minute) + theirs := dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip") + unseen := func() *mariadbtypes.Backup { + b := cleanupOperatorBackup(t) + b.Name = "op-unseen" + b.CreationTimestamp = metav1.NewTime(started) + b.Status.Conditions = nil + return b + } + cases := map[string]string{"recorded dump": strings.TrimPrefix(theirs, cleanupPrefix), "legacy Backup": ""} + for name, recorded := range cases { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: theirs, LastModified: started.Add(time.Minute)}}} + backup := cleanupBackup(recorded) + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), unseen()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + } +} + +// A run started after the recorded dump was uploaded cannot have written it. +func TestCleanupMariaDBBackup_DeletesItsRecordedDumpDespiteALaterUnfinishedRun(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + running := cleanupOperatorBackup(t) + running.Name = "op-running" + running.CreationTimestamp = metav1.NewTime(cleanupCompleted.Add(time.Hour)) + running.Status.Conditions = nil + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), running, s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if want := []string{"bkt/" + key}; fmt.Sprint(bucket.removed) != fmt.Sprint(want) { + t.Errorf("removed %v, want %v", bucket.removed, want) + } + if operatorBackupExists(t, r.Client) { + t.Error("operator Backup kept, want it deleted once its dump is") + } +} + +func TestCleanupMariaDBBackup_RecordedDumpListingFailureRetries(t *testing.T) { + bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +// Without the list of runs nothing is known about who else could have +// written the dump, so it is kept and the delete retried. +func TestCleanupMariaDBBackup_RecordedDumpWithUnreadableRunsRetries(t *testing.T) { + key := cleanupPrefix + cleanupDumpName + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: key, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.BackupList); ok { + return errors.New("apiserver unavailable") + } + return c.List(ctx, list, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +// The whole sequence: run B uploads its dump and prunes this run's before +// B completes, this run's Backup is created and probed, then deleted. B's +// dump must survive whatever the probe recorded. +func TestMariaDBBackup_ProbeThenCleanupKeepsADumpAnUnfinishedRunUploaded(t *testing.T) { + apps := mariadbapp.GroupName + started := cleanupCreated.Add(time.Minute) + theirs := s3ObjectInfo{Key: dumpKey(cleanupPrefix, started.Add(30*time.Second), "gzip"), Size: 42, LastModified: started.Add(time.Minute)} + rendered := newRenderedMariaDBTemplate() + rendered.Storage.S3.Prefix = "tenant/db" + resolved := &ResolvedBackupConfig{ + StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"}, + Parameters: map[string]string{}, + } + job := &backupsv1alpha1.BackupJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bk"}, + Spec: backupsv1alpha1.BackupJobSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "db", APIGroup: &apps}, + }, + } + other := func() *mariadbtypes.Backup { + b := cleanupOperatorBackup(t) + b.Name = "op-other" + b.CreationTimestamp = metav1.NewTime(started) + b.Status.Conditions = nil + return b + } + + prober := reconcilerWithLister(t, &fakeS3Lister{objects: []s3ObjectInfo{theirs}}, job, other(), s3CredsSecret("tenant", "creds")) + created, err := prober.createMariaDBBackupArtifact(context.Background(), job, resolved, cleanupOperatorBackup(t), rendered) + if err != nil { + t.Fatalf("createMariaDBBackupArtifact: %v", err) + } + if got, want := created.Spec.DriverMetadata[mariadbDumpFileKey], strings.TrimPrefix(theirs.Key, cleanupPrefix); got != want { + t.Fatalf("probe recorded dump %q, want %q: this test covers the recorded path", got, want) + } + + backup := created.DeepCopy() + backup.ResourceVersion = "" + now := metav1.Now() + backup.DeletionTimestamp = &now + backup.Finalizers = []string{backupFinalizer} + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{theirs}} + r, rec := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), other(), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_LegacyListingFailureRetries(t *testing.T) { + bucket := &fakeS3Bucket{listErr: errors.New("connection refused")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup("")) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted, want it kept") + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_ReleasesWhatItCannotAttribute(t *testing.T) { + t.Run("operator Backup is gone", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } + }) + t.Run("Backup records no operator Backup", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), tenantNamespace()) + backup := cleanupBackup(cleanupDumpName) + delete(backup.Spec.DriverMetadata, mariadbBackupNameKey) + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup's Job failed", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions[0].Reason = mariadbtypes.ConditionReasonJobFailed + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup has no Complete condition", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions = nil + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("operator Backup never completed", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Status.Conditions[0].Status = metav1.ConditionFalse + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) + t.Run("dump is on a volume", func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage = mariadbtypes.BackupStorage{} + r, rec := cleanupReconciler(t, bucket, op, tenantNamespace()) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) + }) +} + +func TestCleanupMariaDBBackup_SkipAnnotationDeletesNothing(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + backup := cleanupBackup(cleanupDumpName) + backup.Annotations = map[string]string{mariadbSkipArtifactCleanupAnnotation: "true"} + + res, err := r.cleanupMariaDBBackup(context.Background(), backup) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted", bucket.removed, operatorBackupExists(t, r.Client)) + } +} + +func restoreJobOf(name, backupName string, phase backupsv1alpha1.RestoreJobPhase) *backupsv1alpha1.RestoreJob { + return &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: name}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: backupName}}, + Status: backupsv1alpha1.RestoreJobStatus{Phase: phase}, + } +} + +// operatorRestoreOf is a k8s.mariadb.com/Restore reading the dump of the +// operator Backup backupRef, with the given Complete condition. +func operatorRestoreOf(name, backupRef string, complete *metav1.Condition) *mariadbtypes.Restore { + restore := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: name}} + if backupRef != "" { + restore.Spec.BackupRef = &mariadbtypes.BackupReference{Name: backupRef} + } + if complete != nil { + restore.Status.Conditions = []metav1.Condition{*complete} + } + return restore +} + +func restoreComplete(status metav1.ConditionStatus, reason string) *metav1.Condition { + return &metav1.Condition{Type: mariadbtypes.ConditionTypeComplete, Status: status, Reason: reason, LastTransitionTime: metav1.Now()} +} + +// The RestoreJob's phase is not the restore: one failed on its deadline +// leaves its operator Restore running, and that Restore still reads the +// dump. Until the operator marks it Complete, as it does a finished or a +// failed Job, the dump stays. +func TestCleanupMariaDBBackup_WaitsForAnOperatorRestoreOfItsDump(t *testing.T) { + held := map[string]*metav1.Condition{ + "running": restoreComplete(metav1.ConditionFalse, mariadbtypes.ConditionReasonJobRunning), + "suspended": restoreComplete(metav1.ConditionFalse, mariadbtypes.ConditionReasonJobSuspended), + "not yet reconciled": nil, + } + for name, cond := range held { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("rj", "bk", backupsv1alpha1.RestoreJobPhaseFailed), operatorRestoreOf("rj-abcde", "op-bk", cond)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the operator Restore runs", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Normal ArtifactCleanupWaiting") || !strings.Contains(strings.Join(events, "\n"), "k8s.mariadb.com/Restore rj-abcde") { + t.Errorf("events = %v, want a Normal ArtifactCleanupWaiting naming the operator Restore", events) + } + }) + } + released := map[string]*mariadbtypes.Restore{ + "completed": operatorRestoreOf("rj-abcde", "op-bk", restoreComplete(metav1.ConditionTrue, mariadbtypes.ConditionReasonJobComplete)), + "failed": operatorRestoreOf("rj-abcde", "op-bk", restoreComplete(metav1.ConditionTrue, mariadbtypes.ConditionReasonJobFailed)), + "of another operator Backup": operatorRestoreOf("rj-abcde", "op-other", nil), + "from storage, not a Backup": operatorRestoreOf("rj-abcde", "", nil), + } + for name, restore := range released { + t.Run(name, func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), restore) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + }) + } + t.Run("one the cache has not seen yet", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, operatorRestoreOf("rj-abcde", "op-bk", nil)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held", res, err, bucket.removed) + } + }) + t.Run("a failed listing deletes nothing", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil || len(bucket.removed) != 0 { + t.Fatalf("err=%v removed=%v, want the error returned and nothing deleted", err, bucket.removed) + } + }) +} + +func TestCleanupMariaDBBackup_WaitsForARestoreFromIt(t *testing.T) { + for _, phase := range []backupsv1alpha1.RestoreJobPhase{backupsv1alpha1.RestoreJobPhaseEmpty, backupsv1alpha1.RestoreJobPhasePending, backupsv1alpha1.RestoreJobPhaseRunning} { + t.Run(string(phase), func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("rj", "bk", phase)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held while the restore runs", res, err) + } + if len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("removed=%v operatorBackupExists=%v, want nothing deleted during the restore", bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Normal ArtifactCleanupWaiting") || !strings.Contains(strings.Join(events, "\n"), mariadbSkipArtifactCleanupAnnotation) { + t.Errorf("events = %v, want a Normal ArtifactCleanupWaiting naming the skip annotation", events) + } + }) + } + // A restore checks that its Backup is not being deleted before it + // starts, so one that started first is on the API server before the + // delete, though not necessarily in this controller's cache yet. + t.Run("a restore the cache has not seen yet holds it", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = newMariaDBStrategyTestClient(t, restoreJobOf("rj", "bk", backupsv1alpha1.RestoreJobPhaseRunning)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held while the restore runs", res, err, bucket.removed) + } + }) + t.Run("a failed listing deletes nothing", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.APIReader = interceptor.NewClient(newMariaDBStrategyTestClient(t).(client.WithWatch), interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*backupsv1alpha1.RestoreJobList); ok { + return errors.New("etcdserver: request timed out") + } + return c.List(ctx, list, opts...) + }, + }) + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err == nil || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Fatalf("err=%v removed=%v, want the error returned and nothing deleted", err, bucket.removed) + } + }) + t.Run("finished restores and restores of other Backups do not hold it", func(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + restoreJobOf("done", "bk", backupsv1alpha1.RestoreJobPhaseSucceeded), + restoreJobOf("failed", "bk", backupsv1alpha1.RestoreJobPhaseFailed), + restoreJobOf("other", "other-bk", backupsv1alpha1.RestoreJobPhaseRunning)) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + }) +} + +// platformCredsConfig points the reconciler at the platform's credentials +// Secret, the source the per-tenant Secret is projected from. The projected +// Secret is "creds", the one cleanupOperatorBackup names, as the default +// strategy names cozy-backups-creds. +func platformCredsConfig(endpoint string) BackupCredentialsConfig { + return BackupCredentialsConfig{ + SourceNamespace: "cozy-system", + SourceSecretName: "cozy-backups-credentials", + TargetSecretName: "creds", + Endpoint: endpoint, + Region: "us-east-1", + } +} + +func platformCredsSecret(bucketName, endpoint string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "cozy-system", Name: "cozy-backups-credentials"}, + Data: map[string][]byte{ + "accessKey": []byte("PLATFORM"), + "secretKey": []byte("platform-secret"), + "endpoint": []byte(endpoint), + "bucketName": []byte(bucketName), + }, + } +} + +func TestCleanupMariaDBBackup_SystemBucketUsesTheCurrentEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + for _, conn := range bucket.conns { + if conn.Endpoint != "s3.new.example" || !conn.Secure || conn.AccessKey != "PLATFORM" || conn.Region != "us-east-1" { + t.Errorf("connection = %+v, want the platform's current endpoint over TLS with its keys", conn) + } + } +} + +func TestCleanupMariaDBBackup_OtherBucketUsesTheOperatorBackupsEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("another-bucket", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + for _, conn := range bucket.conns { + if conn.Endpoint != "s3.example:9000" || conn.AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", conn) + } + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) { + for _, bucketName := range []string{"bkt", "tenant-own-bucket"} { + t.Run(bucketName, func(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.Bucket = bucketName + // No platform Secret: the projection source is missing. + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted with the operator Backup's own credentials", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0]) + } + }) + } +} + +// A missing or malformed platform Secret is a standing state, and the +// operator Backup's own credentials are the best left. A failed read says +// nothing about the Secret: falling back then can list a moved endpoint's +// old address, find no dump there and report it deleted. +func TestCleanupMariaDBBackup_TransientPlatformCredentialsReadIsRetried(t *testing.T) { + bucket := &fakeS3Bucket{objects: []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}}} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.Secret); ok && key.Name == "cozy-backups-credentials" { + return errors.New("apiserver unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held and retried", res, err) + } + if len(bucket.conns) != 0 || len(bucket.removed) != 0 || !operatorBackupExists(t, r.Client) { + t.Errorf("conns=%v removed=%v operatorBackupExists=%v, want the bucket untouched", bucket.conns, bucket.removed, operatorBackupExists(t, r.Client)) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteHold(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup held", res, err) + } + if !operatorBackupExists(t, r.Client) { + t.Error("operator Backup deleted although its dump was not") + } + events := drainEvents(rec) + if hasEvent(events, "Warning ArtifactNotDeleted") || !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed and the Backup not released", events) + } +} + +func TestCleanupMariaDBBackup_UnreadablePlatformCredentialsAndAFailingDeleteReleaseInATerminatingNamespace(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("dial tcp: no such host")} + ns := tenantNamespace() + now := metav1.Now() + ns.DeletionTimestamp = &now + ns.Finalizers = []string{"kubernetes"} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +func TestCleanupMariaDBBackup_SystemBucketKeepsTheOperatorBackupsCA(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + ca := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "ca"}, Data: map[string][]byte{"ca.crt": []byte("PEM")}} + r, _ := cleanupReconciler(t, bucket, op, ca, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if string(bucket.conns[0].CAPEM) != "PEM" || bucket.conns[0].Endpoint != "s3.new.example" { + t.Errorf("connection = %+v, want the current endpoint trusting the operator Backup's CA", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_SystemBucketWithAnUnreadableCAIsHeld(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + r, rec := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || !requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want the Backup held with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactCleanupFailed") { + t.Errorf("events = %v, want a Warning ArtifactCleanupFailed", events) + } +} + +func TestCleanupMariaDBBackup_PlainHTTPPlatformEndpointNeedsNoCA(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.TLS = &mariadbtypes.S3TLS{Enabled: true, CASecretKeyRef: &mariadbtypes.SecretKeySelector{Name: "ca", Key: "ca.crt"}} + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "creds"), tenantNamespace(), platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("http://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted without reading a CA plain HTTP does not use", res, err, bucket.removed) + } +} + +func TestCleanupMariaDBBackup_PlatformEndpointFallsBackToTheConfiguredOne(t *testing.T) { + bucket := &fakeS3Bucket{} + platform := platformCredsSecret("bkt", "") + delete(platform.Data, "endpoint") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform) + r.CredentialsConfig = platformCredsConfig("https://s3.configured.example") + + if _, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)); err != nil { + t.Fatal(err) + } + if bucket.conns[0].Endpoint != "s3.configured.example" { + t.Errorf("connection = %+v, want BACKUP_STORAGE_ENDPOINT without its scheme", bucket.conns[0]) + } +} + +// blockingBucket lists the dump it holds and answers the call named by +// blockOn only when its context ends, as an unreachable endpoint does. +type blockingBucket struct { + fakeS3Bucket + blockOn string + entered chan struct{} +} + +func (b *blockingBucket) block(ctx context.Context, call string) bool { + if b.blockOn != call { + return false + } + close(b.entered) + <-ctx.Done() + return true +} + +func (b *blockingBucket) ListObjects(ctx context.Context, bucket, prefix string, recursive bool) ([]s3ObjectInfo, error) { + if b.block(ctx, "list") { + return nil, ctx.Err() + } + return b.fakeS3Bucket.ListObjects(ctx, bucket, prefix, recursive) +} + +func (b *blockingBucket) RemoveObject(ctx context.Context, bucket, key string) error { + if b.block(ctx, "remove") { + return ctx.Err() + } + return b.fakeS3Bucket.RemoveObject(ctx, bucket, key) +} + +func TestCleanupMariaDBBackup_BoundsAnUnreachableEndpoint(t *testing.T) { + prev := mariadbDumpCleanupTimeout + mariadbDumpCleanupTimeout = 50 * time.Millisecond + t.Cleanup(func() { mariadbDumpCleanupTimeout = prev }) + for _, call := range []string{"list", "remove"} { + t.Run(call, func(t *testing.T) { + bucket := &blockingBucket{blockOn: call, entered: make(chan struct{})} + bucket.objects = []s3ObjectInfo{{Key: cleanupPrefix + cleanupDumpName, LastModified: cleanupCreated.Add(2 * time.Minute)}} + c := newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + r := &BackupReconciler{Client: c, Scheme: c.Scheme(), Recorder: record.NewFakeRecorder(20), + newS3ObjectLister: func(s3Connection) (s3ObjectLister, error) { return bucket, nil }, + newS3ObjectRemover: func(s3Connection) (s3ObjectRemover, error) { return bucket, nil }} + + done := make(chan ctrl.Result, 1) + go func() { + res, _ := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + done <- res + }() + select { + case res := <-done: + select { + case <-bucket.entered: + default: + t.Fatalf("cleanup returned without reaching the blocking %s", call) + } + if !requeues(res) { + t.Errorf("res = %+v, want the Backup held and retried", res) + } + case <-time.After(5 * time.Second): + t.Fatalf("cleanup did not give up on an unreachable endpoint blocking %s", call) + } + }) + } +} + +// Without the mariadb-operator CRDs every k8s.mariadb.com read fails, the +// listing of operator Restores included: no Restore can run then, so the +// Backup is released rather than held for good. +func TestCleanupMariaDBBackup_ReleasesWhenTheOperatorCRDIsGone(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return &apimeta.NoKindMatchError{GroupKind: mariadbtypes.GroupVersion.WithKind("Backup").GroupKind()} + } + return c.Get(ctx, key, obj, opts...) + }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*mariadbtypes.RestoreList); ok { + return &apimeta.NoKindMatchError{GroupKind: mariadbtypes.GroupVersion.WithKind("Restore").GroupKind()} + } + return c.List(ctx, list, opts...) + }, + }) + r, rec := cleanupReconcilerWithClient(c, bucket) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 0 { + t.Fatalf("got res=%+v err=%v removed=%v, want released with nothing deleted", res, err, bucket.removed) + } + if events := drainEvents(rec); !hasEvent(events, "Warning ArtifactNotDeleted") { + t.Errorf("events = %v, want a Warning ArtifactNotDeleted", events) + } +} + +func TestCleanupMariaDBBackup_OperatorBackupAlreadyDeletedIsDone(t *testing.T) { + bucket := &fakeS3Bucket{} + c := interceptor.NewClient(newMariaDBStrategyTestClient(t, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()).(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*mariadbtypes.Backup); ok { + return apierrors.NewNotFound(mariadbtypes.GroupVersion.WithResource("backups").GroupResource(), obj.GetName()) + } + return c.Delete(ctx, obj, opts...) + }, + }) + r, _ := cleanupReconcilerWithClient(c, bucket) + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) { + t.Fatalf("got res=%+v err=%v, want the Backup released", res, err) + } +} + +// A strategy of its own that happens to use the platform bucket's name, as +// an admin-chosen name on external S3 can, is reached as recorded: only the +// projected Secret marks a Backup as written to the platform bucket. +func TestCleanupMariaDBBackup_SameBucketNameWithOwnCredentialsUsesTheOperatorBackupsEndpoint(t *testing.T) { + bucket := &fakeS3Bucket{} + op := cleanupOperatorBackup(t) + op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name = "own-creds" + op.Spec.Storage.S3.SecretAccessKeySecretKeyRef.Name = "own-creds" + r, _ := cleanupReconciler(t, bucket, op, s3CredsSecret("tenant", "own-creds"), tenantNamespace(), + platformCredsSecret("bkt", "s3.new.example")) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" || bucket.conns[0].AccessKey != "AKID" { + t.Errorf("connection = %+v, want the operator Backup's endpoint and Secret", bucket.conns[0]) + } +} + +// With no endpoint anywhere the platform credentials are unusable, which +// falls back like any other unreadable credentials. +func TestCleanupMariaDBBackup_PlatformCredentialsWithoutAnEndpointFallBack(t *testing.T) { + bucket := &fakeS3Bucket{} + platform := platformCredsSecret("bkt", "") + delete(platform.Data, "endpoint") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), platform) + r.CredentialsConfig = platformCredsConfig("") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_MalformedPlatformCredentialsFallBackToTheOperatorBackup(t *testing.T) { + bucket := &fakeS3Bucket{} + malformed := platformCredsSecret("bkt", "s3.new.example") + delete(malformed.Data, "secretKey") + r, _ := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace(), malformed) + r.CredentialsConfig = platformCredsConfig("https://s3.new.example") + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + if err != nil || requeues(res) || len(bucket.removed) != 1 { + t.Fatalf("got res=%+v err=%v removed=%v, want the dump deleted", res, err, bucket.removed) + } + if bucket.conns[0].Endpoint != "s3.example:9000" { + t.Errorf("connection = %+v, want the operator Backup's own endpoint", bucket.conns[0]) + } +} + +func TestCleanupMariaDBBackup_TerminatingNamespaceReleasesOnS3Failure(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("connection refused")} + ns := tenantNamespace() + now := metav1.Now() + ns.DeletionTimestamp = &now + ns.Finalizers = []string{"kubernetes"} + r, rec := cleanupReconciler(t, bucket, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), ns) + + res, err := r.cleanupMariaDBBackup(context.Background(), cleanupBackup(cleanupDumpName)) + assertReleasedUntouched(t, r, rec, bucket, res, err) +} + +// Reconcile keeps the finalizer while the dump cannot be deleted, and drops +// it once both the dump and the operator Backup are gone. +func TestBackupReconcile_MariaDBFinalizerFollowsTheDumpDelete(t *testing.T) { + bucket := &fakeS3Bucket{removeErr: errors.New("503")} + backup := cleanupBackup(cleanupDumpName) + r, _ := cleanupReconciler(t, bucket, backup, cleanupOperatorBackup(t), s3CredsSecret("tenant", "creds"), tenantNamespace()) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(backup)} + + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); err != nil { + t.Fatalf("Backup gone while its dump could not be deleted: %v", err) + } + + bucket.removeErr = nil + if _, err := r.Reconcile(context.Background(), req); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if err := r.Get(context.Background(), req.NamespacedName, &backupsv1alpha1.Backup{}); !apierrors.IsNotFound(err) { + t.Errorf("Backup still present (err=%v), want its finalizer removed", err) + } +} + +// The minio adapter deletes the exact key, with the colons of the dump's +// timestamp escaped in the path; SeaweedFS refuses them raw. +func TestMinioObjectRemover_DeletesTheEscapedKey(t *testing.T) { + var method, rawPath, auth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + method, rawPath, auth = req.Method, req.URL.EscapedPath(), req.Header.Get("Authorization") + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"}) + if err != nil { + t.Fatal(err) + } + if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "tenant#a/db/backup.2026-10-21T10:01:00Z.gzip.sql"); err != nil { + t.Fatalf("RemoveObject: %v", err) + } + if method != http.MethodDelete || rawPath != "/bkt/tenant%23a/db/backup.2026-10-21T10%3A01%3A00Z.gzip.sql" { + t.Errorf("request %s %s, want DELETE of the escaped key", method, rawPath) + } + if !strings.Contains(auth, "Credential=AKID/") { + t.Errorf("request not signed with the access key: %q", auth) + } +} + +func TestMinioObjectRemover_ReturnsTheServersError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/xml") + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `AccessDeniedAccess Denied`) + })) + defer srv.Close() + + remover, err := newMinioObjectLister(s3Connection{Endpoint: strings.TrimPrefix(srv.URL, "http://"), Region: "us-east-1", AccessKey: "AKID", SecretKey: "s"}) + if err != nil { + t.Fatal(err) + } + if err := remover.(s3ObjectRemover).RemoveObject(context.Background(), "bkt", "k"); err == nil { + t.Fatal("RemoveObject = nil, want the AccessDenied error") + } +} + +func (f *fakeS3Bucket) StatObject(_ context.Context, _, key string) (int64, error) { + for _, o := range f.objects { + if o.Key == key { + return o.Size, nil + } + } + return 0, errors.New("not found") +} diff --git a/internal/backupcontroller/mariadbstrategy_controller.go b/internal/backupcontroller/mariadbstrategy_controller.go index 9fd6780db..a8fd29a01 100644 --- a/internal/backupcontroller/mariadbstrategy_controller.go +++ b/internal/backupcontroller/mariadbstrategy_controller.go @@ -4,6 +4,7 @@ package backupcontroller import ( "context" "encoding/json" + "errors" "fmt" "time" @@ -30,8 +31,6 @@ import ( // --------------------------------------------------------------------------- const ( - mariadbFieldManager = "cozystack-mariadb-backup-driver" - // mariadb-operator's MariaDB ApplicationDefinition (release.prefix). The // cozystack mariadb-rd renders the HelmRelease with releaseName = // "mariadb-" + appName (see packages/system/mariadb-rd/cozyrds/mariadb.yaml, @@ -48,6 +47,15 @@ const ( mariadbBackupNamespaceKey = "k8s.mariadb.com/backup-namespace" mariadbStorageURIKey = "k8s.mariadb.com/storage-uri" + // The dump the operator Backup wrote: its file name under the storage + // prefix, and the RFC3339 timestamp in that name. Every run of an + // application writes under the same prefix, so only this name tells its + // dump from the others, and a restore asks for exactly that dump. + // Recorded only when findMariaDBDump proved the dump belongs to the + // operator Backup. + mariadbDumpFileKey = "k8s.mariadb.com/dump-file" + mariadbDumpTimeKey = "k8s.mariadb.com/dump-time" + // Polling cadence for the operator Backup/Restore lifecycle. mariadbPollInterval = 5 * time.Second @@ -61,6 +69,12 @@ const ( // to terminate. Tenants override via spec.options.restoreTimeoutSeconds. mariadbDefaultRestoreDeadline = 30 * time.Minute + // Retry cadence while the dump a restore asks for cannot be checked. + // Longer than mariadbPollInterval because each attempt can hold the + // RestoreJob controller's single worker for + // mariadbRestoreDumpCheckTimeout. + mariadbDumpCheckRetryInterval = 30 * time.Second + // mariadbBackupSnapshotKind is the Kind stamped onto the snapshot // persisted in Backup.status.underlyingResources. The same shape carries // the storage descriptor and the rendered strategy parameters used at @@ -74,6 +88,12 @@ const ( // existing API surface. var mariadbBackupSnapshotAPIVersion = backupsv1alpha1.GroupVersion.String() +// mariadbRestoreDumpCheckTimeout bounds the object-store round trips a +// restore makes before it creates the operator Restore. The RestoreJob +// controller has a single worker, so an unreachable endpoint stalls every +// other RestoreJob for this long. A variable so tests can shorten it. +var mariadbRestoreDumpCheckTimeout = 10 * time.Second + // mariadbNameForApp returns the k8s.mariadb.com/MariaDB CR name for a // cozystack MariaDB application instance. The mapping mirrors the mariadb // ApplicationDefinition (release.prefix=mariadb-). @@ -422,7 +442,11 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact( if uri := driverMD[mariadbStorageURIKey]; uri != "" { status.Artifact = &backupsv1alpha1.BackupArtifact{URI: uri} if probe { - status.Artifact.SizeBytes = r.mariadbArtifactSizeBytes(ctx, mdbBackup, takenAt.Time) + if dump, ok := r.attributedMariaDBDump(ctx, mdbBackup, takenAt.Time); ok { + status.Artifact.SizeBytes = dump.Size + driverMD[mariadbDumpFileKey] = dump.Name + driverMD[mariadbDumpTimeKey] = dump.Timestamp.UTC().Format(time.RFC3339) + } } } backup := &backupsv1alpha1.Backup{ @@ -594,9 +618,26 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest return ctrl.Result{}, err } - mdbRestore, err := r.ensureMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName) + mdbRestore, err := r.findMariaDBRestoreForJob(ctx, restoreJob) if err != nil { - return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) + // Retried: failing the RestoreJob here could fail it under an + // operator Restore that is running. + return ctrl.Result{}, fmt.Errorf("find k8s.mariadb.com/Restore of RestoreJob %s/%s: %w", restoreJob.Namespace, restoreJob.Name, err) + } + // The dump is chosen once, as the operator Restore is created; one that + // already exists has made its choice. + if mdbRestore == nil { + targetTime, err := r.mariadbRestoreTargetTime(ctx, restoreJob, backup, srcBackup) + var refused mariadbRestoreRefused + if errors.As(err, &refused) { + return r.markRestoreJobFailed(ctx, restoreJob, refused.Error()) + } + if err != nil { + return r.awaitMariaDBDumpCheck(ctx, restoreJob, options, err) + } + if mdbRestore, err = r.createMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName, targetTime); err != nil { + return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err)) + } } cond := apimeta.FindStatusCondition(mdbRestore.Status.Conditions, mariadbtypes.ConditionTypeComplete) @@ -630,12 +671,114 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest } } -// ensureMariaDBRestore creates a k8s.mariadb.com/Restore CR labelled with -// the RestoreJob, or returns the existing one if a previous reconcile -// already created it. -func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName string) (*mariadbtypes.Restore, error) { +// mariadbRestoreRefused is why the dump a RestoreJob asks for cannot be +// restored. It fails the RestoreJob, where any other error from choosing +// the dump is retried. +type mariadbRestoreRefused string + +func (e mariadbRestoreRefused) Error() string { return string(e) } + +// mariadbRestoreTargetTime returns the targetRecoveryTime of the operator +// Restore for backup: the timestamp in the name of the dump backup +// records. mariadb-operator restores the dump whose timestamp equals the +// target and, when there is none, the nearest one, so the dump is first +// checked to still exist. A Backup that records no dump is resolved by +// legacyMariaDBRestoreTargetTime. +func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) { + // Deleting a Backup deletes its dump, and the operator restores the + // nearest other dump once its own is gone, so a restore must not start + // from a Backup being deleted. The copy in hand can predate the delete; + // the cleanup holds the Backup only for restores that exist when it + // lists them. + live := &backupsv1alpha1.Backup{} + if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(backup), live); err != nil { + if apierrors.IsNotFound(err) { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is gone, and its dump may be gone with it", backup.Namespace, backup.Name)) + } + return "", fmt.Errorf("read Backup %s/%s: %w", backup.Namespace, backup.Name, err) + } + if !live.DeletionTimestamp.IsZero() { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is being deleted, and its dump with it", backup.Namespace, backup.Name)) + } + // A failed Job is Complete=True too, with reason JobFailed, and a + // BackupJob that took Complete=True for success made a Backup of it. + if cond := apimeta.FindStatusCondition(src.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed { + return "", mariadbRestoreRefused(fmt.Sprintf( + "the backup run of k8s.mariadb.com/Backup %s/%s failed (Complete=%s/%s) and left no dump to restore; restoring would bring back another run's dump", + src.Namespace, src.Name, cond.Status, cond.Reason)) + } + ctx, cancel := context.WithTimeout(ctx, mariadbRestoreDumpCheckTimeout) + defer cancel() + name := backup.Spec.DriverMetadata[mariadbDumpFileKey] + if name == "" { + return r.legacyMariaDBRestoreTargetTime(ctx, rj, backup, src), nil + } + ts, ok := parseMariaDBLogicalBackupFile(name) + if !ok { + return "", mariadbRestoreRefused(fmt.Sprintf("Backup driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)) + } + exists, err := mariadbDumpExists(ctx, r.objectStore(), src, name) + if err != nil { + return "", fmt.Errorf("cannot check that dump %s of k8s.mariadb.com/Backup %s/%s still exists in storage: %w", name, src.Namespace, src.Name, err) + } + if !exists { + s3 := src.Spec.Storage.S3 + return "", mariadbRestoreRefused(fmt.Sprintf( + "dump %s of k8s.mariadb.com/Backup %s/%s no longer exists in storage (s3://%s/%s); restoring now would bring back a different dump", + name, src.Namespace, src.Name, s3.Bucket, mariadbOperatorS3Prefix(s3.Prefix))) + } + return ts.UTC().Format(time.RFC3339), nil +} + +// legacyMariaDBRestoreTargetTime chooses the dump of a Backup taken before +// Backups recorded theirs: the one findMariaDBDump proves belongs to src, +// with the completion time the size probe was given, which takenAt keeps. +// When none is proven, for any reason, it returns empty, and the operator +// restores the newest dump under the prefix as such a restore always did: +// refusing would leave these Backups unrestorable, and the controller may +// never have been able to reach their storage. A Warning event says the +// dump is a best effort. +func (r *RestoreJobReconciler) legacyMariaDBRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) string { + dump, err := findMariaDBDump(ctx, r.objectStore(), src, backup.Spec.TakenAt.Time) + if err != nil { + getLogger(ctx).Info("no dump proven for a MariaDB Backup that records none; restoring the newest", "restorejob", rj.Name, "backup", backup.Name, "error", err) + r.Recorder.Eventf(rj, corev1.EventTypeWarning, "BestEffortDump", + "Backup %s records no dump and none in storage could be proven to belong to k8s.mariadb.com/Backup %s/%s (%v); mariadb-operator restores the newest dump under its prefix as a best effort, which may be another backup run's", + backup.Name, src.Namespace, src.Name, err) + return "" + } + return dump.Timestamp.UTC().Format(time.RFC3339) +} + +// awaitMariaDBDumpCheck keeps the RestoreJob waiting, with Ready=False +// saying why, while the dump it asks for cannot be checked, and fails it +// with the cause once the restore deadline has passed. +func (r *RestoreJobReconciler) awaitMariaDBDumpCheck(ctx context.Context, rj *backupsv1alpha1.RestoreJob, options MariaDBRestoreOptions, cause error) (ctrl.Result, error) { + deadline := options.effectiveRestoreDeadline() + if rj.Status.StartedAt != nil && time.Since(rj.Status.StartedAt.Time) > deadline { + return r.markRestoreJobFailed(ctx, rj, fmt.Sprintf( + "%v; gave up after %s (override via spec.options.restoreTimeoutSeconds)", cause, deadline)) + } + getLogger(ctx).Info("cannot check the MariaDB dump to restore; retrying", "restorejob", rj.Name, "error", cause) + apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionFalse, + Reason: "DumpCheckPending", + Message: cause.Error(), + }) + if err := r.Status().Update(ctx, rj); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: mariadbDumpCheckRetryInterval}, nil +} + +// findMariaDBRestoreForJob returns the k8s.mariadb.com/Restore labelled +// with the RestoreJob, if a previous reconcile already created one. It is +// read uncached: one the cache has not caught up with would be created +// again, or, once its Backup is being deleted, refused while it runs. +func (r *RestoreJobReconciler) findMariaDBRestoreForJob(ctx context.Context, rj *backupsv1alpha1.RestoreJob) (*mariadbtypes.Restore, error) { list := &mariadbtypes.RestoreList{} - if err := r.List(ctx, list, + if err := r.apiReader().List(ctx, list, client.InNamespace(rj.Namespace), client.MatchingLabels{ backupsv1alpha1.OwningJobNameLabel: rj.Name, @@ -644,10 +787,15 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac ); err != nil { return nil, err } - if len(list.Items) > 0 { - return &list.Items[0], nil + if len(list.Items) == 0 { + return nil, nil } + return &list.Items[0], nil +} +// createMariaDBRestore creates the k8s.mariadb.com/Restore of the +// RestoreJob, labelled with it. targetRecoveryTime is RFC3339, or empty. +func (r *RestoreJobReconciler) createMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName, targetRecoveryTime string) (*mariadbtypes.Restore, error) { obj := &mariadbtypes.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: rj.Namespace, @@ -663,6 +811,7 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac Kind: mariadbtypes.BackupKind, Name: sourceBackupName, }, + TargetRecoveryTime: targetRecoveryTime, }, } if err := r.Create(ctx, obj); err != nil { diff --git a/internal/backupcontroller/mariadbstrategy_controller_test.go b/internal/backupcontroller/mariadbstrategy_controller_test.go index f03b67d94..9696d1eac 100644 --- a/internal/backupcontroller/mariadbstrategy_controller_test.go +++ b/internal/backupcontroller/mariadbstrategy_controller_test.go @@ -317,7 +317,7 @@ func TestEnsureMariaDBBackup_IdempotentByLabel(t *testing.T) { // Restore-side ensure idempotency + target rewrite // --------------------------------------------------------------------------- -func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { +func TestFindMariaDBRestoreForJob_FindsTheCreatedRestore(t *testing.T) { c := newMariaDBStrategyTestClient(t) r := &RestoreJobReconciler{Client: c, Scheme: c.Scheme()} rj := newMariaDBRestoreJob("rj-1", "tenant") @@ -325,16 +325,16 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { t.Fatalf("seed RestoreJob: %v", err) } - first, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target") - if err != nil { - t.Fatalf("first ensureMariaDBRestore: %v", err) + if none, err := r.findMariaDBRestoreForJob(context.Background(), rj); err != nil || none != nil { + t.Fatalf("findMariaDBRestoreForJob before create = %v, %v; want none", none, err) } - second, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target") + created, err := r.createMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target", "2026-10-21T10:00:05Z") if err != nil { - t.Fatalf("second ensureMariaDBRestore: %v", err) + t.Fatalf("createMariaDBRestore: %v", err) } - if first.Name != second.Name { - t.Errorf("expected idempotent reuse: first=%q second=%q", first.Name, second.Name) + found, err := r.findMariaDBRestoreForJob(context.Background(), rj) + if err != nil || found == nil || found.Name != created.Name { + t.Fatalf("findMariaDBRestoreForJob = %v, %v; want the Restore just created", found, err) } list := &mariadbtypes.RestoreList{} if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil { @@ -350,6 +350,9 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) { if got.Spec.BackupRef == nil || got.Spec.BackupRef.Name != "src-backup" || got.Spec.BackupRef.Kind != mariadbtypes.BackupKind { t.Errorf("BackupRef mismatch: %#v", got.Spec.BackupRef) } + if got.Spec.TargetRecoveryTime != "2026-10-21T10:00:05Z" { + t.Errorf("TargetRecoveryTime: got %q want 2026-10-21T10:00:05Z", got.Spec.TargetRecoveryTime) + } } // --------------------------------------------------------------------------- diff --git a/internal/backupcontroller/psmdbstrategy_controller.go b/internal/backupcontroller/psmdbstrategy_controller.go index 9caa218a7..ccf3e7735 100644 --- a/internal/backupcontroller/psmdbstrategy_controller.go +++ b/internal/backupcontroller/psmdbstrategy_controller.go @@ -528,8 +528,8 @@ func (r *BackupJobReconciler) reconcileMongoDB(ctx context.Context, j *backupsv1 } } - switch state := mdbBackup.Status.State; { - case state == psmdbtypes.StateReady: + switch state := mdbBackup.Status.State; state { + case psmdbtypes.StateReady: // Backup completed successfully. Materialise the Cozystack artifact and // finalise the BackupJob. Idempotent: reuse the existing artifact if a // previous reconcile created it and then raced on the status update. @@ -570,7 +570,7 @@ func (r *BackupJobReconciler) reconcileMongoDB(ctx context.Context, j *backupsv1 } return ctrl.Result{}, nil - case state == psmdbtypes.StateError || state == psmdbtypes.StateRejected: + case psmdbtypes.StateError, psmdbtypes.StateRejected: message := mdbBackup.Status.Error // The residual half of the injection race: on the useSystemBucket flow the // operator can latch state=error because it resolved the injected storage @@ -1740,8 +1740,8 @@ func (r *RestoreJobReconciler) reconcileMongoDBRestore(ctx context.Context, rest return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure psmdb.percona.com/PerconaServerMongoDBRestore: %v", err)) } - switch state := mdbRestore.Status.State; { - case state == psmdbtypes.StateReady: + switch state := mdbRestore.Status.State; state { + case psmdbtypes.StateReady: now := metav1.Now() restoreJob.Status.CompletedAt = &now restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded @@ -1756,7 +1756,7 @@ func (r *RestoreJobReconciler) reconcileMongoDBRestore(ctx context.Context, rest } return ctrl.Result{}, nil - case state == psmdbtypes.StateError || state == psmdbtypes.StateRejected: + case psmdbtypes.StateError, psmdbtypes.StateRejected: message := mdbRestore.Status.Error if message == "" { message = fmt.Sprintf("psmdb.percona.com/PerconaServerMongoDBRestore %s/%s reported state=%s", mdbRestore.Namespace, mdbRestore.Name, state) @@ -1855,7 +1855,7 @@ func (r *RestoreJobReconciler) resolveMongoDBBackupSource(ctx context.Context, b } if destination == "" { return nil, fmt.Errorf( - "Backup has no restorable destination (the operator PerconaServerMongoDBBackup was reaped and no snapshot destination is persisted); re-run the BackupJob") + "no restorable destination for the Backup (the operator PerconaServerMongoDBBackup was reaped and no snapshot destination is persisted); re-run the BackupJob") } src := &psmdbtypes.BackupSource{ Type: psmdbtypes.BackupTypeLogical, diff --git a/internal/backupcontroller/rabbitmqstrategy_controller.go b/internal/backupcontroller/rabbitmqstrategy_controller.go index adfe11d83..4c3d9a55c 100644 --- a/internal/backupcontroller/rabbitmqstrategy_controller.go +++ b/internal/backupcontroller/rabbitmqstrategy_controller.go @@ -513,7 +513,7 @@ func (r *BackupJobReconciler) createRabbitmqBackupArtifact( // this run's success while describing something else. if existing.Spec.ApplicationRef.Kind != backup.Spec.ApplicationRef.Kind || existing.Spec.ApplicationRef.Name != backup.Spec.ApplicationRef.Name { - return nil, fmt.Errorf("Backup %s/%s already exists for a different application (%s/%s, not %s/%s)", + return nil, fmt.Errorf("existing Backup %s/%s belongs to a different application (%s/%s, not %s/%s)", backup.Namespace, backup.Name, existing.Spec.ApplicationRef.Kind, existing.Spec.ApplicationRef.Name, backup.Spec.ApplicationRef.Kind, backup.Spec.ApplicationRef.Name) diff --git a/internal/backupcontroller/restorejob_controller.go b/internal/backupcontroller/restorejob_controller.go index 1a61f0800..9737eabdc 100644 --- a/internal/backupcontroller/restorejob_controller.go +++ b/internal/backupcontroller/restorejob_controller.go @@ -57,6 +57,11 @@ type RestoreJobReconciler struct { // see a peer's StartedAt before the cache does. Wired in // SetupWithManager. APIReader client.Reader + + // newS3ObjectLister overrides the S3 client used to check that a + // backup's object still exists; nil means a real minio client. Set by + // unit tests. + newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error) } func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -204,6 +209,8 @@ func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) return r.reconcileRedisRestore(ctx, restoreJob, backup) case strategyv1alpha1.KafkaStrategyKind: return r.reconcileKafkaRestore(ctx, restoreJob, backup) + case strategyv1alpha1.KubernetesStrategyKind: + return r.reconcileKubernetesRestore(ctx, restoreJob, backup) default: return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("StrategyRef.Kind not supported: %s", backup.Spec.StrategyRef.Kind)) } @@ -382,6 +389,12 @@ func (r *RestoreJobReconciler) cleanupOnDelete(ctx context.Context, restoreJob * case strategyv1alpha1.VeleroStrategyKind: r.cleanupVeleroRestore(ctx, restoreJob) + case strategyv1alpha1.KubernetesStrategyKind: + // Its Velero Restores are labelled like the Velero driver's, so the + // same cleanup reaps them; a live cluster it stopped is resumed. + r.cleanupVeleroRestore(ctx, restoreJob) + r.cleanupKubernetesRestore(ctx, restoreJob) + case strategyv1alpha1.CNPGStrategyKind, strategyv1alpha1.JobStrategyKind, strategyv1alpha1.AltinityStrategyKind, strategyv1alpha1.MariaDBStrategyKind, strategyv1alpha1.MongoDBStrategyKind, strategyv1alpha1.FoundationDBStrategyKind, strategyv1alpha1.EtcdStrategyKind, strategyv1alpha1.RabbitmqStrategyKind, strategyv1alpha1.RedisStrategyKind, strategyv1alpha1.KafkaStrategyKind: // Nothing to clean up: these drivers don't materialise namespaced // artifacts that outlive the RestoreJob. (Etcd: the operator-side diff --git a/internal/controller/fluxplunger/drain_outcome_test.go b/internal/controller/fluxplunger/drain_outcome_test.go index cb74e0177..152091dec 100644 --- a/internal/controller/fluxplunger/drain_outcome_test.go +++ b/internal/controller/fluxplunger/drain_outcome_test.go @@ -65,12 +65,14 @@ func (s captureSink) WithName(string) logr.LogSink { return s } func TestRunThenDrain_FencesLeftBehindAreAnErrorNamingThem(t *testing.T) { logger, entries := newCaptureLogger() ctx := log.IntoContext(context.Background(), logger) - RunThenDrain(ctx, + if err := RunThenDrain(ctx, func(context.Context) error { return nil }, func(context.Context) (int, error) { return 1, errors.Join(context.DeadlineExceeded, errors.New("still fenced: tenant-root/vm-disk-0")) }, - time.Minute) + time.Minute); err != nil { + t.Fatalf("RunThenDrain = %v, want the run's nil error", err) + } for _, e := range *entries { if e.err != nil && strings.Contains(e.err.Error(), "tenant-root/vm-disk-0") { return @@ -84,10 +86,12 @@ func TestRunThenDrain_FencesLeftBehindAreAnErrorNamingThem(t *testing.T) { func TestRunThenDrain_AnUnansweredListingIsNotACountOfMinusOne(t *testing.T) { logger, entries := newCaptureLogger() ctx := log.IntoContext(context.Background(), logger) - RunThenDrain(ctx, + if err := RunThenDrain(ctx, func(context.Context) error { return nil }, func(context.Context) (int, error) { return -1, context.DeadlineExceeded }, - time.Minute) + time.Minute); err != nil { + t.Fatalf("RunThenDrain = %v, want the run's nil error", err) + } found := false for _, e := range *entries { for i := 0; i+1 < len(e.kv); i += 2 { @@ -108,10 +112,12 @@ func TestRunThenDrain_AnUnansweredListingIsNotACountOfMinusOne(t *testing.T) { func TestRunThenDrain_ACompleteDrainSaysSo(t *testing.T) { logger, entries := newCaptureLogger() ctx := log.IntoContext(context.Background(), logger) - RunThenDrain(ctx, + if err := RunThenDrain(ctx, func(context.Context) error { return nil }, func(context.Context) (int, error) { return 0, nil }, - time.Minute) + time.Minute); err != nil { + t.Fatalf("RunThenDrain = %v, want the run's nil error", err) + } for _, e := range *entries { if e.err == nil && strings.Contains(e.msg, "Released every uninstall fence") { return diff --git a/internal/controller/fluxplunger/flux_plunger.go b/internal/controller/fluxplunger/flux_plunger.go index d53ec3eef..22775837b 100644 --- a/internal/controller/fluxplunger/flux_plunger.go +++ b/internal/controller/fluxplunger/flux_plunger.go @@ -486,11 +486,12 @@ func ownsSuspensionManagedFields(hr *helmv2.HelmRelease) bool { // fieldsManageSuspend reports whether a managed-fields set includes spec.suspend, // encoded by the apiserver as {"f:spec":{"f:suspend":{}}}. func fieldsManageSuspend(fs *metav1.FieldsV1) bool { - if fs == nil || len(fs.Raw) == 0 { + raw := fs.GetRawBytes() + if len(raw) == 0 { return false } var top map[string]json.RawMessage - if err := json.Unmarshal(fs.Raw, &top); err != nil { + if err := json.Unmarshal(raw, &top); err != nil { return false } specRaw, ok := top["f:spec"] diff --git a/internal/controller/fluxplunger/flux_plunger_test.go b/internal/controller/fluxplunger/flux_plunger_test.go index 365250621..fc1f8adad 100644 --- a/internal/controller/fluxplunger/flux_plunger_test.go +++ b/internal/controller/fluxplunger/flux_plunger_test.go @@ -83,7 +83,7 @@ func suspendManagedFields(manager string, op metav1.ManagedFieldsOperationType) Operation: op, APIVersion: helmv2.GroupVersion.String(), FieldsType: "FieldsV1", - FieldsV1: &metav1.FieldsV1{Raw: []byte(`{"f:spec":{"f:suspend":{}}}`)}, + FieldsV1: metav1.NewFieldsV1(`{"f:spec":{"f:suspend":{}}}`), }} } @@ -286,7 +286,7 @@ func TestOwnsSuspensionManagedFields(t *testing.T) { {"no managed fields", nil, false}, {"plunger apply of other fields", []metav1.ManagedFieldsEntry{{ Manager: suspendFieldManager, Operation: metav1.ManagedFieldsOperationApply, - FieldsType: "FieldsV1", FieldsV1: &metav1.FieldsV1{Raw: []byte(`{"f:spec":{"f:chart":{}}}`)}, + FieldsType: "FieldsV1", FieldsV1: metav1.NewFieldsV1(`{"f:spec":{"f:chart":{}}}`), }}, false}, } for _, tc := range cases { diff --git a/internal/controller/kubeovnplunger/kubeovn_plunger.go b/internal/controller/kubeovnplunger/kubeovn_plunger.go index a45fba073..30aa42c00 100644 --- a/internal/controller/kubeovnplunger/kubeovn_plunger.go +++ b/internal/controller/kubeovnplunger/kubeovn_plunger.go @@ -3,8 +3,10 @@ package kubeovnplunger import ( "bytes" "context" + "errors" "fmt" "io" + "net/http" "strings" "time" @@ -153,12 +155,9 @@ func (r *KubeOVNPlunger) SetupWithManager(mgr ctrl.Manager, kubeOVNNamespace, ap Complete(r); err != nil { return err } - _ = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { - go srv.ListenAndServe() - <-ctx.Done() - _ = srv.Shutdown(context.Background()) - return nil - })) + if err := mgr.Add(serveStatus(srv)); err != nil { + return err + } return mgr.Add(manager.RunnableFunc(func(ctx context.Context) error { ticker := time.NewTicker(rescanInterval) defer ticker.Stop() @@ -180,6 +179,25 @@ func (r *KubeOVNPlunger) SetupWithManager(mgr ctrl.Manager, kubeOVNNamespace, ap })) } +// serveStatus runs the status stream for as long as the manager does. A server +// that cannot start fails the manager, as the metrics server does. +func serveStatus(s *sse.Server) manager.RunnableFunc { + return func(ctx context.Context) error { + served := make(chan error, 1) + go func() { served <- s.ListenAndServe() }() + select { + case err := <-served: + return err + case <-ctx.Done(): + } + _ = s.Shutdown(context.Background()) + if err := <-served; !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil + } +} + type ExecOptions struct { Namespace string Pod string diff --git a/internal/controller/kubeovnplunger/metrics.go b/internal/controller/kubeovnplunger/metrics.go index 1330db814..523467fcd 100644 --- a/internal/controller/kubeovnplunger/metrics.go +++ b/internal/controller/kubeovnplunger/metrics.go @@ -219,7 +219,7 @@ func (r *KubeOVNPlunger) WriteClusterMetrics(db string, snaps []ovnstatus.Health r.metrics.allAgree.WithLabelValues(db, cid).Set(b2f(ecv.AllAgree)) r.metrics.membersExpected.WithLabelValues(db, cid).Set(float64(expectedReplicas)) r.metrics.membersObserved.WithLabelValues(db, cid).Set(float64(ecv.MembersCount)) - r.metrics.ipsExpected.WithLabelValues(db, cid).Set(float64(len(ecv.ConsensusResult.TruthView.Members))) // optional; or len(hints.ExpectedIPs) + r.metrics.ipsExpected.WithLabelValues(db, cid).Set(float64(len(ecv.TruthView.Members))) // optional; or len(hints.ExpectedIPs) r.metrics.ipsObserved.WithLabelValues(db, cid).Set(float64(ecv.DistinctIPCount)) r.metrics.excessMembers.WithLabelValues(db, cid).Set(float64(ecv.ExpectedExcess)) r.metrics.missingMembers.WithLabelValues(db, cid).Set(float64(ecv.ExpectedShortfall)) diff --git a/internal/controller/kubeovnplunger/status_server_test.go b/internal/controller/kubeovnplunger/status_server_test.go new file mode 100644 index 000000000..c894ecf74 --- /dev/null +++ b/internal/controller/kubeovnplunger/status_server_test.go @@ -0,0 +1,51 @@ +package kubeovnplunger + +import ( + "context" + "io" + "log" + "net" + "testing" + "time" + + "github.com/cozystack/cozystack/internal/sse" +) + +func runStatusServer(ctx context.Context, t *testing.T, addr string) (bool, error) { + t.Helper() + ctx, cancel := context.WithCancel(ctx) + t.Cleanup(cancel) + s := sse.New(sse.Options{Addr: addr, Logger: log.New(io.Discard, "", 0)}) + done := make(chan error, 1) + go func() { done <- serveStatus(s)(ctx) }() + select { + case err := <-done: + return true, err + case <-time.After(2 * time.Second): + return false, nil + } +} + +// A status server that cannot listen fails its runnable, and with it the +// manager, rather than leaving the plunger running without it. +func TestStatusServerReportsListenFailure(t *testing.T) { + taken, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = taken.Close() }) + returned, err := runStatusServer(context.Background(), t, taken.Addr().String()) + if !returned || err == nil { + t.Fatalf("serveStatus on an occupied address = %v (returned %v), want an error", err, returned) + } +} + +// Stopping the manager stops the server, and that is not an error. +func TestStatusServerStopsWithoutError(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(100*time.Millisecond, cancel) + returned, err := runStatusServer(ctx, t, "127.0.0.1:0") + if !returned || err != nil { + t.Fatalf("serveStatus after cancel = %v (returned %v), want nil", err, returned) + } +} diff --git a/internal/controller/workload_controller.go b/internal/controller/workload_controller.go index d0d9e7430..052358a05 100644 --- a/internal/controller/workload_controller.go +++ b/internal/controller/workload_controller.go @@ -3,7 +3,6 @@ package controller import ( "context" "strings" - "time" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -15,10 +14,6 @@ import ( cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1" ) -const ( - deletionRequeueDelay = 30 * time.Second -) - // WorkloadMonitorReconciler reconciles a WorkloadMonitor object type WorkloadReconciler struct { client.Client diff --git a/internal/controller/workloadmonitor_controller.go b/internal/controller/workloadmonitor_controller.go index 984ab7b21..d50f1d6e0 100644 --- a/internal/controller/workloadmonitor_controller.go +++ b/internal/controller/workloadmonitor_controller.go @@ -28,7 +28,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "k8s.io/client-go/util/workqueue" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" @@ -374,6 +374,25 @@ func (r *WorkloadMonitorReconciler) isBucketClaimReady(bc *cosiv1alpha1.BucketCl return bc.Status.BucketReady } +// bucketClaimsMessage names every BucketClaim that has not been provisioned, +// with its bucket class, and is empty when there is none. +func (r *WorkloadMonitorReconciler) bucketClaimsMessage(claims []cosiv1alpha1.BucketClaim) string { + var pending []string + for i := range claims { + bc := &claims[i] + if r.isBucketClaimReady(bc) { + continue + } + entry := fmt.Sprintf("BucketClaim %s is not provisioned", bc.Name) + if bc.Spec.BucketClassName != "" { + entry = fmt.Sprintf("BucketClaim %s of bucket class %s is not provisioned", bc.Name, bc.Spec.BucketClassName) + } + pending = append(pending, entry) + } + sort.Strings(pending) + return strings.Join(pending, "; ") +} + // isServiceReady checks if the service has an external IP bound func (r *WorkloadMonitorReconciler) isServiceReady(svc *corev1.Service) bool { return len(svc.Status.LoadBalancer.Ingress) > 0 @@ -423,8 +442,8 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) { Name: monitor.GetName(), UID: monitor.GetUID(), // Set Controller to false to avoid conflict as multiple controllers are not allowed - Controller: pointer.BoolPtr(false), - BlockOwnerDeletion: pointer.BoolPtr(true), + Controller: ptr.To(false), + BlockOwnerDeletion: ptr.To(true), } owners = append(owners, newOwnerRef) } @@ -525,10 +544,10 @@ func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, p if err != nil { return nil, fmt.Errorf("querying Prometheus: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Prometheus returned status %d", resp.StatusCode) + return nil, fmt.Errorf("unexpected status %d from Prometheus", resp.StatusCode) } body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) @@ -549,7 +568,7 @@ func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, p return nil, fmt.Errorf("parsing Prometheus response: %w", err) } if promResp.Status != "success" { - return nil, fmt.Errorf("Prometheus query finished with status %q", promResp.Status) + return nil, fmt.Errorf("the Prometheus query finished with status %q", promResp.Status) } for _, r := range promResp.Data.Result { @@ -1157,6 +1176,8 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } + bucketsMessage := r.bucketClaimsMessage(bucketClaimList.Items) + // Update WorkloadMonitor status based on observed pods monitor.Status.ObservedReplicas = observedReplicas monitor.Status.AvailableReplicas = availableReplicas @@ -1174,22 +1195,30 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ // Default to operational = true, but check MinReplicas if set. // Use fresh.Spec to avoid making decisions based on a stale cached copy // when the spec was updated between the initial read and this retry. - fresh.Status.Operational = pointer.Bool(true) + fresh.Status.Operational = ptr.To(true) if fresh.Spec.MinReplicas != nil && availableReplicas < *fresh.Spec.MinReplicas { - fresh.Status.Operational = pointer.Bool(false) + fresh.Status.Operational = ptr.To(false) } - // Only the DataVolume check writes Message and Reason, so the stored ones - // are the last DataVolume verdict: kept while no reader is installed yet, so a - // controller restart does not report a stuck disk as ready. + // A stored DataVolume verdict is the last one: kept while no reader is + // installed yet, so a controller restart does not report a stuck disk as + // ready. BucketClaims are listed on every pass, so their verdict is + // recomputed and only fills fields no DataVolume verdict holds. if dataVolumesRead { fresh.Status.Message = dataVolumesMessage fresh.Status.Reason = "" if dataVolumesMessage != "" { fresh.Status.Reason = cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady } + } else if fresh.Status.Reason == cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady { + fresh.Status.Message = "" + fresh.Status.Reason = "" + } + if fresh.Status.Message == "" && bucketsMessage != "" { + fresh.Status.Message = bucketsMessage + fresh.Status.Reason = cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady } if fresh.Status.Message != "" { - fresh.Status.Operational = pointer.Bool(false) + fresh.Status.Operational = ptr.To(false) } if equality.Semantic.DeepEqual(stored, &fresh.Status) { return nil diff --git a/internal/controller/workloadmonitor_controller_test.go b/internal/controller/workloadmonitor_controller_test.go index 353066c4b..70bf5aa0f 100644 --- a/internal/controller/workloadmonitor_controller_test.go +++ b/internal/controller/workloadmonitor_controller_test.go @@ -870,7 +870,7 @@ func TestReconcileNoBucketClaimSkips(t *testing.T) { func TestQueryAllBucketMetrics(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[ + _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[ {"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"bucket-aaa"},"value":[1713000000,"10485864"]}, {"metric":{"__name__":"SeaweedFS_s3_bucket_physical_size_bytes","bucket":"bucket-aaa"},"value":[1713000000,"20971728"]}, {"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"bucket-bbb"},"value":[1713000000,"0"]} @@ -909,7 +909,7 @@ func TestQueryAllBucketMetrics(t *testing.T) { func TestQueryAllBucketMetricsEmpty(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) + _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) })) defer srv.Close() @@ -953,7 +953,7 @@ func TestQueryAllBucketMetricsPathPrefixAndUserinfo(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { gotPath = r.URL.Path gotUser, gotPass, gotOK = r.BasicAuth() - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) + _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) })) defer srv.Close() @@ -1219,7 +1219,7 @@ func TestReconcileBucketMetricsFromEndpointOverride(t *testing.T) { if r.URL.Path != "/vm/select/0/prometheus/api/v1/query" { t.Errorf("unexpected query path %q", r.URL.Path) } - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[ + _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[ {"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"cosi-abc123"},"value":[1713000000,"4096"]} ]}}`) })) @@ -1331,7 +1331,7 @@ func TestReconcileRetainsLastKnownSizesOnQueryFailure(t *testing.T) { func TestReconcileRetainsLastKnownSizesWhenBucketMissingFromResult(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) + _, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) })) defer srv.Close() @@ -2975,3 +2975,135 @@ func TestReconcileBucketApp_LegacySizeWinsOverOwnerlessNewWorkload(t *testing.T) t.Errorf("size = %s, want the legacy 689Gi over the stale 10Gi", got.String()) } } + +func newBucketClaim(name string, labels map[string]string, class string, ready bool) *cosiv1alpha1.BucketClaim { + return &cosiv1alpha1.BucketClaim{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Labels: labels}, + Spec: cosiv1alpha1.BucketClaimSpec{ + BucketClassName: class, + Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3}, + }, + Status: cosiv1alpha1.BucketClaimStatus{BucketReady: ready}, + } +} + +func reconcileBucketMonitor(t *testing.T, c client.Client, s *runtime.Scheme, withReader bool) *cozyv1alpha1.WorkloadMonitor { + t.Helper() + reconciler := &WorkloadMonitorReconciler{Client: c, Scheme: s} + if withReader { + reconciler.DataVolumeReader = c + } + req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "my-bucket", Namespace: "default"}} + if _, err := reconciler.Reconcile(context.TODO(), req); err != nil { + t.Fatalf("Reconcile returned error: %v", err) + } + updated := &cozyv1alpha1.WorkloadMonitor{} + if err := c.Get(context.TODO(), req.NamespacedName, updated); err != nil { + t.Fatalf("Failed to get updated WorkloadMonitor: %v", err) + } + if updated.Status.Operational == nil { + t.Fatal("Expected Operational to be set, got nil") + } + return updated +} + +func newBucketMonitorClient(s *runtime.Scheme, objs ...client.Object) client.Client { + monitor := &cozyv1alpha1.WorkloadMonitor{ + ObjectMeta: metav1.ObjectMeta{Name: "my-bucket", Namespace: "default"}, + Spec: cozyv1alpha1.WorkloadMonitorSpec{ + Kind: "bucket", + Type: "s3", + Selector: map[string]string{"app.kubernetes.io/instance": "my-bucket"}, + Replicas: ptr.To[int32](0), + MinReplicas: ptr.To[int32](0), + }, + } + return fake.NewClientBuilder(). + WithScheme(s). + WithObjects(append([]client.Object{monitor}, objs...)...). + WithStatusSubresource(monitor). + Build() +} + +func TestReconcile_BucketClaimDecidesOperational(t *testing.T) { + selected := map[string]string{"app.kubernetes.io/instance": "my-bucket"} + cases := []struct { + name string + claim *cosiv1alpha1.BucketClaim + operational bool + message string + }{ + {"provisioned", newBucketClaim("my-bucket", selected, "tenant-root", true), true, ""}, + {"not provisioned", newBucketClaim("my-bucket", selected, "tenant-root", false), false, "BucketClaim my-bucket of bucket class tenant-root is not provisioned"}, + {"not provisioned without a class", newBucketClaim("my-bucket", selected, "", false), false, "BucketClaim my-bucket is not provisioned"}, + {"outside the selector", newBucketClaim("other", map[string]string{"app.kubernetes.io/instance": "other"}, "tenant-root", false), true, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := newTestScheme() + got := reconcileBucketMonitor(t, newBucketMonitorClient(s, tc.claim), s, true) + if *got.Status.Operational != tc.operational { + t.Errorf("Operational = %v, want %v", *got.Status.Operational, tc.operational) + } + if got.Status.Message != tc.message { + t.Errorf("Message = %q, want %q", got.Status.Message, tc.message) + } + wantReason := "" + if tc.message != "" { + wantReason = cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady + } + if got.Status.Reason != wantReason { + t.Errorf("Reason = %q, want %q", got.Status.Reason, wantReason) + } + }) + } +} + +func TestReconcile_ProvisionedBucketClaimClearsTheVerdict(t *testing.T) { + for _, withReader := range []bool{true, false} { + t.Run(fmt.Sprintf("dataVolumeReader=%v", withReader), func(t *testing.T) { + s := newTestScheme() + claim := newBucketClaim("my-bucket", map[string]string{"app.kubernetes.io/instance": "my-bucket"}, "tenant-root", false) + c := newBucketMonitorClient(s, claim) + + got := reconcileBucketMonitor(t, c, s, withReader) + if *got.Status.Operational || got.Status.Reason != cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady { + t.Fatalf("before provisioning: Operational = %v, Reason = %q", *got.Status.Operational, got.Status.Reason) + } + + stored := &cosiv1alpha1.BucketClaim{} + if err := c.Get(context.TODO(), types.NamespacedName{Name: "my-bucket", Namespace: "default"}, stored); err != nil { + t.Fatalf("Failed to get BucketClaim: %v", err) + } + stored.Status.BucketReady = true + if err := c.Update(context.TODO(), stored); err != nil { + t.Fatalf("Failed to update BucketClaim: %v", err) + } + + got = reconcileBucketMonitor(t, c, s, withReader) + if !*got.Status.Operational || got.Status.Message != "" || got.Status.Reason != "" { + t.Errorf("after provisioning: Operational = %v, Message = %q, Reason = %q", *got.Status.Operational, got.Status.Message, got.Status.Reason) + } + }) + } +} + +func TestReconcile_DataVolumeVerdictOutranksBucketClaim(t *testing.T) { + s := newTestScheme() + selected := map[string]string{"app.kubernetes.io/instance": "my-bucket"} + phase := "ImportInProgress" + c := newBucketMonitorClient(s, + newBucketClaim("my-bucket", selected, "tenant-root", false), + newDataVolume("my-bucket", selected, &phase), + ) + got := reconcileBucketMonitor(t, c, s, true) + if *got.Status.Operational { + t.Error("Operational = true, want false") + } + if got.Status.Reason != cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady { + t.Errorf("Reason = %q, want %q", got.Status.Reason, cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady) + } + if got.Status.Message != "DataVolume my-bucket is ImportInProgress" { + t.Errorf("Message = %q", got.Status.Message) + } +} diff --git a/internal/crdinstall/install.go b/internal/crdinstall/install.go index 5760beb6b..12706060b 100644 --- a/internal/crdinstall/install.go +++ b/internal/crdinstall/install.go @@ -40,7 +40,7 @@ func Install(ctx context.Context, k8sClient client.Client, writeEmbeddedManifest if err != nil { return fmt.Errorf("failed to create temp directory: %w", err) } - defer os.RemoveAll(tmpDir) + defer func() { _ = os.RemoveAll(tmpDir) }() manifestsDir := filepath.Join(tmpDir, "manifests") if err := os.MkdirAll(manifestsDir, 0755); err != nil { diff --git a/internal/fluxinstall/install.go b/internal/fluxinstall/install.go index d16d7d854..63c977803 100644 --- a/internal/fluxinstall/install.go +++ b/internal/fluxinstall/install.go @@ -41,7 +41,7 @@ func Install(ctx context.Context, k8sClient client.Client, writeEmbeddedManifest if err != nil { return fmt.Errorf("failed to create temp directory: %w", err) } - defer os.RemoveAll(tmpDir) + defer func() { _ = os.RemoveAll(tmpDir) }() // Extract embedded manifests (generated by cozypkg) manifestsDir := filepath.Join(tmpDir, "manifests") @@ -171,7 +171,7 @@ func extractNamespace(objects []*unstructured.Unstructured) (string, error) { if obj.GetKind() == "Namespace" { namespace := obj.GetName() if namespace == "" { - return "", fmt.Errorf("Namespace object has no name") + return "", fmt.Errorf("the Namespace object has no name") } return namespace, nil } diff --git a/internal/lineagecontrollerwebhook/webhook.go b/internal/lineagecontrollerwebhook/webhook.go index ea8b0d01b..c3efd84bd 100644 --- a/internal/lineagecontrollerwebhook/webhook.go +++ b/internal/lineagecontrollerwebhook/webhook.go @@ -30,8 +30,8 @@ import ( ) var ( - NoAncestors = errors.New("no managed apps found in lineage") - AncestryAmbiguous = errors.New("object ancestry is ambiguous") + ErrNoAncestors = errors.New("no managed apps found in lineage") + ErrAncestryAmbiguous = errors.New("object ancestry is ambiguous") ) const ( @@ -115,9 +115,9 @@ func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Req owner, err := h.getOwner(ctx, obj) switch { - case err != nil && errors.Is(err, AncestryAmbiguous): + case err != nil && errors.Is(err, ErrAncestryAmbiguous): warn = append(warn, "object ancestry ambiguous, using first ancestor found") - case err != nil && errors.Is(err, NoAncestors): + case err != nil && errors.Is(err, ErrNoAncestors): // not a problem, mark object as unmanaged case err != nil: logger.Error(err, "error computing lineage labels") @@ -147,14 +147,14 @@ func (h *LineageControllerWebhook) Handle(ctx context.Context, req admission.Req func (h *LineageControllerWebhook) getOwner(ctx context.Context, o *unstructured.Unstructured) (*unstructured.Unstructured, error) { owners := lineage.WalkOwnershipGraphWithCache(ctx, h.dynClient, h.mapper, h, h.ownerCache, o) if len(owners) == 0 { - return nil, NoAncestors + return nil, ErrNoAncestors } obj, err := owners[0].GetUnstructuredCached(ctx, h.dynClient, h.mapper, h.ownerCache) if err != nil { return nil, err } if len(owners) > 1 { - err = AncestryAmbiguous + err = ErrAncestryAmbiguous } return obj, err } @@ -176,7 +176,7 @@ func (h *LineageControllerWebhook) computeLabels(ctx context.Context, obj *unstr return s } s = s[:63] - for b := s[62]; !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9')); s = s[:len(s)-1] { + for b := s[62]; (b < 'a' || b > 'z') && (b < 'A' || b > 'Z') && (b < '0' || b > '9'); s = s[:len(s)-1] { b = s[len(s)-1] } return s diff --git a/internal/migrationcontroller/inventory.go b/internal/migrationcontroller/inventory.go index d56bec2da..52a5e4cd8 100644 --- a/internal/migrationcontroller/inventory.go +++ b/internal/migrationcontroller/inventory.go @@ -146,7 +146,7 @@ func (c *inventoryClient) VMs(ctx context.Context, providerUID string) ([]invent defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Forklift inventory returned %s listing VMs", resp.Status) + return nil, fmt.Errorf("listing VMs: the Forklift inventory returned %s", resp.Status) } var vms []inventoryVMRef @@ -195,7 +195,7 @@ func (c *inventoryClient) VM(ctx context.Context, providerUID, vmID string) (*in case http.StatusNotFound: return nil, ¬FoundError{vmID: vmID} default: - return nil, fmt.Errorf("Forklift inventory returned %s for VM %q", resp.Status, vmID) + return nil, fmt.Errorf("VM %q: the Forklift inventory returned %s", vmID, resp.Status) } vm := &inventoryVM{} diff --git a/internal/observabilityproxy/appkinds.go b/internal/observabilityproxy/appkinds.go new file mode 100644 index 000000000..d58368a13 --- /dev/null +++ b/internal/observabilityproxy/appkinds.go @@ -0,0 +1,101 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "errors" + "fmt" + "slices" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/cache" +) + +// ApplicationDefinitionsResource is where the plural of every application +// kind is declared (api/v1alpha1/applicationdefinitions_types.go). +var ApplicationDefinitionsResource = schema.GroupVersionResource{Group: "cozystack.io", Version: "v1alpha1", Resource: "applicationdefinitions"} + +// ErrKindsNotSynced means the ApplicationDefinitions cache has not been +// filled yet, so an unknown plural cannot be told from a missing one. +var ErrKindsNotSynced = errors.New("application definitions not synced") + +const pluralIndex = "plural" + +// KindLookup maps an application plural, as it appears in paths and RBAC, +// to the kind the lineage labels carry. An unknown plural is "", nil. +type KindLookup interface { + Kind(plural string) (string, error) +} + +// ApplicationPluralIndexers indexes ApplicationDefinitions by +// spec.application.plural; the informer feeding ApplicationKinds needs it. +func ApplicationPluralIndexers() cache.Indexers { + return cache.Indexers{pluralIndex: func(obj any) ([]string, error) { + u, ok := obj.(*unstructured.Unstructured) + if !ok { + return nil, nil + } + plural, _, _ := unstructured.NestedString(u.Object, "spec", "application", "plural") + if plural == "" { + return nil, nil + } + return []string{plural}, nil + }} +} + +// ApplicationKinds is the one place the proxy maps a plural to its kind; +// both application roots use it. +type ApplicationKinds struct { + indexer cache.Indexer + hasSynced func() bool +} + +func NewApplicationKinds(indexer cache.Indexer, hasSynced func() bool) *ApplicationKinds { + return &ApplicationKinds{indexer: indexer, hasSynced: hasSynced} +} + +func (k *ApplicationKinds) Kind(plural string) (string, error) { + if !k.hasSynced() { + return "", ErrKindsNotSynced + } + objs, err := k.indexer.ByIndex(pluralIndex, plural) + if err != nil { + return "", err + } + var kinds []string + for _, obj := range objs { + u, ok := obj.(*unstructured.Unstructured) + if !ok { + continue + } + kind, _, _ := unstructured.NestedString(u.Object, "spec", "application", "kind") + if kind != "" && !slices.Contains(kinds, kind) { + kinds = append(kinds, kind) + } + } + switch len(kinds) { + case 0: + return "", nil + case 1: + return kinds[0], nil + } + // Picking one would scope the request to whichever definition the + // cache happens to list first. + slices.Sort(kinds) + return "", fmt.Errorf("plural %q is declared for kinds %v", plural, kinds) +} diff --git a/internal/observabilityproxy/appkinds_test.go b/internal/observabilityproxy/appkinds_test.go new file mode 100644 index 000000000..6eb8d60bc --- /dev/null +++ b/internal/observabilityproxy/appkinds_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "errors" + "testing" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/tools/cache" +) + +func appDefinition(name, kind, plural string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "cozystack.io/v1alpha1", + "kind": "ApplicationDefinition", + "metadata": map[string]any{"name": name}, + "spec": map[string]any{"application": map[string]any{"kind": kind, "plural": plural, "singular": name}}, + }} +} + +func newTestKinds(t *testing.T, synced bool, defs ...*unstructured.Unstructured) *ApplicationKinds { + t.Helper() + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, ApplicationPluralIndexers()) + for _, d := range defs { + if err := indexer.Add(d); err != nil { + t.Fatal(err) + } + } + return NewApplicationKinds(indexer, func() bool { return synced }) +} + +func TestApplicationKinds(t *testing.T) { + kinds := newTestKinds(t, true, + appDefinition("postgres", "Postgres", "postgreses"), + appDefinition("vm-instance", "VMInstance", "vminstances"), + appDefinition("broken", "", "brokens"), + ) + for plural, want := range map[string]string{"postgreses": "Postgres", "vminstances": "VMInstance", "postgres": "", "brokens": "", "": ""} { + got, err := kinds.Kind(plural) + if err != nil || got != want { + t.Errorf("Kind(%q) = %q, %v; want %q", plural, got, err, want) + } + } +} + +func TestApplicationKindsNotSynced(t *testing.T) { + kinds := newTestKinds(t, false, appDefinition("postgres", "Postgres", "postgreses")) + if _, err := kinds.Kind("postgreses"); !errors.Is(err, ErrKindsNotSynced) { + t.Fatalf("Kind before sync: err = %v, want ErrKindsNotSynced", err) + } +} + +func TestApplicationKindsAmbiguous(t *testing.T) { + kinds := newTestKinds(t, true, appDefinition("a", "Postgres", "postgreses"), appDefinition("b", "Postgresql", "postgreses")) + if kind, err := kinds.Kind("postgreses"); err == nil { + t.Fatalf("two kinds for one plural resolved to %q", kind) + } + same := newTestKinds(t, true, appDefinition("a", "Postgres", "postgreses"), appDefinition("b", "Postgres", "postgreses")) + if kind, err := same.Kind("postgreses"); err != nil || kind != "Postgres" { + t.Fatalf("duplicate definitions of one kind: %q, %v", kind, err) + } +} diff --git a/internal/observabilityproxy/appscope.go b/internal/observabilityproxy/appscope.go new file mode 100644 index 000000000..659b55052 --- /dev/null +++ b/internal/observabilityproxy/appscope.go @@ -0,0 +1,217 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +// AppScopeLimits bound the pod and PVC lookup of the application metrics +// root. +type AppScopeLimits struct { + // Lookback is how far before the window the lookup reaches at most, so + // pods gone before the window but inside an expression's range count. + Lookback time.Duration + MaxPods int + MaxPVCs int +} + +// maxLookupBytes bounds a lookup answer; the caps keep a valid one far +// below it. +const maxLookupBytes = 8 << 20 + +// The lineage labels as the scrape class writes them on series, and as +// kube-state-metrics writes them on kube_pod_labels and +// kube_persistentvolumeclaim_labels. +const ( + seriesKindLabel = "apps_cozystack_io_application_kind" + seriesNameLabel = "apps_cozystack_io_application_name" + ksmKindLabel = "label_" + seriesKindLabel + ksmNameLabel = "label_" + seriesNameLabel +) + +func identitySelector(ns, kindLabel, nameLabel, kind, name string) string { + return "{namespace=" + strconv.Quote(ns) + "," + kindLabel + "=" + strconv.Quote(kind) + "," + nameLabel + "=" + strconv.Quote(name) + "}" +} + +// nameBranch matches label against any of names within ns. With no names +// it is "": a branch of pod=~"^()$" would match every series without a pod +// label. +func nameBranch(ns, label string, names []string) string { + if len(names) == 0 { + return "" + } + quoted := make([]string, len(names)) + for i, n := range names { + quoted[i] = regexp.QuoteMeta(n) + } + return "{namespace=" + strconv.Quote(ns) + "," + label + "=~" + strconv.Quote("^("+strings.Join(quoted, "|")+")$") + "}" +} + +// applicationSelectors is the application scope: series stamped with the +// application's identity, plus series of its pods and PVCs for those not +// stamped (cAdvisor, kubelet volume stats). Every branch carries the +// namespace, so their OR stays inside it. +// +// Within the namespace this narrows, it does not isolate: the identity and +// the pod and PVC lists come from lineage labels that the lineage webhook +// keeps as set on an object it finds no owner for, and the pod and PVC +// branches match by name, so a name another application held within the +// lookup window pulls that application's series in. Route.Reviews makes +// the application roots require read access to the namespace root too. +func applicationSelectors(ns, kind, name string, pods, pvcs []string) []string { + out := []string{identitySelector(ns, seriesKindLabel, seriesNameLabel, kind, name)} + for _, b := range []string{nameBranch(ns, "pod", pods), nameBranch(ns, "persistentvolumeclaim", pvcs)} { + if b != "" { + out = append(out, b) + } + } + return out +} + +// applicationScope looks the application's pods and PVCs up in store over +// the samples the request reads: back by the query's own lookback, and +// forward to the latest sample a negative offset or a subquery's +// evaluation past its end reads. A query that reads +// further back than the configured lookback, or whose earliest or latest +// sample is unknown, a failed lookup, and a list over its cap all fail the +// request, because a successful answer would silently miss the +// application's series. +func (s *Server) applicationScope(ctx context.Context, r *http.Request, store string, req metricsRequest, ns, kind, name string) ([]string, *HTTPError) { + if req.last.IsZero() { + return nil, errorf(http.StatusUnprocessableEntity, "the query reads samples at a time the application scope cannot cover (an @ anywhere in the query, a backslash or a comment outside a string, rollup_candlestick, WITH with a negative offset, an offset that is not a plain duration, or a subquery step that is not one); use the namespace root") + } + lim := s.cfg.Metrics.App + if req.start.IsZero() || req.start.Before(req.from.Add(-lim.Lookback)) { + return nil, errorf(http.StatusUnprocessableEntity, "the query reads further back than the application scope looks up pods (%s before the window, or an unknown lookback); use a shorter window or the namespace root", lim.Lookback) + } + start := req.start + var lists [2][]string + for i, l := range []struct { + label, metric string + max int + }{{"pod", "kube_pod_labels", lim.MaxPods}, {"persistentvolumeclaim", "kube_persistentvolumeclaim_labels", lim.MaxPVCs}} { + match := l.metric + identitySelector(ns, ksmKindLabel, ksmNameLabel, kind, name) + names, err := s.lookupNames(ctx, r, store, l.label, match, ns, start, req.last, l.max) + if err != nil { + slog.Warn("application lookup failed", "label", l.label, "namespace", ns, "kind", kind, "name", name, "error", err) + // The deadline can also expire while the answer is read, after + // post has handed it over. + if errors.Is(context.Cause(ctx), context.DeadlineExceeded) { + return nil, errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", l.label) + } + if herr, ok := err.(*HTTPError); ok && (herr.Code == http.StatusUnprocessableEntity || herr.Code == http.StatusGatewayTimeout) { + return nil, herr + } + return nil, errorf(http.StatusBadGateway, "the application's %s lookup failed", l.label) + } + if len(names) > l.max { + return nil, errorf(http.StatusUnprocessableEntity, "the application has more than %d %ss in this window; use a shorter range", l.max, l.label) + } + lists[i] = names + } + return applicationSelectors(ns, kind, name, lists[0], lists[1]), nil +} + +// lookupNames asks for at most limit+1 values, enough to tell a list over +// the cap without reading all of it. +func (s *Server) lookupNames(ctx context.Context, r *http.Request, store, label, match, ns string, start, end time.Time, limit int) ([]string, error) { + params := url.Values{} + params.Set("match[]", match) + params.Set("extra_filters[]", namespaceSelector(ns)) + params.Set("start", formatTime(start)) + params.Set("end", formatTime(end)) + params.Set("limit", strconv.Itoa(limit+1)) + resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/api/v1/label/"+label+"/values", params, "metrics store", nil) + if herr != nil { + return nil, herr + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusUnprocessableEntity { + return nil, lookupRefusal(label, resp.Body) + } + if resp.StatusCode != http.StatusOK { + return nil, errorf(resp.StatusCode, "metrics store answered %s", resp.Status) + } + var body struct { + Status string `json:"status"` + IsPartial bool `json:"isPartial"` + Data []string `json:"data"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, maxLookupBytes)).Decode(&body); err != nil { + return nil, err + } + if body.Status != "success" { + return nil, errorf(http.StatusBadGateway, "metrics store answered status %q", body.Status) + } + if len(body.Data) > limit { + // Over the cap before dropping empty values: the store cut the + // list at limit+1, so a shorter filtered list could hide names. + // A partial list over the cap settles it too. + return body.Data, nil + } + // A storage node that did not answer leaves names out of the list. + if body.IsPartial { + return nil, errorf(http.StatusBadGateway, "metrics store answered a partial %s list", label) + } + names := body.Data[:0] + for _, n := range body.Data { + if n != "" { + names = append(names, n) + } + } + return names, nil +} + +// seriesLimitError is the message vmstorage gives when a search matches +// more series than the -search.max* flags allow (lib/storage/index_db.go); +// cluster vmselect wraps it in its own context, hence a substring match. +const seriesLimitError = "the number of matching timeseries exceeds" + +// lookupRefusal reads a vmselect 422. vmselect answers 422 to any error +// without a status of its own (httpserver.SendPrometheusError), storage +// failures and its own timeout included, so only the series limit, which +// a narrower window fixes, stays a 422 for the client. +func lookupRefusal(label string, body io.Reader) *HTTPError { + var answer struct { + Error string `json:"error"` + } + _ = json.NewDecoder(io.LimitReader(body, 64<<10)).Decode(&answer) + switch { + case strings.Contains(answer.Error, seriesLimitError): + // From the limit on: what precedes it is cluster vmselect's context, + // which can name vmstorage nodes. + limit := answer.Error[strings.Index(answer.Error, seriesLimitError):] + return errorf(http.StatusUnprocessableEntity, "the application's %s lookup: %s", label, limit) + case strings.Contains(answer.Error, "timeout exceeded"): + return errorf(http.StatusGatewayTimeout, "the application's %s lookup did not answer in time", label) + default: + // The text may name vmstorage nodes, so it stays in the log. + slog.Warn("application lookup refused by the store", "label", label, "error", answer.Error) + return errorf(http.StatusBadGateway, "the application's %s lookup failed", label) + } +} diff --git a/internal/observabilityproxy/appscope_test.go b/internal/observabilityproxy/appscope_test.go new file mode 100644 index 000000000..c4b8e5d69 --- /dev/null +++ b/internal/observabilityproxy/appscope_test.go @@ -0,0 +1,129 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "net/http" + "net/url" + "regexp" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +const wantIdentityBranch = `{namespace="tenant-a",apps_cozystack_io_application_kind="Postgres",apps_cozystack_io_application_name="db.main"}` + +func TestApplicationSelectors(t *testing.T) { + tests := []struct { + name string + pods, pvcs []string + want []string + }{ + {"identity only", nil, nil, []string{wantIdentityBranch}}, + {"pods", []string{"db-main-1", "db.main-2"}, nil, []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(db-main-1|db\\.main-2)$"}`, + }}, + {"pvcs", nil, []string{"data-db-main-1"}, []string{ + wantIdentityBranch, + `{namespace="tenant-a",persistentvolumeclaim=~"^(data-db-main-1)$"}`, + }}, + {"all three", []string{"p"}, []string{"v"}, []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(p)$"}`, + `{namespace="tenant-a",persistentvolumeclaim=~"^(v)$"}`, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := applicationSelectors("tenant-a", "Postgres", "db.main", tt.pods, tt.pvcs); !slices.Equal(got, tt.want) { + t.Fatalf("got %q\nwant %q", got, tt.want) + } + }) + } +} + +// DNS-1123 keeps most metacharacters out of names, but the names come +// back from the store, so the encoder must not rely on that. +func TestNameBranchEscapes(t *testing.T) { + names := []string{`a.b`, `x|.*`, `)|(.*`, `q"} or {namespace=~".+`, `back\slash`, `$^+?[]{}`} + sel := nameBranch("tenant-a", "pod", names) + m := regexp.MustCompile(`^\{namespace="tenant-a",pod=~(".*")\}$`).FindStringSubmatch(sel) + if m == nil { + t.Fatalf("selector %s is not one namespace and one pod matcher", sel) + } + pattern, err := strconv.Unquote(m[1]) + if err != nil { + t.Fatalf("pod matcher %s does not unquote: %v", m[1], err) + } + re := regexp.MustCompile(pattern) + for _, n := range names { + if !re.MatchString(n) { + t.Errorf("%q does not match its own branch", n) + } + } + for _, other := range []string{"axb", "x", "anything", "tenant-b", "", `back\\slash`} { + if re.MatchString(other) { + t.Errorf("branch matches %q", other) + } + } + if nameBranch("tenant-a", "pod", nil) != "" { + t.Error("an empty list produced a branch, which would match series without the label") + } +} + +func TestMetricsRequestScope(t *testing.T) { + branches := []string{"{a}", "{b}"} + labels := buildMetrics(t, "api/v1/labels", url.Values{}) + labels.setScope(branches) + if got := labels.params["extra_filters[]"]; !slices.Equal(got, branches) { + t.Errorf("extra_filters[] = %q", got) + } + if got := labels.params["match[]"]; !slices.Equal(got, branches) { + t.Errorf("default match[] = %q, want the branches", got) + } + client := buildMetrics(t, "api/v1/labels", url.Values{"match[]": {"up"}}) + client.setScope(branches) + if got := client.params["match[]"]; !slices.Equal(got, []string{"up"}) { + t.Errorf("client match[] replaced: %q", got) + } + query := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}) + query.setScope(branches) + if query.params.Has("match[]") || !slices.Equal(query.params["extra_filters[]"], branches) { + t.Errorf("query params = %v", query.params) + } + if !query.from.Equal(testNow) || !query.to.Equal(testNow) { + t.Errorf("instant window = [%s, %s], want now", query.from, query.to) + } + rng := buildMetrics(t, "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}) + if !rng.from.Equal(testNow.Add(-time.Hour)) || !rng.to.Equal(testNow) { + t.Errorf("range window = [%s, %s]", rng.from, rng.to) + } +} + +// TestLookupRefusalKeepsStoreErrorsInternal: a store error other than the +// series limit may name vmstorage nodes; it goes to the log, not into a +// message that could reach the client. +func TestLookupRefusalKeepsStoreErrorsInternal(t *testing.T) { + body := strings.NewReader(`{"status":"error","error":"cannot fetch from vmstorage-longterm-0.vmstorage:8401: connection refused"}`) + herr := lookupRefusal("pod", body) + if herr.Code != http.StatusBadGateway || strings.Contains(herr.Message, "vmstorage") { + t.Fatalf("lookupRefusal = %d %q, want 502 without the store's text", herr.Code, herr.Message) + } +} diff --git a/internal/observabilityproxy/metricslookback.go b/internal/observabilityproxy/metricslookback.go new file mode 100644 index 000000000..3571afc4c --- /dev/null +++ b/internal/observabilityproxy/metricslookback.go @@ -0,0 +1,321 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "regexp" + "strconv" + "strings" + "time" +) + +// instantDefaultStep is the step vmselect uses for an instant query without +// one (defaultStep in app/vmselect/prometheus); it is also the implicit +// rollup window of a bare selector. +const instantDefaultStep = 5 * time.Minute + +// silenceInterval is how far before a rollup or subquery window vmselect +// starts reading (maxSilenceInterval in app/vmselect/promql: the value of +// -search.minStalenessInterval, 5m when unset; cozystack does not set it). +const silenceInterval = 5 * time.Minute + +// latencyOffset is how far vmselect moves an instant query evaluated within +// it of the current time back (-search.latencyOffset, 30s by default and +// not set by cozystack); the proxy forwards neither nocache nor +// latency_offset. +const latencyOffset = 30 * time.Second + +// queryDurationPart is the subset of the MetricsQL duration grammar +// (scanSingleDuration in metricsql) the scan understands. Step multiples +// (5i, $__interval) and negative parts are left out on purpose: anything +// else makes the bound unknown. Offsets are scanned lower-cased, as +// metricsql reads units in any case; windows and steps in brackets are +// not, so an upper-case unit there is unknown. +var ( + queryDurationPart = regexp.MustCompile(`(\d+(?:\.\d+)?)(ms|s|m|h|d|w|y)`) + queryDuration = regexp.MustCompile(`^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d|w|y))+$`) + querySeconds = regexp.MustCompile(`^\d+(?:\.\d+)?$`) +) + +// offsetTerminators may follow an offset's duration: whitespace, a +// closing bracket or separator, or a binary operator other than minus. +const offsetTerminators = " \t\n\r)]},+*/^%<>=!" + +var queryDurationUnits = map[string]time.Duration{ + "ms": time.Millisecond, "s": time.Second, "m": time.Minute, "h": time.Hour, + "d": 24 * time.Hour, "w": 7 * 24 * time.Hour, "y": 365 * 24 * time.Hour, +} + +// queryStart is the earliest instant a query evaluated from start may read; +// zero when the bound is unknown. +func queryStart(query string, start time.Time, step time.Duration) time.Time { + lookback, ok := queryLookback(query, step) + if !ok { + return time.Time{} + } + return start.Add(-lookback) +} + +// queryLookback is an upper bound on how far before its evaluation window +// a query reads, from a lexical scan that never parses the query. Every +// [...], offset and "(" counts, including ones inside quoted label values: +// an extra hit only makes the bound larger or unknown, never smaller. +// Windows and offsets are summed rather than maxed, which covers nesting; +// WITH is refused because a template applies one window several times. +// vmselect starts a rollup max(window, step) plus the silence interval +// before the first point, a subquery window+step+silence before it and +// aligns both down to a step. A rollup function or offset over anything +// but a selector is an implicit subquery (getRollupExprArg), so every "(" +// is counted as one. Each step term uses the largest step seen. +func queryLookback(query string, step time.Duration) (time.Duration, bool) { + if strings.Contains(query, "@") { + return 0, false + } + var sum float64 + subqueries := 0 + maxStep := float64(step) + for rest := query; ; { + open := strings.IndexByte(rest, '[') + if open < 0 { + break + } + closing := strings.IndexByte(rest[open:], ']') + if closing < 0 { + return 0, false + } + window, subStep, isSubquery := strings.Cut(rest[open+1:open+closing], ":") + rest = rest[open+closing+1:] + if isSubquery { + subqueries++ + if strings.TrimSpace(window) == "" { + window = "0s" + } + if strings.TrimSpace(subStep) != "" { + d, ok := parseQueryDuration(subStep) + if !ok { + return 0, false + } + maxStep = max(maxStep, d) + } + } + d, ok := parseQueryDuration(window) + if !ok { + return 0, false + } + sum += d + } + lower := strings.ToLower(query) + back, _, ok := scanOffsets(lower) + if !ok { + return 0, false + } + sum += back + if hasKeyword(lower, "with") { + return 0, false + } + n := float64(subqueries + strings.Count(query, "(")) + sum += (2+3*n)*maxStep + (1+n)*float64(silenceInterval) + if sum > float64(maxMetricsDuration) { + return 0, false + } + return time.Duration(sum), true +} + +// queryLookahead is an upper bound on how far after its evaluation window +// a query reads, from the same lexical scan, with step the query's step. +// A negative offset reads forward; the negative offsets are summed, which +// covers nesting. vmselect also evaluates a subquery up to its end plus +// its step, rounded up to the step (evalRollupFuncWithSubquery), and +// range_* transforms and keep_next_value carry those later values back +// onto earlier points, so each subquery level adds two steps; as for the +// lookback, every "(" counts as a possible implicit subquery and each +// term uses the largest step seen. Any @ makes the bound unknown, as it +// does for queryLookback, and so does WITH, which can apply one offset +// several times, and a backslash or a comment outside a string, which can +// hide what the scan looks for. vmselect also evaluates +// rollup_candlestick one step ahead of the evaluation time +// (evalRollupFuncWithoutAt), so it makes the bound unknown too. +func queryLookahead(query string, step time.Duration) (time.Duration, bool) { + if strings.Contains(query, "@") || unscannableOutsideString(query) { + return 0, false + } + lower := strings.ToLower(query) + if hasKeyword(lower, "rollup_candlestick") { + return 0, false + } + _, ahead, ok := scanOffsets(lower) + if !ok || ahead > 0 && hasKeyword(lower, "with") { + return 0, false + } + subqueries := 0 + maxStep := float64(step) + for rest := query; ; { + open := strings.IndexByte(rest, '[') + if open < 0 { + break + } + closing := strings.IndexByte(rest[open:], ']') + if closing < 0 { + return 0, false + } + _, subStep, isSubquery := strings.Cut(rest[open+1:open+closing], ":") + rest = rest[open+closing+1:] + if !isSubquery { + continue + } + subqueries++ + if strings.TrimSpace(subStep) != "" { + d, ok := parseQueryDuration(subStep) + if !ok { + return 0, false + } + maxStep = max(maxStep, d) + } + } + ahead += 2 * float64(subqueries+strings.Count(query, "(")) * maxStep + if ahead > float64(maxMetricsDuration) { + return 0, false + } + return time.Duration(ahead), true +} + +// scanOffsets sums every offset of the lowercased query, the positive ones +// into back and the negative ones into ahead, in nanoseconds; ok is false +// when one is not a plain duration. +func scanOffsets(lower string) (back, ahead float64, ok bool) { + for i := 0; ; { + k := strings.Index(lower[i:], "offset") + if k < 0 { + return back, ahead, true + } + k += i + i = k + len("offset") + if !isKeyword(lower, k, i) { + continue + } + arg := strings.TrimLeft(lower[i:], " \t\n\r") + negative := strings.HasPrefix(arg, "-") + if negative { + arg = strings.TrimLeft(arg[1:], " \t\n\r") + } + end := strings.IndexFunc(arg, func(r rune) bool { return !inDuration(r) }) + if end < 0 { + end = len(arg) + } + // metricsql lexes 1h-30m as one duration whose parts after the + // first negative one all subtract (DurationValue), so -1h-30m is + // -90m and 1h-2h is -1h; the scan does not read such compounds. + // metricsql continues a duration token in more forms than the scan + // reads: after a minus (-30m, -1.h, -.5h) and with a trailing + // $__interval, all summed by DurationValue. Only what certainly + // ends the token is accepted after the duration. + if end < len(arg) && !strings.ContainsRune(offsetTerminators, rune(arg[end])) { + return 0, 0, false + } + d, ok := parseQueryDuration(arg[:end]) + if !ok { + return 0, 0, false + } + if negative { + ahead += d + } else { + back += d + } + } +} + +// unscannableOutsideString reports a backslash or a "#" outside a string +// literal. metricsql unescapes identifiers (unescapeIdent), so +// rollup\_candlestick or offs\et name what the lexical scan looks for +// without its spelling; "#" starts a comment to the end of the line, and +// quotes or brackets inside one would mislead the string tracking here and +// the bracket and offset scans. Inside any of the three quotes a backslash +// skips the next character, as metricsql's scanString does: an escaped +// closing quote does not end the string, backquotes included (a quoted +// label name keeps such an escape). +func unscannableOutsideString(query string) bool { + var quote byte + for i := 0; i < len(query); i++ { + c := query[i] + switch { + case quote == 0 && (c == '\\' || c == '#'): + return true + case quote == 0 && (c == '"' || c == '\'' || c == '`'): + quote = c + case quote != 0 && c == '\\': + i++ + case quote != 0 && c == quote: + quote = 0 + } + } + return false +} + +// isKeyword reports whether lower[start:end] stands on its own rather than +// inside an identifier. +func isKeyword(lower string, start, end int) bool { + return (start == 0 || !continuesIdent(lower[start-1], false)) && + (end == len(lower) || !continuesIdent(lower[end], true)) +} + +func inDuration(r rune) bool { + return r >= '0' && r <= '9' || r >= 'a' && r <= 'z' || r == '.' || r == '_' +} + +func hasKeyword(lower, word string) bool { + for i := 0; ; { + k := strings.Index(lower[i:], word) + if k < 0 { + return false + } + k += i + i = k + len(word) + if isKeyword(lower, k, i) { + return true + } + } +} + +// continuesIdent reports whether c, next to "offset", makes it part of an +// identifier rather than the keyword. Digits and dots only continue an +// identifier after it, so before it they still count as a keyword hit. +func continuesIdent(c byte, after bool) bool { + return c >= 'a' && c <= 'z' || c == '_' || c == ':' || c >= 0x80 || + after && (c >= '0' && c <= '9' || c == '.' || c == '\\') +} + +// parseQueryDuration returns nanoseconds as a float so that a sum of huge +// durations cannot wrap; the caller compares it against the bound. +func parseQueryDuration(s string) (float64, bool) { + s = strings.TrimSpace(s) + if querySeconds.MatchString(s) { + f, err := strconv.ParseFloat(s, 64) + return f * float64(time.Second), err == nil + } + if !queryDuration.MatchString(s) { + return 0, false + } + var d float64 + for _, m := range queryDurationPart.FindAllStringSubmatch(s, -1) { + f, err := strconv.ParseFloat(m[1], 64) + if err != nil { + return 0, false + } + d += f * float64(queryDurationUnits[m[2]]) + } + return d, true +} diff --git a/internal/observabilityproxy/metricsparams.go b/internal/observabilityproxy/metricsparams.go new file mode 100644 index 000000000..c40bc058f --- /dev/null +++ b/internal/observabilityproxy/metricsparams.go @@ -0,0 +1,360 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "fmt" + "math" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" +) + +// MetricsLimits bound the window, resolution and timeout a metrics request +// asks vmselect for. Windows and offsets inside the expression are not +// limited here: they cannot read past the store's retention, vmselect caps +// the series and samples one query may touch, and MaxTimeout bounds the +// work of rollups over long overlapping windows. +type MetricsLimits struct { + MaxRange time.Duration + MaxPoints int + MaxTimeout time.Duration +} + +// MetricsConfig names the vmselect stores of the root tenant and the +// retention that decides between them. +type MetricsConfig struct { + RootTenant string + ShortTermStore string + LongTermStore string + ShortTermRetention time.Duration + Limits MetricsLimits + App AppScopeLimits + // UpstreamURL maps a store name to its Prometheus API base URL, without + // a trailing slash; nil means the in-cluster vmselect Service. + UpstreamURL func(store string) string +} + +// store picks the short-term store when everything from start on is still +// within its retention, so a live graph never waits on the downsampled one. +func (c MetricsConfig) store(start, now time.Time) string { + if now.Sub(start) <= c.ShortTermRetention { + return c.ShortTermStore + } + return c.LongTermStore +} + +// maxMetricsMatchers bounds match[]: vmselect runs one index search per +// matcher and has no cap of its own. +const maxMetricsMatchers = 20 + +// labelsDefaultWindow is vmselect's own default for the label and series +// APIs (defaultStep in app/vmselect/prometheus); the proxy sends it +// explicitly so the store choice and the range check see the same window. +const labelsDefaultWindow = 5 * time.Minute + +var labelName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) + +type metricsRequest struct { + path string + params url.Values + // start is the earliest instant the request reads, for the store choice; + // zero when that is unknown, which picks the long-term store. + start time.Time + // from and to are the window the query is evaluated over, without the + // lookback of its expression. + from, to time.Time + // last is the latest instant the request reads, past to by a negative + // offset; zero when that is unknown. + last time.Time + // scopedMatch is set when match[] holds the scope, not client matchers. + scopedMatch bool +} + +// setScope replaces the namespace scope with the given selectors, which +// vmselect ORs. +func (req *metricsRequest) setScope(selectors []string) { + req.params["extra_filters[]"] = selectors + if req.scopedMatch { + req.params["match[]"] = selectors + } +} + +// namespaceSelector quotes ns the way MetricsQL unquotes a label value +// (strconv.Unquote in metricsql's extractStringValue). +func namespaceSelector(ns string) string { + return "{namespace=" + strconv.Quote(ns) + "}" +} + +// BuildMetricsParams rebuilds the upstream parameters from an allowlist and +// adds the namespace scope as the only extra_filters[] value. vmselect ORs +// repeated extra_filters, so a client value would widen the scope; it ANDs +// the single one into every selector of the query and every match[], which +// is what lets the query pass through unparsed. +func BuildMetricsParams(endpoint string, in url.Values, namespace string, now time.Time, lim MetricsLimits) (metricsRequest, *HTTPError) { + req := metricsRequest{path: endpoint, params: url.Values{}} + out := req.params + out.Set("extra_filters[]", namespaceSelector(namespace)) + timeout := lim.MaxTimeout + if v := in.Get("timeout"); v != "" { + d, herr := parseMetricsDuration("timeout", v) + if herr != nil { + return req, herr + } + timeout = min(d, lim.MaxTimeout) + } + out.Set("timeout", formatSeconds(timeout)) + + switch endpoint { + case "api/v1/query": + if herr := copyQuery(out, in); herr != nil { + return req, herr + } + at := now + if v := in.Get("time"); v != "" { + t, herr := parseMetricsTime("time", v) + if herr != nil { + return req, herr + } + at = t + } + // The default is forwarded too: the application lookup is sized + // for at, and vmselect's own clock reads later, by as long as the + // lookups took. + out.Set("time", formatTime(at)) + step := instantDefaultStep + if v := in.Get("step"); v != "" { + var herr *HTTPError + if step, herr = parseStep(v, lim); herr != nil { + return req, herr + } + out.Set("step", formatSeconds(step)) + } + // vmselect evaluates an instant query whose time is within the + // latency offset of its own clock at that clock minus the offset + // (queryOffset in prometheus.go), so the evaluation instant lies in + // (time-2*offset, time]; the earlier end needs no clock to agree. + req.start = queryStart(out.Get("query"), at.Add(-2*latencyOffset), step) + req.from, req.to = at, at + if ahead, ok := queryLookahead(out.Get("query"), step); ok { + req.last = at.Add(ahead) + } + return req, nil + + case "api/v1/query_range": + if herr := copyQuery(out, in); herr != nil { + return req, herr + } + for _, k := range []string{"start", "end", "step"} { + if in.Get(k) == "" { + return req, errorf(http.StatusBadRequest, "%s is required", k) + } + } + start, end, herr := metricsWindow(in, now, lim) + if herr != nil { + return req, herr + } + step, herr := parseStep(in.Get("step"), lim) + if herr != nil { + return req, herr + } + if points := end.Sub(start)/step + 1; points > time.Duration(lim.MaxPoints) { + return req, errorf(http.StatusUnprocessableEntity, "%d points per series exceed the limit of %d; raise step", points, lim.MaxPoints) + } + req.start = queryStart(out.Get("query"), start, step) + req.from, req.to = start, end + // vmselect rounds end up to a whole step (AdjustStartEnd). + if ahead, ok := queryLookahead(out.Get("query"), step); ok { + req.last = end.Add(step + ahead) + } + out.Set("start", formatTime(start)) + out.Set("end", formatTime(end)) + out.Set("step", formatSeconds(step)) + return req, nil + } + + name, isLabelValues := strings.CutPrefix(endpoint, "api/v1/label/") + if isLabelValues { + var ok bool + if name, ok = strings.CutSuffix(name, "/values"); !ok || strings.Contains(name, "/") { + return req, errorf(http.StatusNotFound, "not found") + } + if !labelName.MatchString(name) { + return req, errorf(http.StatusBadRequest, "invalid label name %q", name) + } + } else if endpoint != "api/v1/series" && endpoint != "api/v1/labels" { + return req, errorf(http.StatusNotFound, "not found") + } + start, end, herr := metricsWindow(in, now, lim) + if herr != nil { + return req, herr + } + req.start, req.from, req.to, req.last = start, start, end, end + out.Set("start", formatTime(start)) + out.Set("end", formatTime(end)) + + matchers := in["match[]"] + if len(matchers) > maxMetricsMatchers { + return req, errorf(http.StatusUnprocessableEntity, "at most %d match[] are allowed", maxMetricsMatchers) + } + for _, m := range matchers { + if m == "" { + return req, errorf(http.StatusBadRequest, "empty match[]") + } + out.Add("match[]", m) + } + if len(matchers) == 0 { + if endpoint == "api/v1/series" { + return req, errorf(http.StatusBadRequest, "match[] is required") + } + // vmselect started with -search.ignoreExtraFiltersAtLabelsAPI drops + // extra_filters[] on the label APIs unless a match[] is present + // (getCommonParamsInternal), so the scope is repeated here. + out.Set("match[]", namespaceSelector(namespace)) + req.scopedMatch = true + } + return req, nil +} + +func copyQuery(out, in url.Values) *HTTPError { + q := in.Get("query") + if q == "" { + return errorf(http.StatusBadRequest, "query is required") + } + out.Set("query", q) + return nil +} + +// metricsWindow reads start and end with vmselect's label API defaults: +// end is now and start is five minutes before end. +func metricsWindow(in url.Values, now time.Time, lim MetricsLimits) (time.Time, time.Time, *HTTPError) { + end := now + if v := in.Get("end"); v != "" { + t, herr := parseMetricsTime("end", v) + if herr != nil { + return time.Time{}, time.Time{}, herr + } + end = t + } + start := end.Add(-labelsDefaultWindow) + if v := in.Get("start"); v != "" { + t, herr := parseMetricsTime("start", v) + if herr != nil { + return time.Time{}, time.Time{}, herr + } + start = t + } + if start.After(end) { + return time.Time{}, time.Time{}, errorf(http.StatusBadRequest, "start is after end") + } + if end.Sub(start) > lim.MaxRange { + return time.Time{}, time.Time{}, errorf(http.StatusUnprocessableEntity, "range %s exceeds the limit of %s", end.Sub(start), lim.MaxRange) + } + return start, end, nil +} + +// parseStep refuses a step above the range limit: vmselect aligns the +// window outward to whole steps, and an implicit rollup window is one step, +// so a huge step reads far more than the range it was sent with. +func parseStep(v string, lim MetricsLimits) (time.Duration, *HTTPError) { + step, herr := parseMetricsDuration("step", v) + if herr != nil { + return 0, herr + } + if step > lim.MaxRange { + return 0, errorf(http.StatusUnprocessableEntity, "step %s exceeds the limit of %s", step, lim.MaxRange) + } + return step, nil +} + +// maxMetricsDuration is vmselect's own bound on step and timeout +// (maxDurationMsecs in lib/httputil); it also keeps the rounding below +// from overflowing. +const maxMetricsDuration = 100 * 365 * 24 * time.Hour + +// parseMetricsDuration takes Prometheus float seconds or a duration such as +// 1m30s. vmselect works in whole milliseconds and refuses anything shorter; +// the value is rounded up to the millisecond so the limits are checked +// against what vmselect will use, not against a fraction it drops. +func parseMetricsDuration(key, v string) (time.Duration, *HTTPError) { + d, err := parseDuration(v) + if err != nil || d < time.Millisecond || d > maxMetricsDuration { + return 0, errorf(http.StatusBadRequest, "invalid %s %q", key, v) + } + return (d + time.Millisecond - 1).Truncate(time.Millisecond), nil +} + +// parseMetricsTime takes the two forms the Prometheus API defines: float +// unix seconds and RFC3339. vmselect would also guess milliseconds from the +// digit count and accept relative times; a guessed unit silently moves the +// window, so those are refused. The result is cut to the millisecond, as +// formatTime sends it, so the limits see the window vmselect will use. +func parseMetricsTime(key, v string) (time.Time, *HTTPError) { + if sec, frac, ok := strings.Cut(v, "."); isDigits(sec) && (!ok || isDigits(frac)) { + if len(sec) > 10 { + return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q: not unix seconds", key, v) + } + secs, _ := strconv.ParseInt(sec, 10, 64) + var nanos int64 + if ok { + nanos, _ = strconv.ParseInt((frac + "000000000")[:9], 10, 64) + } + return metricsTimeInRange(key, v, time.Unix(secs, nanos)) + } + t, err := time.Parse(time.RFC3339Nano, v) + if err != nil { + return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q", key, v) + } + return metricsTimeInRange(key, v, t) +} + +// maxMetricsTime is the last instant vmselect can read: it takes an +// RFC3339 time through time.UnixNano (timeutil.ParseTimeAt). +var maxMetricsTime = time.Unix(0, math.MaxInt64) + +func metricsTimeInRange(key, v string, t time.Time) (time.Time, *HTTPError) { + if t.Before(time.Unix(0, 0)) || !t.Before(maxMetricsTime) { + return time.Time{}, errorf(http.StatusBadRequest, "invalid %s %q: outside 1970..2262", key, v) + } + return time.UnixMilli(t.UnixMilli()).UTC(), nil +} + +// formatTime writes RFC3339 with milliseconds. vmselect reads a number +// without "-" as a unix time whose unit it guesses from its size +// (timeutil.getUnixTimestampNanoseconds): "1000000.000" is taken as +// 1000000000 seconds, and anything past 2038 in milliseconds as +// microseconds. A dated form never reaches that guess. +func formatTime(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.000Z07:00") +} + +// formatSeconds writes a whole-millisecond duration so that vmselect reads +// back exactly those milliseconds. vmselect multiplies the float seconds +// by 1e3 and truncates (httputil.GetDuration), so "1.001" arrives as +// 1000ms; where the shortest form truncates low, half a millisecond more +// lands the truncation on the intended value without reaching the next. +func formatSeconds(d time.Duration) string { + ms := d.Milliseconds() + s := strconv.FormatFloat(float64(ms)/1000, 'f', -1, 64) + if secs, _ := strconv.ParseFloat(s, 64); int64(secs*1e3) != ms { + s = fmt.Sprintf("%d.%03d5", ms/1000, ms%1000) + } + return s +} diff --git a/internal/observabilityproxy/metricsparams_test.go b/internal/observabilityproxy/metricsparams_test.go new file mode 100644 index 000000000..ba14a9781 --- /dev/null +++ b/internal/observabilityproxy/metricsparams_test.go @@ -0,0 +1,566 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "net/http" + "net/url" + "strconv" + "strings" + "testing" + "time" +) + +func testMetricsLimits() MetricsLimits { + return MetricsLimits{MaxRange: 30 * 24 * time.Hour, MaxPoints: 11000, MaxTimeout: 30 * time.Second} +} + +const wantMetricsScope = `{namespace="tenant-a"}` + +func unix(t time.Time) string { return strconv.FormatInt(t.Unix(), 10) } + +func buildMetrics(t *testing.T, endpoint string, in url.Values) metricsRequest { + t.Helper() + req, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits()) + if err != nil { + t.Fatalf("BuildMetricsParams(%s, %v): %v", endpoint, in, err) + } + return req +} + +func metricsStatus(endpoint string, in url.Values) int { + _, err := BuildMetricsParams(endpoint, in, "tenant-a", testNow, testMetricsLimits()) + if err == nil { + return http.StatusOK + } + return err.Code +} + +func TestBuildMetricsParamsAllowlist(t *testing.T) { + hostile := url.Values{ + "extra_filters": {`{namespace=~".+"}`}, + "extra_filters[]": {`{namespace="tenant-b"}`, `{namespace=~".*"}`}, + "extra_label": {"namespace=tenant-b"}, + "match": {`{namespace="tenant-b"}`}, + "nocache": {"1"}, + "limit": {"1"}, + "AccountID": {"7"}, + "latency_offset": {"1h"}, + "round_digits": {"2"}, + } + cases := map[string]url.Values{ + "api/v1/query": {"query": {"up"}, "time": {unix(testNow)}, "step": {"30s"}}, + "api/v1/query_range": {"query": {"up"}, "start": {unix(testNow.Add(-time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}, + "api/v1/series": {"match[]": {"up"}}, + "api/v1/labels": {"match[]": {"up"}}, + "api/v1/label/pod/values": {"match[]": {"up"}}, + } + allowed := map[string]bool{"query": true, "start": true, "end": true, "step": true, "time": true, "match[]": true, "timeout": true, "extra_filters[]": true} + for endpoint, in := range cases { + t.Run(endpoint, func(t *testing.T) { + for k, v := range hostile { + in[k] = v + } + req := buildMetrics(t, endpoint, in) + if req.path != endpoint { + t.Errorf("path = %q", req.path) + } + for k := range req.params { + if !allowed[k] { + t.Errorf("parameter %q forwarded: %v", k, req.params) + } + } + if got := req.params["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope { + t.Errorf("extra_filters[] = %q, want exactly [%s]", got, wantMetricsScope) + } + for _, m := range req.params["match[]"] { + if strings.Contains(m, "tenant-b") { + t.Errorf("client match without brackets forwarded as match[]: %q", m) + } + } + }) + } +} + +func TestNamespaceSelectorEscapes(t *testing.T) { + for _, ns := range []string{"tenant-a", `a"} or {namespace=~".+`, `back\slash`, "new\nline"} { + sel := namespaceSelector(ns) + lit, ok := strings.CutPrefix(sel, "{namespace=") + lit, ok2 := strings.CutSuffix(lit, "}") + if !ok || !ok2 { + t.Fatalf("selector %q has the wrong shape", sel) + } + // MetricsQL reads a double-quoted label value with strconv.Unquote + // (metricsql extractStringValue). + got, err := strconv.Unquote(lit) + if err != nil || got != ns { + t.Errorf("selector %q decodes to %q, %v; want %q", sel, got, err, ns) + } + } +} + +func TestBuildMetricsParamsLabelEndpointsAlwaysMatch(t *testing.T) { + for _, endpoint := range []string{"api/v1/labels", "api/v1/label/__name__/values"} { + req := buildMetrics(t, endpoint, url.Values{}) + if got := req.params["match[]"]; len(got) != 1 || got[0] != wantMetricsScope { + t.Errorf("%s without match[] sends %q, want the namespace selector", endpoint, got) + } + req = buildMetrics(t, endpoint, url.Values{"match[]": {"up", `{job="x"}`}}) + if got := req.params["match[]"]; len(got) != 2 || got[0] != "up" || got[1] != `{job="x"}` { + t.Errorf("%s client match[] = %q, want them kept", endpoint, got) + } + } + if got := metricsStatus("api/v1/series", url.Values{}); got != http.StatusBadRequest { + t.Errorf("series without match[] = %d, want 400", got) + } + if got := metricsStatus("api/v1/series", url.Values{"match[]": {""}}); got != http.StatusBadRequest { + t.Errorf("empty match[] = %d, want 400", got) + } + many := url.Values{} + for i := range maxMetricsMatchers + 1 { + many.Add("match[]", "up"+strconv.Itoa(i)) + } + if got := metricsStatus("api/v1/labels", many); got != http.StatusUnprocessableEntity { + t.Errorf("%d match[] = %d, want 422", maxMetricsMatchers+1, got) + } +} + +func TestBuildMetricsParamsLimits(t *testing.T) { + ago := func(d time.Duration) string { return unix(testNow.Add(-d)) } + day := 24 * time.Hour + rng := func(start, end, step string) url.Values { + return url.Values{"query": {"up"}, "start": {start}, "end": {end}, "step": {step}} + } + tests := []struct { + name string + endpoint string + in url.Values + want int + }{ + {"range at the limit", "api/v1/query_range", rng(ago(30*day), ago(0), "1h"), http.StatusOK}, + {"range above the limit", "api/v1/query_range", rng(ago(31*day), ago(0), "1h"), http.StatusUnprocessableEntity}, + {"points at the limit", "api/v1/query_range", rng(ago(10999*time.Minute), ago(0), "1m"), http.StatusOK}, + {"points above the limit", "api/v1/query_range", rng(ago(11000*time.Minute), ago(0), "1m"), http.StatusUnprocessableEntity}, + {"points above the limit after rounding the step up", "api/v1/query_range", rng(ago(22*time.Second), ago(0), "0.0019"), http.StatusUnprocessableEntity}, + // 10.9999s admits 11000 points, but vmselect gets both bounds cut to + // the millisecond: 11.000s, 11001 points. + {"points above the limit once the bounds are cut to milliseconds", "api/v1/query_range", rng("2026-11-04T11:59:49.0009Z", "2026-11-04T12:00:00.0008Z", "0.001"), http.StatusUnprocessableEntity}, + {"overflowing step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "2562047h47m16.854775807s"), http.StatusBadRequest}, + {"overflowing timeout", "api/v1/query", url.Values{"query": {"up"}, "timeout": {"2562047h47m16.854775807s"}}, http.StatusBadRequest}, + {"step above the range limit", "api/v1/query_range", rng(ago(time.Hour), ago(0), "31d"), http.StatusUnprocessableEntity}, + {"zero step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0"), http.StatusBadRequest}, + {"sub-millisecond step", "api/v1/query_range", rng(ago(time.Hour), ago(0), "0.0001"), http.StatusBadRequest}, + {"missing step", "api/v1/query_range", url.Values{"query": {"up"}, "start": {ago(time.Hour)}, "end": {ago(0)}}, http.StatusBadRequest}, + {"missing start", "api/v1/query_range", url.Values{"query": {"up"}, "end": {ago(0)}, "step": {"1m"}}, http.StatusBadRequest}, + {"missing query", "api/v1/query_range", url.Values{"start": {ago(time.Hour)}, "end": {ago(0)}, "step": {"1m"}}, http.StatusBadRequest}, + {"inverted range", "api/v1/query_range", rng(ago(0), ago(time.Hour), "1m"), http.StatusBadRequest}, + {"millisecond timestamp", "api/v1/query_range", rng(ago(time.Hour)+"000", ago(0)+"000", "1m"), http.StatusBadRequest}, + {"relative time", "api/v1/query_range", rng("-1h", "now", "1m"), http.StatusBadRequest}, + {"rfc3339", "api/v1/query_range", rng("2026-11-04T11:00:00Z", "2026-11-04T12:00:00.5Z", "15s"), http.StatusOK}, + {"instant without time", "api/v1/query", url.Values{"query": {"up"}}, http.StatusOK}, + {"instant step above the range limit", "api/v1/query", url.Values{"query": {"up"}, "step": {"31d"}}, http.StatusUnprocessableEntity}, + {"instant bad time", "api/v1/query", url.Values{"query": {"up"}, "time": {"yesterday"}}, http.StatusBadRequest}, + {"negative timeout", "api/v1/query", url.Values{"query": {"up"}, "timeout": {"-1s"}}, http.StatusBadRequest}, + {"series range above the limit", "api/v1/series", url.Values{"match[]": {"up"}, "start": {ago(31 * day)}}, http.StatusUnprocessableEntity}, + {"labels inverted range", "api/v1/labels", url.Values{"start": {ago(0)}, "end": {ago(time.Hour)}}, http.StatusBadRequest}, + {"invalid label name", "api/v1/label/a-b/values", url.Values{}, http.StatusBadRequest}, + {"unknown endpoint", "api/v1/status/tsdb", url.Values{}, http.StatusNotFound}, + {"export", "api/v1/export", url.Values{"match[]": {"up"}}, http.StatusNotFound}, + {"label without values", "api/v1/label/pod", url.Values{}, http.StatusNotFound}, + {"root", "", url.Values{}, http.StatusNotFound}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := metricsStatus(tt.endpoint, tt.in); got != tt.want { + t.Fatalf("status = %d, want %d", got, tt.want) + } + }) + } +} + +func TestBuildMetricsParamsCanonicalForms(t *testing.T) { + req := buildMetrics(t, "api/v1/query_range", url.Values{ + "query": {"up"}, "start": {"2026-11-04T11:00:00.25Z"}, "end": {unix(testNow)}, "step": {"1m30s"}, + }) + want := url.Values{ + "query": {"up"}, "start": {"2026-11-04T11:00:00.250Z"}, "end": {"2026-11-04T12:00:00.000Z"}, "step": {"90"}, + "timeout": {"30"}, "extra_filters[]": {wantMetricsScope}, + } + if req.params.Encode() != want.Encode() { + t.Errorf("params = %v\nwant %v", req.params, want) + } + + // vmselect drops a sub-millisecond remainder; the points check has to + // see the step it will use. + for in, want := range map[string]string{"60.0001": "60.001", "0.0019": "0.002"} { + req = buildMetrics(t, "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-20 * time.Second))}, "end": {unix(testNow)}, "step": {in}}) + if got := req.params.Get("step"); got != want { + t.Errorf("step %s forwarded as %q, want %s", in, got, want) + } + } + + for in, want := range map[string]string{"": "30", "10s": "10", "5m": "30", "1.5": "1.5"} { + v := url.Values{"query": {"up"}} + if in != "" { + v.Set("timeout", in) + } + if got := buildMetrics(t, "api/v1/query", v).params.Get("timeout"); got != want { + t.Errorf("timeout %q forwarded as %q, want %q", in, got, want) + } + } + + req = buildMetrics(t, "api/v1/labels", url.Values{}) + if req.params.Get("end") != "2026-11-04T12:00:00.000Z" || req.params.Get("start") != "2026-11-04T11:55:00.000Z" { + t.Errorf("labels window = %s..%s, want the last 5m", req.params.Get("start"), req.params.Get("end")) + } + if q := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}}); q.params.Get("time") != "2026-11-04T12:00:00.000Z" || !q.start.Equal(testNow.Add(-15*time.Minute-2*latencyOffset)) { + t.Errorf("instant query without time: params %v, start %v", q.params, q.start) + } +} + +func TestMetricsStoreByRange(t *testing.T) { + cfg := MetricsConfig{ShortTermStore: "shortterm", LongTermStore: "longterm", ShortTermRetention: 72 * time.Hour} + tests := []struct { + name string + endpoint string + in url.Values + want string + }{ + {"instant now", "api/v1/query", url.Values{"query": {"up"}}, "shortterm"}, + {"instant within retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-71 * time.Hour))}}, "shortterm"}, + {"instant past retention", "api/v1/query", url.Values{"query": {"up"}, "time": {unix(testNow.Add(-73 * time.Hour))}}, "longterm"}, + {"live graph", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-6 * time.Hour))}, "end": {unix(testNow)}, "step": {"60"}}, "shortterm"}, + {"range fits with lookback", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-71 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "shortterm"}, + {"range lookback past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-72 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"}, + {"range past retention", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-7 * 24 * time.Hour))}, "end": {unix(testNow)}, "step": {"600"}}, "longterm"}, + {"old window ending long ago", "api/v1/query_range", url.Values{"query": {"up"}, "start": {unix(testNow.Add(-5 * 24 * time.Hour))}, "end": {unix(testNow.Add(-4 * 24 * time.Hour))}, "step": {"600"}}, "longterm"}, + {"labels default window", "api/v1/labels", url.Values{}, "shortterm"}, + {"series past retention", "api/v1/series", url.Values{"match[]": {"up"}, "start": {unix(testNow.Add(-4 * 24 * time.Hour))}}, "longterm"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := buildMetrics(t, tt.endpoint, tt.in) + if got := cfg.store(req.start, testNow); got != tt.want { + t.Fatalf("store = %s, want %s", got, tt.want) + } + }) + } +} + +func TestQueryLookback(t *testing.T) { + tests := []struct { + query string + step time.Duration + want time.Duration + }{ + {`up`, 5 * time.Minute, 15 * time.Minute}, + {`increase(x[7d])`, 5 * time.Minute, 7*24*time.Hour + 35*time.Minute}, + {`rate(x[300])`, time.Minute, 20 * time.Minute}, + {`rate(x[ 1h30m ])`, time.Minute, 105 * time.Minute}, + {`max_over_time(x[1h:5m])`, time.Minute, 115 * time.Minute}, + {`max_over_time(x[1h:])`, time.Minute, 83 * time.Minute}, + {`max_over_time(max_over_time(rate(x[5m])[1h:1m])[1d:1h])`, time.Minute, 42*time.Hour + 35*time.Minute}, + {`rate(rate(x))`, time.Hour, 2*(3*time.Hour+5*time.Minute) + 2*time.Hour + 5*time.Minute}, + {`x offset 10d`, 5 * time.Minute, 10*24*time.Hour + 15*time.Minute}, + {`sum(x OFFSET 2h)`, 5 * time.Minute, 155 * time.Minute}, + {`rate(x[5m] offset 1h)`, time.Minute, 80 * time.Minute}, + {`x offset -1h`, 5 * time.Minute, 15 * time.Minute}, + {`x offset - 1h`, 5 * time.Minute, 15 * time.Minute}, + {`node_offset_seconds + offset_total + with_x`, 5 * time.Minute, 15 * time.Minute}, + {`x{a="[1h]"}`, 5 * time.Minute, 75 * time.Minute}, + {`x{a="]"}[7d]`, 5 * time.Minute, 7*24*time.Hour + 15*time.Minute}, + } + for _, tt := range tests { + t.Run(tt.query, func(t *testing.T) { + got, ok := queryLookback(tt.query, tt.step) + if !ok || got != tt.want { + t.Fatalf("queryLookback = %s, %t; want %s, true", got, ok, tt.want) + } + }) + } +} + +func TestQueryLookbackUnknown(t *testing.T) { + for _, q := range []string{ + `x @ 123`, + `x @ start()`, + `x{a="a@b"}`, + `rate(x[5i])`, + `rate(x[$__interval])`, + `x[1h-5m]`, + `x[7D]`, + `x[1.d]`, + `x[]`, + `rate(x[7d)`, + `x offset`, + `x offset w`, + `WITH (w = 7d) rate(x[w])`, + `x{a="["}`, + `x{a="[5m"}[7d]`, + `x{path="/offset/1"}`, + `x[99999999999y]`, + `x offset 99999999999y`, + `x offset 3_000_000`, + `WITH (f(q) = max_over_time(q[2d:1h])) f(f(x))`, + `with (s(m) = m offset 2d) s(s(x))`, + } { + t.Run(q, func(t *testing.T) { + if got, ok := queryLookback(q, 5*time.Minute); ok { + t.Fatalf("queryLookback = %s, true; want unknown", got) + } + }) + } +} + +func TestMetricsStoreByLookback(t *testing.T) { + cfg := MetricsConfig{ShortTermStore: "shortterm", LongTermStore: "longterm", ShortTermRetention: 72 * time.Hour} + rangeFrom := func(q string, ago time.Duration, step string) url.Values { + return url.Values{"query": {q}, "start": {unix(testNow.Add(-ago))}, "end": {unix(testNow)}, "step": {step}} + } + tests := []struct { + name string + endpoint string + in url.Values + want string + }{ + {"instant short window", "api/v1/query", url.Values{"query": {"rate(x[5m])"}}, "shortterm"}, + {"instant 7d window", "api/v1/query", url.Values{"query": {"increase(x[7d])"}}, "longterm"}, + {"instant offset 10d", "api/v1/query", url.Values{"query": {"x offset 10d"}}, "longterm"}, + {"instant negative offset", "api/v1/query", url.Values{"query": {"x offset -1h"}}, "shortterm"}, + {"instant at timestamp", "api/v1/query", url.Values{"query": {"x @ 123"}}, "longterm"}, + {"instant at start", "api/v1/query", url.Values{"query": {"x @ start()"}}, "longterm"}, + {"instant step window", "api/v1/query", url.Values{"query": {"rate(x[5i])"}}, "longterm"}, + {"instant step past retention", "api/v1/query", url.Values{"query": {"up"}, "step": {"259200"}}, "longterm"}, + {"instant latency offset at the edge", "api/v1/query", url.Values{"query": {"x[71h54m59s]"}, "step": {"0.001"}}, "longterm"}, + {"instant subquery past retention", "api/v1/query", url.Values{"query": {"max_over_time(rate(x[5m])[4d:1h])"}}, "longterm"}, + {"instant offset with underscores", "api/v1/query", url.Values{"query": {"x offset 3_000_000"}}, "longterm"}, + {"instant WITH template", "api/v1/query", url.Values{"query": {"WITH (f(q) = max_over_time(q[2d:1h])) f(f(x))"}}, "longterm"}, + {"instant implicit subqueries", "api/v1/query", url.Values{"query": {"rate(rate(rate(rate(rate(rate(x))))))"}, "step": {"43200"}}, "longterm"}, + {"quoted at sign", "api/v1/query", url.Values{"query": {`x{a="a@b"}`}}, "longterm"}, + {"quoted open bracket hides window", "api/v1/query", url.Values{"query": {`rate(x{a="[5m"}[7d])`}}, "longterm"}, + {"quoted close bracket before window", "api/v1/query", url.Values{"query": {`rate(x{a="]"}[7d])`}}, "longterm"}, + {"quoted offset before offset", "api/v1/query", url.Values{"query": {`x{a="offset 1m"} offset 10d`}}, "longterm"}, + {"range short window", "api/v1/query_range", rangeFrom("rate(x[1h])", 6*time.Hour, "60"), "shortterm"}, + {"range 7d window", "api/v1/query_range", rangeFrom("increase(x[7d])", 6*time.Hour, "60"), "longterm"}, + {"range offset past retention", "api/v1/query_range", rangeFrom("x offset 3d", time.Hour, "60"), "longterm"}, + {"range lookback fits", "api/v1/query_range", rangeFrom("rate(x[5m])", 70*time.Hour, "600"), "shortterm"}, + {"range lookback crosses retention", "api/v1/query_range", rangeFrom("rate(x[5m])", 71*time.Hour+50*time.Minute, "600"), "longterm"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := buildMetrics(t, tt.endpoint, tt.in) + if got := cfg.store(req.start, testNow); got != tt.want { + t.Fatalf("store = %s, want %s", got, tt.want) + } + if q := req.params.Get("query"); q != tt.in.Get("query") { + t.Fatalf("query forwarded as %q, want it unchanged", q) + } + }) + } +} + +// vmselectMillis repeats how vmselect reads a duration argument +// (httputil.GetDuration in lib/httputil/duration.go, v1.136.22 and +// v1.139.0): float seconds, multiplied by 1e3 and truncated. +func vmselectMillis(t *testing.T, v string) int64 { + t.Helper() + secs, err := strconv.ParseFloat(v, 64) + if err != nil { + t.Fatalf("vmselect cannot parse %q: %v", v, err) + } + return int64(secs * 1e3) +} + +// TestFormatSecondsVmselectExact sweeps every step up to 2000s and the +// large end of the range: vmselect must read back the milliseconds the +// limits were checked with. +func TestFormatSecondsVmselectExact(t *testing.T) { + check := func(ms int64) { + if got := vmselectMillis(t, formatSeconds(time.Duration(ms)*time.Millisecond)); got != ms { + t.Fatalf("%dms formatted as %q reaches vmselect as %dms", ms, formatSeconds(time.Duration(ms)*time.Millisecond), got) + } + } + for ms := int64(1); ms <= 2_000_000; ms++ { + check(ms) + } + for ms := maxMetricsDuration.Milliseconds() - 100_000; ms <= maxMetricsDuration.Milliseconds(); ms++ { + check(ms) + } + // A spread of the range in between, on a stride no power of ten divides. + for ms := int64(2_000_001); ms < maxMetricsDuration.Milliseconds(); ms += 3_153_599_999 { + check(ms) + } + // Whole seconds keep their plain form. + if got := formatSeconds(90 * time.Second); got != "90" { + t.Errorf("90s formatted as %q", got) + } +} + +// TestBuildMetricsParamsPointsAsVmselectCounts: a 1.001s step is 1001ms +// to the proxy; vmselect must count the same 11000 points at the limit. +func TestBuildMetricsParamsPointsAsVmselectCounts(t *testing.T) { + end := testNow + step := 1001 * time.Millisecond + rng := func(points int64) url.Values { + start := end.Add(-time.Duration(points-1) * step) + return url.Values{"query": {"up"}, "start": {start.Format(time.RFC3339Nano)}, "end": {end.Format(time.RFC3339Nano)}, "step": {"1.001"}} + } + req := buildMetrics(t, "api/v1/query_range", rng(11000)) + stepMs := vmselectMillis(t, req.params.Get("step")) + startMs, endMs := vmselectTimeMillis(t, req.params.Get("start")), vmselectTimeMillis(t, req.params.Get("end")) + if points := (endMs-startMs)/stepMs + 1; stepMs != 1001 || points != 11000 { + t.Fatalf("vmselect reads step %dms and %d points, want 1001ms and 11000", stepMs, points) + } + if got := metricsStatus("api/v1/query_range", rng(11001)); got != http.StatusUnprocessableEntity { + t.Fatalf("11001 points answered %d, want 422", got) + } +} + +func TestQueryLookahead(t *testing.T) { + tests := []struct { + query string + want time.Duration + }{ + {`up`, 0}, + {`x offset 1h`, 0}, + {`x offset -1h`, time.Hour}, + {`x OFFSET - 90m`, 90 * time.Minute}, + // Each "(" may be an implicit subquery, which vmselect evaluates + // up to two steps past its end; here the step is 1m. + {`rate(x[5m] offset -30m) offset -1h`, 92 * time.Minute}, + {`x offset -1h + y offset 2h`, time.Hour}, + {`node_offset_seconds - 1`, 0}, + // An escape inside a string literal is only part of a label value. + {`x{a=~"a\\.b"}`, 0}, + {`x{a="\"rollup\\_candlestick\""}`, 0}, + // metricsql lexes a duration as one token, so a sign with spaces, + // or a plus, is an operator after the offset, not part of it. + {`x offset 1h+30m`, 0}, + {`x offset 1h - 30m`, 0}, + {`x offset -1h + 30m`, time.Hour}, + {`x offset -1h30m`, 90 * time.Minute}, + } + for _, tt := range tests { + t.Run(tt.query, func(t *testing.T) { + got, ok := queryLookahead(tt.query, time.Minute) + if !ok || got != tt.want { + t.Fatalf("queryLookahead = %s, %v; want %s", got, ok, tt.want) + } + }) + } + for _, q := range []string{ + `x @ 1700000000`, + // The lookback scan cannot bound any @, so the application root + // refuses every one; the lookahead does not tell them apart. + `x @ end()`, + `x @ start() offset -1h`, + `x @ (end() - 1h)`, + `x{a="a@b"}`, + `x offset -5i`, + // Compound durations with a negative part (metricsql DurationValue): + // -1h-30m is -90m and 1h-2h is -1h, neither of which the scan reads. + `x offset -1h-30m`, + `x offset - 1h-30m`, + `x offset 1h-2h`, + `x offset 1h-30m`, + // metricsql continues a duration with -1.h or -.5h too, so any minus + // right after the duration is refused rather than told apart. + `x offset 5m-1.h`, + `x offset 5m-.5h`, + `x offset 1h-1`, + `x offset 5m-y`, + // metricsql also reads a trailing $__interval as part of the + // duration: -1h$__interval is -(1h + step). + `x offset -1h$__interval`, + `rate(x[5m]) offset -1h$__interval`, + // A subquery step the scan cannot read, an unclosed bracket, and a + // sum past the duration range. + `rate(x[1h:5i])`, + `rate(x[1h:$__interval])`, + `rate(x[1h`, + `max_over_time(x[1h:100y])`, + `rate(x[5m] offset 1h-30m)`, + // metricsql unescapes identifiers (unescapeIdent), so an escaped + // rollup_candlestick, or any escape outside a string, could hide + // what the scan looks for. + `rollup\_candlestick(x + y)`, + `rollup_candle\stick(x)`, + `rollup\x5fcandlestick(x)`, + `x offs\et -1h`, + // A comment runs to the end of the line, so quotes in it would + // confuse the string tracking and brackets in it the scans. + "x # '\n+ rollup\\_candlestick(y) # '", + "x # \"\n+ rollup\\_candlestick(y) # \"", + "rate(x[5m]) # comment", + // metricsql lets a backslash escape the closing quote in backquotes + // too (scanString), and quoted label names keep such escapes, so + // a backquoted string must not end at an escaped backquote. + "container_cpu_usage_seconds_total{`\\``=\"\"} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)", + "{`\\`a`} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)", + // vmselect evaluates rollup_candlestick one step ahead + // (evalRollupFuncWithoutAt), whatever the offset. + `rollup_candlestick(x)`, + `ROLLUP_CANDLESTICK(x[5m])`, + `sum(rollup_candlestick(x[1h:5m]))`, + `x offset -$__interval`, + `with (f(a) = a offset -1h) f(f(x))`, + } { + if got, ok := queryLookahead(q, time.Minute); ok { + t.Errorf("queryLookahead(%q) = %s, want unknown", q, got) + } + } +} + +// TestQueryLookaheadSubquery: vmselect evaluates a subquery up to its end +// plus one subquery step, rounded up to the step (evalRollupFuncWithSubquery), +// and range_last and the other range_* transforms copy the last of those +// values back onto earlier points, so a series that starts after the +// evaluation time can reach the answer. +func TestQueryLookaheadSubquery(t *testing.T) { + // One explicit subquery and one "(": two levels of two steps, at the + // subquery's step, or at the query's step when the subquery has none. + for q, want := range map[string]time.Duration{ + `max_over_time(x[1h:1h])`: 4 * time.Hour, + `max_over_time(x[1h:])`: 4 * time.Minute, + } { + if got, ok := queryLookahead(q, time.Minute); !ok || got != want { + t.Errorf("queryLookahead(%q) = %s, %v; want %s", q, got, ok, want) + } + } + for _, q := range []string{`max_over_time(range_last(x)[1h:1h])`, `max_over_time(keep_next_value(x)[1h:1h])`} { + got, ok := queryLookahead(q, time.Minute) + if !ok || got < 2*time.Hour { + t.Errorf("queryLookahead(%q) = %s, %v; want at least two subquery steps", q, got, ok) + } + } +} + +// TestBuildMetricsParamsInstantStartCoversLatencyOffset: vmselect evaluates +// an instant query whose time is within 30s of its clock at that clock +// minus 30s, anywhere in (time-60s, time], so the earliest read the store +// choice and the lookup rely on starts from time-60s. +func TestBuildMetricsParamsInstantStartCoversLatencyOffset(t *testing.T) { + at := testNow.Add(29 * time.Second) + req := buildMetrics(t, "api/v1/query", url.Values{"query": {"x"}, "time": {at.Format(time.RFC3339)}, "step": {"1s"}}) + lookback, ok := queryLookback("x", time.Second) + if !ok { + t.Fatal("lookback of x unknown") + } + if want := at.Add(-2*latencyOffset - lookback); req.start.After(want) { + t.Fatalf("start = %s, want at most %s", req.start, want) + } +} diff --git a/internal/observabilityproxy/route.go b/internal/observabilityproxy/route.go index 56ee9e496..bc12c6276 100644 --- a/internal/observabilityproxy/route.go +++ b/internal/observabilityproxy/route.go @@ -131,8 +131,28 @@ func ParseRoute(escapedPath string) (Route, *HTTPError) { return r, nil } -// ResourceAttributes is the single SubjectAccessReview a route requires. It -// depends on the path only, so a POST query authorizes exactly like a GET. +// Reviews lists every access a request on the route needs. An application +// root also needs read access to its namespace root, because its scope +// narrows within the namespace but does not isolate: on both roots the +// identity comes from lineage labels that the lineage webhook keeps as set +// on an object it finds no owner for, so anyone who can create a pod there +// can label it as the application; on the metrics root the pod and PVC +// branches also match by name across the window, so a name another +// application held earlier reaches that application's series. A grant +// narrowed by resourceNames would read more than it names. +func (r Route) Reviews() []authorizationv1.ResourceAttributes { + switch r.Kind { + case ApplicationLog: + return []authorizationv1.ResourceAttributes{Route{Kind: NamespaceLogs, Namespace: r.Namespace}.ResourceAttributes(), r.ResourceAttributes()} + case ApplicationMetrics: + return []authorizationv1.ResourceAttributes{Route{Kind: NamespaceMetrics, Namespace: r.Namespace}.ResourceAttributes(), r.ResourceAttributes()} + } + return []authorizationv1.ResourceAttributes{r.ResourceAttributes()} +} + +// ResourceAttributes is the SubjectAccessReview of the route's own +// resource. It depends on the path only, so a POST query authorizes +// exactly like a GET. func (r Route) ResourceAttributes() authorizationv1.ResourceAttributes { switch r.Kind { case NamespaceLogs: diff --git a/internal/observabilityproxy/route_test.go b/internal/observabilityproxy/route_test.go index f34311e0f..a8e85b377 100644 --- a/internal/observabilityproxy/route_test.go +++ b/internal/observabilityproxy/route_test.go @@ -18,6 +18,7 @@ package observabilityproxy import ( "net/http" + "slices" "testing" authorizationv1 "k8s.io/api/authorization/v1" @@ -130,3 +131,26 @@ func TestRouteResourceAttributes(t *testing.T) { } } } + +func TestRouteReviews(t *testing.T) { + nsLogs := authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "logs", Namespace: "a"} + nsMetrics := authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "metrics", Namespace: "a"} + tests := []struct { + route Route + want []authorizationv1.ResourceAttributes + }{ + {Route{Kind: NamespaceLogs, Namespace: "a"}, []authorizationv1.ResourceAttributes{nsLogs}}, + {Route{Kind: NamespaceMetrics, Namespace: "a"}, []authorizationv1.ResourceAttributes{nsMetrics}}, + {Route{Kind: ApplicationLog, Namespace: "a", Plural: "postgreses", Name: "db"}, []authorizationv1.ResourceAttributes{ + nsLogs, {Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "log", Name: "db", Namespace: "a"}, + }}, + {Route{Kind: ApplicationMetrics, Namespace: "a", Plural: "postgreses", Name: "db"}, []authorizationv1.ResourceAttributes{ + nsMetrics, {Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "metrics", Name: "db", Namespace: "a"}, + }}, + } + for _, tt := range tests { + if got := tt.route.Reviews(); !slices.Equal(got, tt.want) { + t.Errorf("%v.Reviews() = %+v, want %+v", tt.route.Kind, got, tt.want) + } + } +} diff --git a/internal/observabilityproxy/server.go b/internal/observabilityproxy/server.go index e980818d9..cf873af72 100644 --- a/internal/observabilityproxy/server.go +++ b/internal/observabilityproxy/server.go @@ -25,6 +25,7 @@ import ( "log/slog" "math" "net/http" + "net/url" "os" "strconv" "strings" @@ -56,6 +57,7 @@ type Config struct { // means the in-cluster Service of the "generic" logs storage. UpstreamURL func(store string) string Logs LogsLimits + Metrics MetricsConfig Limiter LimiterConfig RequestTimeout time.Duration TailIdleTimeout time.Duration @@ -71,23 +73,31 @@ type Server struct { cfg Config auth *Authorizer stores StoreLookup + kinds KindLookup client *http.Client limits *limiter now func() time.Time requests *prometheus.CounterVec } -func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, reg prometheus.Registerer) *Server { +func NewServer(cfg Config, auth *Authorizer, stores StoreLookup, kinds KindLookup, reg prometheus.Registerer) *Server { if cfg.UpstreamURL == nil { domain := cfg.ClusterDomain cfg.UpstreamURL = func(store string) string { return fmt.Sprintf("http://vlselect-generic.%s.svc.%s:9471", store, domain) } } + if cfg.Metrics.UpstreamURL == nil { + domain, root := cfg.ClusterDomain, cfg.Metrics.RootTenant + cfg.Metrics.UpstreamURL = func(store string) string { + return fmt.Sprintf("http://vmselect-%s.%s.svc.%s:8481/select/0/prometheus", store, root, domain) + } + } s := &Server{ cfg: cfg, auth: auth, stores: stores, + kinds: kinds, // No client timeout: it would cut live tails. Non-tail requests are // bounded by a context deadline instead. client: &http.Client{Transport: http.DefaultTransport.(*http.Transport).Clone()}, @@ -164,12 +174,7 @@ func writeError(w http.ResponseWriter, e *HTTPError) { http.Error(w, e.Message, e.Code) } -// serve answers a root it has no handler for before authenticating, so an -// unknown path costs no review calls. func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTTPError { - if route.Kind != NamespaceLogs { - return errorf(http.StatusNotFound, "not found") - } user, herr := s.authorize(r, route) if herr != nil { return herr @@ -182,6 +187,15 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT w.Header().Set("Retry-After", strconv.FormatInt(int64(math.Ceil(wait.Seconds())), 10)) return errorf(http.StatusTooManyRequests, "request rate limit exceeded") } + var kind string + if route.Kind == ApplicationLog || route.Kind == ApplicationMetrics { + if kind, herr = s.applicationKind(route.Plural); herr != nil { + return herr + } + } + if route.Kind == NamespaceMetrics || route.Kind == ApplicationMetrics { + return s.serveMetrics(w, r, route, user, kind) + } endpoint, ok := strings.CutPrefix(route.Endpoint, "select/logsql/") if !ok { return errorf(http.StatusNotFound, "not found") @@ -203,6 +217,9 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT if herr != nil { return herr } + if kind != "" { + params.Add("extra_stream_filters", applicationStreamFilter(kind, route.Name)) + } store, err := s.stores.Resolve(route.Namespace) if err != nil { @@ -226,29 +243,17 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT ctx, stop = context.WithTimeout(ctx, limit) defer stop() } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, - s.cfg.UpstreamURL(store)+"/select/logsql/"+endpoint, strings.NewReader(params.Encode())) - if err != nil { - return errorf(http.StatusInternalServerError, "building upstream request: %v", err) - } - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("AccountID", "0") - req.Header.Set("ProjectID", "0") + tenant := http.Header{} + tenant.Set("AccountID", "0") + tenant.Set("ProjectID", "0") if tail { // vlselect holds even the headers of a gzip-encoded tail until the // first line; an explicit value stops the transport from asking for gzip. - req.Header.Set("Accept-Encoding", "identity") + tenant.Set("Accept-Encoding", "identity") } - resp, err := s.client.Do(req) - switch { - case err == nil: - case r.Context().Err() != nil: - return errorf(http.StatusBadGateway, "client went away") - case errors.Is(context.Cause(ctx), context.DeadlineExceeded): - return errorf(http.StatusGatewayTimeout, "log store did not answer within %s", limit) - default: - slog.Warn("log store unreachable", "store", store, "error", err) - return errorf(http.StatusBadGateway, "log store unreachable") + resp, herr := s.post(ctx, r, s.cfg.UpstreamURL(store)+"/select/logsql/"+endpoint, params, "log store", tenant) + if herr != nil { + return herr } defer func() { _ = resp.Body.Close() }() @@ -280,6 +285,104 @@ func (s *Server) serve(w http.ResponseWriter, r *http.Request, route Route) *HTT return nil } +// serveMetrics scopes to the namespace, or with kind set to the application +// route.Name of that kind within it. +func (s *Server) serveMetrics(w http.ResponseWriter, r *http.Request, route Route, user, kind string) *HTTPError { + release, ok := s.limits.acquire(user) + if !ok { + return tooManyConcurrent(w) + } + defer release() + if herr := s.readForm(w, r); herr != nil { + return herr + } + now := s.now() + req, herr := BuildMetricsParams(route.Endpoint, r.Form, route.Namespace, now, s.cfg.Metrics.Limits) + if herr != nil { + return herr + } + + ctx := r.Context() + if s.cfg.RequestTimeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, s.cfg.RequestTimeout) + defer cancel() + } + store := s.cfg.Metrics.store(req.start, now) + if kind != "" { + selectors, herr := s.applicationScope(ctx, r, store, req, route.Namespace, kind, route.Name) + if herr != nil { + return herr + } + req.setScope(selectors) + } + resp, herr := s.post(ctx, r, s.cfg.Metrics.UpstreamURL(store)+"/"+req.path, req.params, "metrics store", nil) + if herr != nil { + return herr + } + defer func() { _ = resp.Body.Close() }() + if ct := resp.Header.Get("Content-Type"); ct != "" { + w.Header().Set("Content-Type", ct) + } + // As for a bounded logs answer: the client must take it by the request + // deadline, and a break-off cuts the connection. + if deadline, ok := ctx.Deadline(); ok { + _ = http.NewResponseController(w).SetWriteDeadline(deadline) + } + w.WriteHeader(resp.StatusCode) + if _, err := io.Copy(w, resp.Body); err != nil { + slog.Debug("copying upstream response", "error", err) + return errAborted + } + return nil +} + +func (s *Server) applicationKind(plural string) (string, *HTTPError) { + kind, err := s.kinds.Kind(plural) + if err != nil { + slog.Error("mapping application plural to kind", "plural", plural, "error", err) + return "", errorf(http.StatusServiceUnavailable, "cannot resolve the application kind") + } + if kind == "" { + return "", errorf(http.StatusNotFound, "unknown application resource %q", plural) + } + return kind, nil +} + +// applicationStreamFilter selects the streams Fluent Bit labels with the +// pod's lineage. vlselect ANDs each extra_stream_filters value into the +// query and every subquery (Query.AddExtraFilters in lib/logstorage), so +// it narrows the namespace filter sent next to it. +func applicationStreamFilter(kind, name string) string { + return "{apps_cozystack_io_application_kind=" + strconv.Quote(kind) + ",apps_cozystack_io_application_name=" + strconv.Quote(name) + "}" +} + +// post sends params as a form with only the given headers, so nothing the +// client sent reaches the store, and tells an unreachable store (502) from +// one that missed its deadline or, for a tail, its idle timeout (504). +func (s *Server) post(ctx context.Context, r *http.Request, target string, params url.Values, what string, header http.Header) (*http.Response, *HTTPError) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, strings.NewReader(params.Encode())) + if err != nil { + return nil, errorf(http.StatusInternalServerError, "building upstream request: %v", err) + } + for k, v := range header { + req.Header[k] = v + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + resp, err := s.client.Do(req) + switch { + case err == nil: + return resp, nil + case r.Context().Err() != nil: + return nil, errorf(http.StatusBadGateway, "client went away") + case errors.Is(context.Cause(ctx), context.DeadlineExceeded): + return nil, errorf(http.StatusGatewayTimeout, "%s did not answer in time", what) + default: + slog.Warn(what+" unreachable", "url", target, "error", err) + return nil, errorf(http.StatusBadGateway, "%s unreachable", what) + } +} + // concurrencyRetryAfter is a hint only: a slot frees when another request // of the user ends, which the proxy cannot predict, and one second keeps // a polling client from retrying in a tight loop. @@ -348,13 +451,15 @@ func (s *Server) authorize(r *http.Request, route Route) (string, *HTTPError) { slog.Error("token review failed", "error", err) return "", errorf(http.StatusServiceUnavailable, "authentication unavailable") } - allowed, err := s.auth.Authorize(ctx, user, route.ResourceAttributes()) - if err != nil { - slog.Error("subject access review failed", "error", err) - return "", errorf(http.StatusServiceUnavailable, "authorization unavailable") - } - if !allowed { - return "", errorf(http.StatusForbidden, "forbidden") + for _, attrs := range route.Reviews() { + allowed, err := s.auth.Authorize(ctx, user, attrs) + if err != nil { + slog.Error("subject access review failed", "error", err) + return "", errorf(http.StatusServiceUnavailable, "authorization unavailable") + } + if !allowed { + return "", errorf(http.StatusForbidden, "forbidden") + } } return user.Username, nil } diff --git a/internal/observabilityproxy/server_app_test.go b/internal/observabilityproxy/server_app_test.go new file mode 100644 index 000000000..056e1a555 --- /dev/null +++ b/internal/observabilityproxy/server_app_test.go @@ -0,0 +1,668 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "encoding/json" + "io" + "net/http" + "net/url" + "slices" + "strings" + "testing" + "time" + + authorizationv1 "k8s.io/api/authorization/v1" +) + +const appRoot = "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/" + +// appPolicy lets alice read every postgres in tenant-a and the namespace +// roots, as the view role does. bob and carol may read only postgres "a", +// as a grant narrowed with resourceNames does; carol may also read the +// namespace roots. +func appPolicy(user string, a *authorizationv1.ResourceAttributes) bool { + if a.Namespace != "tenant-a" { + return false + } + if a.Verb == "list" && a.Group == "cozystack.io" && (a.Resource == "logs" || a.Resource == "metrics") && a.Name == "" && a.Subresource == "" { + return user == "alice" || user == "carol" + } + if a.Verb != "get" || a.Group != "apps.cozystack.io" || a.Resource != "postgreses" || + (a.Subresource != "log" && a.Subresource != "metrics") { + return false + } + return user == "alice" || a.Name == "a" +} + +func TestServerApplicationLogsScope(t *testing.T) { + h := newPolicyHarness(t, Config{}, nil, appPolicy, nil) + hostile := url.Values{ + "query": {"error"}, + "field": {"level"}, + "extra_filters": {`{kubernetes_namespace_name="tenant-b"}`}, + "extra_stream_filters": {`{apps_cozystack_io_application_name=~".*"}`}, + "AccountID": {"9"}, + } + headers := map[string]string{"AccountID": "9", "ProjectID": "9"} + endpoints := []string{"query", "hits", "stats_query_range", "tail", "field_values", "stream_field_values", "facets"} + for _, ep := range endpoints { + resp := h.do(http.MethodPost, appRoot+"db.main/log/select/logsql/"+ep+"?extra_stream_filters=x", "good", hostile, headers) + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("%s answered %d", ep, resp.StatusCode) + } + } + want := authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "log", Name: "db.main", Namespace: "tenant-a"} + if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want { + t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want) + } + reqs := h.requests() + if len(reqs) != len(endpoints) { + t.Fatalf("upstream saw %d requests, want %d", len(reqs), len(endpoints)) + } + // Two values: vlselect ANDs every extra_stream_filters value into the + // query and each of its subqueries. + wantFilters := []string{wantScope, `{apps_cozystack_io_application_kind="Postgres",apps_cozystack_io_application_name="db.main"}`} + for i, r := range reqs { + if r.path != "/select/logsql/"+endpoints[i] { + t.Errorf("upstream path = %s, want %s", r.path, endpoints[i]) + } + if got := r.form["extra_stream_filters"]; !slices.Equal(got, wantFilters) { + t.Errorf("%s: extra_stream_filters = %q, want %q", endpoints[i], got, wantFilters) + } + if r.form.Has("extra_filters") || r.form.Has("AccountID") || r.headers.Get("AccountID") != "0" || r.headers.Get("ProjectID") != "0" { + t.Errorf("%s: client scope reached the store: %v %v", endpoints[i], r.form, r.headers) + } + } +} + +func TestServerApplicationKinds(t *testing.T) { + tests := []struct { + name string + kinds KindLookup + path string + want int + }{ + {"unknown plural logs", nil, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/mysqls/a/log/select/logsql/query", http.StatusNotFound}, + {"unknown plural metrics", nil, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/mysqls/a/metrics/api/v1/query?query=up", http.StatusNotFound}, + {"not synced logs", newTestKinds(t, false), appRoot + "a/log/select/logsql/query", http.StatusServiceUnavailable}, + {"not synced metrics", newTestKinds(t, false), appRoot + "a/metrics/api/v1/query?query=up", http.StatusServiceUnavailable}, + {"ambiguous plural", newTestKinds(t, true, appDefinition("x", "Postgres", "postgreses"), appDefinition("y", "Pg", "postgreses")), appRoot + "a/log/select/logsql/query", http.StatusServiceUnavailable}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := newPolicyHarness(t, Config{}, nil, func(string, *authorizationv1.ResourceAttributes) bool { return true }, tt.kinds) + if got := h.status(http.MethodGet, tt.path, "good"); got != tt.want { + t.Fatalf("status = %d, want %d", got, tt.want) + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests", n) + } + }) + } + // The kind is looked up only for an authorized caller. + h := newPolicyHarness(t, Config{}, nil, appPolicy, newTestKinds(t, false)) + if got := h.status(http.MethodGet, appRoot+"b/log/select/logsql/query", "bob-token"); got != http.StatusForbidden { + t.Fatalf("denied caller answered %d, want 403", got) + } +} + +const wantPodLookup = `kube_pod_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}` + +// appMetricsUpstream answers the pod and PVC lookups with the given names, +// or with status when it is not 200, and every other path like vmselect. +func appMetricsUpstream(pods, pvcs []string, podStatus int) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + names := map[string][]string{"/api/v1/label/pod/values": pods, "/api/v1/label/persistentvolumeclaim/values": pvcs} + list, lookup := names[r.URL.Path] + if !lookup { + promUpstream(w, r) + return + } + if r.URL.Path == "/api/v1/label/pod/values" && podStatus != http.StatusOK { + http.Error(w, "unavailable", podStatus) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"status": "success", "data": list}) + } +} + +func TestServerApplicationMetricsScope(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"db-main-0", "db.main-1"}, []string{"data-db-main-0"}, http.StatusOK), appPolicy, nil) + hostile := url.Values{ + "query": {"sum(rate(container_cpu_usage_seconds_total[5m]))"}, + "start": {unix(testNow.Add(-time.Hour))}, + "end": {unix(testNow)}, + "step": {"60"}, + "extra_filters": {`{namespace=~".+"}`}, + "extra_filters[]": {`{namespace="tenant-a"}`}, + "extra_label": {"apps_cozystack_io_application_name=b"}, + } + resp := h.do(http.MethodPost, appRoot+"db.main/metrics/api/v1/query_range", "good", hostile, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("answered %d %q", resp.StatusCode, body) + } + want := authorizationv1.ResourceAttributes{Verb: "get", Group: "apps.cozystack.io", Resource: "postgreses", Subresource: "metrics", Name: "db.main", Namespace: "tenant-a"} + if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want { + t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want) + } + reqs := h.requests() + if len(reqs) != 3 { + t.Fatalf("upstream saw %d requests, want two lookups and the query", len(reqs)) + } + lookups := map[string]string{ + "/api/v1/label/pod/values": wantPodLookup, + "/api/v1/label/persistentvolumeclaim/values": `kube_persistentvolumeclaim_labels{namespace="tenant-a",label_apps_cozystack_io_application_kind="Postgres",label_apps_cozystack_io_application_name="db.main"}`, + } + for _, r := range reqs[:2] { + if got := r.form["match[]"]; len(got) != 1 || got[0] != lookups[r.path] { + t.Errorf("%s match[] = %q", r.path, got) + } + if got := r.form["extra_filters[]"]; !slices.Equal(got, []string{wantMetricsScope}) { + t.Errorf("%s extra_filters[] = %q", r.path, got) + } + // The query's own lookback is known and shorter than the 24h cap. + if want := formatTime(buildMetrics(t, "api/v1/query_range", hostile).start); r.form.Get("start") != want { + t.Errorf("%s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) + } + // One step past end, which vmselect rounds up to a whole step, and + // two steps for each "(" that may be an implicit subquery. + if r.form.Get("end") != formatTime(testNow.Add(5*time.Minute)) || r.form.Get("limit") != "101" { + t.Errorf("%s end/limit = %s/%s", r.path, r.form.Get("end"), r.form.Get("limit")) + } + } + main := reqs[2] + wantBranches := []string{ + wantIdentityBranch, + `{namespace="tenant-a",pod=~"^(db-main-0|db\\.main-1)$"}`, + `{namespace="tenant-a",persistentvolumeclaim=~"^(data-db-main-0)$"}`, + } + if main.path != "/api/v1/query_range" || !slices.Equal(main.form["extra_filters[]"], wantBranches) { + t.Fatalf("query %s extra_filters[] = %q, want %q", main.path, main.form["extra_filters[]"], wantBranches) + } + if main.form.Has("extra_filters") || main.form.Has("extra_label") { + t.Fatalf("client scope reached the store: %v", main.form) + } + if !slices.Equal(h.stores, []string{"shortterm", "shortterm", "shortterm"}) { + t.Fatalf("stores = %q, want the lookups on the query's store", h.stores) + } +} + +func TestServerApplicationMetricsBranches(t *testing.T) { + podBranch := `{namespace="tenant-a",pod=~"^(p)$"}` + tests := []struct { + name string + pods, pvcs []string + podStatus int + podBody string + cfg Config + path string + wantCode int + want []string + }{ + {name: "no pods or pvcs", path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, + // A failed lookup would leave the application's own series out of + // a successful answer, so it fails the request instead. + {name: "pod lookup fails", pods: []string{"p"}, pvcs: []string{"v"}, podStatus: http.StatusServiceUnavailable, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, + {name: "pod lookup times out at the store", pods: []string{"p"}, podStatus: http.StatusGatewayTimeout, path: "api/v1/query?query=up", wantCode: http.StatusGatewayTimeout}, + {name: "pods over the cap", pods: []string{"p", "q"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, + path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, + {name: "over the cap with an empty name", pods: []string{"", "p"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, + path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, + {name: "pvcs over the cap", pods: []string{"p"}, pvcs: []string{"v", "w"}, cfg: Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}}, + path: "api/v1/query?query=up", wantCode: http.StatusUnprocessableEntity}, + {name: "labels get the branches as match[]", pods: []string{"p"}, path: "api/v1/labels", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, + {name: "series needs match[]", path: "api/v1/series", wantCode: http.StatusBadRequest}, + // An empty name would make pod=~"^(|p)$", which matches every + // series without a pod label. + {name: "empty pod name dropped", pods: []string{"", "p"}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch, podBranch}}, + {name: "only empty pod names", pods: []string{""}, path: "api/v1/query?query=up", wantCode: http.StatusOK, want: []string{wantIdentityBranch}}, + {name: "lookup status error", podBody: `{"status":"error","data":["p"]}`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, + {name: "truncated lookup answer", podBody: `{"status":"success","data":["p"`, path: "api/v1/query?query=up", wantCode: http.StatusBadGateway}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.podStatus == 0 { + tt.podStatus = http.StatusOK + } + upstream := appMetricsUpstream(tt.pods, tt.pvcs, tt.podStatus) + if tt.podBody != "" { + upstream = func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/label/pod/values" { + _, _ = io.WriteString(w, tt.podBody) + return + } + appMetricsUpstream(nil, nil, http.StatusOK)(w, r) + } + } + h := newPolicyHarness(t, tt.cfg, upstream, appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tt.path, "good"); got != tt.wantCode { + t.Fatalf("status = %d, want %d", got, tt.wantCode) + } + var main *upstreamRequest + for _, r := range h.requests() { + if !strings.HasPrefix(r.path, "/api/v1/label/pod/") && !strings.HasPrefix(r.path, "/api/v1/label/persistentvolumeclaim/") { + main = &r + } + } + if tt.want == nil { + if main != nil { + t.Fatalf("query reached the store: %s %v", main.path, main.form) + } + return + } + if main == nil || !slices.Equal(main.form["extra_filters[]"], tt.want) { + t.Fatalf("extra_filters[] = %v, want %q", main, tt.want) + } + if strings.HasSuffix(tt.path, "labels") && !slices.Equal(main.form["match[]"], tt.want) { + t.Fatalf("match[] = %q, want the branches", main.form["match[]"]) + } + }) + } +} + +// A grant narrowed to application "a" by resourceNames must not reach "b" +// through any endpoint of either root. Without read access to the +// namespace roots it does not reach "a" either: an ownerless pod keeps the +// lineage labels its creator set, so it can claim to be "a" on both roots, +// and on the metrics root the pod and PVC branches match by name, so a name +// another application used earlier in the window would pull its series in. +func TestServerApplicationNarrowedGrant(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream([]string{"a-0"}, nil, http.StatusOK), appPolicy, nil) + paths := []string{ + "metrics/api/v1/query?query=up", + "metrics/api/v1/query_range?query=up&start=" + unix(testNow.Add(-time.Hour)) + "&end=" + unix(testNow) + "&step=60", + "metrics/api/v1/series?match[]=up", + "metrics/api/v1/labels", + "metrics/api/v1/label/pod/values", + "log/select/logsql/query", + "log/select/logsql/hits", + "log/select/logsql/stats_query_range?query=" + url.QueryEscape("* | stats count()"), + "log/select/logsql/tail", + "log/select/logsql/field_values?field=level", + "log/select/logsql/stream_field_values?field=level", + "log/select/logsql/facets", + } + for _, p := range paths { + for _, denied := range []struct{ app, token string }{{"b", "bob-token"}, {"b", "carol-token"}, {"a", "bob-token"}} { + if got := h.status(http.MethodGet, appRoot+denied.app+"/"+p, denied.token); got != http.StatusForbidden { + t.Errorf("%s on %s/%s answered %d, want 403", denied.token, denied.app, p, got) + } + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for a denied application", n) + } + for _, p := range []string{"metrics/api/v1/labels", "log/select/logsql/query"} { + if got := h.status(http.MethodGet, appRoot+"a/"+p, "carol-token"); got != http.StatusOK { + t.Errorf("a/%s answered %d, want 200", p, got) + } + } +} + +// TestServerApplicationLookbackBeyondCap: the lookup reaches back at most +// --metrics-app-lookback, so a query that reads further back, or whose +// lookback is unknown, would miss pods that existed only earlier; it is +// refused rather than answered without their series. +func TestServerApplicationLookbackBeyondCap(t *testing.T) { + cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 100, MaxPVCs: 100}}} + h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + // max_over_time(x[2h]) reads just past the 1h cap; rate(x[5i]) has an + // unknown lookback. + for _, query := range []string{"rate(x[7d])", "max_over_time(x[2h])", "rate(x[5i])"} { + q := url.Values{"query": {query}, "time": {unix(testNow)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for refused queries", n) + } + q := url.Values{"query": {"rate(x[5m])"}, "time": {unix(testNow)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("rate(x[5m]) answered %d, want 200", got) + } + r := h.requests()[0] + want := formatTime(buildMetrics(t, "api/v1/query", q).start) + if r.path != "/api/v1/label/pod/values" || r.form.Get("start") != want { + t.Fatalf("lookup %s start = %s, want the query's earliest read %s", r.path, r.form.Get("start"), want) + } +} + +// TestServerApplicationLookupCoversNegativeOffset evaluates two hours ago +// a selector shifted an hour forward: the pod lookup must reach the time +// of the samples, or a pod created in between is left out. +func TestServerApplicationLookupCoversNegativeOffset(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + at := testNow.Add(-2 * time.Hour) + q := url.Values{"query": {"container_cpu_usage_seconds_total offset -1h"}, "time": {unix(at)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + r := h.requests()[0] + if want := formatTime(at.Add(time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("end") != want { + t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want) + } +} + +// TestServerApplicationForwardsDefaultTime: without time the lookup is +// sized at the proxy's clock, so that instant is forwarded. Otherwise +// vmselect evaluates at its own later clock, past the end of the lookup by +// however long the lookups took, and leaves out pods first sampled since. +func TestServerApplicationForwardsDefaultTime(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + q := url.Values{"query": {"container_cpu_usage_seconds_total"}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + reqs := h.requests() + lookup, final := reqs[0], reqs[len(reqs)-1] + want := formatTime(testNow) + if lookup.path != "/api/v1/label/pod/values" || lookup.form.Get("end") != want { + t.Fatalf("lookup %s end = %s, want %s", lookup.path, lookup.form.Get("end"), want) + } + if final.path != "/api/v1/query" || final.form.Get("time") != want { + t.Fatalf("query %s time = %q, want the lookup's instant %s", final.path, final.form.Get("time"), want) + } +} + +// TestServerApplicationRefusesUnboundedSampleTime refuses a query whose +// samples may lie at a time the lookup cannot be sized for, rather than +// answer with series silently left out. +func TestServerApplicationRefusesUnboundedSampleTime(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{"up @ 1700000000", "up offset -5i"} { + q := url.Values{"query": {query}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for a refused query", n) + } + // The namespace root has no lookup to size and still serves it. + if got := h.status(http.MethodGet, metricsRoot+"api/v1/query?"+url.Values{"query": {"up @ 1700000000"}}.Encode(), "good"); got != http.StatusOK { + t.Fatalf("namespace root answered %d", got) + } +} + +// TestServerApplicationLookupCoversAlignedEnd: vmselect rounds a range +// query's end up to a whole step, so the lookup reaches one step past it. +func TestServerApplicationLookupCoversAlignedEnd(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + end := testNow.Add(-90 * time.Minute) + q := url.Values{"query": {"up"}, "start": {unix(end.Add(-6 * time.Hour))}, "end": {unix(end)}, "step": {"3600"}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query_range?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + r := h.requests()[0] + if want := formatTime(end.Add(time.Hour)); r.path != "/api/v1/label/pod/values" || r.form.Get("end") != want { + t.Fatalf("lookup %s end = %s, want %s", r.path, r.form.Get("end"), want) + } +} + +// TestServerApplicationLookupDeadline: a lookup that misses the request +// deadline answers 504, whether the store sent nothing or stalled after +// its headers, and the query itself is not sent. +func TestServerApplicationLookupDeadline(t *testing.T) { + for _, headers := range []bool{false, true} { + h := newPolicyHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/label/pod/values" { + appMetricsUpstream(nil, nil, http.StatusOK)(w, r) + return + } + if headers { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"status":"success","data":[`) + w.(http.Flusher).Flush() + } + <-r.Context().Done() + }, appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusGatewayTimeout { + t.Fatalf("headers sent %v: status = %d, want 504", headers, got) + } + for _, r := range h.requests() { + if !strings.HasPrefix(r.path, "/api/v1/label/") { + t.Fatalf("headers sent %v: the query reached the store after a failed lookup", headers) + } + } + } +} + +// TestServerApplicationDefaultLookbackServesLongPanels: the lookback is +// only the refusal threshold, and the lookback estimate is a loose upper +// bound, so the default must serve a day's increase and a month-long +// panel at the step a dashboard picks for it. +func TestServerApplicationDefaultLookbackServesLongPanels(t *testing.T) { + cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: 30 * 24 * time.Hour, MaxPods: 100, MaxPVCs: 100}}} + h := newPolicyHarness(t, cfg, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + month := url.Values{"start": {unix(testNow.Add(-29 * 24 * time.Hour))}, "end": {unix(testNow)}} + for _, tc := range []struct{ path, query, step string }{ + {"api/v1/query", "increase(x[24h])", ""}, + {"api/v1/query_range", "rate(x[5m])", "21600"}, + {"api/v1/query_range", "histogram_quantile(0.99, sum by (le) (rate(x[5m])))", "7200"}, + } { + q := url.Values{"query": {tc.query}} + if tc.step != "" { + q.Set("step", tc.step) + q.Set("start", month.Get("start")) + q.Set("end", month.Get("end")) + } + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/"+tc.path+"?"+q.Encode(), "good"); got != http.StatusOK { + t.Errorf("%s step %s answered %d, want 200", tc.query, tc.step, got) + } + } +} + +// TestServerApplicationLookupStoreErrors: vmselect answers 422 to any error +// without a status of its own (SendPrometheusError), so only its series +// limit, which a narrower window fixes, reaches the client as 422. +func TestServerApplicationLookupStoreErrors(t *testing.T) { + for _, tc := range []struct { + label, message string + want int + }{ + {"pod", "the number of matching timeseries exceeds 300000; either narrow down the search or increase -search.max* command-line flag values", http.StatusUnprocessableEntity}, + {"persistentvolumeclaim", "the number of matching timeseries exceeds 300000; either narrow down the search", http.StatusUnprocessableEntity}, + {"pod", "cannot fetch label values from vmstorage nodes: the number of matching timeseries exceeds 300000", http.StatusUnprocessableEntity}, + {"pod", "timeout exceeded before starting the query processing: 30s", http.StatusGatewayTimeout}, + {"pod", "error when searching for label values: cannot obtain data from vmstorage", http.StatusBadGateway}, + } { + h := newPolicyHarness(t, Config{}, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/label/"+tc.label+"/values" { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnprocessableEntity) + _ = json.NewEncoder(w).Encode(map[string]string{"status": "error", "errorType": "422", "error": tc.message}) + return + } + appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r) + }, appPolicy, nil) + resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good", nil, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != tc.want { + t.Errorf("%s lookup %q answered %d, want %d", tc.label, tc.message, resp.StatusCode, tc.want) + } + if tc.want == http.StatusUnprocessableEntity && !strings.Contains(string(body), "the number of matching timeseries exceeds 300000") { + t.Errorf("the store's limit message did not reach the client: %q", body) + } + if strings.Contains(string(body), `"errorType"`) { + t.Errorf("the store's raw JSON reached the client: %q", body) + } + // Other store errors may name vmstorage nodes; only the limit + // message, which the client can act on, is passed on. + if strings.Contains(string(body), "vmstorage") { + t.Errorf("the store's error text reached the client: %q", body) + } + } +} + +// TestServerApplicationRefusesPartialLookup: vmselect answers a label +// values lookup with isPartial when a vmstorage node is down, and a short +// pod list would drop the missing pods' series without a word. +func TestServerApplicationRefusesPartialLookup(t *testing.T) { + for _, label := range []string{"pod", "persistentvolumeclaim"} { + h := newPolicyHarness(t, Config{}, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/label/"+label+"/values" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"status":"success","isPartial":true,"data":["db-0"]}`) + return + } + appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r) + }, appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusBadGateway { + t.Errorf("partial %s lookup answered %d, want 502", label, got) + } + for _, req := range h.requests() { + if req.path == "/api/v1/query" { + t.Errorf("partial %s lookup still sent the query", label) + } + } + } + // A partial list already over the cap settles it: the full one is longer. + cfg := Config{Metrics: MetricsConfig{App: AppScopeLimits{Lookback: time.Hour, MaxPods: 1, MaxPVCs: 1}}} + h := newPolicyHarness(t, cfg, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/v1/label/pod/values" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"status":"success","isPartial":true,"data":["db-0","db-1"]}`) + return + } + appMetricsUpstream([]string{"p"}, nil, http.StatusOK)(w, r) + }, appPolicy, nil) + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?query=up", "good"); got != http.StatusUnprocessableEntity { + t.Errorf("partial pod list over the cap answered %d, want 422", got) + } +} + +// TestServerApplicationRefusesEveryAt: no @ form can be bounded, and the +// answer says so rather than naming forms that would pass. +func TestServerApplicationRefusesEveryAt(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{"up @ end()", "up @ start()", "up @ 1700000000"} { + resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+url.Values{"query": {query}}.Encode(), "good", nil, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, resp.StatusCode) + } + if strings.Contains(string(body), "start() or end()") { + t.Errorf("%s: the refusal names @ forms as allowed: %q", query, body) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for refused queries", n) + } +} + +// TestServerApplicationRefusesForwardReads: queries that read past the +// evaluation time by an amount the scan cannot bound answer 422 and send +// nothing upstream, rather than answer 200 without pods created in that +// time. +func TestServerApplicationRefusesForwardReads(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{ + "container_cpu_usage_seconds_total offset -1h-30m", + "container_cpu_usage_seconds_total offset 1h-2h", + "container_cpu_usage_seconds_total offset -1h$__interval", + "rollup_candlestick(container_cpu_usage_seconds_total)", + } { + q := url.Values{"query": {query}, "time": {unix(testNow.Add(-2 * time.Hour))}} + resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good", nil, nil) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, resp.StatusCode) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for refused queries", n) + } +} + +// TestServerApplicationLookupCoversSubquery: a pod created during the two +// subquery steps past the evaluation time can show up through range_last, +// so the lookup reaches that far. +func TestServerApplicationLookupCoversSubquery(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + at := testNow.Add(-6 * time.Hour) + q := url.Values{"query": {"max_over_time(range_last(container_cpu_usage_seconds_total)[1h:1h])"}, "time": {unix(at)}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + r := h.requests()[0] + end, err := time.Parse(time.RFC3339, r.form.Get("end")) + if err != nil || r.path != "/api/v1/label/pod/values" || end.Before(at.Add(2*time.Hour)) { + t.Fatalf("lookup %s end = %s, want at least %s", r.path, r.form.Get("end"), formatTime(at.Add(2*time.Hour))) + } +} + +// TestServerApplicationNamesUnreadableSubqueryStep: the refusal names the +// cause when a subquery step is what the scan cannot read. +func TestServerApplicationNamesUnreadableSubqueryStep(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + resp := h.do(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+url.Values{"query": {"max_over_time(x[1h:5i])"}}.Encode(), "good", nil, nil) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusUnprocessableEntity || !strings.Contains(string(body), "subquery step") { + t.Fatalf("answered %d %q, want 422 naming the subquery step", resp.StatusCode, body) + } +} + +// TestServerApplicationLookupCoversLatencyOffset: vmselect evaluates an +// instant query whose time is within 30s of its own clock at its clock +// minus 30s (prometheus.go, queryOffset), so for a time slightly in the +// future it reads from earlier than time minus 30s. The lookup starts +// early enough for any evaluation instant in (time-60s, time]. +func TestServerApplicationLookupCoversLatencyOffset(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + at := testNow.Add(29 * time.Second) + q := url.Values{"query": {"container_cpu_usage_seconds_total"}, "time": {at.Format(time.RFC3339)}, "step": {"1s"}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusOK { + t.Fatalf("status = %d", got) + } + // The earliest read of the query evaluated at the earliest instant. + lookback, _ := queryLookback("container_cpu_usage_seconds_total", time.Second) + want := at.Add(-2*latencyOffset - lookback) + r := h.requests()[0] + start, err := time.Parse(time.RFC3339, r.form.Get("start")) + if err != nil || start.After(want) { + t.Fatalf("lookup start = %s, want at most %s", r.form.Get("start"), formatTime(want)) + } +} + +// TestServerApplicationRefusesBackquoteEscapeTail: an escaped backquote +// inside a quoted label name does not end the string in metricsql, so the +// escaped rollup_candlestick after it must still be refused. +func TestServerApplicationRefusesBackquoteEscapeTail(t *testing.T) { + h := newPolicyHarness(t, Config{}, appMetricsUpstream(nil, nil, http.StatusOK), appPolicy, nil) + for _, query := range []string{ + "container_cpu_usage_seconds_total{`\\``=\"\"} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)", + "{`\\`a`} or rollup\\_candlestick(container_cpu_usage_seconds_total offset -5m+1h)", + } { + q := url.Values{"query": {query}, "time": {unix(testNow.Add(-6 * time.Hour))}} + if got := h.status(http.MethodGet, appRoot+"db.main/metrics/api/v1/query?"+q.Encode(), "good"); got != http.StatusUnprocessableEntity { + t.Errorf("%s answered %d, want 422", query, got) + } + } + if n := len(h.requests()); n != 0 { + t.Fatalf("upstream saw %d requests for refused queries", n) + } +} diff --git a/internal/observabilityproxy/server_deadline_test.go b/internal/observabilityproxy/server_deadline_test.go index 7415b46fa..5723365b3 100644 --- a/internal/observabilityproxy/server_deadline_test.go +++ b/internal/observabilityproxy/server_deadline_test.go @@ -421,15 +421,22 @@ func stalledAPIServer(t *testing.T, stallTokens bool) kubernetes.Interface { // a bounded review call ends the request. func TestServerStalledReviewAnswersInTime(t *testing.T) { for _, tc := range []struct { - name string + name, path string stallTokens bool - }{{"token review", true}, {"subject access review", false}} { + }{ + {"logs token review", logsRoot + "query", true}, + {"logs subject access review", logsRoot + "query", false}, + {"metrics token review", metricsRoot + "api/v1/query", true}, + {"metrics subject access review", metricsRoot + "api/v1/query", false}, + {"application logs subject access review", appRoot + "db/log/select/logsql/query", false}, + {"application metrics subject access review", appRoot + "db/metrics/api/v1/query", false}, + } { t.Run(tc.name, func(t *testing.T) { h := newHarness(t, Config{RequestTimeout: 200 * time.Millisecond}, nil) h.server.auth = NewAuthorizer(stalledAPIServer(t, tc.stallTokens), 10*time.Second) got := make(chan int, 1) go func() { - resp := h.do(http.MethodPost, logsRoot+"query", "good", url.Values{"query": {"*"}}, nil) + resp := h.do(http.MethodPost, tc.path, "good", url.Values{"query": {"up"}}, nil) _ = resp.Body.Close() got <- resp.StatusCode }() @@ -444,3 +451,56 @@ func TestServerStalledReviewAnswersInTime(t *testing.T) { }) } } + +// TestServerMetricsRequestLifetime repeats the body, write and break-off +// checks on the metrics root, which has its own handler. +func TestServerMetricsRequestLifetime(t *testing.T) { + query := metricsRoot + "api/v1/query?query=up" + t.Run("slow body", func(t *testing.T) { + h := newHarness(t, Config{ + RequestTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1}, + }, promUpstream) + pipe := servePipe(t, h.server) + conn := pipe.send(t, "POST "+metricsRoot+"api/v1/query HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n"+ + "Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 100\r\n\r\nquery=") + eventually(t, time.Second, func() bool { return slotsHeld(h) == 1 }, + "a metrics request with an unfinished body does not hold a concurrency slot") + if resp := readResponse(t, conn); resp.StatusCode != http.StatusRequestTimeout { + t.Fatalf("unfinished body answered %d, want 408", resp.StatusCode) + } + eventually(t, time.Second, func() bool { return slotsHeld(h) == 0 }, "the slot of an unfinished body was not released") + }) + t.Run("stalled client", func(t *testing.T) { + // Larger than the server's write buffer, so the handler itself + // blocks on the client. + big := `{"status":"success","data":"` + strings.Repeat("x", 64<<10) + `"}` + h := newHarness(t, Config{ + RequestTimeout: 200 * time.Millisecond, + Limiter: LimiterConfig{MaxConcurrentPerUser: 1}, + }, func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, big) }) + pipe := servePipe(t, h.server) + pipe.send(t, "GET "+query+" HTTP/1.1\r\nHost: proxy\r\nAuthorization: Bearer good\r\n\r\n") + eventually(t, time.Second, func() bool { return slotsHeld(h) == 1 }, "the stalled metrics request never held its slot") + eventually(t, 2*time.Second, func() bool { return slotsHeld(h) == 0 }, + "a metrics client that stopped reading keeps its concurrency slot") + }) + t.Run("truncated answer", func(t *testing.T) { + h := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", "100") + _, _ = io.WriteString(w, `{"status":"success",`) + }) + // Nothing has been flushed yet when the copy fails, so the + // connection may close before the status line. + req, _ := http.NewRequest(http.MethodGet, h.proxy.URL+query, nil) + req.Header.Set("Authorization", "Bearer good") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return + } + defer func() { _ = resp.Body.Close() }() + if _, err := io.ReadAll(resp.Body); err == nil { + t.Fatal("a truncated store answer reached the client as a complete response") + } + }) +} diff --git a/internal/observabilityproxy/server_metrics_test.go b/internal/observabilityproxy/server_metrics_test.go new file mode 100644 index 000000000..1a3d1de10 --- /dev/null +++ b/internal/observabilityproxy/server_metrics_test.go @@ -0,0 +1,210 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "io" + "net/http" + "net/url" + "sync" + "testing" + "time" + + authorizationv1 "k8s.io/api/authorization/v1" +) + +const metricsRoot = "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/" + +func promUpstream(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) +} + +func TestServerMetricsRebuildsUpstreamRequest(t *testing.T) { + h := newHarness(t, Config{}, promUpstream) + hostile := url.Values{ + "query": {"sum(rate(container_cpu_usage_seconds_total[5m]))"}, + "start": {unix(testNow.Add(-time.Hour))}, + "end": {unix(testNow)}, + "step": {"60"}, + "extra_filters": {`{namespace=~".+"}`}, + "extra_filters[]": {`{namespace="tenant-b"}`, `{namespace=~".*"}`}, + "extra_label": {"namespace=tenant-b"}, + "nocache": {"1"}, + } + headers := map[string]string{"AccountID": "9", "X-Forwarded-For": "1.2.3.4", "Cookie": "kc-access=secret"} + + resp := h.do(http.MethodGet, metricsRoot+"api/v1/query_range?"+hostile.Encode(), "good", nil, headers) + _ = resp.Body.Close() + resp = h.do(http.MethodPost, metricsRoot+"api/v1/query_range?extra_filters[]="+url.QueryEscape(`{namespace="x"}`), "good", hostile, headers) + body, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Type") != "application/json" { + t.Fatalf("POST answered %d %s %q", resp.StatusCode, resp.Header.Get("Content-Type"), body) + } + + reqs := h.requests() + if len(reqs) != 2 { + t.Fatalf("upstream saw %d requests", len(reqs)) + } + if reqs[0].form.Encode() != reqs[1].form.Encode() { + t.Fatalf("GET and POST built different upstream requests:\n%v\n%v", reqs[0].form, reqs[1].form) + } + if n := h.kube.sars.Load(); n != 1 { + t.Fatalf("SubjectAccessReviews = %d, want 1: POST must reuse the GET decision", n) + } + want := authorizationv1.ResourceAttributes{Verb: "list", Group: "cozystack.io", Resource: "metrics", Namespace: "tenant-a"} + if got := h.kube.lastSAR.Load().ResourceAttributes; *got != want { + t.Fatalf("SubjectAccessReview = %+v, want %+v", *got, want) + } + for _, r := range reqs { + if r.path != "/api/v1/query_range" { + t.Errorf("upstream path = %s", r.path) + } + if got := r.form["extra_filters[]"]; len(got) != 1 || got[0] != wantMetricsScope { + t.Errorf("extra_filters[] = %q", got) + } + if r.form.Has("extra_filters") || r.form.Has("extra_label") || r.form.Has("nocache") { + t.Errorf("client parameter forwarded: %v", r.form) + } + for _, k := range []string{"Authorization", "AccountID", "X-Forwarded-For", "Cookie"} { + if r.headers.Get(k) != "" { + t.Errorf("client header %s forwarded", k) + } + } + } +} + +func TestServerMetricsStoreByRange(t *testing.T) { + h := newHarness(t, Config{}, promUpstream) + for _, path := range []string{ + "api/v1/query?query=up", + "api/v1/query_range?query=up&step=600&start=" + unix(testNow.Add(-7*24*time.Hour)) + "&end=" + unix(testNow), + "api/v1/label/pod/values", + } { + if got := h.status(http.MethodGet, metricsRoot+path, "good"); got != http.StatusOK { + t.Fatalf("%s answered %d", path, got) + } + } + h.mu.Lock() + defer h.mu.Unlock() + if want := []string{"shortterm", "longterm", "shortterm"}; len(h.stores) != 3 || h.stores[0] != want[0] || h.stores[1] != want[1] || h.stores[2] != want[2] { + t.Fatalf("stores = %v, want %v", h.stores, want) + } + if r := h.got[2]; r.path != "/api/v1/label/pod/values" || r.form.Get("match[]") != wantMetricsScope { + t.Fatalf("label values upstream = %s %v", r.path, r.form) + } +} + +func TestServerMetricsStatusCodes(t *testing.T) { + h := newHarness(t, Config{}, promUpstream) + tooLong := "api/v1/query_range?query=up&step=3600&start=" + unix(testNow.Add(-31*24*time.Hour)) + "&end=" + unix(testNow) + if got := h.status(http.MethodGet, metricsRoot+tooLong, "good"); got != http.StatusUnprocessableEntity { + t.Errorf("range above the limit answered %d, want 422", got) + } + if got := h.status(http.MethodGet, metricsRoot+"api/v1/series", "good"); got != http.StatusBadRequest { + t.Errorf("series without match[] answered %d, want 400", got) + } + if got := h.status(http.MethodPut, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusMethodNotAllowed { + t.Errorf("PUT answered %d, want 405", got) + } + if n := len(h.requests()); n != 0 { + t.Errorf("upstream saw %d requests for refused ones", n) + } + + h2 := newHarness(t, Config{}, func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, `{"status":"error","errorType":"422"}`, http.StatusUnprocessableEntity) + }) + if got := h2.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusUnprocessableEntity { + t.Errorf("upstream 422 answered as %d", got) + } + h2.upstream.Close() + if got := h2.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusBadGateway { + t.Errorf("unreachable upstream answered %d, want 502", got) + } + + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + h3 := newHarness(t, Config{RequestTimeout: 50 * time.Millisecond}, func(_ http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-r.Context().Done(): + } + }) + if got := h3.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good"); got != http.StatusGatewayTimeout { + t.Errorf("slow upstream answered %d, want 504", got) + } + + h4 := newHarness(t, Config{Limiter: LimiterConfig{RatePerUser: 0.001, RateBurst: 1}}, promUpstream) + h4.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good") + limited := h4.do(http.MethodGet, metricsRoot+"api/v1/query?query=up", "good", nil, nil) + _ = limited.Body.Close() + if limited.StatusCode != http.StatusTooManyRequests || limited.Header.Get("Retry-After") != "1000" { + t.Errorf("rate-limited request answered %d with Retry-After %q, want 429 with 1000", + limited.StatusCode, limited.Header.Get("Retry-After")) + } +} + +func TestServerMetricsConcurrency(t *testing.T) { + entered, release := make(chan struct{}, 3), make(chan struct{}) + h := newHarness(t, Config{Limiter: LimiterConfig{MaxConcurrentPerUser: 1}}, func(w http.ResponseWriter, _ *http.Request) { + entered <- struct{}{} + <-release + promUpstream(w, nil) + }) + // Registered after the harness so it runs before the servers close. + var once sync.Once + t.Cleanup(func() { once.Do(func() { close(release) }) }) + held := make(chan int, 2) + hold := func(token string) { + go func() { held <- h.status(http.MethodGet, metricsRoot+"api/v1/query?query=up", token) }() + select { + case <-entered: + case <-time.After(2 * time.Second): + t.Fatalf("request of %s did not reach the upstream", token) + } + } + hold("good") + refused := make(chan *http.Response, 1) + go func() { + resp := h.do(http.MethodPost, metricsRoot+"api/v1/query?query=up", "good", nil, nil) + _ = resp.Body.Close() + refused <- resp + }() + select { + case got := <-refused: + if got.StatusCode != http.StatusTooManyRequests || got.Header.Get("Retry-After") != "1" { + t.Fatalf("concurrent request answered %d with Retry-After %q, want 429 with 1", got.StatusCode, got.Header.Get("Retry-After")) + } + case <-time.After(2 * time.Second): + t.Fatal("concurrent request of the same user reached the upstream") + } + hold("bob-token") + once.Do(func() { close(release) }) + for range 2 { + if got := <-held; got != http.StatusOK { + t.Fatalf("held request answered %d", got) + } + } +} + +func TestDefaultMetricsUpstreamURL(t *testing.T) { + s := NewServer(Config{ClusterDomain: "cozy.local", Metrics: MetricsConfig{RootTenant: "tenant-root"}}, nil, nil, nil, nil) + if got, want := s.cfg.Metrics.UpstreamURL("shortterm"), "http://vmselect-shortterm.tenant-root.svc.cozy.local:8481/select/0/prometheus"; got != want { + t.Fatalf("upstream = %s, want %s", got, want) + } +} diff --git a/internal/observabilityproxy/server_test.go b/internal/observabilityproxy/server_test.go index df3e23bc4..c60ed49d6 100644 --- a/internal/observabilityproxy/server_test.go +++ b/internal/observabilityproxy/server_test.go @@ -18,6 +18,7 @@ package observabilityproxy import ( "bufio" + "errors" "io" "net/http" "net/http/httptest" @@ -52,13 +53,22 @@ type mapStores map[string]string func (m mapStores) Resolve(ns string) (string, error) { return m[ns], nil } // newHarness wires the proxy to a fake apiserver that knows tokens "good" -// (alice) and "bob-token" (bob), both allowed list logs in tenant-a only, -// and to an upstream served by handler. +// (alice) and "bob-token" (bob), both allowed list logs and metrics in +// tenant-a only, and to an upstream served by handler. func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { + return newPolicyHarness(t, cfg, handler, func(_ string, a *authorizationv1.ResourceAttributes) bool { + return a.Namespace == "tenant-a" && (a.Resource == "logs" || a.Resource == "metrics") + }, nil) +} + +// newPolicyHarness is newHarness with the given SubjectAccessReview policy +// and plural-to-kind mapping; nil kinds knows only postgreses. +func newPolicyHarness(t *testing.T, cfg Config, handler http.HandlerFunc, allow func(string, *authorizationv1.ResourceAttributes) bool, kinds KindLookup) *harness { h := &harness{t: t} - h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob"}, func(_ string, a *authorizationv1.ResourceAttributes) bool { - return a.Namespace == "tenant-a" && a.Resource == "logs" - }) + h.kube = newFakeKube(map[string]string{"good": "alice", "bob-token": "bob", "carol-token": "carol"}, allow) + if kinds == nil { + kinds = newTestKinds(t, true, appDefinition("postgres", "Postgres", "postgreses")) + } h.upstream = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { t.Errorf("upstream ParseForm: %v", err) @@ -80,10 +90,20 @@ func newHarness(t *testing.T, cfg Config, handler http.HandlerFunc) *harness { h.mu.Unlock() return h.upstream.URL } + cfg.Metrics.UpstreamURL = cfg.UpstreamURL + if cfg.Metrics.ShortTermStore == "" { + cfg.Metrics.ShortTermStore, cfg.Metrics.LongTermStore, cfg.Metrics.ShortTermRetention = "shortterm", "longterm", 72*time.Hour + } + if cfg.Metrics.Limits == (MetricsLimits{}) { + cfg.Metrics.Limits = testMetricsLimits() + } + if cfg.Metrics.App == (AppScopeLimits{}) { + cfg.Metrics.App = AppScopeLimits{Lookback: 24 * time.Hour, MaxPods: 100, MaxPVCs: 100} + } if cfg.Logs == (LogsLimits{}) { cfg.Logs = testLimits() } - h.server = NewServer(cfg, NewAuthorizer(h.kube.client, 10*time.Second), mapStores{"tenant-a": "tenant-a-store"}, nil) + h.server = NewServer(cfg, NewAuthorizer(h.kube.client, 10*time.Second), mapStores{"tenant-a": "tenant-a-store"}, kinds, nil) h.server.now = func() time.Time { return testNow } h.proxy = httptest.NewServer(h.server) t.Cleanup(h.proxy.Close) @@ -144,9 +164,11 @@ func TestServerStatusCodes(t *testing.T) { {"other namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/logs/select/logsql/query", "good", http.StatusForbidden}, {"invalid namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/Tenant/logs/select/logsql/query", "good", http.StatusBadRequest}, {"unknown root", http.MethodGet, "/api/v1/namespaces/tenant-a/pods", "good", http.StatusNotFound}, - {"namespace metrics not served yet", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query", "good", http.StatusNotFound}, - {"application log not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusNotFound}, - {"application metrics not served yet", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusNotFound}, + {"unknown metrics endpoint", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/status/tsdb", "good", http.StatusNotFound}, + {"metrics in another namespace", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-b/metrics/api/v1/query?query=up", "good", http.StatusForbidden}, + {"metrics without token", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/api/v1/query?query=up", "", http.StatusUnauthorized}, + {"application log without a grant", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/select/logsql/query", "good", http.StatusForbidden}, + {"application metrics without a grant", http.MethodGet, "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query", "good", http.StatusForbidden}, {"tenant_ids not proxied", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/select/tenant_ids", "good", http.StatusNotFound}, {"unknown logsql endpoint", http.MethodGet, logsRoot + "stats_query", "good", http.StatusNotFound}, {"logs root itself", http.MethodGet, "/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/", "good", http.StatusNotFound}, @@ -413,7 +435,12 @@ func TestServerTailCapsAndIdle(t *testing.T) { closed <- err }() select { - case <-closed: + case err := <-closed: + // The server cuts an idle tail off mid-stream rather than ending the + // response, so the client reads an unexpected EOF; a clean end is fine too. + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatalf("idle tail ended with %v, want the stream to end or be cut", err) + } case <-time.After(3 * time.Second): t.Fatal("idle tail not closed") } diff --git a/internal/observabilityproxy/vmselect_parse_test.go b/internal/observabilityproxy/vmselect_parse_test.go new file mode 100644 index 000000000..1fb23d18f --- /dev/null +++ b/internal/observabilityproxy/vmselect_parse_test.go @@ -0,0 +1,119 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observabilityproxy + +import ( + "math" + "net/url" + "strconv" + "strings" + "testing" + "time" +) + +// vmselectTimeMillis repeats how vmselect v1.136.0-cluster reads a time +// argument (httputil.GetTime -> timeutil.ParseTimeMsec -> ParseTimeAt in +// lib/timeutil/time.go) for the forms a client or the proxy may send: a +// value without "-" is a number whose unit vmselect guesses from its size +// (getUnixTimestampNanoseconds), anything else with a zone is RFC3339 read +// through time.UnixNano. Relative and partial dates are left out; the +// proxy refuses them. +func vmselectTimeMillis(t *testing.T, s string) int64 { + t.Helper() + if strings.Contains(s, "-") { + ts, err := time.Parse(time.RFC3339, s) + if err != nil { + t.Fatalf("vmselect cannot parse %q: %v", s, err) + } + return int64(math.Round(float64(ts.UnixNano()) / 1e6)) + } + intPart, frac, _ := strings.Cut(s, ".") + n, err := strconv.ParseInt(intPart+frac, 10, 64) + if err != nil { + t.Fatalf("vmselect cannot parse %q: %v", s, err) + } + for e := len(frac); e%3 != 0; e++ { + n *= 10 + } + var nanos int64 + switch { + case n < 1<<31: + nanos = n * 1e9 + case n < 1e3*(1<<31): + nanos = n * 1e6 + case n < 1e6*(1<<31): + nanos = n * 1e3 + default: + nanos = n + } + return int64(math.Round(float64(nanos) / 1e6)) +} + +// TestVmselectReadsProxyTimes sends times across the range the proxy +// accepts, in both client forms, and checks that vmselect reads the +// forwarded value as the same instant the proxy validated. A number such +// as "1000000.000" is seconds to the proxy but milliseconds-as-seconds +// guesswork to vmselect, so the proxy must not forward one. +func TestVmselectReadsProxyTimes(t *testing.T) { + for _, client := range []string{ + "0", "1000000", "1762400", "2147483", "2147484", "1000000.5", + "1793793600", "1793793600.123", "2147483647", "2147483648", "9223372035", + "1970-01-01T00:00:00Z", "1970-01-12T13:46:40.001Z", "2026-11-04T12:00:00.123Z", "2262-01-01T00:00:00Z", + } { + want, herr := parseMetricsTime("time", client) + if herr != nil { + t.Fatalf("%q refused: %v", client, herr) + } + req := buildMetrics(t, "api/v1/query", url.Values{"query": {"up"}, "time": {client}}) + sent := req.params.Get("time") + if got := vmselectTimeMillis(t, sent); got != want.UnixMilli() { + t.Errorf("client time %q forwarded as %q reaches vmselect as %dms, want %dms", client, sent, got, want.UnixMilli()) + } + } +} + +// TestVmselectReadsSecondsWithFraction is the window that passed the proxy as +// 8.8 days and 30 points while vmselect, reading "1000000.000" as a +// thousand times that many seconds, saw 2001..2025 and 30000 points. +func TestVmselectReadsSecondsWithFraction(t *testing.T) { + in := url.Values{"query": {"up"}, "start": {"1000000"}, "end": {"1762400"}, "step": {"25414"}} + req := buildMetrics(t, "api/v1/query_range", in) + start, end := vmselectTimeMillis(t, req.params.Get("start")), vmselectTimeMillis(t, req.params.Get("end")) + step := vmselectMillis(t, req.params.Get("step")) + if start != 1000000*1000 || end != 1762400*1000 { + t.Fatalf("vmselect reads %dms..%dms, want 1000000s..1762400s", start, end) + } + if points := (end-start)/step + 1; points != 30 { + t.Fatalf("vmselect counts %d points, want the 30 the proxy checked", points) + } + labels := buildMetrics(t, "api/v1/labels", url.Values{"start": {"1000000"}, "end": {"1000300"}}) + if got := vmselectTimeMillis(t, labels.params.Get("start")); got != 1000000*1000 { + t.Fatalf("label API start reaches vmselect as %dms, want 1000000s", got) + } +} + +func TestParseMetricsTimeRange(t *testing.T) { + // vmselect reads an RFC3339 time through time.UnixNano, which is + // undefined from 2262-04-11 on. + // Before 1970 vmselect clamps to 0, which would quietly answer for a + // window other than the one asked for. + for _, v := range []string{"9223372037", "9999999999", "2262-04-12T00:00:00Z", "9999-01-01T00:00:00Z", "1969-12-31T23:59:59Z", "1969-12-31T23:59:59.999Z"} { + if _, herr := parseMetricsTime("time", v); herr == nil { + t.Errorf("%q accepted, want refused", v) + } + } +} diff --git a/internal/operator/package_reconciler_test.go b/internal/operator/package_reconciler_test.go index c9923e7dc..fd8a15fe5 100644 --- a/internal/operator/package_reconciler_test.go +++ b/internal/operator/package_reconciler_test.go @@ -790,15 +790,6 @@ func scanRecordingClient(scheme *runtime.Scheme, listed *[]string, writes *int, }).Build() } -func listedKind(listed []string, kind string) bool { - for _, l := range listed { - if strings.Contains(l, kind) { - return true - } - } - return false -} - // The comparison guarding that write has to be semantic rather than structural, or a // LimitRange written in a different but equivalent notation would re-apply on every // reconcile — invisible in a test that only looked at the resulting object, since every one diff --git a/internal/sse/server.go b/internal/sse/server.go index 0c55032ec..3c121a08a 100644 --- a/internal/sse/server.go +++ b/internal/sse/server.go @@ -51,6 +51,9 @@ type Server struct { clientsMu sync.RWMutex clients map[*client]struct{} + // closing is set by Shutdown under clientsMu, so no stream registers + // after the clients it stops. + closing bool // latest holds the most recent payload (sent to new clients on connect). latestMu sync.RWMutex @@ -60,6 +63,7 @@ type Server struct { type client struct { ch chan string closeCh chan struct{} + closeOnce sync.Once flusher http.Flusher w http.ResponseWriter req *http.Request @@ -121,8 +125,9 @@ func (s *Server) ListenAndServe() error { // Shutdown gracefully stops the server. func (s *Server) Shutdown(ctx context.Context) error { s.clientsMu.Lock() + s.closing = true for c := range s.clients { - close(c.closeCh) + c.stop() } s.clientsMu.Unlock() return s.http.Shutdown(ctx) @@ -183,11 +188,24 @@ func (s *Server) handleStream(w http.ResponseWriter, r *http.Request) { // Register client s.clientsMu.Lock() + if s.closing { + s.clientsMu.Unlock() + http.Error(w, "server is shutting down", http.StatusServiceUnavailable) + return + } s.clients[c] = struct{}{} s.clientsMu.Unlock() + defer func() { + s.clientsMu.Lock() + delete(s.clients, c) + s.clientsMu.Unlock() + }() + // Initial comment to open the stream for some proxies - fmt.Fprintf(w, ": connected %s\n\n", time.Now().Format(time.RFC3339)) + if _, err := fmt.Fprintf(w, ": connected %s\n\n", time.Now().Format(time.RFC3339)); err != nil { + return + } flusher.Flush() // Send latest if any @@ -195,21 +213,31 @@ func (s *Server) handleStream(w http.ResponseWriter, r *http.Request) { latest := s.latest s.latestMu.RUnlock() if latest != "" { - writeSSE(w, latest) + if err := writeSSE(w, latest); err != nil { + return + } flusher.Flush() } - // Start pump - go c.pump() + // A write the client no longer takes ends the pump; the handler then + // gives the client up without waiting for the request to end. + pumped := make(chan struct{}) + go func() { + c.pump() + close(pumped) + }() + select { + case <-r.Context().Done(): + case <-pumped: + } + c.stop() + <-pumped +} - // Block until client disconnects - <-r.Context().Done() - - // Unregister client - close(c.closeCh) - s.clientsMu.Lock() - delete(s.clients, c) - s.clientsMu.Unlock() +// stop ends the client's pump. Shutdown and the client's own handler both +// stop it, in either order. +func (c *client) stop() { + c.closeOnce.Do(func() { close(c.closeCh) }) } func (c *client) pump() { @@ -220,23 +248,30 @@ func (c *client) pump() { case <-c.closeCh: return case msg := <-c.ch: - writeSSE(c.w, msg) + if err := writeSSE(c.w, msg); err != nil { + return + } c.flusher.Flush() case <-t.C: // heartbeat comment (keeps connections alive through proxies) - fmt.Fprint(c.w, ": hb\n\n") + if _, err := fmt.Fprint(c.w, ": hb\n\n"); err != nil { + return + } c.flusher.Flush() } } } -func writeSSE(w http.ResponseWriter, msg string) { +func writeSSE(w http.ResponseWriter, msg string) error { // Split on lines; each needs its own "data:" field per the SSE spec lines := strings.SplitSeq(strings.TrimRight(msg, "\n"), "\n") for ln := range lines { - fmt.Fprintf(w, "data: %s\n", ln) + if _, err := fmt.Fprintf(w, "data: %s\n", ln); err != nil { + return err + } } - fmt.Fprint(w, "\n") + _, err := fmt.Fprint(w, "\n") + return err } // Minimal index page with live updates diff --git a/internal/sse/server_test.go b/internal/sse/server_test.go new file mode 100644 index 000000000..577d19308 --- /dev/null +++ b/internal/sse/server_test.go @@ -0,0 +1,275 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sse + +import ( + "context" + "errors" + "io" + "log" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +// streamWriter accepts the first okWrites writes and fails every one after. +type streamWriter struct { + mu sync.Mutex + header http.Header + okWrites int + body strings.Builder +} + +func (w *streamWriter) Header() http.Header { return w.header } +func (w *streamWriter) WriteHeader(int) {} +func (w *streamWriter) Flush() {} + +func (w *streamWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.okWrites == 0 { + return 0, errors.New("connection reset") + } + w.okWrites-- + return w.body.Write(p) +} + +func (w *streamWriter) String() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.body.String() +} + +// serve runs the stream handler for a client that never disconnects and +// reports whether the handler gave the client up within a second. +func serve(t *testing.T, s *Server, w *streamWriter, publish string) bool { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx) + done := make(chan struct{}) + go func() { + s.handleStream(w, r) + close(done) + }() + if publish != "" { + deadline := time.Now().Add(time.Second) + for len(s.snapshotClients()) == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + // Onto the client's queue, which only the pump reads, so the failing + // write is the pump's and not the handler's write of latest. + for _, c := range s.snapshotClients() { + c.ch <- publish + } + } + select { + case <-done: + return len(s.snapshotClients()) == 0 + case <-time.After(time.Second): + return false + } +} + +func (s *Server) snapshotClients() []*client { + s.clientsMu.RLock() + defer s.clientsMu.RUnlock() + var out []*client + for c := range s.clients { + out = append(out, c) + } + return out +} + +func newTestServer(heartbeat time.Duration) *Server { + return New(Options{Heartbeat: heartbeat, Logger: log.New(io.Discard, "", 0)}) +} + +// A client whose connection stops taking writes is dropped at once, at +// whichever write first fails, instead of holding its handler until the +// request context ends. +func TestStreamDropsClientOnWriteFailure(t *testing.T) { + for _, tc := range []struct { + name string + okWrites int + latest string + publish string + heartbeat time.Duration + }{ + {name: "connect comment", okWrites: 0, heartbeat: time.Hour}, + {name: "latest payload", okWrites: 1, latest: "state", heartbeat: time.Hour}, + {name: "published event", okWrites: 1, publish: "event", heartbeat: time.Hour}, + {name: "heartbeat", okWrites: 1, heartbeat: 10 * time.Millisecond}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newTestServer(tc.heartbeat) + if tc.latest != "" { + s.Publish(tc.latest) + } + if !serve(t, s, &streamWriter{header: http.Header{}, okWrites: tc.okWrites}, tc.publish) { + t.Fatal("the handler kept a client whose writes fail") + } + }) + } +} + +// Each line of a payload is its own data field, and the event ends with an +// empty line. +func TestStreamFramesMultilinePayload(t *testing.T) { + s := newTestServer(time.Hour) + s.Publish("one\ntwo\n") + w := &streamWriter{header: http.Header{}, okWrites: 100} + ctx, cancel := context.WithCancel(context.Background()) + r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx) + done := make(chan struct{}) + go func() { + s.handleStream(w, r) + close(done) + }() + deadline := time.Now().Add(time.Second) + for !strings.Contains(w.String(), "data: two\n\n") && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + cancel() + <-done + if got := w.String(); !strings.Contains(got, "data: one\ndata: two\n\n") { + t.Fatalf("stream = %q, want both lines framed as one event", got) + } + if len(s.snapshotClients()) != 0 { + t.Fatal("a disconnected client stayed registered") + } +} + +// Shutdown ends every open stream, and the handler that then returns does +// not close the client a second time. +func TestShutdownEndsOpenStreams(t *testing.T) { + s := newTestServer(time.Hour) + w := &streamWriter{header: http.Header{}, okWrites: 100} + r := httptest.NewRequest(http.MethodGet, "/stream", nil) + done := make(chan struct{}) + go func() { + s.handleStream(w, r) + close(done) + }() + deadline := time.Now().Add(time.Second) + for len(s.snapshotClients()) == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if err := s.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("Shutdown left the stream open") + } +} + +// blockingWriter holds every write after the first until release is closed. +type blockingWriter struct { + streamWriter + writing chan struct{} + release chan struct{} + returned chan struct{} +} + +func (w *blockingWriter) Write(p []byte) (int, error) { + if w.String() == "" { + return w.streamWriter.Write(p) + } + select { + case w.writing <- struct{}{}: + default: + } + <-w.release + return w.streamWriter.Write(p) +} + +// The handler does not return while its pump is still writing: net/http does +// not allow a write to a response after ServeHTTP has returned. +func TestStreamWaitsForPumpBeforeReturning(t *testing.T) { + s := newTestServer(time.Hour) + w := &blockingWriter{ + streamWriter: streamWriter{header: http.Header{}, okWrites: 100}, + writing: make(chan struct{}, 1), + release: make(chan struct{}), + returned: make(chan struct{}), + } + ctx, cancel := context.WithCancel(context.Background()) + r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx) + go func() { + s.handleStream(w, r) + close(w.returned) + }() + deadline := time.Now().Add(time.Second) + for len(s.snapshotClients()) == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + // Straight onto the client's queue, which only the pump reads: Publish + // would also set latest, which the handler itself may still write. + s.snapshotClients()[0].ch <- "event" + select { + case <-w.writing: + case <-time.After(time.Second): + t.Fatal("the pump never wrote the event") + } + cancel() + select { + case <-w.returned: + close(w.release) + t.Fatal("the handler returned while its pump was writing") + case <-time.After(100 * time.Millisecond): + } + close(w.release) + select { + case <-w.returned: + case <-time.After(time.Second): + t.Fatal("the handler did not return once its pump finished") + } +} + +// A stream that opens once Shutdown has run is refused instead of being +// registered after the clients were stopped, where nothing would end it. +func TestStreamOpenedDuringShutdownIsRefused(t *testing.T) { + s := newTestServer(time.Hour) + if err := s.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + rec := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/stream", nil).WithContext(ctx) + done := make(chan struct{}) + go func() { + s.handleStream(rec, r) + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("a stream opened after Shutdown stayed open") + } + if rec.Code != http.StatusServiceUnavailable { + t.Errorf("status = %d, want %d", rec.Code, http.StatusServiceUnavailable) + } + if len(s.snapshotClients()) != 0 { + t.Error("a stream opened after Shutdown was registered") + } +} diff --git a/internal/telemetry/collector.go b/internal/telemetry/collector.go index f4bb8dd88..636bbc85a 100644 --- a/internal/telemetry/collector.go +++ b/internal/telemetry/collector.go @@ -118,11 +118,11 @@ func (c *Collector) collect(ctx context.Context) { // Write application count metrics for kind, count := range appKindCounts { - metrics.WriteString(fmt.Sprintf( + fmt.Fprintf(&metrics, "cozy_application_count{kind=\"%s\"} %d\n", kind, count, - )) + ) } // Send metrics only if there's something to send @@ -147,7 +147,7 @@ func (c *Collector) sendMetrics(clusterID, metrics string) error { if err != nil { return fmt.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return fmt.Errorf("unexpected status code: %d", resp.StatusCode) diff --git a/internal/telemetry/operator_collector.go b/internal/telemetry/operator_collector.go index 8b4ec3028..9f2f9fcbe 100644 --- a/internal/telemetry/operator_collector.go +++ b/internal/telemetry/operator_collector.go @@ -133,11 +133,11 @@ func (c *OperatorCollector) collect(ctx context.Context) { var metrics strings.Builder // Add cluster info metric - metrics.WriteString(fmt.Sprintf( + fmt.Fprintf(&metrics, "cozy_cluster_info{cozystack_version=\"%s\",kubernetes_version=\"%s\"} 1\n", version.Version, k8sVersion.GitVersion, - )) + ) // Collect node metrics grouped by OS and kernel nodeOSCount := make(map[string]map[string]int) // os -> kernel -> count @@ -153,12 +153,12 @@ func (c *OperatorCollector) collect(ctx context.Context) { for osKey, kernels := range nodeOSCount { for kernel, count := range kernels { - metrics.WriteString(fmt.Sprintf( + fmt.Fprintf(&metrics, "cozy_nodes_count{os=\"%s\",kernel=\"%s\"} %d\n", osKey, kernel, count, - )) + ) } } @@ -174,11 +174,11 @@ func (c *OperatorCollector) collect(ctx context.Context) { } for resourceName, total := range capacityTotals { - metrics.WriteString(fmt.Sprintf( + fmt.Fprintf(&metrics, "cozy_cluster_capacity{resource=\"%s\"} %d\n", resourceName, total, - )) + ) } // Collect LoadBalancer services metrics @@ -192,7 +192,7 @@ func (c *OperatorCollector) collect(ctx context.Context) { lbCount++ } } - metrics.WriteString(fmt.Sprintf("cozy_loadbalancers_count %d\n", lbCount)) + fmt.Fprintf(&metrics, "cozy_loadbalancers_count %d\n", lbCount) } // Collect PV metrics grouped by driver and size @@ -224,12 +224,12 @@ func (c *OperatorCollector) collect(ctx context.Context) { for size, drivers := range pvMetrics { for driver, count := range drivers { - metrics.WriteString(fmt.Sprintf( + fmt.Fprintf(&metrics, "cozy_pvs_count{driver=\"%s\",size=\"%s\"} %d\n", driver, size, count, - )) + ) } } } @@ -244,11 +244,11 @@ func (c *OperatorCollector) collect(ctx context.Context) { if variant == "" { variant = "default" } - metrics.WriteString(fmt.Sprintf( + fmt.Fprintf(&metrics, "cozy_package_info{name=\"%s\",variant=\"%s\"} 1\n", pkg.Name, variant, - )) + ) } } @@ -272,7 +272,7 @@ func (c *OperatorCollector) sendMetrics(clusterID, metrics string) error { if err != nil { return fmt.Errorf("failed to send request: %w", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return fmt.Errorf("unexpected status code: %d", resp.StatusCode) diff --git a/packages/apps/bucket/README.md b/packages/apps/bucket/README.md index 13e52d108..4db481810 100644 --- a/packages/apps/bucket/README.md +++ b/packages/apps/bucket/README.md @@ -4,11 +4,12 @@ ### Parameters -| Name | Description | Type | Value | -| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ------- | -| `locking` | Provisions bucket from the `-lock` BucketClass (with object lock enabled). | `bool` | `false` | -| `storagePool` | Selects a specific BucketClass by storage pool name. | `string` | `""` | -| `ui` | Deploys the web UI (s3manager) with its Service and its Ingress or HTTPRoute at `.`. Set to `false` to run the bucket without them. | `bool` | `true` | -| `users` | Users configuration map. | `map[string]object` | `{}` | -| `users[name].readonly` | Whether the user has read-only access. | `bool` | `false` | +| Name | Description | Type | Value | +| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ------- | +| `locking` | Provisions bucket from the `-lock` BucketClass (with object lock enabled). | `bool` | `false` | +| `storagePool` | Selects a specific BucketClass by storage pool name. | `string` | `""` | +| `ui` | Deploys the web UI (s3manager) with its Service and its Ingress or HTTPRoute at `.`. Set to `false` to run the bucket without them. | `bool` | `true` | +| `publicRead` | Serves the bucket's objects to anyone without credentials at `//`. Anonymous callers can read objects but cannot list the bucket or write to it. | `bool` | `false` | +| `users` | Users configuration map. | `map[string]object` | `{}` | +| `users[name].readonly` | Whether the user has read-only access. | `bool` | `false` | diff --git a/packages/apps/bucket/templates/bucketclaim.yaml b/packages/apps/bucket/templates/bucketclaim.yaml index 0639916f0..8ce03d454 100644 --- a/packages/apps/bucket/templates/bucketclaim.yaml +++ b/packages/apps/bucket/templates/bucketclaim.yaml @@ -22,3 +22,18 @@ spec: credentialsSecretName: {{ $.Release.Name }}-{{ $name }} protocol: s3 {{- end }} +{{- if .Values.publicRead }} +{{- if hasKey .Values.users "public-read" }} +{{- fail "publicRead creates the BucketAccess -public-read, which the user \"public-read\" already owns: rename that user" }} +{{- end }} +--- +apiVersion: objectstorage.k8s.io/v1alpha1 +kind: BucketAccess +metadata: + name: {{ .Release.Name }}-public-read +spec: + bucketAccessClassName: {{ $seaweedfs }}{{- if $pool }}-{{ $pool }}{{- end }}-public + bucketClaimName: {{ .Release.Name }} + credentialsSecretName: {{ .Release.Name }}-public-read + protocol: s3 +{{- end }} diff --git a/packages/apps/bucket/templates/hooks/cleanup-acme.yaml b/packages/apps/bucket/templates/hooks/cleanup-acme.yaml index 94814a2c1..7680df52e 100644 --- a/packages/apps/bucket/templates/hooks/cleanup-acme.yaml +++ b/packages/apps/bucket/templates/hooks/cleanup-acme.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: {{ .Release.Name }}-cleanup-acme + name: {{ printf "%s-cleanup-acme" .Release.Name | trunc 63 | trimSuffix "-" }} labels: app.kubernetes.io/instance: {{ .Release.Name }} annotations: diff --git a/packages/apps/bucket/tests/job_name_budget_test.yaml b/packages/apps/bucket/tests/job_name_budget_test.yaml new file mode 100644 index 000000000..94a295b5a --- /dev/null +++ b/packages/apps/bucket/tests/job_name_budget_test.yaml @@ -0,0 +1,80 @@ +suite: bucket cleanup Job name budget + +# The API server copies a Job's name into the job-name label of its pods, and a +# label value takes 63 characters at most, while a release name may take 53. +# The ServiceAccount, Role and RoleBinding have no such limit and keep the full +# name, so a hook that ran before keeps the same objects. + +templates: + - templates/hooks/cleanup-acme.yaml + +set: + _namespace: + host: example.org + +tests: + - it: keeps -cleanup-acme for a short release name + release: + name: my-bucket + namespace: tenant-root + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: my-bucket-cleanup-acme + + - it: keeps the name when it lands exactly on 63 characters + release: + name: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefg + namespace: tenant-root + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefg-cleanup-acme + + - it: cuts the name to 63 characters for a 53-character release name + release: + name: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij + namespace: tenant-root + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-a + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ + - equal: + path: spec.template.spec.serviceAccountName + value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-acme + + - it: leaves the RBAC objects on the full name for a 53-character release name + release: + name: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij + namespace: tenant-root + asserts: + - equal: + path: metadata.name + value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-acme + documentSelector: + path: kind + value: ServiceAccount + - equal: + path: roleRef.name + value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-acme + documentSelector: + path: kind + value: RoleBinding + - equal: + path: subjects[0].name + value: bucket-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-cleanup-acme + documentSelector: + path: kind + value: RoleBinding diff --git a/packages/apps/bucket/tests/public_read_test.yaml b/packages/apps/bucket/tests/public_read_test.yaml new file mode 100644 index 000000000..1bdf5d82b --- /dev/null +++ b/packages/apps/bucket/tests/public_read_test.yaml @@ -0,0 +1,82 @@ +suite: bucket publicRead +templates: + - templates/bucketclaim.yaml + +release: + name: bucket-my-bucket + namespace: tenant-root + +set: + _namespace: + seaweedfs: tenant-root + +tests: + - it: renders no public BucketAccess by default + asserts: + - hasDocuments: + count: 1 + - isKind: + of: BucketClaim + + - it: renders a public-read BucketAccess on the -public class + set: + publicRead: true + documentSelector: + path: metadata.name + value: bucket-my-bucket-public-read + asserts: + - isKind: + of: BucketAccess + - equal: + path: spec.bucketAccessClassName + value: tenant-root-public + - equal: + path: spec.bucketClaimName + value: bucket-my-bucket + - equal: + path: spec.credentialsSecretName + value: bucket-my-bucket-public-read + + - it: uses the storage pool's -public class + set: + publicRead: true + storagePool: fast + documentSelector: + path: metadata.name + value: bucket-my-bucket-public-read + asserts: + - equal: + path: spec.bucketAccessClassName + value: tenant-root-fast-public + + - it: keeps the access class free of -lock on a locked bucket + set: + publicRead: true + locking: true + documentSelector: + path: metadata.name + value: bucket-my-bucket-public-read + asserts: + - equal: + path: spec.bucketAccessClassName + value: tenant-root-public + + - it: keeps the users' BucketAccesses next to the public one + set: + publicRead: true + users: + app: {} + reader: + readonly: true + asserts: + - hasDocuments: + count: 4 + + - it: refuses a user that would own the public BucketAccess name + set: + publicRead: true + users: + public-read: {} + asserts: + - failedTemplate: + errorPattern: user "public-read" already owns diff --git a/packages/apps/bucket/values.schema.json b/packages/apps/bucket/values.schema.json index 249169395..641822c51 100644 --- a/packages/apps/bucket/values.schema.json +++ b/packages/apps/bucket/values.schema.json @@ -20,6 +20,11 @@ "type": "boolean", "default": true }, + "publicRead": { + "description": "Serves the bucket's objects to anyone without credentials at `\u003cendpoint\u003e/\u003cbucket\u003e/\u003ckey\u003e`. Anonymous callers can read objects but cannot list the bucket or write to it.", + "type": "boolean", + "default": false + }, "users": { "description": "Users configuration map.", "type": "object", diff --git a/packages/apps/bucket/values.yaml b/packages/apps/bucket/values.yaml index 3b79909fc..a5d87b9b3 100644 --- a/packages/apps/bucket/values.yaml +++ b/packages/apps/bucket/values.yaml @@ -8,6 +8,9 @@ storagePool: "" ## @param {bool} ui=true - Deploys the web UI (s3manager) with its Service and its Ingress or HTTPRoute at `.`. Set to `false` to run the bucket without them. ui: true +## @param {bool} publicRead=false - Serves the bucket's objects to anyone without credentials at `//`. Anonymous callers can read objects but cannot list the bucket or write to it. +publicRead: false + ## @typedef {struct} User - Bucket user configuration. ## @field {bool} [readonly] - Whether the user has read-only access. diff --git a/packages/apps/clickhouse/templates/backup-cronjob.yaml b/packages/apps/clickhouse/templates/backup-cronjob.yaml index f792e8f9d..2a0bbd0b2 100644 --- a/packages/apps/clickhouse/templates/backup-cronjob.yaml +++ b/packages/apps/clickhouse/templates/backup-cronjob.yaml @@ -18,11 +18,19 @@ */}} {{- if and .Values.backup.enabled .Values.backup.schedule (not .Values.backup.useSystemBucket) }} {{ $image := .Files.Get "images/backup.json" | fromJson }} +{{- /* The CronJob controller starts each Job as -, so the API + refuses a CronJob name over 52 characters. Cutting the release name to + fit could give two releases one CronJob, so a name that does not fit + keeps a hash of the full release name instead. */}} +{{- $cronJobName := printf "%s-backup" .Release.Name }} +{{- if gt (len $cronJobName) 52 }} +{{- $cronJobName = printf "%s-%s-backup" (.Release.Name | trunc 38 | trimSuffix "-") (.Release.Name | sha256sum | trunc 6) }} +{{- end }} apiVersion: batch/v1 kind: CronJob metadata: - name: {{ .Release.Name }}-backup + name: {{ $cronJobName }} spec: schedule: "{{ .Values.backup.schedule }}" concurrencyPolicy: Forbid diff --git a/packages/apps/clickhouse/templates/clickhouse.yaml b/packages/apps/clickhouse/templates/clickhouse.yaml index 5d0c1c6ab..bc439d40b 100644 --- a/packages/apps/clickhouse/templates/clickhouse.yaml +++ b/packages/apps/clickhouse/templates/clickhouse.yaml @@ -222,6 +222,18 @@ spec: key: backup - name: ALLOW_EMPTY_BACKUPS value: "true" + {{- /* + create_remote keeps the local backup when its upload fails, + and the upstream defaults never prune local backups (0) and + resume unfinished uploads at start, which pushes a failed + attempt to the bucket as an archive no Backup refers to and + holds the API lock (423) while it does. -1 keeps only the + newest local backup and removes it once its upload succeeds. + */}} + - name: BACKUPS_TO_KEEP_LOCAL + value: "-1" + - name: API_COMPLETE_RESUMABLE_AFTER_RESTART + value: "false" - name: REMOTE_STORAGE value: s3 {{- /* diff --git a/packages/apps/clickhouse/tests/backup_test.yaml b/packages/apps/clickhouse/tests/backup_test.yaml index 7954d5d3d..a01e10ea5 100644 --- a/packages/apps/clickhouse/tests/backup_test.yaml +++ b/packages/apps/clickhouse/tests/backup_test.yaml @@ -302,3 +302,42 @@ tests: - notMatchRegex: path: stringData["backup.sh"] pattern: 'echo DB_LIST=' + + # A strategy attempt that fails mid-upload leaves its local backup behind, + # and with the upstream defaults the sidecar would keep it forever and push + # it to the bucket on its next start as an archive no Backup refers to. The + # sidecar must neither resume uploads at start nor keep local copies once an + # upload succeeds, on either S3 flow. + - it: "sidecar drops local backups after upload and does not resume uploads at start" + template: templates/clickhouse.yaml + documentSelector: + path: kind + value: ClickHouseInstallation + set: + backup: + enabled: true + useSystemBucket: true + asserts: + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="BACKUPS_TO_KEEP_LOCAL")].value + value: "-1" + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="API_COMPLETE_RESUMABLE_AFTER_RESTART")].value + value: "false" + + - it: "legacy flow: sidecar drops local backups after upload and does not resume uploads at start" + template: templates/clickhouse.yaml + documentSelector: + path: kind + value: ClickHouseInstallation + set: + backup: + enabled: true + useSystemBucket: false + asserts: + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="BACKUPS_TO_KEEP_LOCAL")].value + value: "-1" + - equal: + path: spec.templates.podTemplates[0].spec.containers[?(@.name=="clickhouse-backup")].env[?(@.name=="API_COMPLETE_RESUMABLE_AFTER_RESTART")].value + value: "false" diff --git a/packages/apps/clickhouse/tests/cronjob_name_budget_test.yaml b/packages/apps/clickhouse/tests/cronjob_name_budget_test.yaml new file mode 100644 index 000000000..0ff3e2305 --- /dev/null +++ b/packages/apps/clickhouse/tests/cronjob_name_budget_test.yaml @@ -0,0 +1,94 @@ +suite: clickhouse legacy backup CronJob name budget + +# The API refuses a CronJob name over 52 characters, because the controller +# names each Job it starts -, while a release name may take 53. +# Cutting the release name alone would give two long names one CronJob, so a +# name that does not fit is the release name cut to 38 characters, a hash of +# the full release name and -backup. + +templates: + - templates/backup-cronjob.yaml + - templates/backup-script.yaml + - templates/backup-secret.yaml + +set: + _cluster: {} + backup: + enabled: true + schedule: "0 2 * * *" + +tests: + - it: keeps -backup for a short release name + release: + name: ch-test + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: ch-test-backup + + - it: keeps the name when it lands exactly on 52 characters + release: + name: clickhouse-abcdefghijklmnopqrstuvwxyz01234567 + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: clickhouse-abcdefghijklmnopqrstuvwxyz01234567-backup + + - it: shortens the name one character past the budget + release: + name: clickhouse-abcdefghijklmnopqrstuvwxyz012345678 + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: clickhouse-abcdefghijklmnopqrstuvwxyz0-bec95e-backup + + - it: fits a 53-character release name into 52 characters + release: + name: clickhouse-abcdefghijklmnopqrstuvwxyz0123456789abcdef + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: clickhouse-abcdefghijklmnopqrstuvwxyz0-3e8cf9-backup + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,50}[a-z0-9])?$ + + - it: gives a release name differing only in its last character another CronJob + release: + name: clickhouse-abcdefghijklmnopqrstuvwxyz0123456789abcdex + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: clickhouse-abcdefghijklmnopqrstuvwxyz0-bd3283-backup + + - it: does not end the cut release name with a dash + release: + name: clickhouse-abcdefghijklmnopqrstuvwxyz-0123456789 + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: clickhouse-abcdefghijklmnopqrstuvwxyz-07ba9e-backup diff --git a/packages/apps/kafka/templates/migration-hook.yaml b/packages/apps/kafka/templates/migration-hook.yaml index 9877f699b..997b28ea1 100644 --- a/packages/apps/kafka/templates/migration-hook.yaml +++ b/packages/apps/kafka/templates/migration-hook.yaml @@ -79,7 +79,7 @@ subjects: apiVersion: batch/v1 kind: Job metadata: - name: {{ .Release.Name }}-kafka-migration + name: {{ printf "%s-kafka-migration" .Release.Name | trunc 63 | trimSuffix "-" }} annotations: helm.sh/hook: pre-install,pre-upgrade helm.sh/hook-weight: "2" diff --git a/packages/apps/kafka/tests/job_name_budget_test.yaml b/packages/apps/kafka/tests/job_name_budget_test.yaml new file mode 100644 index 000000000..908ef75ed --- /dev/null +++ b/packages/apps/kafka/tests/job_name_budget_test.yaml @@ -0,0 +1,66 @@ +suite: kafka migration Job name budget + +# The API server copies a Job's name into the job-name label of its pods, and a +# label value takes 63 characters at most. The Kafka name cap leaves a release +# name of up to 50 characters, and the migration hook runs on install, so an +# overlong Job name would fail every install of the longest names. The +# ServiceAccount, Role and RoleBinding have no such limit and keep the full +# name, so the ones an earlier attempt left behind are replaced, not orphaned. + +templates: + - templates/migration-hook.yaml + +tests: + - it: keeps -kafka-migration for a short release name + release: + name: test-kafka + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: test-kafka-kafka-migration + + - it: keeps the name when it lands exactly on 63 characters + release: + name: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcde + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcde-kafka-migration + + - it: cuts the name to 63 characters for the longest release name the cap admits + release: + name: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh-kafka-migrat + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ + - equal: + path: spec.template.spec.serviceAccountName + value: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh-kafka-migration + + - it: leaves the ServiceAccount on the full name for the longest release name + release: + name: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh + namespace: tenant-test + documentSelector: + path: kind + value: ServiceAccount + asserts: + - equal: + path: metadata.name + value: kafka-abcdefghijklmnopqrstuvwxyz0123456789abcdefgh-kafka-migration diff --git a/packages/apps/kubernetes-nodes/.helmignore b/packages/apps/kubernetes-nodes/.helmignore index cdada6673..1ea0ae84f 100644 --- a/packages/apps/kubernetes-nodes/.helmignore +++ b/packages/apps/kubernetes-nodes/.helmignore @@ -1,5 +1,3 @@ .helmignore /logos /Makefile -/hack -/images/*/* diff --git a/packages/apps/kubernetes-nodes/Makefile b/packages/apps/kubernetes-nodes/Makefile index 701d81eaa..9c3dcb3de 100644 --- a/packages/apps/kubernetes-nodes/Makefile +++ b/packages/apps/kubernetes-nodes/Makefile @@ -9,16 +9,15 @@ generate: cozyvalues-gen -m 'kubernetesnodes' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/kubernetesnodes/types.go ../../../hack/update-crd.sh -# Keep the worker-image inputs (Talos<->Kubernetes support matrix and the -# kubectl.tag used by the talos-reconcile Job) in lockstep with the parent -# kubernetes chart. They are copied, not symlinked, so `helm package` bundles -# them; run this after the parent chart's `make update`. +# Copy the Kubernetes version map (files/versions.yaml, minor -> patch release) +# and the kubectl.tag used by the talos-reconcile Job from the kubernetes +# chart, which owns both. They are copied, not symlinked, so `helm package` +# bundles them; run this after either changes there. # -# _talosconfigtemplate.tpl is not copied from anywhere: the parent chart no -# longer carries worker node pools, so this chart owns the worker machine -# config outright. It stays here because the TalosConfigTemplate it renders is -# content-hash named and both consumers in this chart - nodegroup.yaml, which -# writes bootstrap.configRef, and talos-reconcile-job.yaml, which creates the +# _talosconfigtemplate.tpl is not copied from anywhere: this chart owns the +# worker machine config. The TalosConfigTemplate it renders is content-hash +# named and both consumers in this chart - nodegroup.yaml, which writes +# bootstrap.configRef, and talos-reconcile-job.yaml, which creates the # object - have to hash the same bytes. update: cp ../kubernetes/files/versions.yaml files/versions.yaml diff --git a/packages/apps/kubernetes-nodes/README.md b/packages/apps/kubernetes-nodes/README.md index c11e38f6f..3b8bdd36e 100644 --- a/packages/apps/kubernetes-nodes/README.md +++ b/packages/apps/kubernetes-nodes/README.md @@ -1,93 +1,94 @@ # Kubernetes Worker Node Pool -`kubernetes-nodes` manages a single worker node pool for a managed Kubernetes -cluster. It is the worker half of the Kubernetes-app split (see -[cozystack/community#8](https://github.com/cozystack/community/pull/8)): the -control plane lives in the `kubernetes` chart, and each pool is a separate -`kubernetes-nodes` release attached to a parent cluster by name. +`kubernetes-nodes` manages one worker node pool of a managed Kubernetes cluster. The control plane and the cluster addons come from the `kubernetes` application; every pool is a separate `KubernetesNodes` resource that attaches to its cluster by name. -A `KubernetesNodes` CR named `-` (e.g. `myk8s-md0`) provisions a -`MachineDeployment` + `KubevirtMachineTemplate` + `MachineHealthCheck` wired to -the parent cluster `kubernetes-` in the same namespace, plus a -`talos-reconcile` Job that produces the pool's `TalosConfigTemplate`. +A `KubernetesNodes` resource named `-` (for example `myk8s-md0`) sets `cluster: ` and renders, in the namespace of the `kubernetes-` cluster: + +- a `MachineDeployment` that the cluster autoscaler scales between `minReplicas` and `maxReplicas`; +- the machine template its workers are created from: a `KubevirtMachineTemplate` on `substrate: kubevirt` (the default), or a `ProxmoxMachineTemplate` on `substrate: proxmox`; +- a `MachineHealthCheck` that replaces workers which do not become Ready or stop being Ready; +- a `WorkloadMonitor` for the pool's workers; +- a `talos-reconcile` Job that creates the pool's `TalosConfigTemplate`, the Talos machine config every worker boots with. + +The pool's Kubernetes minor (`version`) may lag the cluster's during an upgrade but may not lead it: upgrade the `Kubernetes` resource first, then each pool. Delete the pools of a cluster before the cluster itself. ## Parameters ### Linkage -| Name | Description | Type | Value | -| --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ----- | -| `cluster` | Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool name (the part after `-`) must not collide with a nodeGroup still managed by the parent kubernetes chart (e.g. the default `md0`), or the render fails on ownership conflict. | `string` | `""` | +| Name | Description | Type | Value | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | ----- | +| `cluster` | Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool's objects are named `kubernetes--`; when a MachineDeployment of that name already exists and belongs to another Helm release, the render fails instead of taking it over. | `string` | `""` | ### Common Parameters -| Name | Description | Type | Value | -| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------ | -| `storageClass` | StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. | `string` | `replicated` | +| Name | Description | Type | Value | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------ | +| `storageClass` | StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`. | `string` | `replicated` | ### Pool Sizing -| Name | Description | Type | Value | -| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | -| `minReplicas` | Minimum number of replicas in this pool. Used as the cluster-autoscaler floor. | `int` | `0` | -| `maxReplicas` | Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling. | `int` | `10` | -| `instanceType` | Virtual machine instance type. | `string` | `u1.medium` | -| `diskSize` | System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. | `quantity` | `20Gi` | -| `roles` | List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller. | `[]string` | `[]` | -| `resources` | Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `"010"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `"010"` reserves what `"10"` does. | `object` | `{}` | -| `resources.cpu` | Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores. | `quantity` | `""` | -| `resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `podCpuLimit` | CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart. | `quantity` | `""` | -| `podCpuRequest` | CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart. | `quantity` | `""` | -| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | -| `gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` | -| `kernelModules` | Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls. | `[]object` | `[]` | -| `kernelModules[i].name` | Module name as `modprobe` takes it, e.g. `nvidia_uvm`. | `string` | `""` | -| `kernelModules[i].parameters` | Module parameters, each as a bare `key=value` string. | `[]string` | `[]` | -| `kubelet` | Kubelet resource reservations for this pool. | `object` | `{}` | -| `kubelet.systemReservedMemory` | Memory reserved for host OS. Auto-computed from instanceType if empty. | `string` | `""` | -| `kubelet.kubeReservedMemory` | Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | -| `kubelet.systemReservedCpu` | CPU reserved for host OS. Auto-computed from instanceType if empty. | `string` | `""` | -| `kubelet.kubeReservedCpu` | CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | -| `kubelet.evictionHardMemory` | Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%). | `string` | `7%` | -| `kubelet.evictionSoftMemory` | Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%). | `string` | `10%` | -| `logSerialConsole` | Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it. | `bool` | `false` | -| `maxUnhealthy` | Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is a percentage. Drop to "0%" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At "50%" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two. | `string` | `50%` | -| `nodeStartupTimeout` | Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning. | `string` | `20m` | -| `readyUnknownTimeout` | How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart. | `string` | `5m` | -| `readyFalseTimeout` | How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart. | `string` | `10m` | -| `maxNodeProvisionTime` | How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours ("30m", "1h30m"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to ("20m0s", "1h0m0s") is accepted. Keep in sync with the parent kubernetes chart. | `string` | `30m` | +| Name | Description | Type | Value | +| ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | +| `minReplicas` | Minimum number of replicas in this pool. Used as the cluster-autoscaler floor. | `int` | `0` | +| `maxReplicas` | Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling. | `int` | `10` | +| `instanceType` | KubeVirt instance type that sizes each worker VM when `resources` is not set. Ignored when `resources.cpu` and `resources.memory` are both set, and therefore on `substrate: proxmox`, which requires them. | `string` | `u1.medium` | +| `diskSize` | System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. Has no effect on `substrate: proxmox`, where the worker disk is a clone of the VM template's disk and keeps its size. | `quantity` | `20Gi` | +| `roles` | List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller. | `[]string` | `[]` | +| `resources` | Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `"010"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `"010"` reserves what `"10"` does. | `object` | `{}` | +| `resources.cpu` | Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores. | `quantity` | `""` | +| `resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `podCpuLimit` | CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. | `quantity` | `""` | +| `podCpuRequest` | CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. | `quantity` | `""` | +| `gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | +| `gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` | +| `kernelModules` | Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. | `[]object` | `[]` | +| `kernelModules[i].name` | Module name as `modprobe` takes it, e.g. `nvidia_uvm`. | `string` | `""` | +| `kernelModules[i].parameters` | Module parameters, each as a bare `key=value` string. | `[]string` | `[]` | +| `kubelet` | Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`. | `object` | `{}` | +| `kubelet.systemReservedMemory` | Memory reserved for host OS. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi. | `string` | `""` | +| `kubelet.kubeReservedMemory` | Memory reserved for kubelet and container runtime. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi. | `string` | `""` | +| `kubelet.systemReservedCpu` | CPU reserved for host OS. When empty, 5% of the worker's CPU, clamped to 50m..500m. | `string` | `""` | +| `kubelet.kubeReservedCpu` | CPU reserved for kubelet and container runtime. When empty, 5% of the worker's CPU, clamped to 50m..500m. | `string` | `""` | +| `kubelet.evictionHardMemory` | Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%). | `string` | `7%` | +| `kubelet.evictionSoftMemory` | Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%). | `string` | `10%` | +| `logSerialConsole` | Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it. | `bool` | `false` | +| `maxUnhealthy` | Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is a percentage. Drop to "0%" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At "50%" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two. | `string` | `50%` | +| `nodeStartupTimeout` | Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (the Talos image import from the Image Factory, or a storage class slow to provision or clone the worker disk). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning. | `string` | `20m` | +| `readyUnknownTimeout` | How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. | `string` | `5m` | +| `readyFalseTimeout` | How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. | `string` | `10m` | +| `maxNodeProvisionTime` | How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours ("30m", "1h30m"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to ("20m0s", "1h0m0s") is accepted. | `string` | `30m` | ### Kubernetes Version -| Name | Description | Type | Value | -| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | -| `version` | Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool -- and must satisfy the Talos<->Kubernetes support matrix against `talos.version`. | `string` | `v1.35` | +| Name | Description | Type | Value | +| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | +| `version` | Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool. The render fails when it is ahead. Its pairing with the Talos release the pool resolves (`talos.version`, or the `osImage` override) is checked against the chart's Talos<->Kubernetes support matrix: for a Talos minor the matrix lists, a Kubernetes minor outside that minor's list is refused; a Talos minor the matrix does not list is not checked. | `string` | `v1.35` | ### Talos Worker Image -| Name | Description | Type | Value | -| --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ------------------------------------------------------------------ | -| `osImage` | Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool. | `object` | `{}` | -| `osImage.builtin` | Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled. | `*object` | `null` | -| `osImage.builtin.schematicID` | Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`. | `string` | `""` | -| `osImage.builtin.version` | Talos version of the golden to clone. Defaults to `talos.version`. | `string` | `""` | -| `osImage.factory` | Import from an Image Factory / mirror (default). | `*object` | `null` | -| `osImage.factory.version` | Talos version. Defaults to `talos.version`. | `string` | `""` | -| `osImage.factory.schematicID` | Talos schematic ID. Defaults to `talos.schematicID`. | `string` | `""` | -| `osImage.factory.imageFactoryURL` | Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. NOTE: this redirects only the worker boot-disk import; the in-guest upgrade installer image is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry. | `string` | `""` | -| `talos` | Talos worker image configuration. Keep in sync with the parent cluster's `talos`. | `object` | `{}` | -| `talos.version` | Default Talos release for this pool's worker OS image and in-guest installer. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix rejects a pairing it lists as unsupported; it lists the Talos minors this chart ships and passes any other minor unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead. | `string` | `v1.13.6` | -| `talos.schematicID` | Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID. | `string` | `ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515` | -| `talos.imageFactoryURL` | Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal. | `string` | `https://factory.talos.dev` | -| `talos.installerRepository` | OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash. | `string` | `factory.talos.dev/installer` | -| `talos.registryMirrors` | Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. | `object` | `{}` | -| `talos.sysctls` | Sysctl overrides rendered into Talos `machine.sysctls` on this pool's guest workers only. Values must be strings (quote numeric values); for example, `user.max_user_namespaces: "11255"` enables the user namespaces required by the Talos gVisor extension. An empty map preserves Talos defaults and the existing bootstrap hash. Adding, changing or removing an override changes the content-hashed TalosConfigTemplate and rolls the pool's workers. Supported names and values depend on the worker's Talos release and kernel; unsafe settings can weaken guest isolation or prevent workers from booting. This does not install gVisor or configure a RuntimeClass, and does not change the management hosts, control plane or other pools. | `map[string]string` | `{}` | -| `talos.imageFactoryCA` | Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there. | `string` | `""` | +| Name | Description | Type | Value | +| --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ------------------------------------------------------------------ | +| `osImage` | Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool. | `object` | `{}` | +| `osImage.builtin` | Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled. | `*object` | `null` | +| `osImage.builtin.schematicID` | Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`. | `string` | `""` | +| `osImage.builtin.version` | Talos version of the golden to clone. Defaults to `talos.version`. | `string` | `""` | +| `osImage.factory` | Import from an Image Factory / mirror (default). | `*object` | `null` | +| `osImage.factory.version` | Talos version. Defaults to `talos.version`. | `string` | `""` | +| `osImage.factory.schematicID` | Talos schematic ID. Defaults to `talos.schematicID`. | `string` | `""` | +| `osImage.factory.imageFactoryURL` | Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. This redirects only the worker boot-disk import; the installer image in the worker machine config is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry. | `string` | `""` | +| `talos` | Talos worker image configuration for this pool. | `object` | `{}` | +| `talos.version` | Default Talos release for this pool's worker OS image and for the installer image written into the worker machine config. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix lists, for each Talos minor the chart knows, the Kubernetes minors it supports and refuses a pool `version` outside that list; a Talos minor it does not list passes unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead. | `string` | `v1.13.6` | +| `talos.schematicID` | Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID. | `string` | `ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515` | +| `talos.imageFactoryURL` | Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal. On `substrate: proxmox` the workers clone a VM template that already carries the OS, so this field has no effect there beyond that format check. | `string` | `https://factory.talos.dev` | +| `talos.installerRepository` | OCI repository prefix for the Talos installer image, written into the worker machine config as `machine.install.image`. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash. | `string` | `factory.talos.dev/installer` | +| `talos.registryMirrors` | Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. | `object` | `{}` | +| `talos.sysctls` | Sysctl overrides rendered into Talos `machine.sysctls` on this pool's guest workers only. Values must be strings (quote numeric values); for example, `user.max_user_namespaces: "11255"` enables the user namespaces required by the Talos gVisor extension. An empty map preserves Talos defaults and the existing bootstrap hash. Adding, changing or removing an override changes the content-hashed TalosConfigTemplate and rolls the pool's workers. Supported names and values depend on the worker's Talos release and kernel; unsafe settings can weaken guest isolation or prevent workers from booting. This does not install gVisor or configure a RuntimeClass, and does not change the management hosts, control plane or other pools. | `map[string]string` | `{}` | +| `talos.imageFactoryCA` | Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there. On `substrate: proxmox` it has no effect on the workers either: the ConfigMap is still rendered, but nothing references it. | `string` | `""` | ### Substrate diff --git a/packages/apps/kubernetes-nodes/templates/NOTES.txt b/packages/apps/kubernetes-nodes/templates/NOTES.txt index 197b45006..b82912e60 100644 --- a/packages/apps/kubernetes-nodes/templates/NOTES.txt +++ b/packages/apps/kubernetes-nodes/templates/NOTES.txt @@ -7,12 +7,11 @@ This pool is a separate release from the control plane. Three operational notes: cluster that is being torn down. Delete the KubernetesNodes pools first, then the parent Kubernetes cluster. -2. Version skew. Keep this pool's `version` and `talos.*` in sync with the parent - Kubernetes cluster. A pool on a Kubernetes/Talos combination the parent does - not run can produce workers the control plane will not admit. +2. Version skew. This pool's `version` may lag the parent cluster's Kubernetes + minor during an upgrade but may not lead it: a worker kubelet ahead of the + apiserver is refused at render time. Upgrade the parent Kubernetes CR first, + then each pool. -3. Pool name. The pool name (the part after `-` in the KubernetesNodes - name) must not collide with a nodeGroup still managed by the parent kubernetes - chart — most importantly the implicit default `md0`. A collision makes this - release fail on a Helm ownership conflict against the parent-owned objects. - The chart guards this at render time with a clear error. +3. Pool name. The pool's objects are named `kubernetes--`. When a + MachineDeployment of that name already exists and belongs to another Helm + release, this release fails at render time rather than taking it over. diff --git a/packages/apps/kubernetes-nodes/templates/_helpers.tpl b/packages/apps/kubernetes-nodes/templates/_helpers.tpl index 08a74b7ce..5e317f8e6 100644 --- a/packages/apps/kubernetes-nodes/templates/_helpers.tpl +++ b/packages/apps/kubernetes-nodes/templates/_helpers.tpl @@ -165,10 +165,9 @@ The pool attaches to the parent Kubernetes CR named .Values.cluster, whose HelmRelease (and therefore CAPI Cluster / KamajiControlPlane / KubevirtCluster and every worker object it owns) is `kubernetes-`. This chart does NOT use its own .Release.Name for CAPI wiring — the pool lives in a separate -HelmRelease from the control plane, so every reference that the monolithic -chart made through $.Release.Name is reconstructed here as kubernetes- -instead. Linkage is by name convention (mirrors vm-instance -> vm-disk), not -ownerReference or lookup-gated render. +HelmRelease from the control plane, so every reference to the cluster's objects +is built from kubernetes-. Linkage is by name convention (mirrors +vm-instance -> vm-disk), not ownerReference or lookup-gated render. */}} {{- define "kubernetes-nodes.clusterName" -}} {{- if not .Values.cluster -}} @@ -194,9 +193,9 @@ The node-group name for this pool, derived from the release name. A KubernetesNodes CR is named - and gets the release prefix `kubernetes-nodes-`, so the release name is `kubernetes-nodes--`. The group name is the suffix. Enforcing the `-` segment keeps -every rendered object named `kubernetes--` — byte-identical to -what the monolithic chart rendered for the same group — and prevents two -clusters in one namespace from colliding on a pool named e.g. `md0`. +every rendered object named `kubernetes--` — the names worker +objects adopted by migration 56 already carry — and prevents two clusters in +one namespace from colliding on a pool named e.g. `md0`. */}} {{- define "kubernetes-nodes.groupName" -}} {{- $prefix := printf "kubernetes-nodes-%s-" .Values.cluster -}} @@ -208,13 +207,13 @@ clusters in one namespace from colliding on a pool named e.g. `md0`. {{/* Fail early with a clear message if this pool's MachineDeployment already exists -under a different Helm release — i.e. the pool name collides with a nodeGroup -still managed by the parent kubernetes chart (most likely the default `md0`). +under a different Helm release: a worker pool migration 56 left with the parent +cluster's release, or another pool whose - split reconstructs +the same object name (cluster `a` with pool `b-c`, cluster `a-b` with pool `c`). Without this guard the collision surfaces as a cryptic Helm "invalid ownership metadata" error at install time. Inert under `helm template`/unittest (lookup -returns nil with no cluster) and a no-op during the Phase 2b adoption -migration, which re-annotates the MachineDeployment onto this release before -it reconciles. +returns nil with no cluster) and a no-op for a pool migration 56 adopted, which +re-annotates the MachineDeployment onto this release before it reconciles. */}} {{- define "kubernetes-nodes.assertNoForeignPool" -}} {{- $clusterName := include "kubernetes-nodes.clusterName" . -}} @@ -224,7 +223,7 @@ it reconciles. {{- if $existing -}} {{- $owner := dig "annotations" "meta.helm.sh/release-name" "" $existing.metadata -}} {{- if and $owner (ne $owner .Release.Name) -}} -{{- fail (printf "kubernetes-nodes: MachineDeployment %q in namespace %q is already managed by release %q, not this pool release %q — the pool name collides with a nodeGroup still managed by the parent kubernetes chart. Rename the pool or remove it from the parent Kubernetes CR's nodeGroups first." $mdName .Release.Namespace $owner .Release.Name) -}} +{{- fail (printf "kubernetes-nodes: MachineDeployment %q in namespace %q is already managed by release %q, not this pool release %q — two pools cannot share the object name kubernetes--: give this pool a name whose object name is free, or remove the existing pool from release %q first." $mdName .Release.Namespace $owner .Release.Name $owner) -}} {{- end -}} {{- /* clusterName-drift guard. The release name kubernetes-nodes-- does not encode where ends and begins: for one release @@ -253,8 +252,7 @@ control plane is supported (up to n-3 upstream) and is the normal state during a rolling upgrade — the parent Kubernetes CR is bumped first, then each pool. A symmetric equality check would flip every pool of a cluster to render-failure the moment the parent is bumped, blocking all pool operations until each pool is -hand-edited, so the check is directional. The split removed the single `version` -that used to feed both control plane and workers. Looks up the parent +hand-edited, so the check is directional. Looks up the parent KamajiControlPlane (named like the reconstructed clusterName) and compares its spec.version minor against .Values.version. Skipped when the lookup is empty (helm template / unittest, or the parent not yet present) so it validates only diff --git a/packages/apps/kubernetes-nodes/templates/_talosconfigtemplate.tpl b/packages/apps/kubernetes-nodes/templates/_talosconfigtemplate.tpl index c33d43f16..d4439c0a3 100644 --- a/packages/apps/kubernetes-nodes/templates/_talosconfigtemplate.tpl +++ b/packages/apps/kubernetes-nodes/templates/_talosconfigtemplate.tpl @@ -87,7 +87,7 @@ {{- /* Management cluster DNS domain. Tenant Kubernetes workers run as KubeVirt VMs inside the management cluster's pod network and need to resolve management-cluster service short names ("..svc") - the same way an Ubuntu+kubeadm worker did via cloud-init DHCP. The + the same way a pod of the management cluster does. The kubevirt-csi NFS mount path is the load-bearing case: kubelet on the worker host calls `mount.nfs linstor-csi-nfs.cozy-linstor.svc` (no domain suffix), and without searchDomains in /etc/resolv.conf diff --git a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml index 18ec0e16b..7ebf42033 100644 --- a/packages/apps/kubernetes-nodes/templates/nodegroup.yaml +++ b/packages/apps/kubernetes-nodes/templates/nodegroup.yaml @@ -1,17 +1,16 @@ {{- /* Worker node pool for a managed Kubernetes cluster. - This chart is the second half of the Kubernetes-app split (cozystack/community#8): - the control plane (Cluster / KamajiControlPlane / KubevirtCluster / PKI) lives in - the `kubernetes` chart, and each worker pool is a separate `kubernetes-nodes` - release. The pool objects (KubevirtMachineTemplate, MachineDeployment, - MachineHealthCheck, worker WorkloadMonitor) and the talos-reconcile Job that - produces the pool's TalosConfigTemplate are rendered here, wired to the parent - cluster `kubernetes-` by name convention (see _helpers.tpl). Every - reference the monolithic chart made through $.Release.Name is reconstructed here - as the parent clusterName so the objects are byte-identical to what the parent - rendered for the same group — the KubevirtMachineTemplate is content-hash-named, - so any drift would roll every live worker VM. + The control plane (Cluster / KamajiControlPlane / KubevirtCluster / PKI) lives + in the `kubernetes` chart, and each worker pool is a separate + `kubernetes-nodes` release. The pool objects (machine template, + MachineDeployment, MachineHealthCheck, worker WorkloadMonitor) and the + talos-reconcile Job that produces the pool's TalosConfigTemplate are rendered + here, wired to the parent cluster `kubernetes-` by name convention + (see _helpers.tpl). Every reference to the cluster is built from that + clusterName. Pools adopted by migration 56 keep running on objects of exactly + these names and this content: the machine template is content-hash-named, so + any drift in its text would roll every live worker VM. Talos -> Kubernetes support matrix, against the pool default. The pool can also reach a different Talos release through osImage.builtin.version / @@ -346,7 +345,7 @@ spec: fails the whole release of a pool whose disks are all on the right class and which changed nothing. Only a pool with no disk yet resolves to today's default, which is the class its first clone will actually get. */}} - {{- $cloneWorkerSC = .group.storageClass | default $.Values.storageClass | toString }} + {{- $cloneWorkerSC = .group.storageClass | toString }} {{- if and (not $cloneWorkerSC) $poolDiskName }} {{- $poolPVC := lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $poolDiskName }} {{- if $poolPVC }} @@ -527,12 +526,10 @@ spec: Worker VMs live-migrate, so their disks need ReadWriteMany. The RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — it is not set on the DataVolume here. linstor-csi - rejects RWX volumes that are not on a DRBD-backed StorageClass, so - an empty nodeGroup storageClass falls back to the application-level - storageClass (a replicated/DRBD class) instead of the cluster - default, which may be a non-DRBD class that the CSI driver would - reject. Both empty still omits the field so the cluster default - applies. + rejects RWX volumes that are not on a DRBD-backed StorageClass, + which is why the pool's storageClass defaults to a replicated/DRBD + class rather than to the cluster default. An empty storageClass + omits the field so the cluster default applies. A builtin pool instead renders the class the guard above resolved for it. Omitting the field there would let the disk bind on whatever the @@ -541,12 +538,12 @@ spec: after the pool's earlier disks were bound turns the next clone into a host-assisted copy over the pod network, the transfer the guard exists to refuse, while the guard reports no drift because it compared the - class those earlier disks sit on. With both levels empty and no cluster - to resolve against the field is still omitted, so the HTTP path renders - exactly as before and the KubevirtMachineTemplate hash named after it - does not move. + class those earlier disks sit on. With the class empty and no cluster + to resolve against the field is still omitted, so the HTTP path does + not render one and the KubevirtMachineTemplate hash named after it + does not depend on the cluster default. */}} - {{- with $cloneWorkerSC | default (.group.storageClass | default $.Values.storageClass) }} + {{- with $cloneWorkerSC | default .group.storageClass }} storageClassName: {{ . | quote }} {{- end }} template: @@ -1066,19 +1063,17 @@ spec: {{- /* MHC spec.maxUnhealthy is an IntOrString. CAPI admission accepts a bare integer (0, 1, ...) or a percentage string ("0%", "50%"), but rejects a quoted plain integer ("0"). Quote only when the operator - supplied a percentage; otherwise render the value as a bare int so - the admission webhook is happy with the chart-default "0". */}} + supplied a percentage; otherwise render the value as a bare int. */}} {{- $maxUnhealthy := toString $.Values.maxUnhealthy }} - {{- $muSource := "maxUnhealthy" }} {{- if hasSuffix "%" $maxUnhealthy }} {{- if not (regexMatch "^[0-9]+%$" $maxUnhealthy) }} - {{- fail (printf "%s must be a bare integer (e.g. 0, 1) or an integer percentage (e.g. 50%%), got %q" $muSource $maxUnhealthy) }} + {{- fail (printf "maxUnhealthy must be a bare integer (e.g. 0, 1) or an integer percentage (e.g. 50%%), got %q" $maxUnhealthy) }} {{- end }} maxUnhealthy: {{ $maxUnhealthy | quote }} {{- else if regexMatch "^[0-9]+$" $maxUnhealthy }} maxUnhealthy: {{ $maxUnhealthy | int }} {{- else }} - {{- fail (printf "%s must be a bare integer (e.g. 0, 1) or an integer percentage (e.g. 50%%), got %q" $muSource $maxUnhealthy) }} + {{- fail (printf "maxUnhealthy must be a bare integer (e.g. 0, 1) or an integer percentage (e.g. 50%%), got %q" $maxUnhealthy) }} {{- end }} nodeStartupTimeout: {{ $.Values.nodeStartupTimeout }} selector: diff --git a/packages/apps/kubernetes-nodes/tests/clustername_drift_test.yaml b/packages/apps/kubernetes-nodes/tests/clustername_drift_test.yaml index 57c4f79ff..8d0b56cf6 100644 --- a/packages/apps/kubernetes-nodes/tests/clustername_drift_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/clustername_drift_test.yaml @@ -1,7 +1,7 @@ suite: worker pool clusterName-drift guard (assertNoForeignPool) # assertNoForeignPool reconstructs the pool MachineDeployment name -# kubernetes-- and looks it up. Beyond the pre-existing +# kubernetes-- and looks it up. Beyond the # foreign-owner branch, it refuses the render when a live MD of that name is # owned by this release but its spec.clusterName disagrees with the clusterName # this pool renders: that is the fingerprint of a post-create .Values.cluster @@ -126,3 +126,28 @@ tests: value: MachineDeployment exists: path: metadata.name + + # The other branch of the same guard: a live MachineDeployment of this name + # that another release owns, here the cluster's own release, which is what a + # pool migration 56 did not adopt still carries. The message has to point at + # a remedy the operator can act on; the cluster's Kubernetes CR renders no + # worker pool, so editing it cannot clear the conflict. + - it: refuses to render over a MachineDeployment another release owns + kubernetesProvider: + scheme: *scheme + objects: + - *kcp + - *schemeMachineSet + - *schemeKmt + - kind: MachineDeployment + apiVersion: cluster.x-k8s.io/v1beta1 + metadata: + name: kubernetes-myk8s-md0 + namespace: tenant-test + annotations: + meta.helm.sh/release-name: kubernetes-myk8s + spec: + clusterName: kubernetes-myk8s + asserts: + - failedTemplate: + errorPattern: 'already managed by release "kubernetes-myk8s", not this pool release "kubernetes-nodes-myk8s-md0" — two pools cannot share the object name kubernetes--: give this pool a name whose object name is free, or remove the existing pool from release "kubernetes-myk8s" first\.' diff --git a/packages/apps/kubernetes-nodes/tests/kubelet_reservation_test.yaml b/packages/apps/kubernetes-nodes/tests/kubelet_reservation_test.yaml index 57ae0b476..1974c97b2 100644 --- a/packages/apps/kubernetes-nodes/tests/kubelet_reservation_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/kubelet_reservation_test.yaml @@ -1,10 +1,9 @@ suite: kubelet reservation validation -# The kubelet-reservation branch (quantity-format regexes, positive-value and -# eviction-threshold invariants) moved verbatim from the parent chart into -# nodegroup.yaml. These failure-path guards reject malformed input at render -# time before a broken machineconfig reaches a worker. Ported from the parent -# chart's kubelet_reservation_test. +# The kubelet-reservation branch of nodegroup.yaml (quantity-format regexes, +# positive-value and eviction-threshold invariants). These failure-path guards +# reject malformed input at render time before a broken machineconfig reaches +# a worker. templates: - templates/nodegroup.yaml diff --git a/packages/apps/kubernetes-nodes/tests/mhc_rendering_test.yaml b/packages/apps/kubernetes-nodes/tests/mhc_rendering_test.yaml index 92603df9b..50be9041f 100644 --- a/packages/apps/kubernetes-nodes/tests/mhc_rendering_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/mhc_rendering_test.yaml @@ -4,8 +4,6 @@ suite: MachineHealthCheck maxUnhealthy int-vs-percent rendering # but rejects a bare numeric STRING ("0"): a percentage must be a quoted string # and a count must be an unquoted int. nodegroup.yaml renders it as # maxUnhealthy: {{ if hasSuffix "%" ... }}{{ . | quote }}{{ else }}{{ . | int }}{{ end }} -# Ported from the parent chart's mhc_rendering_test after the pool moved to -# kubernetes-nodes. templates: - templates/nodegroup.yaml diff --git a/packages/apps/kubernetes-nodes/tests/mhc_timeouts_test.yaml b/packages/apps/kubernetes-nodes/tests/mhc_timeouts_test.yaml index ecea4ec2e..31fa9e3f2 100644 --- a/packages/apps/kubernetes-nodes/tests/mhc_timeouts_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/mhc_timeouts_test.yaml @@ -16,9 +16,8 @@ suite: MachineHealthCheck timeouts # # `nodeStartupTimeout` bounds the phase before any of that: it is the window a # Machine has to produce a Node at all, and the unhealthyConditions entries are -# evaluated only once one exists. Its default is pinned here because the parity -# gate compares this chart against its counterpart, so a default moved in both -# at once passes there and only fails on this assertion. +# evaluated only once one exists. Its default is pinned here because a moved +# default changes every pool that does not set the field. templates: - templates/nodegroup.yaml diff --git a/packages/apps/kubernetes-nodes/tests/pod_cpu_limit_test.yaml b/packages/apps/kubernetes-nodes/tests/pod_cpu_limit_test.yaml index d7fdb5bd9..c172ac15d 100644 --- a/packages/apps/kubernetes-nodes/tests/pod_cpu_limit_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/pod_cpu_limit_test.yaml @@ -1,7 +1,6 @@ suite: worker pool pod CPU limit -# Mirror of the parent kubernetes chart's suite, on the pool-shaped values of -# this chart. `podCpuLimit` is the only knob that sets +# `podCpuLimit` is the only knob that sets # `domain.resources.limits.cpu` on a worker VM. KubeVirt otherwise derives that # limit from the guest vCPU count whenever the namespace carries a ResourceQuota # that limits CPU, and it stops deriving as soon as the VMI declares the limit @@ -79,8 +78,7 @@ tests: asserts: # Absent, not empty: the KubevirtMachineTemplate is content-hash named, so # a `cpu:` key rendered for a pool that asked for nothing would rename the - # template and roll every live worker VM of the pool on upgrade. The - # render-parity gate compares those bytes against the parent chart. + # template and roll every live worker VM of the pool on upgrade. - notExists: path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.resources.limits.cpu - equal: diff --git a/packages/apps/kubernetes-nodes/tests/pod_cpu_request_test.yaml b/packages/apps/kubernetes-nodes/tests/pod_cpu_request_test.yaml index af5d71041..9f6bf801d 100644 --- a/packages/apps/kubernetes-nodes/tests/pod_cpu_request_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/pod_cpu_request_test.yaml @@ -1,7 +1,6 @@ suite: worker pool pod CPU request -# Mirror of the parent kubernetes chart's suite, on the pool-shaped values of -# this chart. `podCpuRequest` is the only knob that sets +# `podCpuRequest` is the only knob that sets # `domain.resources.requests.cpu` on a worker VM, and it is the other half of # `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the # VM declares none, so without this field the ceiling and the scheduler @@ -100,8 +99,7 @@ tests: asserts: # Absent, not empty: the KubevirtMachineTemplate is content-hash named, so # a `cpu:` key rendered for a pool that asked for nothing would rename the - # template and roll every live worker VM of the pool on upgrade. The - # render-parity gate compares those bytes against the parent chart. + # template and roll every live worker VM of the pool on upgrade. - notExists: path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.resources.requests.cpu - equal: diff --git a/packages/apps/kubernetes-nodes/tests/render_snapshot_test.yaml b/packages/apps/kubernetes-nodes/tests/render_snapshot_test.yaml index 4e51d0f93..5ac60b0f3 100644 --- a/packages/apps/kubernetes-nodes/tests/render_snapshot_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/render_snapshot_test.yaml @@ -2,12 +2,9 @@ suite: worker node pool golden render snapshot # Freezes the full rendered pool (KubevirtMachineTemplate, MachineDeployment, # MachineHealthCheck, worker WorkloadMonitor) for a representative node group. -# In PR-2a this render was proven byte-identical to the monolithic kubernetes -# chart's output via tests/render-parity.sh; PR-2b removed the pool from the -# parent, so cross-chart parity can no longer run. This snapshot takes over as -# the drift guard: any change to the rendered pool objects (which would rename -# the content-hashed KubevirtMachineTemplate and churn live worker VMs on -# upgrade) fails here until the snapshot is regenerated deliberately with +# It is the drift guard for those objects: any change to them (which would +# rename the content-hashed KubevirtMachineTemplate and churn live worker VMs +# on upgrade) fails here until the snapshot is regenerated deliberately with # `helm unittest -u`. # # Coverage note: every case sets instanceType: "" with explicit resources, @@ -44,10 +41,10 @@ tests: asserts: - matchSnapshot: {} - # GPU and kubelet-override branches live on in nodegroup.yaml after the split - # and are content-hash inputs to the KubevirtMachineTemplate; pin them too so - # a change to either branch that would rename the KMT (and roll live GPU or - # kubelet-tuned worker VMs) fails here. Value sets mirror the PR-2a parity gate. + # The GPU and kubelet-override branches of nodegroup.yaml are content-hash + # inputs to the KubevirtMachineTemplate; pin them too so a change to either + # branch that would rename the KMT (and roll live GPU or kubelet-tuned worker + # VMs) fails here. - it: matches the golden render of a GPU pool set: roles: [] diff --git a/packages/apps/kubernetes-nodes/tests/talos_k8s_support_matrix_test.yaml b/packages/apps/kubernetes-nodes/tests/talos_k8s_support_matrix_test.yaml new file mode 100644 index 000000000..ea1aa5ce3 --- /dev/null +++ b/packages/apps/kubernetes-nodes/tests/talos_k8s_support_matrix_test.yaml @@ -0,0 +1,62 @@ +suite: Talos to Kubernetes support-matrix guard (assertTalosSupportsKubernetes) + +# The guard is keyed by Talos minor and consulted twice in nodegroup.yaml: once +# for talos.version and once for the release osImage resolves to. These cases +# pin the pairings it has to let through: each one fails if the matrix row loses +# the Kubernetes version under test. +# +# The refusals are in hack/kubernetes-nodes-talos-matrix.bats. The values schema +# offers only Kubernetes versions the v1.13 row lists, so no schema-valid input +# reaches a refusal, and helm-unittest always validates the schema; the bats file +# renders with --skip-schema-validation instead. + +templates: + - templates/nodegroup.yaml + +release: + name: kubernetes-nodes-myk8s-md0 + namespace: tenant-test + +set: + cluster: myk8s + _cluster: + cluster-domain: cozy.local + minReplicas: 1 + maxReplicas: 3 + instanceType: "" + diskSize: 20Gi + storageClass: replicated + roles: [ingress-nginx] + resources: + cpu: "2" + memory: 4Gi + +tests: + - it: renders the oldest Kubernetes version the schema offers against the default Talos + set: + version: "v1.31" + asserts: + - notFailedTemplate: {} + - hasDocuments: + count: 4 + + - it: renders the newest Kubernetes version the schema offers against the default Talos + set: + version: "v1.35" + asserts: + - notFailedTemplate: {} + + - it: renders an in-window pairing reached through osImage.factory.version + set: + version: "v1.31" + talos: + version: "v1.12.0" + osImage: + factory: + version: "v1.13.0" + asserts: + - notFailedTemplate: {} + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.dataVolumeTemplates[0].spec.source.http.url + value: https://factory.talos.dev/image/ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515/v1.13.0/openstack-amd64.raw.xz + documentSelector: {path: kind, value: KubevirtMachineTemplate} diff --git a/packages/apps/kubernetes-nodes/tests/talos_worker_config_test.yaml b/packages/apps/kubernetes-nodes/tests/talos_worker_config_test.yaml index 10bff88f8..bcc387dd6 100644 --- a/packages/apps/kubernetes-nodes/tests/talos_worker_config_test.yaml +++ b/packages/apps/kubernetes-nodes/tests/talos_worker_config_test.yaml @@ -7,7 +7,6 @@ suite: Talos worker machineconfig (GPU labels + kubelet reservations) # - GPU pools must inject nodeLabels gpu=on so HAMi's device plugin # (nodeSelector gpu=on) schedules and advertises nvidia.com/gpu; # - kubelet reservation overrides must reach the machineconfig kubelet config. -# Ported from the parent chart's gpu_node_labels_test / kubelet_reservation_test. templates: - templates/talos-reconcile-job.yaml diff --git a/packages/apps/kubernetes-nodes/values.schema.json b/packages/apps/kubernetes-nodes/values.schema.json index 18efc647e..bbc1ddb8d 100644 --- a/packages/apps/kubernetes-nodes/values.schema.json +++ b/packages/apps/kubernetes-nodes/values.schema.json @@ -3,7 +3,7 @@ "type": "object", "properties": { "cluster": { - "description": "Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-\u003ccluster\u003e`; the KubernetesNodes CR must be named `\u003ccluster\u003e-\u003cpool\u003e`. The pool name (the part after `\u003ccluster\u003e-`) must not collide with a nodeGroup still managed by the parent kubernetes chart (e.g. the default `md0`), or the render fails on ownership conflict.", + "description": "Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-\u003ccluster\u003e`; the KubernetesNodes CR must be named `\u003ccluster\u003e-\u003cpool\u003e`. The pool's objects are named `kubernetes-\u003ccluster\u003e-\u003cpool\u003e`; when a MachineDeployment of that name already exists and belongs to another Helm release, the render fails instead of taking it over.", "type": "string", "default": "", "x-kubernetes-validations": [ @@ -14,7 +14,7 @@ ] }, "storageClass": { - "description": "StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster.", + "description": "StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`.", "type": "string", "default": "replicated", "x-kubernetes-validations": [ @@ -38,7 +38,7 @@ "default": 10 }, "instanceType": { - "description": "Virtual machine instance type.", + "description": "KubeVirt instance type that sizes each worker VM when `resources` is not set. Ignored when `resources.cpu` and `resources.memory` are both set, and therefore on `substrate: proxmox`, which requires them.", "type": "string", "default": "u1.medium", "x-cozystack-options": { @@ -46,7 +46,7 @@ } }, "diskSize": { - "description": "System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events.", + "description": "System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. Has no effect on `substrate: proxmox`, where the worker disk is a clone of the VM template's disk and keeps its size.", "default": "20Gi", "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", "anyOf": [ @@ -101,7 +101,7 @@ } }, "podCpuLimit": { - "description": "CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.", + "description": "CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content.", "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", "anyOf": [ { @@ -114,7 +114,7 @@ "x-kubernetes-int-or-string": true }, "podCpuRequest": { - "description": "CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.", + "description": "CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content.", "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", "anyOf": [ { @@ -147,7 +147,7 @@ } }, "kernelModules": { - "description": "Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls.", + "description": "Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.", "type": "array", "items": { "type": "object", @@ -170,7 +170,7 @@ } }, "kubelet": { - "description": "Kubelet resource reservations for this pool.", + "description": "Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`.", "type": "object", "default": {}, "properties": { @@ -185,19 +185,19 @@ "default": "10%" }, "kubeReservedCpu": { - "description": "CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.", + "description": "CPU reserved for kubelet and container runtime. When empty, 5% of the worker's CPU, clamped to 50m..500m.", "type": "string" }, "kubeReservedMemory": { - "description": "Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.", + "description": "Memory reserved for kubelet and container runtime. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.", "type": "string" }, "systemReservedCpu": { - "description": "CPU reserved for host OS. Auto-computed from instanceType if empty.", + "description": "CPU reserved for host OS. When empty, 5% of the worker's CPU, clamped to 50m..500m.", "type": "string" }, "systemReservedMemory": { - "description": "Memory reserved for host OS. Auto-computed from instanceType if empty.", + "description": "Memory reserved for host OS. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.", "type": "string" } } @@ -213,27 +213,27 @@ "default": "50%" }, "nodeStartupTimeout": { - "description": "Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.", + "description": "Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (the Talos image import from the Image Factory, or a storage class slow to provision or clone the worker disk). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.", "type": "string", "default": "20m" }, "readyUnknownTimeout": { - "description": "How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.", + "description": "How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.", "type": "string", "default": "5m" }, "readyFalseTimeout": { - "description": "How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.", + "description": "How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.", "type": "string", "default": "10m" }, "maxNodeProvisionTime": { - "description": "How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted. Keep in sync with the parent kubernetes chart.", + "description": "How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted.", "type": "string", "default": "30m" }, "version": { - "description": "Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool -- and must satisfy the Talos\u003c-\u003eKubernetes support matrix against `talos.version`.", + "description": "Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool. The render fails when it is ahead. Its pairing with the Talos release the pool resolves (`talos.version`, or the `osImage` override) is checked against the chart's Talos\u003c-\u003eKubernetes support matrix: for a Talos minor the matrix lists, a Kubernetes minor outside that minor's list is refused; a Talos minor the matrix does not list is not checked.", "type": "string", "default": "v1.35", "enum": [ @@ -268,7 +268,7 @@ "type": "object", "properties": { "imageFactoryURL": { - "description": "Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. NOTE: this redirects only the worker boot-disk import; the in-guest upgrade installer image is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.", + "description": "Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. This redirects only the worker boot-disk import; the installer image in the worker machine config is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.", "type": "string" }, "schematicID": { @@ -284,7 +284,7 @@ } }, "talos": { - "description": "Talos worker image configuration. Keep in sync with the parent cluster's `talos`.", + "description": "Talos worker image configuration for this pool.", "type": "object", "default": {}, "required": [ @@ -296,17 +296,17 @@ ], "properties": { "imageFactoryCA": { - "description": "Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there.", + "description": "Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there. On `substrate: proxmox` it has no effect on the workers either: the ConfigMap is still rendered, but nothing references it.", "type": "string", "default": "" }, "imageFactoryURL": { - "description": "Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal.", + "description": "Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal. On `substrate: proxmox` the workers clone a VM template that already carries the OS, so this field has no effect there beyond that format check.", "type": "string", "default": "https://factory.talos.dev" }, "installerRepository": { - "description": "OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `\u003cinstallerRepository\u003e/\u003cschematicID\u003e:\u003cversion\u003e` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.", + "description": "OCI repository prefix for the Talos installer image, written into the worker machine config as `machine.install.image`. Resolved as `\u003cinstallerRepository\u003e/\u003cschematicID\u003e:\u003cversion\u003e` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.", "type": "string", "default": "factory.talos.dev/installer" }, @@ -330,7 +330,7 @@ } }, "version": { - "description": "Default Talos release for this pool's worker OS image and in-guest installer. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos\u003c-\u003eKubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix rejects a pairing it lists as unsupported; it lists the Talos minors this chart ships and passes any other minor unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.", + "description": "Default Talos release for this pool's worker OS image and for the installer image written into the worker machine config. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos\u003c-\u003eKubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix lists, for each Talos minor the chart knows, the Kubernetes minors it supports and refuses a pool `version` outside that list; a Talos minor it does not list passes unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.", "type": "string", "default": "v1.13.6" } diff --git a/packages/apps/kubernetes-nodes/values.yaml b/packages/apps/kubernetes-nodes/values.yaml index c3fad197b..5ca1d8c50 100644 --- a/packages/apps/kubernetes-nodes/values.yaml +++ b/packages/apps/kubernetes-nodes/values.yaml @@ -2,7 +2,7 @@ ## @section Linkage ## -## @param {string} cluster - Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool name (the part after `-`) must not collide with a nodeGroup still managed by the parent kubernetes chart (e.g. the default `md0`), or the render fails on ownership conflict. +## @param {string} cluster - Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool's objects are named `kubernetes--`; when a MachineDeployment of that name already exists and belongs to another Helm release, the render fails instead of taking it over. ## @immutable cluster: "" @@ -22,7 +22,7 @@ cluster: "" ## @section Common Parameters ## -## @param {string} storageClass - StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. +## @param {string} storageClass - StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`. ## @x-cozystack-options {source: storageclass} ## @immutable storageClass: replicated @@ -44,10 +44,10 @@ storageClass: replicated ## @field {[]string} [parameters] - Module parameters, each as a bare `key=value` string. ## @typedef {struct} Kubelet - Kubelet resource reservations for worker nodes. -## @field {string} [systemReservedMemory] - Memory reserved for host OS. Auto-computed from instanceType if empty. -## @field {string} [kubeReservedMemory] - Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty. -## @field {string} [systemReservedCpu] - CPU reserved for host OS. Auto-computed from instanceType if empty. -## @field {string} [kubeReservedCpu] - CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty. +## @field {string} [systemReservedMemory] - Memory reserved for host OS. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi. +## @field {string} [kubeReservedMemory] - Memory reserved for kubelet and container runtime. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi. +## @field {string} [systemReservedCpu] - CPU reserved for host OS. When empty, 5% of the worker's CPU, clamped to 50m..500m. +## @field {string} [kubeReservedCpu] - CPU reserved for kubelet and container runtime. When empty, 5% of the worker's CPU, clamped to 50m..500m. ## @field {string} [evictionHardMemory]="7%" - Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%). ## @field {string} [evictionSoftMemory]="10%" - Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%). @@ -57,11 +57,11 @@ minReplicas: 0 ## @param {int} maxReplicas=10 - Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling. maxReplicas: 10 -## @param {string} instanceType="u1.medium" - Virtual machine instance type. +## @param {string} instanceType="u1.medium" - KubeVirt instance type that sizes each worker VM when `resources` is not set. Ignored when `resources.cpu` and `resources.memory` are both set, and therefore on `substrate: proxmox`, which requires them. ## @x-cozystack-options {source: instancetype} instanceType: u1.medium -## @param {quantity} diskSize="20Gi" - System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. +## @param {quantity} diskSize="20Gi" - System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. Has no effect on `substrate: proxmox`, where the worker disk is a clone of the VM template's disk and keeps its size. diskSize: 20Gi ## @param {[]string} roles - List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller. @@ -70,16 +70,16 @@ roles: [] ## @param {Resources} [resources] - Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `"010"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `"010"` reserves what `"10"` does. resources: {} -## @param {quantity} [podCpuLimit] - CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart. +## @param {quantity} [podCpuLimit] - CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. -## @param {quantity} [podCpuRequest] - CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart. +## @param {quantity} [podCpuRequest] - CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. ## @param {[]GPU} gpus - List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). gpus: [] -## @param {[]KernelModule} [kernelModules] - Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls. +## @param {[]KernelModule} [kernelModules] - Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. -## @param {Kubelet} [kubelet] - Kubelet resource reservations for this pool. +## @param {Kubelet} [kubelet] - Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`. kubelet: {} ## @param {bool} logSerialConsole=false - Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it. @@ -88,16 +88,16 @@ logSerialConsole: false ## @param {string} maxUnhealthy="50%" - Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is a percentage. Drop to "0%" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At "50%" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two. maxUnhealthy: "50%" -## @param {string} nodeStartupTimeout="20m" - Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning. +## @param {string} nodeStartupTimeout="20m" - Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (the Talos image import from the Image Factory, or a storage class slow to provision or clone the worker disk). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning. nodeStartupTimeout: "20m" -## @param {string} readyUnknownTimeout="5m" - How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart. +## @param {string} readyUnknownTimeout="5m" - How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. readyUnknownTimeout: "5m" -## @param {string} readyFalseTimeout="10m" - How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart. +## @param {string} readyFalseTimeout="10m" - How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. readyFalseTimeout: "10m" -## @param {string} maxNodeProvisionTime="30m" - How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours ("30m", "1h30m"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to ("20m0s", "1h0m0s") is accepted. Keep in sync with the parent kubernetes chart. +## @param {string} maxNodeProvisionTime="30m" - How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours ("30m", "1h30m"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to ("20m0s", "1h0m0s") is accepted. maxNodeProvisionTime: "30m" ## @@ -111,23 +111,23 @@ maxNodeProvisionTime: "30m" ## @value v1.32 ## @value v1.31 -## @param {Version} version - Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool -- and must satisfy the Talos<->Kubernetes support matrix against `talos.version`. +## @param {Version} version - Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool. The render fails when it is ahead. Its pairing with the Talos release the pool resolves (`talos.version`, or the `osImage` override) is checked against the chart's Talos<->Kubernetes support matrix: for a Talos minor the matrix lists, a Kubernetes minor outside that minor's list is refused; a Talos minor the matrix does not list is not checked. version: "v1.35" ## ## @section Talos Worker Image ## -## @typedef {struct} WorkerImageBuiltin - Boot this pool's workers by CDI clone of a golden image from the opt-in worker image catalog (packages/system/kubernetes-worker-image) in cozy-public. The catalog package must be enabled (bundles.enabledPackages) and hold a matching (schematicID, version) entry, otherwise the render fails. A clone is a storage-layer CSI copy, so N workers no longer each stream the OS image over HTTP. Pair this with a StorageClass whose volumes are reachable from any node -- in practice a replicated/DRBD class. The copy runs between two volumes that a node-pinned class requires to be co-located, and nothing here arranges that: the golden in cozy-public has no consumer, so it binds as soon as it imports, on whichever node CDI's importer landed on, while a worker disk binds where its VM is scheduled. The platform also ships `cloneStrategyOverride: csi-clone`, which rules out the host-assisted `copy` strategy that would cross nodes. What a node-pinned class (`allowRemoteVolumeAccess: false`, WaitForFirstConsumer) actually does here has not been measured, so treat `builtin` on one as unsupported rather than slow. +## @typedef {struct} WorkerImageBuiltin - Boot this pool's workers by CDI clone of a golden image from the opt-in worker image catalog (packages/system/kubernetes-worker-image) in cozy-public. The catalog package must be enabled (bundles.enabledPackages) and hold a matching (schematicID, version) entry, otherwise the render fails. A clone is a storage-layer CSI copy, so the workers do not each stream the OS image over HTTP. Pair this with a StorageClass whose volumes are reachable from any node -- in practice a replicated/DRBD class. The copy runs between two volumes that a node-pinned class requires to be co-located, and nothing here arranges that: the golden in cozy-public has no consumer, so it binds as soon as it imports, on whichever node CDI's importer landed on, while a worker disk binds where its VM is scheduled. The platform also ships `cloneStrategyOverride: csi-clone`, which rules out the host-assisted `copy` strategy that would cross nodes. What a node-pinned class (`allowRemoteVolumeAccess: false`, WaitForFirstConsumer) actually does here has not been measured, so treat `builtin` on one as unsupported rather than slow. ## @field {string} [schematicID] - Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`. ## @field {string} [version] - Talos version of the golden to clone. Defaults to `talos.version`. ## @typedef {struct} WorkerImageFactory - Boot this pool's workers by importing the Talos OS image over HTTP from an Image Factory (public, self-hosted, or caching mirror). This is the bring-your-own path and the default when `osImage` is omitted. ## @field {string} [version] - Talos version. Defaults to `talos.version`. ## @field {string} [schematicID] - Talos schematic ID. Defaults to `talos.schematicID`. -## @field {string} [imageFactoryURL] - Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. NOTE: this redirects only the worker boot-disk import; the in-guest upgrade installer image is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry. +## @field {string} [imageFactoryURL] - Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. This redirects only the worker boot-disk import; the installer image in the worker machine config is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry. -## @typedef {struct} WorkerImage - Worker OS image source for this pool. Set exactly one of `builtin` or `factory`; when the whole field is omitted the workers import over HTTP using the pool's `talos.*` defaults (equivalent to an empty `factory`). The disk-system source is derived entirely from this field, so a pool with `osImage` omitted renders identically to a chart without this feature — adopting the feature never rolls existing workers. Editing this field on an existing pool changes the rendered source and therefore re-images (rolls) that pool, the same as changing any other image reference. +## @typedef {struct} WorkerImage - Worker OS image source for this pool. Set exactly one of `builtin` or `factory`; when the whole field is omitted the workers import over HTTP using the pool's `talos.*` defaults (equivalent to an empty `factory`). The disk-system source is derived entirely from this field. Editing this field on an existing pool changes the rendered source and therefore re-images (rolls) that pool, the same as changing any other image reference. ## @field {*WorkerImageBuiltin} [builtin] - Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled. ## @field {*WorkerImageFactory} [factory] - Import from an Image Factory / mirror (default). @@ -135,15 +135,15 @@ version: "v1.35" osImage: {} ## @typedef {struct} Talos - Talos worker image configuration. -## @field {string} version - Default Talos release for this pool's worker OS image and in-guest installer. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix rejects a pairing it lists as unsupported; it lists the Talos minors this chart ships and passes any other minor unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead. +## @field {string} version - Default Talos release for this pool's worker OS image and for the installer image written into the worker machine config. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix lists, for each Talos minor the chart knows, the Kubernetes minors it supports and refuses a pool `version` outside that list; a Talos minor it does not list passes unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead. ## @field {string} schematicID - Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID. -## @field {string} imageFactoryURL - Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal. -## @field {string} installerRepository - OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash. +## @field {string} imageFactoryURL - Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal. On `substrate: proxmox` the workers clone a VM template that already carries the OS, so this field has no effect there beyond that format check. +## @field {string} installerRepository - OCI repository prefix for the Talos installer image, written into the worker machine config as `machine.install.image`. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash. ## @field {object} registryMirrors - Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. ## @field {map[string]string} sysctls - Sysctl overrides rendered into Talos `machine.sysctls` on this pool's guest workers only. Values must be strings (quote numeric values); for example, `user.max_user_namespaces: "11255"` enables the user namespaces required by the Talos gVisor extension. An empty map preserves Talos defaults and the existing bootstrap hash. Adding, changing or removing an override changes the content-hashed TalosConfigTemplate and rolls the pool's workers. Supported names and values depend on the worker's Talos release and kernel; unsafe settings can weaken guest isolation or prevent workers from booting. This does not install gVisor or configure a RuntimeClass, and does not change the management hosts, control plane or other pools. -## @field {string} [imageFactoryCA] - Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there. +## @field {string} [imageFactoryCA] - Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there. On `substrate: proxmox` it has no effect on the workers either: the ConfigMap is still rendered, but nothing references it. -## @param {Talos} talos - Talos worker image configuration. Keep in sync with the parent cluster's `talos`. +## @param {Talos} talos - Talos worker image configuration for this pool. talos: version: "v1.13.6" schematicID: "ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515" diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index 2ac5cfc45..f7d1062b1 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -8,11 +8,9 @@ Cozystack deploys and manages Kubernetes-as-a-service as standalone applications In Cozystack, such clusters are named tenant Kubernetes clusters, while the base Cozystack cluster is called a management or root cluster. Tenant clusters are fully separated from the management cluster and are intended for deploying tenant-specific or customer-developed applications. -Within a tenant cluster, users can take advantage of LoadBalancer services and easily provision physical volumes as needed. -The control-plane operates within containers, while the worker nodes are deployed as virtual machines, all seamlessly managed by the application. +Within a tenant cluster, users can take advantage of LoadBalancer services and easily provision physical volumes as needed. The control plane runs in containers in the management cluster, and the worker nodes are virtual machines, grouped into pools that are separate `KubernetesNodes` resources. -Kubernetes version in tenant clusters is independent of Kubernetes in the management cluster. -Users can select the supported patch versions from 1.31 to 1.35. +Kubernetes version in tenant clusters is independent of Kubernetes in the management cluster. Users choose the Kubernetes minor version, from 1.31 to 1.35, and the chart deploys the patch release it pins for that minor. ## Why Use a Managed Kubernetes Cluster? @@ -58,14 +56,9 @@ Deploying it involves the following components: Each control plane pod includes essential components like `kube-apiserver`, `controller-manager`, and `scheduler`, allowing for efficient multi-tenancy and resource utilization. -- **Etcd Cluster**: A dedicated etcd cluster is deployed using the cozystack [etcd-operator](https://github.com/cozystack/etcd-operator) (`etcd-operator.cozystack.io/v1alpha2`). - It provides reliable and scalable key-value storage for the Kubernetes control plane. +- **Etcd**: the control plane keeps its state in the tenant's etcd, which this application does not deploy. It is the etcd the Tenant runs when `etcd` is enabled on it or on one of its parent tenants, built with the cozystack [etcd-operator](https://github.com/cozystack/etcd-operator), and every tenant Kubernetes cluster that uses it shares it. Until that etcd exists, the application creates only a `-awaiting-etcd` ConfigMap that says so, and provisions the cluster once the etcd is there. -- **Worker Nodes**: Virtual Machines are provisioned to serve as worker nodes using KubeVirt. - These nodes are configured to join the tenant Kubernetes cluster, enabling the deployment and management of workloads. - Worker node disks automatically detect and match the underlying volume's block size - (`blockSize.matchVolume`) to ensure compatibility with storage backends that use - non-512-byte sectors, such as LINSTOR DRBD with 4Kn drives. +- **Worker Nodes**: worker nodes are virtual machines running Talos Linux that join the tenant cluster. They come in pools, and each pool is a separate `KubernetesNodes` resource that names this cluster and sets its own size, disk, Talos image and scaling bounds; this application creates no worker nodes itself. With `substrate: kubevirt` the virtual machines run on KubeVirt in the management cluster, and with `substrate: proxmox` on an external Proxmox VE cluster. - **Cluster API**: Cozystack is using the [Kubernetes Cluster API](https://cluster-api.sigs.k8s.io/) to provision the components of a cluster. @@ -83,23 +76,11 @@ See the reference for components utilized in this service: - [github.com/kubernetes-sigs/cluster-api-provider-kubevirt](https://github.com/kubernetes-sigs/cluster-api-provider-kubevirt) - [github.com/kubevirt/csi-driver](https://github.com/kubevirt/csi-driver) -## Breaking Changes +The top-level `storageClass` is the default StorageClass inside the tenant cluster and cannot be changed after the cluster is created; see [storageClass immutability](https://cozystack.io/docs/v1/operations/storage-class-immutability/) for the contract and which consumers enforce it. Each worker pool sets the StorageClass of its own disks on its `KubernetesNodes` resource. -- **`ephemeralStorage` renamed to `diskSize`** (v1.4): The `nodeGroups[name].ephemeralStorage` field has been renamed to `nodeGroups[name].diskSize`. Existing clusters are migrated transparently by platform migration 41 during the pre-upgrade hook — no manual action is required. Newly written values should use `diskSize`. Existing VMs are rolling-updated via CAPI when the new values are applied. With the Talos worker rollover in this release the field now sizes the **single system disk** (Talos OS image streamed from `factory.talos.dev`, kubelet state, containerd image cache, local-path PVCs) — the pre-Talos `disk-kubelet` PVC layout was removed. State on that single disk persists across same-VM reboots (virt-launcher restart, guest reboot, node failure); VM replacement by CAPI (e.g. nodeGroup field change, MachineHealthCheck remediation) provisions a fresh disk. +## Upgrading -- **Air-gapped tenant workers (Phase 1 Talos rollover)**: tenant worker bootstrap moves from Ubuntu containerDisk + kubeadm to a Talos OS disk image fetched over HTTP by CDI and a `.../installer/...` installer reference. Both sources are now overridable — set `talos.imageFactoryURL` (default `https://factory.talos.dev`) to a self-hosted Image Factory, a caching mirror, or an internal HTTP file server, and `talos.installerRepository` (default `factory.talos.dev/installer`) to a mirrored registry. The Helm-rendered `*-patch-containerd` Secret that previously plumbed the platform-wide `registries.mirrors` config to tenant workers (via the kubeadm template's containerd certs.d mount) has no consumer in the Talos machineconfig, so the Secret is removed together with the platform `registries` value that fed it. A Package that still sets `registries` keeps installing, the key is ignored. Registry mirrors for tenant workers, **in-guest container-image pulls** (pods pulling from registries inside the tenant cluster) included, are configured per worker pool through `talos.registryMirrors` of the `KubernetesNodes` resource. - -- **Worker MachineHealthCheck remediation is now enabled by default** (Phase 1 Talos rollover): the worker MHC `maxUnhealthy` moved from a hard-coded `0` (remediation effectively **off**) to a configurable `nodeHealthCheck.maxUnhealthy` defaulting to `"50%"`. CAPI now auto-remediates (deletes and replaces) unhealthy worker Machines while at least 50% of a group stays healthy. `"50%"` deliberately leaves headroom for transient unhealthy nodes during the kubeadm→Talos rollover and slow first boots from `factory.talos.dev`; set `nodeHealthCheck.maxUnhealthy: "0%"` to keep the previous remediation-off behaviour until your fleet is stable on Talos workers. - -- **GitOps-managed tenant `Kubernetes` CRs must bump `spec.version` in Git** (Phase 1 Talos rollover): platform migration 46 patches live `kuberneteses.apps.cozystack.io` objects from `v1.30` to `v1.31` (v1.30 left the Talos↔Kubernetes support matrix). When the tenant `Kubernetes` CR is reconciled from Git (Flux/Argo), the next source reconcile re-applies `version: v1.30` over migration 46's patch, which then trips the chart's `_versions.tpl` guard and the HelmRelease fails. Update `spec.version` to `v1.31` (or newer) in your Git source before/with the platform upgrade. Tenants managed via the API or dashboard need no action — migration 46 handles them. - -- **Remote-accessible LINSTOR StorageClasses are auto-propagated to tenants** (v1.5): infra-cluster LINSTOR StorageClasses whose `linstor.csi.linbit.com/allowRemoteVolumeAccess` is not `"false"` are created inside each tenant under the same name (provisioned by `csi.kubevirt.io`). **Upgrade caveat:** delete any *manually created* tenant StorageClass whose name collides with a propagated infra class (e.g. a hand-made `replicated`) before upgrading, or the propagated class cannot be created and the tenant CSI release will not converge. Infra classes that must stay node-local need `allowRemoteVolumeAccess: "false"` set explicitly — an absent annotation is treated as remote-accessible. Propagation is evaluated at HelmRelease render time, so classes added or removed on the infra cluster after a tenant exists propagate only on that tenant's next reconcile. - -- **Worker pools split into `KubernetesNodes` (Phase 2)**: `spec.nodeGroups`, `spec.nodeHealthCheck` and `spec.maxNodeProvisionTime` are removed from the `Kubernetes` CR — worker pools are now separate `KubernetesNodes` resources (one HelmRelease per pool; see the `kubernetes-nodes` chart), on which `spec.cluster` and `spec.storageClass` are immutable (`self == oldSelf`). Existing pools are adopted automatically on upgrade by platform migration 56 with no worker-VM churn. Consequences: (1) the `Kubernetes` CR's `WorkloadsReady` condition is now **control-plane-only** — worker health is reported on each `KubernetesNodes` resource, so a dashboard or `kubectl wait` watching the parent no longer reflects worker readiness; (2) enabling `addons.ingressNginx` now requires a pool carrying `roles: [ingress-nginx]` (the implicit default `md0` is gone — without such a pool the controller DaemonSet schedules nothing); (3) do **not** edit `Kubernetes` CRs during a platform upgrade — a mid-window parent render can transiently fail Helm ownership validation while migration 56 re-annotates worker objects, and self-heals once the new chart artifact lands; (4) when deleting a cluster, delete its `KubernetesNodes` pools **before** the parent `Kubernetes` resource, so each pool's pre-delete unpin hook can run while the tenant apiserver and the CAPI CRDs still exist — deleting the parent first can leave the pool release reconciling CAPI objects against a cluster that is being torn down (in the pre-split monolith the pools were part of the parent release and died with it; they are now separate resources with their own lifecycle); (5) the removed fields (`nodeGroups`, `nodeHealthCheck`, `maxNodeProvisionTime`) are still **accepted and stored** on an already-upgraded `Kubernetes` CR — the adoption migration deliberately leaves them in place rather than rewriting parent values mid-upgrade — but they have **no effect**: editing them (for example scaling via `spec.nodeGroups..minReplicas`) does nothing, and the API returns an admission warning pointing you at the `KubernetesNodes` resources. A cluster name is also capped at 32 characters so every pool's `KubernetesNodes` release name fits the 53-character Helm limit. ComputePlane clusters (the `computeplane` module) are affected the same way: their worker pools are rendered as separate per-pool `KubernetesNodes` releases too (materialised directly by the module rather than adopted), so the same pool-name length limit applies to their `nodeGroups` keys. - -- **`controlPlane.apiServer.extraArgs` refuses flags the tenant control plane owns** (Kamaji 26.9.4-edge): the tenant control plane now sets a list of kube-apiserver flags itself and drops an `extraArgs` entry naming one, where the previous build let the entry win. The entry stops taking effect once the new build reconciles the control plane, whatever the chart renders. Two of those flags have a field the control plane renders them from, and the chart moves an entry written as `--flag=value` into it while the field is empty: `--enable-admission-plugins` into `controlPlane.apiServer.admissionControllers`, `--kubelet-preferred-address-types` into `controlPlane.kubelet.preferredAddressTypes`. The KamajiControlPlane CRD accepts only the plugin names it lists, so a plugin outside that list that the apiserver leaves off by default can no longer be enabled, and a moved name outside it fails when the KamajiControlPlane is applied, with an error that names it, rather than at render. The chart refuses to render any other entry naming one of the flags, so a tenant carrying one stops reconciling until the entry is removed; the description of `controlPlane.apiServer.extraArgs` lists the flags. The same Kamaji update passes every `extraArgs` entry to the apiserver, so repeats of a flag no longer collapse to the last one: naming the kube feature gates both bare and `kube:`-prefixed across entries, or giving two `--emulated-version` entries that name kube, now stops the apiserver from starting. The chart refuses neither, in any `spec.oidc.mode`: both reach the apiserver as written. Keep one spelling for kube gate terms and one kube emulated version. - -> The top-level `storageClass` field is annotated as immutable in the chart schema — see [storageClass immutability](https://cozystack.io/docs/v1/operations/storage-class-immutability/) for the contract and which consumers enforce it. The per-pool worker `storageClass` now lives on the `KubernetesNodes` resource (see the `kubernetes-nodes` chart). +Changes that need action when Cozystack is upgraded are described in the upgrade notes of each release, for example [Upgrading from v1.6 to v1.7](https://cozystack.io/docs/v1.7/operations/cluster/upgrade-to-v1.7/). ## Proxmox substrate: credentials, privileges and tenant scope @@ -133,36 +114,36 @@ A storage shared between tenants cannot be scoped: `Datastore.Allocate` on it is ### Application-specific Parameters -| Name | Description | Type | Value | -| --------- | ---------------------------------------------------------------------------------------------- | -------- | ------- | -| `version` | Kubernetes major.minor version to deploy | `string` | `v1.35` | -| `host` | External hostname for Kubernetes cluster. Defaults to `.` if empty. | `string` | `""` | +| Name | Description | Type | Value | +| --------- | --------------------------------------------------------------------------------------------------------- | -------- | ------- | +| `version` | Kubernetes major.minor version to deploy | `string` | `v1.35` | +| `host` | External hostname for Kubernetes cluster. Defaults to `kubernetes-.` if empty. | `string` | `""` | ### Substrate -| Name | Description | Type | Value | -| --------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ---------- | -| `substrate` | Which infrastructure provider backs this cluster's worker VMs. `kubevirt` runs them inside this cluster; `proxmox` runs them on an external Proxmox VE cluster through capmox. The control plane is Kamaji either way — this selects the infrastructure half only. Must match the `substrate` of every KubernetesNodes pool attached to this cluster: the pools reference this cluster's infrastructure object by kind, and a mismatch leaves their Machines unreconciled. Switching an existing cluster is not supported; create a new one. | `string` | `kubevirt` | -| `proxmox` | Proxmox substrate settings. | `object` | `{}` | -| `proxmox.allowedNodes` | Proxmox nodes capmox may place VMs on. Empty means every node in the Proxmox cluster. | `[]string` | `[]` | -| `proxmox.dnsServers` | Nameservers written into each worker's network config. Required by the ProxmoxCluster schema. | `[]string` | `[]` | -| `proxmox.insecure` | Skip verification of the Proxmox API server certificate in the CCM and the CSI controller, both of which run in the management cluster. Default false: the certificate is verified unless an operator turns that off. A stock Proxmox VE install serves a self-signed one, so set true for it, or install a certificate the management cluster trusts. | `bool` | `false` | -| `proxmox.ipv4Config` | Address pool for workers. capmox assigns static addresses and has no DHCP mode, so this is required rather than optional. | `object` | `{}` | -| `proxmox.ipv4Config.addresses` | Ranges or CIDRs, e.g. `["10.0.0.120-10.0.0.170"]`. capmox turns these into an InClusterIPPool, so they must not overlap any DHCP range on the same L2 or two workers will answer to one address. | `[]string` | `[]` | -| `proxmox.ipv4Config.gateway` | Default gateway for the workers. | `string` | `""` | -| `proxmox.ipv4Config.prefix` | Netmask prefix length. | `int` | `24` | -| `proxmox.ccm` | Proxmox cloud-controller-manager settings. | `object` | `{}` | -| `proxmox.ccm.credentialsSecretName` | Secret in THIS namespace holding the controller's Proxmox API credentials under the keys `url`, `token_id`, `token_secret` and `region`. May name the same Secret as `csi.credentialsSecretName`; it is a separate knob so the two can hold separate tokens, since this one only reads VM inventory while the driver attaches and detaches disks. | `string` | `""` | -| `proxmox.csi` | Proxmox CSI driver settings. | `object` | `{}` | -| `proxmox.csi.enabled` | Install the driver. Without it a Proxmox-backed tenant has no way to provision PersistentVolumes at all. | `bool` | `true` | -| `proxmox.csi.credentialsSecretName` | Secret in THIS namespace holding the driver's Proxmox API credentials under the keys `url`, `token_id`, `token_secret` and `region`. The parent chart composes them into the controller's config file here, in the management cluster; nothing reaches the tenant, and no token is written into a rendered manifest. Distinct from the capmox credentials on purpose: the driver attaches and detaches disks, which is a different blast radius from creating VMs. | `string` | `""` | -| `proxmox.csi.storageClasses` | StorageClasses to create in the tenant. | `[]object` | `[]` | -| `proxmox.csi.storageClasses[i].name` | StorageClass name inside the tenant cluster. | `string` | `""` | -| `proxmox.csi.storageClasses[i].storage` | Proxmox storage id the volumes are created on (as it appears in `pvesm status`). | `string` | `""` | -| `proxmox.csi.storageClasses[i].fstype` | Filesystem the driver formats volumes with: ext4 or xfs. | `string` | `""` | -| `proxmox.csi.storageClasses[i].ssd` | Mark volumes as SSD-backed, which changes the discard/cache defaults Proxmox applies. | `bool` | `false` | -| `proxmox.csi.storageClasses[i].reclaimPolicy` | Delete or Retain. | `string` | `""` | +| Name | Description | Type | Value | +| --------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ---------- | +| `substrate` | Which infrastructure provider backs this cluster's worker VMs. `kubevirt` runs them inside this cluster; `proxmox` runs them on an external Proxmox VE cluster through capmox. The control plane is Kamaji either way — this selects the infrastructure half only. Every KubernetesNodes pool of this cluster must set the same `substrate`: a pool set to the other one refuses to render once this cluster exists. Switching an existing cluster is not supported; create a new one. | `string` | `kubevirt` | +| `proxmox` | Proxmox substrate settings. | `object` | `{}` | +| `proxmox.allowedNodes` | Proxmox nodes capmox may place VMs on. Empty means every node in the Proxmox cluster. | `[]string` | `[]` | +| `proxmox.dnsServers` | Nameservers written into each worker's network config. Required by the ProxmoxCluster schema. | `[]string` | `[]` | +| `proxmox.insecure` | Skip verification of the Proxmox API server certificate in the CCM and the CSI controller, both of which run in the management cluster. Default false: the certificate is verified unless an operator turns that off. A stock Proxmox VE install serves a self-signed one, so set true for it, or install a certificate the management cluster trusts. | `bool` | `false` | +| `proxmox.ipv4Config` | Address pool for workers. capmox assigns static addresses and has no DHCP mode, so this is required rather than optional. | `object` | `{}` | +| `proxmox.ipv4Config.addresses` | Ranges or CIDRs, e.g. `["10.0.0.120-10.0.0.170"]`. capmox turns these into an InClusterIPPool, so they must not overlap any DHCP range on the same L2 or two workers will answer to one address. | `[]string` | `[]` | +| `proxmox.ipv4Config.gateway` | Default gateway for the workers. | `string` | `""` | +| `proxmox.ipv4Config.prefix` | Netmask prefix length. | `int` | `24` | +| `proxmox.ccm` | Proxmox cloud-controller-manager settings. | `object` | `{}` | +| `proxmox.ccm.credentialsSecretName` | Secret in THIS namespace holding the controller's Proxmox API credentials under the keys `url`, `token_id`, `token_secret` and `region`. May name the same Secret as `csi.credentialsSecretName`; it is a separate knob so the two can hold separate tokens, since this one only reads VM inventory while the driver attaches and detaches disks. | `string` | `""` | +| `proxmox.csi` | Proxmox CSI driver settings. | `object` | `{}` | +| `proxmox.csi.enabled` | Install the driver. Without it a Proxmox-backed tenant has no way to provision PersistentVolumes at all. | `bool` | `true` | +| `proxmox.csi.credentialsSecretName` | Secret in THIS namespace holding the driver's Proxmox API credentials under the keys `url`, `token_id`, `token_secret` and `region`. The parent chart composes them into the controller's config file here, in the management cluster; nothing reaches the tenant, and no token is written into a rendered manifest. Distinct from the capmox credentials on purpose: the driver attaches and detaches disks, which is a different blast radius from creating VMs. | `string` | `""` | +| `proxmox.csi.storageClasses` | StorageClasses to create in the tenant. | `[]object` | `[]` | +| `proxmox.csi.storageClasses[i].name` | StorageClass name inside the tenant cluster. | `string` | `""` | +| `proxmox.csi.storageClasses[i].storage` | Proxmox storage id the volumes are created on (as it appears in `pvesm status`). | `string` | `""` | +| `proxmox.csi.storageClasses[i].fstype` | Filesystem the driver formats volumes with: ext4 or xfs. | `string` | `""` | +| `proxmox.csi.storageClasses[i].ssd` | Mark volumes as SSD-backed, which changes the discard/cache defaults Proxmox applies. | `bool` | `false` | +| `proxmox.csi.storageClasses[i].reclaimPolicy` | Delete or Retain. | `string` | `""` | ### Cluster Addons @@ -205,49 +186,37 @@ A storage shared between tenants cannot be scoped: `Datastore.Allocate` on it is ### Kubernetes Control Plane Configuration -| Name | Description | Type | Value | -| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------- | ----------- | -| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` | -| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` | -| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` | -| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` | -| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `c1.medium` | -| `controlPlane.apiServer.extraArgs` | Extra command-line flags appended to the tenant kube-apiserver, passed through to KamajiControlPlane `spec.apiServer.extraArgs`. For OIDC use `spec.oidc.mode` — this passthrough is the escape hatch for unrelated apiserver flags. The tenant control plane sets each of the following flags itself and drops an entry naming one before the apiserver sees it. The chart moves an `--enable-admission-plugins=` entry into `controlPlane.apiServer.admissionControllers` and a `--kubelet-preferred-address-types=` entry into `controlPlane.kubelet.preferredAddressTypes` while that field is empty, and refuses any other entry naming one of them: `--advertise-address`, `--client-ca-file`, `--enable-admission-plugins`, `--service-cluster-ip-range`, `--kubelet-client-certificate`, `--kubelet-client-key`, `--kubelet-preferred-address-types`, `--proxy-client-cert-file`, `--proxy-client-key-file`, `--requestheader-allowed-names`, `--requestheader-client-ca-file`, `--secure-port`, `--service-account-key-file`, `--service-account-signing-key-file`, `--tls-cert-file`, `--tls-private-key-file`, `--etcd-servers`, `--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, `--etcd-prefix`, `--etcd-compaction-interval`, `--egress-selector-config-file`. When `spec.oidc.mode` is not `None` the chart also renders OIDC flags here, reads what you have written on them, and refuses some combinations outright; the Aggregated API servers section of the Kubernetes OIDC guide on cozystack.io carries which flags, which combinations, and why. Do NOT add legacy `--oidc-*` flags here when `spec.oidc.mode` is not `None`; the chart injects `--authentication-config` and the apiserver refuses to boot with both. Empty by default (no change to current behavior). | `[]string` | `[]` | -| `controlPlane.apiServer.admissionControllers` | Admission plugins the tenant kube-apiserver enables, passed through to KamajiControlPlane `spec.admissionControllers`, which the control plane renders as `--enable-admission-plugins`. A non-empty list replaces the control plane's default set instead of adding to it. The kube-apiserver enables these plugins in addition to its own default ones, so leaving a plugin out does not turn it off; to turn one off, name it in `--disable-admission-plugins` in `controlPlane.apiServer.extraArgs` and keep it out of this list, since the apiserver refuses to start with a plugin in both, and the control plane's default set names several. Empty by default, which keeps that default set, taken from the TenantControlPlane CRD: CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, LimitRanger, MutatingAdmissionWebhook, NamespaceLifecycle, PersistentVolumeClaimResize, Priority, ResourceQuota, RuntimeClass, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. `NodeRestriction` is not in it. | `[]string` | `[]` | -| `controlPlane.apiServer.extraVolumes` | Extra volumes added to the control-plane Deployment, passed through to KamajiControlPlane `spec.deployment.extraVolumes`. Use to mount a ConfigMap or Secret holding an AuthenticationConfiguration file referenced by `extraArgs`. Each item is a core/v1 Volume, but because the control-plane pod runs on the management cluster only `configMap` and `secret` sources are allowed (host-reaching sources like `hostPath`/`csi` and token-projection via `projected` are rejected); each volume must have a unique, non-empty name and exactly one source. The names `talos-ca` and `talos-tls-cert` are reserved by the chart. Empty by default. | `[]object` | `[]` | -| `controlPlane.apiServer.extraVolumeMounts` | Extra volume mounts added to the tenant kube-apiserver container, passed through to KamajiControlPlane `spec.apiServer.extraVolumeMounts`. Each `name` must reference a volume declared in `extraVolumes`; the chart-managed talos secret volumes cannot be mounted. Each item is a core/v1 VolumeMount. Empty by default. | `[]object` | `[]` | -| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` | -| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` | -| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` | -| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` | -| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` | -| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` | -| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` | -| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` | -| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` | -| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` | -| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` | -| `controlPlane.kubelet` | Kubelet connection settings. | `object` | `{}` | -| `controlPlane.kubelet.preferredAddressTypes` | Node address types the tenant kube-apiserver tries, in order, when it connects to a kubelet, passed through to KamajiControlPlane `spec.kubelet.preferredAddressTypes`, which the control plane renders as `--kubelet-preferred-address-types`. The KamajiControlPlane CRD rejects a repeated name. Empty by default, which renders `InternalIP`, `ExternalIP`, or the list of a `--kubelet-preferred-address-types=` entry in `controlPlane.apiServer.extraArgs` when there is one. | `[]string` | `[]` | - - -### Talos Worker Image - -| Name | Description | Type | Value | -| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------------------------------------------------------------------ | -| `talos` | Talos worker image configuration. | `object` | `{}` | -| `talos.version` | Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`. | `string` | `v1.13.6` | -| `talos.schematicID` | Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. | `string` | `ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515` | -| `talos.imageFactoryURL` | Base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. | `string` | `https://factory.talos.dev` | -| `talos.installerRepository` | OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:`. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash. | `string` | `factory.talos.dev/installer` | -| `talos.registryMirrors` | Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. | `object` | `{}` | +| Name | Description | Type | Value | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ----------- | +| `controlPlane` | Kubernetes control-plane configuration. | `object` | `{}` | +| `controlPlane.replicas` | Number of control-plane replicas. | `int` | `2` | +| `controlPlane.apiServer` | API Server configuration. | `object` | `{}` | +| `controlPlane.apiServer.resources` | CPU and memory resources for API Server. | `object` | `{}` | +| `controlPlane.apiServer.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.apiServer.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.apiServer.resourcesPreset` | Preset if `resources` omitted. | `string` | `c1.medium` | +| `controlPlane.apiServer.extraArgs` | Extra command-line flags appended to the tenant kube-apiserver, passed through to KamajiControlPlane `spec.apiServer.extraArgs`. For OIDC use `spec.oidc.mode` — this passthrough is the escape hatch for unrelated apiserver flags. The tenant control plane sets each of the following flags itself and drops an entry naming one before the apiserver sees it. The chart moves an `--enable-admission-plugins=` entry into `controlPlane.apiServer.admissionControllers` and a `--kubelet-preferred-address-types=` entry into `controlPlane.kubelet.preferredAddressTypes` while that field is empty, and refuses any other entry naming one of them: `--advertise-address`, `--client-ca-file`, `--enable-admission-plugins`, `--service-cluster-ip-range`, `--kubelet-client-certificate`, `--kubelet-client-key`, `--kubelet-preferred-address-types`, `--proxy-client-cert-file`, `--proxy-client-key-file`, `--requestheader-allowed-names`, `--requestheader-client-ca-file`, `--secure-port`, `--service-account-key-file`, `--service-account-signing-key-file`, `--tls-cert-file`, `--tls-private-key-file`, `--etcd-servers`, `--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, `--etcd-prefix`, `--etcd-compaction-interval`, `--egress-selector-config-file`. When `spec.oidc.mode` is not `None` the chart also renders OIDC flags here, reads what you have written on them, and refuses some combinations outright; the Aggregated API servers section of the Kubernetes OIDC guide on cozystack.io carries which flags, which combinations, and why. Do NOT add legacy `--oidc-*` flags here when `spec.oidc.mode` is not `None`; the chart injects `--authentication-config` and the apiserver refuses to boot with both. Empty by default. | `[]string` | `[]` | +| `controlPlane.apiServer.admissionControllers` | Admission plugins the tenant kube-apiserver enables, passed through to KamajiControlPlane `spec.admissionControllers`, which the control plane renders as `--enable-admission-plugins`. A non-empty list replaces the control plane's default set instead of adding to it. The kube-apiserver enables these plugins in addition to its own default ones, so leaving a plugin out does not turn it off; to turn one off, name it in `--disable-admission-plugins` in `controlPlane.apiServer.extraArgs` and keep it out of this list, since the apiserver refuses to start with a plugin in both, and the control plane's default set names several. Empty by default, which keeps that default set, taken from the TenantControlPlane CRD: CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, LimitRanger, MutatingAdmissionWebhook, NamespaceLifecycle, PersistentVolumeClaimResize, Priority, ResourceQuota, RuntimeClass, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. `NodeRestriction` is not in it. | `[]string` | `[]` | +| `controlPlane.apiServer.extraVolumes` | Extra volumes added to the control-plane Deployment, passed through to KamajiControlPlane `spec.deployment.extraVolumes`. Use to mount a ConfigMap or Secret holding an AuthenticationConfiguration file referenced by `extraArgs`. Each item is a core/v1 Volume, but because the control-plane pod runs on the management cluster only `configMap` and `secret` sources are allowed (host-reaching sources like `hostPath`/`csi` and token-projection via `projected` are rejected); each volume must have a unique, non-empty name and exactly one source. The names `talos-ca` and `talos-tls-cert` are reserved by the chart. Empty by default. | `[]object` | `[]` | +| `controlPlane.apiServer.extraVolumeMounts` | Extra volume mounts added to the tenant kube-apiserver container, passed through to KamajiControlPlane `spec.apiServer.extraVolumeMounts`. Each `name` must reference a volume declared in `extraVolumes`; the chart-managed talos secret volumes cannot be mounted. Each item is a core/v1 VolumeMount. Empty by default. | `[]object` | `[]` | +| `controlPlane.controllerManager` | Controller Manager configuration. | `object` | `{}` | +| `controlPlane.controllerManager.resources` | CPU and memory resources for Controller Manager. | `object` | `{}` | +| `controlPlane.controllerManager.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.controllerManager.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.controllerManager.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` | +| `controlPlane.scheduler` | Scheduler configuration. | `object` | `{}` | +| `controlPlane.scheduler.resources` | CPU and memory resources for Scheduler. | `object` | `{}` | +| `controlPlane.scheduler.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.scheduler.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.scheduler.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` | +| `controlPlane.konnectivity` | Konnectivity configuration. | `object` | `{}` | +| `controlPlane.konnectivity.server` | Konnectivity Server configuration. | `object` | `{}` | +| `controlPlane.konnectivity.server.resources` | CPU and memory resources for Konnectivity. | `object` | `{}` | +| `controlPlane.konnectivity.server.resources.cpu` | CPU available. | `quantity` | `""` | +| `controlPlane.konnectivity.server.resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `controlPlane.konnectivity.server.resourcesPreset` | Preset if `resources` omitted. | `string` | `t1.micro` | +| `controlPlane.kubelet` | Kubelet connection settings. | `object` | `{}` | +| `controlPlane.kubelet.preferredAddressTypes` | Node address types the tenant kube-apiserver tries, in order, when it connects to a kubelet, passed through to KamajiControlPlane `spec.kubelet.preferredAddressTypes`, which the control plane renders as `--kubelet-preferred-address-types`. The KamajiControlPlane CRD rejects a repeated name. Empty by default, which renders `InternalIP`, `ExternalIP`, or the list of a `--kubelet-preferred-address-types=` entry in `controlPlane.apiServer.extraArgs` when there is one. | `[]string` | `[]` | ### OIDC diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go index 9029c9ff3..b61121758 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller.go @@ -424,7 +424,8 @@ func (w *WrappedControllerService) ControllerUnpublishVolume(ctx context.Context pvc, err := w.infraClient.CoreV1().PersistentVolumeClaims(w.infraNamespace).Get(ctx, dvName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { - return &csi.ControllerUnpublishVolumeResponse{}, nil + // A failed PVC creation can still leave a hotplug entry in the VM. + return w.ControllerService.ControllerUnpublishVolume(ctx, req) } return nil, err } diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go index 46e2e1dce..c6029f17b 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver/controller_test.go @@ -310,15 +310,19 @@ const ( testVolumeName = "pvc-0" ) -// teardownClient models the CAPI/CAPK teardown race: the VM is gone while its -// VMI still reports the hot-plugged volume. The VMI drops the volume once -// RemoveVolumeFromVMI is called, or when the garbage collector deletes a -// VMI whose controlling VM is gone. +// teardownClient keeps hotplug state until removal or garbage collection, +// including when the VMI outlives its VM during CAPI/CAPK teardown. type teardownClient struct { kubevirtclient.Client - vmiOwners []metav1.OwnerReference - vmiRemovals int - lookups int + vm *kubevirtv1.VirtualMachine + vmiOwners []metav1.OwnerReference + detached bool + vmiGone bool + vmiGoneAfterCheck bool + vmiLookups int + vmRemovals int + vmiRemovals int + lookups int } func (c *teardownClient) vmiGarbageCollected() bool { @@ -328,22 +332,46 @@ func (c *teardownClient) vmiGarbageCollected() bool { func (c *teardownClient) GetWorkloadManagingVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachine, error) { c.lookups++ + if c.vm != nil { + return c.vm, nil + } return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachines"}, testVMName) } func (c *teardownClient) GetVirtualMachine(context.Context, string, string) (*kubevirtv1.VirtualMachineInstance, error) { c.lookups++ + c.vmiLookups++ + if c.vmiGone { + return nil, k8serrors.NewNotFound(schema.GroupResource{Group: "kubevirt.io", Resource: "virtualmachineinstances"}, testVMName) + } vmi := &kubevirtv1.VirtualMachineInstance{ ObjectMeta: metav1.ObjectMeta{Name: testVMName, Namespace: testInfraNamespace, OwnerReferences: c.vmiOwners}, } - if c.vmiRemovals == 0 { - vmi.Status.VolumeStatus = []kubevirtv1.VolumeStatus{{Name: testVolumeName, HotplugVolume: &kubevirtv1.HotplugVolumeStatus{}}} + if !c.detached && c.vmRemovals == 0 && c.vmiRemovals == 0 { + vmi.Status.VolumeStatus = []kubevirtv1.VolumeStatus{{Name: testVolumeName, Phase: kubevirtv1.VolumePending, Reason: "FailedPvcNotFound", HotplugVolume: &kubevirtv1.HotplugVolumeStatus{}}} } return vmi, nil } +func (c *teardownClient) EnsureVolumeRemovedVM(context.Context, string, string, string) (bool, error) { + return c.vm == nil || len(c.vm.Spec.Template.Spec.Volumes) == 0, nil +} + func (c *teardownClient) EnsureVolumeRemovedVMI(context.Context, string, string, string) (bool, error) { - return c.vmiRemovals > 0, nil + removed := c.vmiGone || c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0 + if c.vmiGoneAfterCheck { + c.vmiGone = true + } + return removed, nil +} + +func (c *teardownClient) RemoveVolumeFromVM(_ context.Context, namespace, name string, opts *kubevirtv1.RemoveVolumeOptions) error { + if namespace != testInfraNamespace || name != testVMName || opts.Name != testVolumeName { + return errors.New("unexpected hotplug removal target") + } + c.vmRemovals++ + c.vm.Spec.Template.Spec.Volumes = nil + return nil } func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *kubevirtv1.RemoveVolumeOptions) error { @@ -352,7 +380,7 @@ func (c *teardownClient) RemoveVolumeFromVMI(context.Context, string, string, *k } func (c *teardownClient) EnsureVolumeRemoved(context.Context, string, string, string, time.Duration) error { - if c.vmiRemovals > 0 || c.vmiGarbageCollected() { + if c.vmiGone || c.detached || c.vmRemovals > 0 || c.vmiRemovals > 0 || c.vmiGarbageCollected() { return nil } return errors.New("timed out waiting for the VMI to drop the volume") @@ -410,6 +438,58 @@ func TestControllerUnpublishVolumeHonoursVMIHotplugFallback(t *testing.T) { } } +func TestControllerUnpublishVolumeMissingPVC(t *testing.T) { + tests := []struct { + name string + vmGone bool + detached bool + vmiGone bool + vmiGoneAfterCheck bool + wantVMILookups int + wantVMCalls int + wantVMICalls int + }{ + {name: "pending hotplug volume", wantVMCalls: 1}, + {name: "already detached", detached: true}, + {name: "VM gone, VMI still has volume", vmGone: true, wantVMICalls: 1, wantVMILookups: 1}, + {name: "VM gone, no attached volume", vmGone: true, detached: true}, + {name: "VM and VMI gone", vmGone: true, vmiGone: true}, + {name: "VMI disappears after attachment check", vmGone: true, vmiGoneAfterCheck: true, wantVMILookups: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := &teardownClient{detached: tt.detached, vmiGone: tt.vmiGone, vmiGoneAfterCheck: tt.vmiGoneAfterCheck} + if !tt.vmGone { + client.vm = &kubevirtv1.VirtualMachine{Spec: kubevirtv1.VirtualMachineSpec{Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{}}} + if !tt.detached { + client.vm.Spec.Template.Spec.Volumes = []kubevirtv1.Volume{{ + Name: testVolumeName, + VolumeSource: kubevirtv1.VolumeSource{DataVolume: &kubevirtv1.DataVolumeSource{Name: testVolumeName, Hotpluggable: true}}, + }} + } + } + w := &WrappedControllerService{ + ControllerService: service.NewControllerService(client, testInfraNamespace, nil, util.StorageClassEnforcement{AllowAll: true}, true), + infraClient: fake.NewClientset(), + infraNamespace: testInfraNamespace, + } + req := &csi.ControllerUnpublishVolumeRequest{VolumeId: testVolumeName, NodeId: testInfraNamespace + "/" + testVMName} + if _, err := w.ControllerUnpublishVolume(context.Background(), req); err != nil { + t.Fatalf("ControllerUnpublishVolume() error = %v", err) + } + if client.vmRemovals != tt.wantVMCalls || client.vmiRemovals != tt.wantVMICalls { + t.Errorf("VM/VMI removevolume calls = %d/%d, want %d/%d", client.vmRemovals, client.vmiRemovals, tt.wantVMCalls, tt.wantVMICalls) + } + if client.vmiLookups != tt.wantVMILookups { + t.Errorf("GetVirtualMachine calls = %d, want %d", client.vmiLookups, tt.wantVMILookups) + } + if client.vm != nil && len(client.vm.Spec.Template.Spec.Volumes) != 0 { + t.Error("hotplug volume remains in VM spec") + } + }) + } +} + func nfsCNP(owners ...string) *unstructured.Unstructured { refs := make([]interface{}, 0, len(owners)) for _, owner := range owners { diff --git a/packages/apps/kubernetes/templates/_helpers.tpl b/packages/apps/kubernetes/templates/_helpers.tpl index 300b12732..e27f5ccdb 100644 --- a/packages/apps/kubernetes/templates/_helpers.tpl +++ b/packages/apps/kubernetes/templates/_helpers.tpl @@ -5,24 +5,6 @@ Expand the name of the chart. {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "kubernetes.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - {{/* Create chart name and version as used by the chart label. */}} @@ -185,29 +167,6 @@ never boots. {{- printf "has(claims.groups) && dyn(claims.groups).exists(g, g in [\"%s-view\", \"%s-use\", \"%s-admin\", \"%s-super-admin\"])" $ns $ns $ns $ns -}} {{- end }} -{{- /* -Validates and returns a duration destined for a consumer that does not reject -a bad value: the cluster-autoscaler parses its annotation with -time.ParseDuration and silently falls back to its built-in default on a value -it cannot parse, and the CAPI webhook does not validate unhealthyConditions -timeouts at all, so a zero or negative one applies and remediates a Machine the -moment its condition flips. The values schema types these fields as strings and -stops there, which lets "30" and "-5m" reach the template. The accepted shape -is narrower than Go's duration grammar on purpose, because that grammar also -admits a value large enough to overflow the parser -- the silent fallback again --- and a fraction small enough to round to zero. The first segment must be -positive; later segments may be zero, so the canonical rendering of a duration -("20m0s", "1h0m0s" -- what metav1.Duration serializes to) round-trips. Every -string this admits parses to a positive duration. -*/ -}} -{{- define "kubernetes.positiveDuration" -}} -{{- $value := toString .value -}} -{{- if not (regexMatch "^[1-9][0-9]{0,4}[smh]([0-9]{1,5}[smh]){0,2}$" $value) -}} -{{- fail (printf "%s must be a whole number of s, m or h (e.g. 30m, 1h30m), got %q" .field $value) -}} -{{- end -}} -{{- $value -}} -{{- end -}} - {{/* The two kube-apiserver flags the tenant control plane owns that have a KamajiControlPlane field it renders them from, keyed to that field. diff --git a/packages/apps/kubernetes/templates/_resources.tpl b/packages/apps/kubernetes/templates/_resources.tpl deleted file mode 100644 index 6539c99a1..000000000 --- a/packages/apps/kubernetes/templates/_resources.tpl +++ /dev/null @@ -1,49 +0,0 @@ -{{/* -Copyright Broadcom, Inc. All Rights Reserved. -SPDX-License-Identifier: APACHE-2.0 -*/}} - -{{/* vim: set filetype=mustache: */}} - -{{/* -Return a resource request/limit object based on a given preset. -These presets are for basic testing and not meant to be used in production -{{ include "resources.preset" (dict "type" "nano") -}} -*/}} -{{- define "resources.preset" -}} -{{- $presets := dict - "nano" (dict - "requests" (dict "cpu" "100m" "memory" "128Mi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "128Mi" "ephemeral-storage" "2Gi") - ) - "micro" (dict - "requests" (dict "cpu" "250m" "memory" "256Mi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "256Mi" "ephemeral-storage" "2Gi") - ) - "small" (dict - "requests" (dict "cpu" "500m" "memory" "512Mi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "512Mi" "ephemeral-storage" "2Gi") - ) - "medium" (dict - "requests" (dict "cpu" "500m" "memory" "1Gi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "1Gi" "ephemeral-storage" "2Gi") - ) - "large" (dict - "requests" (dict "cpu" "1" "memory" "2Gi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "2Gi" "ephemeral-storage" "2Gi") - ) - "xlarge" (dict - "requests" (dict "cpu" "2" "memory" "4Gi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "4Gi" "ephemeral-storage" "2Gi") - ) - "2xlarge" (dict - "requests" (dict "cpu" "4" "memory" "8Gi" "ephemeral-storage" "50Mi") - "limits" (dict "memory" "8Gi" "ephemeral-storage" "2Gi") - ) - }} -{{- if hasKey $presets .type -}} -{{- index $presets .type | toYaml -}} -{{- else -}} -{{- printf "ERROR: Preset key '%s' invalid. Allowed values are %s" .type (join "," (keys $presets)) | fail -}} -{{- end -}} -{{- end -}} diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index b27582694..e7c9118ef 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -1,29 +1,3 @@ -{{- /* - Talos -> Kubernetes support matrix. Each Talos minor release supports a - bounded window of Kubernetes minor versions; running a kubelet outside - that window produces a silently broken Talos+kubelet combination that - HelmRelease cannot detect. Fail the chart render instead, so the operator - sees the mismatch in Flux/dashboards before workers roll over. - - Upstream declares the window per Talos *minor* release, and patch releases - never move it, so the lookup key is the minor derived from talos.version. - Keying on the full patch version would silently disable this guard on every - patch bump — exactly the mismatch it exists to catch. - - The matrix only fires for Talos minors it knows about; operators who - override talos.version to something outside this list take responsibility - for compatibility themselves (extend the dict or upstream a PR). - Source: https://docs.siderolabs.com/talos/v1.13/getting-started/support-matrix -*/}} -{{- $talosK8sSupportMatrix := dict - "v1.13" (list "v1.31" "v1.32" "v1.33" "v1.34" "v1.35" "v1.36") -}} -{{- $talosMinor := regexFind "^v[0-9]+\\.[0-9]+" .Values.talos.version }} -{{- with index $talosK8sSupportMatrix $talosMinor }} -{{- if not (has $.Values.version .) }} -{{- fail (printf "Kubernetes %s is not supported by Talos %s. Supported versions: %v. Bump tenant 'version' or set talos.version explicitly." $.Values.version $.Values.talos.version .) }} -{{- end }} -{{- end }} {{- $etcd := .Values._namespace.etcd }} {{- /* Substrate selects the infrastructure half of the cluster. The control plane is Kamaji either way, so this changes exactly one object — the @@ -66,20 +40,23 @@ */}} {{- /* Worker node pools (KubevirtMachineTemplate, MachineDeployment, MachineHealthCheck, worker WorkloadMonitor) and the Talos - reconcile Job that produces each pool's TalosConfigTemplate now - live in the separate `kubernetes-nodes` chart (one HelmRelease - per pool). This chart renders the control plane only. The - reconcile Job in the kubernetes-nodes release still re-patches - this KamajiControlPlane's apiserver-Service ClusterIP-SAN and - owns each TalosConfigTemplate via an ownerReference to this KCP, - so garbage collection on control-plane uninstall is unchanged. */}} + reconcile Job that produces each pool's TalosConfigTemplate are + rendered by the `kubernetes-nodes` chart (one HelmRelease per + pool). This chart renders the control plane only. The reconcile + Job in the kubernetes-nodes release re-patches this + KamajiControlPlane's apiserver-Service ClusterIP-SAN and owns + each TalosConfigTemplate via an ownerReference to this KCP, so + uninstalling the control plane garbage-collects them. */}} {{- /* Cluster pod/service CIDRs MUST match between the CAPI Cluster spec.clusterNetwork below and the Talos worker machineconfig's cluster.network — Talos surfaces overlaps against the host pod CIDR as a diagnostic at boot, and divergence between CAPI and Talos would - manifest as silently broken kube-proxy / Cilium routing. Keep them - in one place here. + manifest as silently broken kube-proxy / Cilium routing. The worker + side is a copy of these two literals in + packages/apps/kubernetes-nodes/templates/_talosconfigtemplate.tpl, and + templates/helmreleases/coredns.yaml pins the DNS Service IP inside the + service CIDR, so a change here is a change in all three places. */}} {{- $podCIDR := "10.243.0.0/16" }} {{- $serviceCIDR := "10.95.0.0/16" }} @@ -104,7 +81,7 @@ data: to reconcile - this HelmRelease will pick up the DataStore on its next 5m reconcile loop and provision the cluster. {{- else }} -{{- /* Phase 2 worker-pool adoption guard. Worker pools now live in per-pool +{{- /* Worker-pool adoption guard. Worker pools live in per-pool kubernetes-nodes releases; the platform migration adopts each pre-split pool's live objects onto its child release (re-annotating meta.helm.sh/release-name, pinning helm.sh/resource-policy=keep) as a diff --git a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml index 746b80583..52c178d8f 100644 --- a/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/ingress-nginx.yaml @@ -23,9 +23,8 @@ ingress-nginx: {{- if and .Values.addons.ingressNginx.enabled .Values._namespace.etcd }} {{- /* The ingress-nginx controller DaemonSet pins itself to nodes labelled node-role.kubernetes.io/ingress-nginx (see cozystack.defaultIngressValues - nodeSelector). Since Phase 2 moved worker pools into the separate - kubernetes-nodes chart, that label is produced by a KubernetesNodes pool - with roles: [ingress-nginx]. If no such pool exists the DaemonSet simply + nodeSelector). That label is produced by a KubernetesNodes pool with + roles: [ingress-nginx]. If no such pool exists the DaemonSet simply schedules nothing (degraded, not failed) — the requirement is documented in NOTES.txt and the release note rather than enforced at render time, so an existing cluster is never flipped from Ready to Failed on upgrade. */}} diff --git a/packages/apps/kubernetes/templates/oidc-authn-config.yaml b/packages/apps/kubernetes/templates/oidc-authn-config.yaml index 5a36be06a..8fe2e93b2 100644 --- a/packages/apps/kubernetes/templates/oidc-authn-config.yaml +++ b/packages/apps/kubernetes/templates/oidc-authn-config.yaml @@ -6,8 +6,8 @@ /etc/kubernetes/authentication-config/config.yaml and consumed via --authentication-config=. Compared to the legacy `--oidc-*` flags, the file form supports multiple issuers (the `jwt` block is a list) - and inline private-CA PEM, so future BYO and Phase-2 issuers extend - the same Secret instead of fighting the chart shape. + and inline private-CA PEM, so further issuers extend the same Secret + instead of fighting the chart shape. System mode: Single JWT entry built from `_cluster.root-host` (the cozy realm diff --git a/packages/apps/kubernetes/templates/oidc-keycloak.yaml b/packages/apps/kubernetes/templates/oidc-keycloak.yaml index a8687faf1..99ddcf695 100644 --- a/packages/apps/kubernetes/templates/oidc-keycloak.yaml +++ b/packages/apps/kubernetes/templates/oidc-keycloak.yaml @@ -1,5 +1,5 @@ {{- /* - Phase 1 — System mode: render one per-cluster Keycloak client and a + System mode: render one per-cluster Keycloak client and a per-cluster audience client scope inside the platform `cozy` realm. The clientId is namespace-prefixed so it is globally unique across the diff --git a/packages/apps/kubernetes/templates/talos/bootstrap-token-tenant-job.yaml b/packages/apps/kubernetes/templates/talos/bootstrap-token-tenant-job.yaml index 642ac4691..9e8805003 100644 --- a/packages/apps/kubernetes/templates/talos/bootstrap-token-tenant-job.yaml +++ b/packages/apps/kubernetes/templates/talos/bootstrap-token-tenant-job.yaml @@ -8,10 +8,9 @@ kube-system, which the management cluster can only reach via the Kamaji-issued admin kubeconfig stored as -admin-kubeconfig. - Why a main-phase resource and not a Helm post-install/post-upgrade hook - (the earlier shape): the hook is unreliable on the very failure it was - meant to cover. This release carries - release.cozystack.io/helm-install-disable-wait: "true" (cozystack-api + Why a main-phase resource and not a Helm post-install/post-upgrade hook: + the hook is unreliable on the very failure it was meant to cover. This + release carries release.cozystack.io/helm-install-disable-wait: "true" (cozystack-api sets it on both Install.DisableWait and Upgrade.DisableWait), so there is NO main-resource wait phase in front of the hooks — a post-install hook is not gated on MachineDeployment Ready and does fire on a fresh @@ -25,9 +24,9 @@ -admin-kubeconfig (materialised by Kamaji after the KamajiControlPlane is applied) is not yet available. - As an ordinary main-phase resource (mirroring talos-reconcile-job.yaml) - the Job is applied on every install and every reconcile regardless of - hook phase. It lands together with the KamajiControlPlane and + As an ordinary main-phase resource (like talos-reconcile-job.yaml in the + kubernetes-nodes chart) the Job is applied on every install and every + reconcile regardless of hook phase. It lands together with the KamajiControlPlane and talos-secrets; a wait-for-kubeconfig init container holds the Pod until Kamaji publishes the admin kubeconfig Secret (the Secret volume is optional so kubelet does not FailedMount in the meantime), then the @@ -40,9 +39,9 @@ upgrade that lands while the Job is live (or inside ttlSecondsAfterFinished) with `spec.template: field is immutable`, and RetryOnFailure would loop on it. The Job therefore carries a - content-hash name suffix over its whole rendered spec (mirrors - talos-reconcile-job.yaml and the KubevirtMachineTemplate idiom in - cluster.yaml): an identical render keeps the name and Helm no-ops it; + content-hash name suffix over its whole rendered spec (the idiom of + talos-reconcile-job.yaml and the KubevirtMachineTemplate name in the + kubernetes-nodes chart): an identical render keeps the name and Helm no-ops it; any change that flows into the spec — notably a new images/kubectl.tag — yields a new name so a fresh Job runs and Helm prunes the old one. The hash also folds in the looked-up bootstrap token — both halves, diff --git a/packages/apps/kubernetes/tests/control_plane_only_test.yaml b/packages/apps/kubernetes/tests/control_plane_only_test.yaml index c72c333a4..e89b6e795 100644 --- a/packages/apps/kubernetes/tests/control_plane_only_test.yaml +++ b/packages/apps/kubernetes/tests/control_plane_only_test.yaml @@ -1,8 +1,8 @@ -suite: control-plane-only cluster.yaml render (worker pools moved to kubernetes-nodes) +suite: control-plane-only cluster.yaml render (worker pools are kubernetes-nodes releases) -# Phase 2 moved the worker node pools (KubevirtMachineTemplate, MachineDeployment, +# The worker node pools (KubevirtMachineTemplate, MachineDeployment, # MachineHealthCheck, worker WorkloadMonitor) and the per-pool talos-reconcile Job -# into the separate kubernetes-nodes chart. cluster.yaml now renders the control +# are rendered by the kubernetes-nodes chart. cluster.yaml renders the control # plane only, in source order: Cluster (0), KamajiControlPlane (1), the # control-plane WorkloadMonitor (2), KubevirtCluster (3). This suite pins that # surface so a regression that re-introduces pool objects into the parent chart diff --git a/packages/apps/kubernetes/tests/oidc_test.yaml b/packages/apps/kubernetes/tests/oidc_test.yaml index 8511e392c..55a34aba9 100644 --- a/packages/apps/kubernetes/tests/oidc_test.yaml +++ b/packages/apps/kubernetes/tests/oidc_test.yaml @@ -1,12 +1,11 @@ -suite: tenant kube-apiserver OIDC (Phase 1) +suite: tenant kube-apiserver OIDC values: - values/common.yaml # In cluster.yaml the KamajiControlPlane is document index 1 -# (0: Cluster, 1: KamajiControlPlane, 2: WorkloadMonitor, 3: KubevirtCluster, ...). -# Documents at higher indices are KubevirtMachineTemplate, MachineDeployment, -# MachineHealthCheck, WorkloadMonitor per node group. +# (0: Cluster, 1: KamajiControlPlane, 2: WorkloadMonitor, 3: KubevirtCluster); +# tests/control_plane_only_test.yaml pins that there are no others. tests: ############################################### diff --git a/packages/apps/kubernetes/tests/stored_talos_values_test.yaml b/packages/apps/kubernetes/tests/stored_talos_values_test.yaml new file mode 100644 index 000000000..f95d05d85 --- /dev/null +++ b/packages/apps/kubernetes/tests/stored_talos_values_test.yaml @@ -0,0 +1,41 @@ +suite: stored Kubernetes specs that still carry a talos block + +# The chart declares no `talos` key: worker pools are KubernetesNodes releases +# and read their own `talos`, and the Talos to Kubernetes support matrix is +# checked there. A Kubernetes spec stored before that still carries the block, +# and both the values schema (no additionalProperties: false at the root) and +# the aggregated API accept it, so the chart has to keep rendering it as if it +# were absent. + +templates: + - templates/cluster.yaml + +values: + - values/common.yaml + +release: + name: test-k8s + namespace: tenant-test + +tests: + - it: renders the same control plane for a stored spec carrying every talos key the chart used to declare + set: + talos: + version: "v1.13.6" + schematicID: "ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515" + imageFactoryURL: "https://factory.example.com" + installerRepository: "registry.example.com/installer" + registryMirrors: + ghcr.io: + endpoints: + - https://registry.example.com + version: "v1.31" + asserts: + - hasDocuments: + count: 4 + - isKind: + of: Cluster + documentIndex: 0 + - isKind: + of: KamajiControlPlane + documentIndex: 1 diff --git a/packages/apps/kubernetes/tests/talos_k8s_support_matrix_test.yaml b/packages/apps/kubernetes/tests/talos_k8s_support_matrix_test.yaml deleted file mode 100644 index 176813dd8..000000000 --- a/packages/apps/kubernetes/tests/talos_k8s_support_matrix_test.yaml +++ /dev/null @@ -1,53 +0,0 @@ -suite: Talos to Kubernetes support-matrix guard tests - -# Each Talos minor release supports a bounded window of Kubernetes minor -# versions; running a kubelet outside that window produces a silently -# broken Talos+kubelet combination that HelmRelease cannot detect. The -# chart fails render with a clear message instead, so the operator sees -# the mismatch in Flux/dashboards before workers roll over. Source of -# truth is the $talosK8sSupportMatrix dict at the top of cluster.yaml. -# -# The dict is keyed by Talos minor, and the template derives that minor from -# talos.version, so every patch release of a supported minor stays guarded. -# -# Failure-path coverage is left as a follow-up: values.schema.json -# enforces Version enum (v1.31..v1.35), which is strictly tighter than -# the Talos v1.13 matrix entry (v1.31..v1.36), so the only way to -# trigger the fail() guard via helm-unittest is to feed a Version the -# schema rejects before the template ever runs. Schema-rejection happens -# before template execution, so the chart fail() message is unreachable -# from the test harness until either the matrix gains an exclusion the -# schema enum still allows, or helm-unittest gains a schema-skip toggle. - -templates: - - templates/cluster.yaml - -values: - - values/common.yaml - -tests: - - it: accepts a Kubernetes version inside the Talos v1.13 support window - release: - name: test-k8s - namespace: tenant-test - set: - talos: - version: "v1.13.6" - schematicID: "ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515" - version: "v1.31" - asserts: - - hasDocuments: - count: 4 - - - it: keeps rendering for an older patch release of the same Talos minor - release: - name: test-k8s - namespace: tenant-test - set: - talos: - version: "v1.13.0" - schematicID: "ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515" - version: "v1.31" - asserts: - - hasDocuments: - count: 4 diff --git a/packages/apps/kubernetes/tests/values/common.yaml b/packages/apps/kubernetes/tests/values/common.yaml index 6377414e1..919116caf 100644 --- a/packages/apps/kubernetes/tests/values/common.yaml +++ b/packages/apps/kubernetes/tests/values/common.yaml @@ -19,5 +19,5 @@ controlPlane: server: resources: {} resourcesPreset: "micro" -# Worker pools are managed via KubernetesNodes resources since the Phase 2 split; -# the kubernetes chart renders the control plane only. +# Worker pools are KubernetesNodes resources; the kubernetes chart renders the +# control plane only. diff --git a/packages/apps/kubernetes/tests/worker_adoption_guard_test.yaml b/packages/apps/kubernetes/tests/worker_adoption_guard_test.yaml index b95d6f414..e7bb3996a 100644 --- a/packages/apps/kubernetes/tests/worker_adoption_guard_test.yaml +++ b/packages/apps/kubernetes/tests/worker_adoption_guard_test.yaml @@ -1,4 +1,4 @@ -suite: Phase 2 worker-pool adoption guard +suite: worker-pool adoption guard # The guard in templates/cluster.yaml refuses to render the control-plane-only # chart when a live worker MachineDeployment for this cluster is NOT owned by diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index 3885bccf6..23af2636d 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -3,7 +3,7 @@ "type": "object", "x-cozystack-name": { "type": "string", - "description": "Cluster name. Every worker pool is a separate KubernetesNodes release named `kubernetes-nodes-\u003cname\u003e-\u003cpool\u003e`, so the name must leave room for the default `md0` pool inside the 53-character Helm release name limit.", + "description": "Cluster name. Each worker pool is a KubernetesNodes release named `kubernetes-nodes-\u003cname\u003e-\u003cpool\u003e`, which has to fit the 53-character Helm release name limit, so a cluster name of at most 32 characters leaves at least three characters for the pool name.", "maxLength": 32 }, "properties": { @@ -34,12 +34,12 @@ ] }, "host": { - "description": "External hostname for Kubernetes cluster. Defaults to `\u003ccluster-name\u003e.\u003ctenant-host\u003e` if empty.", + "description": "External hostname for Kubernetes cluster. Defaults to `kubernetes-\u003ccluster-name\u003e.\u003ctenant-host\u003e` if empty.", "type": "string", "default": "" }, "substrate": { - "description": "Which infrastructure provider backs this cluster's worker VMs. `kubevirt` runs them inside this cluster; `proxmox` runs them on an external Proxmox VE cluster through capmox. The control plane is Kamaji either way — this selects the infrastructure half only. Must match the `substrate` of every KubernetesNodes pool attached to this cluster: the pools reference this cluster's infrastructure object by kind, and a mismatch leaves their Machines unreconciled. Switching an existing cluster is not supported; create a new one.", + "description": "Which infrastructure provider backs this cluster's worker VMs. `kubevirt` runs them inside this cluster; `proxmox` runs them on an external Proxmox VE cluster through capmox. The control plane is Kamaji either way — this selects the infrastructure half only. Every KubernetesNodes pool of this cluster must set the same `substrate`: a pool set to the other one refuses to render once this cluster exists. Switching an existing cluster is not supported; create a new one.", "type": "string", "default": "kubevirt", "x-kubernetes-validations": [ @@ -506,7 +506,7 @@ } }, "extraArgs": { - "description": "Extra command-line flags appended to the tenant kube-apiserver, passed through to KamajiControlPlane `spec.apiServer.extraArgs`. For OIDC use `spec.oidc.mode` — this passthrough is the escape hatch for unrelated apiserver flags. The tenant control plane sets each of the following flags itself and drops an entry naming one before the apiserver sees it. The chart moves an `--enable-admission-plugins=` entry into `controlPlane.apiServer.admissionControllers` and a `--kubelet-preferred-address-types=` entry into `controlPlane.kubelet.preferredAddressTypes` while that field is empty, and refuses any other entry naming one of them: `--advertise-address`, `--client-ca-file`, `--enable-admission-plugins`, `--service-cluster-ip-range`, `--kubelet-client-certificate`, `--kubelet-client-key`, `--kubelet-preferred-address-types`, `--proxy-client-cert-file`, `--proxy-client-key-file`, `--requestheader-allowed-names`, `--requestheader-client-ca-file`, `--secure-port`, `--service-account-key-file`, `--service-account-signing-key-file`, `--tls-cert-file`, `--tls-private-key-file`, `--etcd-servers`, `--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, `--etcd-prefix`, `--etcd-compaction-interval`, `--egress-selector-config-file`. When `spec.oidc.mode` is not `None` the chart also renders OIDC flags here, reads what you have written on them, and refuses some combinations outright; the Aggregated API servers section of the Kubernetes OIDC guide on cozystack.io carries which flags, which combinations, and why. Do NOT add legacy `--oidc-*` flags here when `spec.oidc.mode` is not `None`; the chart injects `--authentication-config` and the apiserver refuses to boot with both. Empty by default (no change to current behavior).", + "description": "Extra command-line flags appended to the tenant kube-apiserver, passed through to KamajiControlPlane `spec.apiServer.extraArgs`. For OIDC use `spec.oidc.mode` — this passthrough is the escape hatch for unrelated apiserver flags. The tenant control plane sets each of the following flags itself and drops an entry naming one before the apiserver sees it. The chart moves an `--enable-admission-plugins=` entry into `controlPlane.apiServer.admissionControllers` and a `--kubelet-preferred-address-types=` entry into `controlPlane.kubelet.preferredAddressTypes` while that field is empty, and refuses any other entry naming one of them: `--advertise-address`, `--client-ca-file`, `--enable-admission-plugins`, `--service-cluster-ip-range`, `--kubelet-client-certificate`, `--kubelet-client-key`, `--kubelet-preferred-address-types`, `--proxy-client-cert-file`, `--proxy-client-key-file`, `--requestheader-allowed-names`, `--requestheader-client-ca-file`, `--secure-port`, `--service-account-key-file`, `--service-account-signing-key-file`, `--tls-cert-file`, `--tls-private-key-file`, `--etcd-servers`, `--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, `--etcd-prefix`, `--etcd-compaction-interval`, `--egress-selector-config-file`. When `spec.oidc.mode` is not `None` the chart also renders OIDC flags here, reads what you have written on them, and refuses some combinations outright; the Aggregated API servers section of the Kubernetes OIDC guide on cozystack.io carries which flags, which combinations, and why. Do NOT add legacy `--oidc-*` flags here when `spec.oidc.mode` is not `None`; the chart injects `--authentication-config` and the apiserver refuses to boot with both. Empty by default.", "type": "array", "default": [], "items": { @@ -953,46 +953,6 @@ } } }, - "talos": { - "description": "Talos worker image configuration.", - "type": "object", - "default": {}, - "required": [ - "imageFactoryURL", - "installerRepository", - "registryMirrors", - "schematicID", - "version" - ], - "properties": { - "imageFactoryURL": { - "description": "Base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash.", - "type": "string", - "default": "https://factory.talos.dev" - }, - "installerRepository": { - "description": "OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `\u003cinstallerRepository\u003e/\u003cschematicID\u003e:\u003cversion\u003e`. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.", - "type": "string", - "default": "factory.talos.dev/installer" - }, - "registryMirrors": { - "description": "Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap.", - "type": "object", - "default": {}, - "x-kubernetes-preserve-unknown-fields": true - }, - "schematicID": { - "description": "Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here.", - "type": "string", - "default": "ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515" - }, - "version": { - "description": "Talos release used for worker OS image and installer. Must satisfy the chart's Talos\u003c-\u003eKubernetes support matrix against the chosen `version`.", - "type": "string", - "default": "v1.13.6" - } - } - }, "oidc": { "description": "OIDC authentication and per-user RBAC for the tenant kube-apiserver. See https://cozystack.io/docs/v1.6/kubernetes/oidc-authentication/ for the operator guide.", "type": "object", diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index 007b54a53..2ee0d1580 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -1,4 +1,4 @@ -## @name {string} - Cluster name. Every worker pool is a separate KubernetesNodes release named `kubernetes-nodes--`, so the name must leave room for the default `md0` pool inside the 53-character Helm release name limit. +## @name {string} - Cluster name. Each worker pool is a KubernetesNodes release named `kubernetes-nodes--`, which has to fit the 53-character Helm release name limit, so a cluster name of at most 32 characters leaves at least three characters for the pool name. ## @maxLength 32 ## @@ -79,13 +79,13 @@ storageClass: replicated version: "v1.35" -## @param {string} host - External hostname for Kubernetes cluster. Defaults to `.` if empty. +## @param {string} host - External hostname for Kubernetes cluster. Defaults to `kubernetes-.` if empty. host: "" ## ## @section Substrate -## @param {string} substrate="kubevirt" - Which infrastructure provider backs this cluster's worker VMs. `kubevirt` runs them inside this cluster; `proxmox` runs them on an external Proxmox VE cluster through capmox. The control plane is Kamaji either way — this selects the infrastructure half only. Must match the `substrate` of every KubernetesNodes pool attached to this cluster: the pools reference this cluster's infrastructure object by kind, and a mismatch leaves their Machines unreconciled. Switching an existing cluster is not supported; create a new one. +## @param {string} substrate="kubevirt" - Which infrastructure provider backs this cluster's worker VMs. `kubevirt` runs them inside this cluster; `proxmox` runs them on an external Proxmox VE cluster through capmox. The control plane is Kamaji either way — this selects the infrastructure half only. Every KubernetesNodes pool of this cluster must set the same `substrate`: a pool set to the other one refuses to render once this cluster exists. Switching an existing cluster is not supported; create a new one. ## @immutable substrate: kubevirt @@ -288,7 +288,7 @@ addons: ## @typedef {struct} APIServer - API Server configuration. ## @field {Resources} resources - CPU and memory resources for API Server. ## @field {ResourcesPreset} resourcesPreset="c1.medium" - Preset if `resources` omitted. -## @field {[]string} extraArgs - Extra command-line flags appended to the tenant kube-apiserver, passed through to KamajiControlPlane `spec.apiServer.extraArgs`. For OIDC use `spec.oidc.mode` — this passthrough is the escape hatch for unrelated apiserver flags. The tenant control plane sets each of the following flags itself and drops an entry naming one before the apiserver sees it. The chart moves an `--enable-admission-plugins=` entry into `controlPlane.apiServer.admissionControllers` and a `--kubelet-preferred-address-types=` entry into `controlPlane.kubelet.preferredAddressTypes` while that field is empty, and refuses any other entry naming one of them: `--advertise-address`, `--client-ca-file`, `--enable-admission-plugins`, `--service-cluster-ip-range`, `--kubelet-client-certificate`, `--kubelet-client-key`, `--kubelet-preferred-address-types`, `--proxy-client-cert-file`, `--proxy-client-key-file`, `--requestheader-allowed-names`, `--requestheader-client-ca-file`, `--secure-port`, `--service-account-key-file`, `--service-account-signing-key-file`, `--tls-cert-file`, `--tls-private-key-file`, `--etcd-servers`, `--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, `--etcd-prefix`, `--etcd-compaction-interval`, `--egress-selector-config-file`. When `spec.oidc.mode` is not `None` the chart also renders OIDC flags here, reads what you have written on them, and refuses some combinations outright; the Aggregated API servers section of the Kubernetes OIDC guide on cozystack.io carries which flags, which combinations, and why. Do NOT add legacy `--oidc-*` flags here when `spec.oidc.mode` is not `None`; the chart injects `--authentication-config` and the apiserver refuses to boot with both. Empty by default (no change to current behavior). +## @field {[]string} extraArgs - Extra command-line flags appended to the tenant kube-apiserver, passed through to KamajiControlPlane `spec.apiServer.extraArgs`. For OIDC use `spec.oidc.mode` — this passthrough is the escape hatch for unrelated apiserver flags. The tenant control plane sets each of the following flags itself and drops an entry naming one before the apiserver sees it. The chart moves an `--enable-admission-plugins=` entry into `controlPlane.apiServer.admissionControllers` and a `--kubelet-preferred-address-types=` entry into `controlPlane.kubelet.preferredAddressTypes` while that field is empty, and refuses any other entry naming one of them: `--advertise-address`, `--client-ca-file`, `--enable-admission-plugins`, `--service-cluster-ip-range`, `--kubelet-client-certificate`, `--kubelet-client-key`, `--kubelet-preferred-address-types`, `--proxy-client-cert-file`, `--proxy-client-key-file`, `--requestheader-allowed-names`, `--requestheader-client-ca-file`, `--secure-port`, `--service-account-key-file`, `--service-account-signing-key-file`, `--tls-cert-file`, `--tls-private-key-file`, `--etcd-servers`, `--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, `--etcd-prefix`, `--etcd-compaction-interval`, `--egress-selector-config-file`. When `spec.oidc.mode` is not `None` the chart also renders OIDC flags here, reads what you have written on them, and refuses some combinations outright; the Aggregated API servers section of the Kubernetes OIDC guide on cozystack.io carries which flags, which combinations, and why. Do NOT add legacy `--oidc-*` flags here when `spec.oidc.mode` is not `None`; the chart injects `--authentication-config` and the apiserver refuses to boot with both. Empty by default. ## @field {[]AdmissionController} admissionControllers - Admission plugins the tenant kube-apiserver enables, passed through to KamajiControlPlane `spec.admissionControllers`, which the control plane renders as `--enable-admission-plugins`. A non-empty list replaces the control plane's default set instead of adding to it. The kube-apiserver enables these plugins in addition to its own default ones, so leaving a plugin out does not turn it off; to turn one off, name it in `--disable-admission-plugins` in `controlPlane.apiServer.extraArgs` and keep it out of this list, since the apiserver refuses to start with a plugin in both, and the control plane's default set names several. Empty by default, which keeps that default set, taken from the TenantControlPlane CRD: CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, LimitRanger, MutatingAdmissionWebhook, NamespaceLifecycle, PersistentVolumeClaimResize, Priority, ResourceQuota, RuntimeClass, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. `NodeRestriction` is not in it. ## @field {[]object} extraVolumes - Extra volumes added to the control-plane Deployment, passed through to KamajiControlPlane `spec.deployment.extraVolumes`. Use to mount a ConfigMap or Secret holding an AuthenticationConfiguration file referenced by `extraArgs`. Each item is a core/v1 Volume, but because the control-plane pod runs on the management cluster only `configMap` and `secret` sources are allowed (host-reaching sources like `hostPath`/`csi` and token-projection via `projected` are rejected); each volume must have a unique, non-empty name and exactly one source. The names `talos-ca` and `talos-tls-cert` are reserved by the chart. Empty by default. ## @field {[]object} extraVolumeMounts - Extra volume mounts added to the tenant kube-apiserver container, passed through to KamajiControlPlane `spec.apiServer.extraVolumeMounts`. Each `name` must reference a volume declared in `extraVolumes`; the chart-managed talos secret volumes cannot be mounted. Each item is a core/v1 VolumeMount. Empty by default. @@ -342,25 +342,6 @@ controlPlane: kubelet: preferredAddressTypes: [] -## -## @section Talos Worker Image -## - -## @typedef {struct} Talos - Talos worker image configuration. -## @field {string} version - Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`. -## @field {string} schematicID - Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. -## @field {string} imageFactoryURL - Base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. -## @field {string} installerRepository - OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:`. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash. -## @field {object} registryMirrors - Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. - -## @param {Talos} talos - Talos worker image configuration. -talos: - version: "v1.13.6" - schematicID: "ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515" - imageFactoryURL: "https://factory.talos.dev" - installerRepository: "factory.talos.dev/installer" - registryMirrors: {} - ## ## @section OIDC ## @@ -385,7 +366,7 @@ talos: ## @field {string} [config] - Inline AuthenticationConfiguration YAML (`apiserver.config.k8s.io/v1beta1`). The chart writes the value verbatim into a Secret mounted at `/etc/kubernetes/authentication-config/config.yaml` on the kube-apiserver. Mutually exclusive with `secretRef.name`. ## @field {OIDCSecretRef} [secretRef] - Reference to an existing Secret in the tenant namespace carrying the `AuthenticationConfiguration`. -## @typedef {struct} OIDC - OIDC selector for the tenant cluster (Phase 1 — per-cluster audience binding; no per-tenant Keycloak realm). +## @typedef {struct} OIDC - OIDC selector for the tenant cluster. There is no per-tenant Keycloak realm: `System` mode uses the platform realm with a public client and an audience of its own per cluster. ## @field {OIDCMode} mode - Identity mode. `None`: no OIDC, only the static admin kubeconfig works. `System`: trust the platform Keycloak realm (`cozy` by default) via a per-cluster public client with audience binding; zero-config default. `CustomConfig`: trust a tenant-supplied issuer directly (BYO); `cozy` is not in the path. ## @field {OIDCCustomConfig} [customConfig] - Tenant-supplied AuthenticationConfiguration; consumed only when `mode: CustomConfig`. ## @field {[]OIDCUser} [users] - Users granted access to the tenant cluster; each entry produces one ClusterRoleBinding inside the tenant cluster. Works for both `System` and `CustomConfig` modes. diff --git a/packages/apps/mariadb/templates/backup-cronjob.yaml b/packages/apps/mariadb/templates/backup-cronjob.yaml index d39d5a8dd..1072a7c85 100644 --- a/packages/apps/mariadb/templates/backup-cronjob.yaml +++ b/packages/apps/mariadb/templates/backup-cronjob.yaml @@ -1,10 +1,18 @@ {{- if .Values.backup.enabled }} {{ $image := .Files.Get "images/backup.json" | fromJson }} +{{- /* The CronJob controller starts each Job as -, so the API + refuses a CronJob name over 52 characters. Cutting the release name to + fit could give two releases one CronJob, so a name that does not fit + keeps a hash of the full release name instead. */}} +{{- $cronJobName := printf "%s-backup" .Release.Name }} +{{- if gt (len $cronJobName) 52 }} +{{- $cronJobName = printf "%s-%s-backup" (.Release.Name | trunc 38 | trimSuffix "-") (.Release.Name | sha256sum | trunc 6) }} +{{- end }} apiVersion: batch/v1 kind: CronJob metadata: - name: {{ .Release.Name }}-backup + name: {{ $cronJobName }} spec: schedule: "{{ .Values.backup.schedule }}" concurrencyPolicy: Forbid diff --git a/packages/apps/mariadb/tests/cronjob_name_budget_test.yaml b/packages/apps/mariadb/tests/cronjob_name_budget_test.yaml new file mode 100644 index 000000000..18e5e40cf --- /dev/null +++ b/packages/apps/mariadb/tests/cronjob_name_budget_test.yaml @@ -0,0 +1,92 @@ +suite: mariadb backup CronJob name budget + +# The API refuses a CronJob name over 52 characters, because the controller +# names each Job it starts -, while a release name may take 53. +# Cutting the release name alone would give two long names one CronJob, so a +# name that does not fit is the release name cut to 38 characters, a hash of +# the full release name and -backup. + +templates: + - templates/backup-cronjob.yaml + - templates/backup-script.yaml + - templates/backup-secret.yaml + +set: + backup: + enabled: true + +tests: + - it: keeps -backup for a short release name + release: + name: mariadb-test + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: mariadb-test-backup + + - it: keeps the name when it lands exactly on 52 characters + release: + name: mariadb-abcdefghijklmnopqrstuvwxyz0123456789a + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: mariadb-abcdefghijklmnopqrstuvwxyz0123456789a-backup + + - it: shortens the name one character past the budget + release: + name: mariadb-abcdefghijklmnopqrstuvwxyz0123456789ab + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: mariadb-abcdefghijklmnopqrstuvwxyz0123-57a159-backup + + - it: fits a 53-character release name into 52 characters + release: + name: mariadb-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: mariadb-abcdefghijklmnopqrstuvwxyz0123-b5132c-backup + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,50}[a-z0-9])?$ + + - it: gives a release name differing only in its last character another CronJob + release: + name: mariadb-abcdefghijklmnopqrstuvwxyz0123456789abcdefghx + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: mariadb-abcdefghijklmnopqrstuvwxyz0123-a49389-backup + + - it: does not end the cut release name with a dash + release: + name: mariadb-abcdefghijklmnopqrstuvwxyz012-3456789abc + namespace: tenant-test + documentSelector: + path: kind + value: CronJob + asserts: + - equal: + path: metadata.name + value: mariadb-abcdefghijklmnopqrstuvwxyz012-2985e6-backup diff --git a/packages/apps/postgres/README.md b/packages/apps/postgres/README.md index c508476f4..4e320f814 100644 --- a/packages/apps/postgres/README.md +++ b/packages/apps/postgres/README.md @@ -20,7 +20,7 @@ together rather than picking one: | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | | **Archive plumbing** (chart, legacy) | Renders `spec.plugins` (referencing a barman-cloud `ObjectStore`) on the cnpg.io Cluster from helm-install onwards. CNPG runs the barman-cloud plugin as a WAL-archiver sidecar, every WAL switch ships to object storage, and any backup driver gets a complete WAL chain to start replay from. Renders only when `backup.enabled=true`, `useSystemBucket=false`, `destinationPath` is non-empty, AND inline-or-external creds are supplied. **Skipped in the platform `useSystemBucket=true` mode** — the CNPG backup driver SSA-applies the `ObjectStore` and patches `spec.plugins` onto the live Cluster at first BackupJob time; until that first BackupJob fires, plugin WAL archiving is not active and WAL accumulates on the PVC. Fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. | Legacy: `backup.enabled=true` plus `backup.destinationPath`, `backup.endpointURL`, and either `backup.s3AccessKey`+`backup.s3SecretKey` or `backup.s3CredentialsSecret`. Platform: `backup.enabled=true` plus `backup.useSystemBucket=true` — no S3 fields. | | **Backup orchestration** (recommended) | Drives ad-hoc and scheduled backups, retention, and restores from `backups.cozystack.io` resources that can span multiple Postgres apps in a tenant. The driver SSA-applies its own barman-cloud `ObjectStore` and patches `spec.plugins` with `ForceOwnership` (so compression / target settings flow from the strategy template, while the live Cluster's `serverName` — its WAL-archive S3 prefix — is always preserved); chart-managed values cover the same destination so there is no tug-of-war on the live Cluster. | `strategy.backups.cozystack.io/CNPG` + `BackupClass` + `Plan` (recurring) or `BackupJob` (ad-hoc), restores via `RestoreJob` | -| Legacy chart-emitted scheduled backup | The chart can also emit a `cnpg.io/ScheduledBackup` directly. Superseded by the BackupClass + Plan path, kept around for clusters that did not migrate. **Off by default** - rendered only when both `backup.enabled` and `backup.schedule` are non-empty. With `backup.schedule=""` plugin WAL archiving still runs; only the chart-emitted scheduled backup is silent. | `backup.enabled=true` plus `backup.schedule` (CNPG 6-field cron, e.g. `"0 2 * * * *"`) | +| Legacy chart-emitted scheduled backup | The chart can also emit a `cnpg.io/ScheduledBackup` directly. Superseded by the BackupClass + Plan path, kept around for clusters that did not migrate. **Off by default** - rendered only when both `backup.enabled` and `backup.schedule` are non-empty. With `backup.schedule=""` plugin WAL archiving still runs; only the chart-emitted scheduled backup is silent. | `backup.enabled=true` plus `backup.schedule` (five-field cron such as `"0 2 * * *"`, or six fields with leading seconds) | The **canonical setup** depends on whether the cluster ships the platform `cozy-default` BackupClass. @@ -87,16 +87,13 @@ field. ### How to enable chart-managed scheduled backups (legacy) -The chart can also emit a `cnpg.io/ScheduledBackup` directly, without a -BackupClass. Superseded by the BackupClass + Plan path above and kept -around for clusters that did not migrate. It does not run by default - -`backup.schedule` defaults to an empty string, which gates the chart's -ScheduledBackup template off. To turn it on, fill in a CNPG 6-field cron -expression: +The chart can also emit a `cnpg.io/ScheduledBackup` directly, without a BackupClass. Superseded by the BackupClass + Plan path above and kept around for clusters that did not migrate. It does not run by default - `backup.schedule` defaults to an empty string, which gates the chart's ScheduledBackup template off. To turn it on, set a standard five-field cron such as `0 2 * * *` for every day at 02:00, or the CloudNativePG six-field form with a leading seconds field such as `0 0 2 * * *` for the same time. The chart prepends a `0` seconds field to a five-field value, because CloudNativePG reads the first field as seconds and would otherwise run `0 2 * * *` every hour. + +On upgrade, an existing `backup.schedule: "0 2 * * *"` changes from hourly at minute 2 to daily at 02:00. To keep the previous hourly schedule, set `backup.schedule: "0 2 * * * *"` before upgrading. ```yaml ## @param backup.enabled Enable plugin WAL archiving + render the chart-managed ScheduledBackup -## @param backup.schedule Cron schedule (CNPG 6-field). Empty means no chart-managed ScheduledBackup +## @param backup.schedule Cron schedule, five fields or six with leading seconds. Empty means no chart-managed ScheduledBackup ## @param backup.retentionPolicy Retention policy ## @param backup.destinationPath Path to store the backup (i.e. s3://bucket/path/to/folder) ## @param backup.endpointURL S3 Endpoint used to upload data to the cloud @@ -107,7 +104,7 @@ backup: retentionPolicy: 30d destinationPath: s3://bucket/path/to/folder/ endpointURL: http://minio-gateway-service:9000 - schedule: "0 2 * * * *" # opt in - empty (the default) means no chart-managed schedule + schedule: "0 2 * * *" # opt in - empty (the default) means no chart-managed schedule s3AccessKey: oobaiRus9pah8PhohL1ThaeTa4UVa7gu s3SecretKey: ju3eum4dekeich9ahM1te8waeGai0oog ``` @@ -295,7 +292,7 @@ For `sslmode=verify-full` to work, the CA bundle retrieved above must be saved t | `backup` | Backup configuration. | `object` | `{}` | | `backup.enabled` | Enable regular backups. | `bool` | `false` | | `backup.useSystemBucket` | Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`. | `bool` | `false` | -| `backup.schedule` | Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. | `string` | `""` | +| `backup.schedule` | Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. | `string` | `""` | | `backup.retentionPolicy` | Retention policy (e.g. "30d"). | `string` | `30d` | | `backup.destinationPath` | DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility. | `string` | `s3://bucket/path/to/folder/` | | `backup.endpointURL` | DEPRECATED. See `destinationPath`. | `string` | `http://minio-gateway-service:9000` | diff --git a/packages/apps/postgres/templates/backup.yaml b/packages/apps/postgres/templates/backup.yaml index aa1d888e3..4025ca571 100644 --- a/packages/apps/postgres/templates/backup.yaml +++ b/packages/apps/postgres/templates/backup.yaml @@ -23,15 +23,32 @@ against a Cluster with no backup configuration and fail. This mirrors the ClickHouse `useSystemBucket` gates on the chart-managed backup resources. + + CNPG parses `schedule` with robfig/cron v1 `Parse`, which takes the first + field as seconds and makes day-of-week optional. A standard five-field + cron passes its webhook with only a warning and then runs one field off, + so `5 * * * *` fires every minute instead of hourly. A five-field value + therefore gets a leading `0` seconds field. Six fields and descriptors + such as `@daily` pass through as they are, and any other field count + fails the render, which CNPG would reject at admission anyway. */}} {{- if and .Values.backup.enabled .Values.backup.schedule (not .Values.backup.useSystemBucket) }} +{{- $schedule := .Values.backup.schedule }} +{{- $fields := regexFindAll `\S+` $schedule -1 }} +{{- if not (hasPrefix "@" (trim $schedule)) }} +{{- if eq (len $fields) 5 }} +{{- $schedule = printf "0 %s" (join " " $fields) }} +{{- else if ne (len $fields) 6 }} +{{- fail (printf "postgres: backup.schedule %q is not a cron schedule with 5 or 6 fields (it has %d). Use a standard five-field cron such as \"0 2 * * *\", a six-field cron with leading seconds such as \"0 0 2 * * *\", or a descriptor such as \"@daily\"." $schedule (len $fields)) }} +{{- end }} +{{- end }} --- apiVersion: postgresql.cnpg.io/v1 kind: ScheduledBackup metadata: name: {{ .Release.Name }} spec: - schedule: {{ .Values.backup.schedule | quote }} + schedule: {{ $schedule | quote }} backupOwnerReference: self method: plugin pluginConfiguration: diff --git a/packages/apps/postgres/tests/backup_schedule_test.yaml b/packages/apps/postgres/tests/backup_schedule_test.yaml new file mode 100644 index 000000000..e78526848 --- /dev/null +++ b/packages/apps/postgres/tests/backup_schedule_test.yaml @@ -0,0 +1,80 @@ +suite: chart-managed scheduled backup takes five-field and six-field cron + +# CNPG parses the ScheduledBackup schedule with robfig/cron v1 Parse, which +# takes the first field as seconds and accepts five fields with only a +# warning, so an unconverted "5 * * * *" would run every minute. + +templates: + - templates/backup.yaml + +release: + name: pg-test + namespace: tenant-test + +tests: + - it: prepends a zero seconds field to a standard five-field schedule + set: + backup: + enabled: true + schedule: "5 * * * *" + asserts: + - equal: + path: spec.schedule + value: "0 5 * * * *" + + - it: joins the fields of a five-field schedule with single spaces + set: + backup: + enabled: true + schedule: " 0 2 * *\t* " + asserts: + - equal: + path: spec.schedule + value: "0 0 2 * * *" + + - it: passes a six-field schedule through unchanged + set: + backup: + enabled: true + schedule: "0 5 * * * *" + asserts: + - equal: + path: spec.schedule + value: "0 5 * * * *" + + - it: passes a descriptor through unchanged + set: + backup: + enabled: true + schedule: "@every 6h" + asserts: + - equal: + path: spec.schedule + value: "@every 6h" + + - it: refuses a schedule with four fields + set: + backup: + enabled: true + schedule: "5 * * *" + asserts: + - failedTemplate: + errorMessage: 'postgres: backup.schedule "5 * * *" is not a cron schedule with 5 or 6 fields (it has 4). Use a standard five-field cron such as "0 2 * * *", a six-field cron with leading seconds such as "0 0 2 * * *", or a descriptor such as "@daily".' + + - it: refuses a schedule with seven fields + set: + backup: + enabled: true + schedule: "0 0 2 * * * 2026" + asserts: + - failedTemplate: + errorMessage: 'postgres: backup.schedule "0 0 2 * * * 2026" is not a cron schedule with 5 or 6 fields (it has 7). Use a standard five-field cron such as "0 2 * * *", a six-field cron with leading seconds such as "0 0 2 * * *", or a descriptor such as "@daily".' + + - it: refuses a schedule of only spaces + set: + backup: + enabled: true + schedule: " " + asserts: + - failedTemplate: + errorMessage: 'postgres: backup.schedule " " is not a cron schedule with 5 or 6 fields (it has 0). Use a standard five-field cron such as "0 2 * * *", a six-field cron with leading seconds such as "0 0 2 * * *", or a descriptor such as "@daily".' diff --git a/packages/apps/postgres/tests/barman_resources_test.yaml b/packages/apps/postgres/tests/barman_resources_test.yaml index 905fde2bb..1903fe8b1 100644 --- a/packages/apps/postgres/tests/barman_resources_test.yaml +++ b/packages/apps/postgres/tests/barman_resources_test.yaml @@ -13,7 +13,7 @@ set: s3CredentialsSecret: name: postgres-backup tests: - - it: sets resources for native backup without inheriting the tenant 128Mi limit + - it: sets resources for native backup without inheriting the tenant 128Mi and 250m limits set: backup.enabled: true asserts: @@ -24,15 +24,11 @@ tests: cpu: 100m memory: 256Mi limits: + cpu: "1" memory: 1Gi documentSelector: path: kind value: ObjectStore - - notExists: - path: spec.instanceSidecarConfiguration.resources.limits.cpu - documentSelector: - path: kind - value: ObjectStore - equal: path: spec.configuration.s3Credentials.accessKeyId.name value: postgres-backup diff --git a/packages/apps/postgres/values.schema.json b/packages/apps/postgres/values.schema.json index 4d544ff6b..4b5542d15 100644 --- a/packages/apps/postgres/values.schema.json +++ b/packages/apps/postgres/values.schema.json @@ -427,7 +427,7 @@ "default": "" }, "schedule": { - "description": "Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.", + "description": "Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.", "type": "string", "default": "" }, diff --git a/packages/apps/postgres/values.yaml b/packages/apps/postgres/values.yaml index fb4a67b35..a8d324dc8 100644 --- a/packages/apps/postgres/values.yaml +++ b/packages/apps/postgres/values.yaml @@ -321,7 +321,7 @@ databases: {} ## @typedef {struct} Backup - Backup configuration. ## @field {bool} enabled - Enable regular backups. ## @field {bool} [useSystemBucket] - Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`. -## @field {string} [schedule] - Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. +## @field {string} [schedule] - Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time. ## @field {string} [retentionPolicy] - Retention policy (e.g. "30d"). ## @field {string} [destinationPath] - DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility. ## @field {string} [endpointURL] - DEPRECATED. See `destinationPath`. diff --git a/packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml b/packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml index bbc9547f9..3e2038e91 100644 --- a/packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml +++ b/packages/apps/qdrant/templates/hooks/cleanup-pvc.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: {{ .Release.Name }}-qdrant-cleanup + name: {{ printf "%s-qdrant-cleanup" .Release.Name | trunc 63 | trimSuffix "-" }} labels: app.kubernetes.io/instance: {{ .Release.Name }} annotations: diff --git a/packages/apps/qdrant/tests/job_name_budget_test.yaml b/packages/apps/qdrant/tests/job_name_budget_test.yaml new file mode 100644 index 000000000..76416b387 --- /dev/null +++ b/packages/apps/qdrant/tests/job_name_budget_test.yaml @@ -0,0 +1,64 @@ +suite: qdrant cleanup Job name budget + +# The API server copies a Job's name into the job-name label of its pods, and a +# label value takes 63 characters at most, while a release name may take 53. +# The ServiceAccount, Role and RoleBinding have no such limit and keep the full +# name. + +templates: + - templates/hooks/cleanup-pvc.yaml + +tests: + - it: keeps -qdrant-cleanup for a short release name + release: + name: qdrant-test + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: qdrant-test-qdrant-cleanup + + - it: keeps the name when it lands exactly on 63 characters + release: + name: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcde + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcde-qdrant-cleanup + + - it: cuts the name to 63 characters for a 53-character release name + release: + name: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-qdrant-cl + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ + - equal: + path: spec.template.spec.serviceAccountName + value: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-qdrant-cleanup + + - it: leaves the ServiceAccount on the full name for a 53-character release name + release: + name: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij + namespace: tenant-test + documentSelector: + path: kind + value: ServiceAccount + asserts: + - equal: + path: metadata.name + value: qdrant-abcdefghijklmnopqrstuvwxyz0123456789abcdefghij-qdrant-cleanup diff --git a/packages/apps/redis/README.md b/packages/apps/redis/README.md index 84cb04930..be03bbea3 100644 --- a/packages/apps/redis/README.md +++ b/packages/apps/redis/README.md @@ -39,11 +39,14 @@ Service utilizes the freshworks-oss Redis Operator (a maintained fork of the arc ### Application-specific parameters -| Name | Description | Type | Value | -| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | ------ | -| `authEnabled` | Enable password generation. | `bool` | `true` | -| `minReplicasToWrite` | Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value. | `*int` | `null` | -| `minReplicasMaxLag` | Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it. | `*int` | `null` | +| Name | Description | Type | Value | +| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | ------------ | +| `authEnabled` | Enable password generation. | `bool` | `true` | +| `minReplicasToWrite` | Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value. | `*int` | `null` | +| `minReplicasMaxLag` | Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it. | `*int` | `null` | +| `maxmemory` | Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers. | `quantity` | `""` | +| `maxmemoryPolicy` | Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset. | `string` | `noeviction` | +| `snapshots` | Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty. | `bool` | `true` | ### Sentinel parameters diff --git a/packages/apps/redis/templates/redisfailover.yaml b/packages/apps/redis/templates/redisfailover.yaml index 05a96e3cc..330a2d517 100644 --- a/packages/apps/redis/templates/redisfailover.yaml +++ b/packages/apps/redis/templates/redisfailover.yaml @@ -116,8 +116,41 @@ spec: - loglevel notice {{- if not .Values.size }} - appendonly no + {{- end }} + {{- /* The operator applies these lines with CONFIG SET on every reconcile and never + resets a line that disappears, so each one is always rendered with an explicit + value: switching back to the default must reach the running pods too. The + schedule is the Redis 7 and 8 default. */}} + {{- if and .Values.size .Values.snapshots }} + - save 3600 1 300 100 60 10000 + {{- else }} - save "" {{- end }} + {{- $maxmemory := 0 }} + {{- if not (kindIs "invalid" .Values.maxmemory) }} + {{- $maxmemoryBytes := include "cozy-lib.resources.toFloat" (.Values.maxmemory | toString) | float64 }} + {{- /* Checked before the int64 cast, which would truncate a sub-byte quantity such + as 512m (0.512 bytes) or -512m to 0, and Redis reads 0 as no limit. */}} + {{- if lt $maxmemoryBytes 0.0 }} + {{- fail (printf "maxmemory (%v) must not be negative" .Values.maxmemory) }} + {{- end }} + {{- if ne (floor $maxmemoryBytes) $maxmemoryBytes }} + {{- fail (printf "maxmemory (%v) must be a whole number of bytes, e.g. 512Mi rather than 512m (millibytes)" .Values.maxmemory) }} + {{- end }} + {{- $maxmemory = $maxmemoryBytes | int64 }} + {{- $redisResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.resourcesPreset .Values.resources $) | fromYaml }} + {{- $memoryRequest := include "cozy-lib.resources.toFloat" (dig "requests" "memory" "0" $redisResources | toString) | float64 }} + {{- /* The operator compares maxmemory with 90% of the pod's memory request + (reservedPodMemoryPercent) and silently skips the line when it is larger. */}} + {{- $allowed := mulf $memoryRequest 0.9 | int64 }} + {{- if gt $maxmemory $allowed }} + {{- fail (printf "maxmemory (%v, %d bytes) must be at most 90%% of the Redis memory request (%d bytes): the operator ignores a larger value" .Values.maxmemory $maxmemory $allowed) }} + {{- end }} + {{- end }} + {{- /* The operator sends these in list order, and lowering maxmemory below the + dataset evicts at once, so the policy has to be in place first. */}} + - maxmemory-policy {{ .Values.maxmemoryPolicy | default "noeviction" }} + - maxmemory {{ $maxmemory }} {{- /* A master with `replicas` pods has replicas-1 replicas, so a requirement of `replicas` or more can never be met and the master would refuse every write. */}} {{- if and (not (kindIs "invalid" .Values.minReplicasToWrite)) (ge (.Values.minReplicasToWrite | int) (.Values.replicas | int)) }} diff --git a/packages/apps/redis/tests/cache_config_test.yaml b/packages/apps/redis/tests/cache_config_test.yaml new file mode 100644 index 000000000..1b618bcac --- /dev/null +++ b/packages/apps/redis/tests/cache_config_test.yaml @@ -0,0 +1,150 @@ +suite: Redis cache tuning (maxmemory, eviction, snapshots) + +# The operator applies customConfig with CONFIG SET and never resets a line +# that disappears, so every directive here is rendered with an explicit value +# even at its default: reverting a setting has to reach the running pods. + +release: + name: my-redis + namespace: tenant-test + +set: + _cluster: + cluster-domain: cluster.local + _namespace: + host: example.com + +templates: + - templates/redisfailover.yaml + +tests: + - it: "renders the Redis defaults explicitly" + documentSelector: + path: kind + value: RedisFailover + asserts: + - contains: + path: spec.redis.customConfig + content: save 3600 1 300 100 60 10000 + - contains: + path: spec.redis.customConfig + content: maxmemory 0 + - contains: + path: spec.redis.customConfig + content: maxmemory-policy noeviction + - notContains: + path: spec.redis.customConfig + content: appendonly no + + - it: "disables snapshots while keeping the volume" + set: + snapshots: false + documentSelector: + path: kind + value: RedisFailover + asserts: + - contains: + path: spec.redis.customConfig + content: save "" + - notContains: + path: spec.redis.customConfig + content: save 3600 1 300 100 60 10000 + - equal: + path: spec.redis.storage.persistentVolumeClaim.spec.resources.requests.storage + value: 1Gi + + - it: "keeps snapshots off without a volume even when enabled" + set: + size: null + snapshots: true + documentSelector: + path: kind + value: RedisFailover + asserts: + - contains: + path: spec.redis.customConfig + content: save "" + - contains: + path: spec.redis.customConfig + content: appendonly no + - notExists: + path: spec.redis.storage + + - it: "renders maxmemory in bytes with the chosen policy" + set: + resourcesPreset: t1.large + maxmemory: 1Gi + maxmemoryPolicy: allkeys-lru + documentSelector: + path: kind + value: RedisFailover + asserts: + - contains: + path: spec.redis.customConfig + content: maxmemory 1073741824 + - contains: + path: spec.redis.customConfig + content: maxmemory-policy allkeys-lru + + - it: "sizes the maxmemory ceiling from explicit resources" + set: + resources: + memory: 1Gi + maxmemory: 900Mi + documentSelector: + path: kind + value: RedisFailover + asserts: + - contains: + path: spec.redis.customConfig + content: maxmemory 943718400 + + # The operator compares maxmemory with 90% of the pod's memory request and + # silently skips a larger line, which would leave Redis unbounded. + - it: "refuses maxmemory above 90% of the memory request" + set: + resources: + memory: 1Gi + maxmemory: 1000Mi + asserts: + - failedTemplate: + errorPattern: must be at most 90% of the Redis memory request + + - it: "refuses a negative maxmemory" + set: + maxmemory: -1Mi + asserts: + - failedTemplate: + errorPattern: must not be negative + + # A sub-byte quantity would truncate to maxmemory 0, which Redis reads as no + # limit, so the bound the tenant asked for would silently disappear. + - it: "refuses a maxmemory that is not a whole number of bytes" + set: + maxmemory: 512m + asserts: + - failedTemplate: + errorPattern: must be a whole number of bytes + + - it: "refuses a negative sub-byte maxmemory" + set: + maxmemory: -512m + asserts: + - failedTemplate: + errorPattern: must not be negative + + - it: "applies the eviction policy before the memory bound" + set: + resourcesPreset: t1.large + maxmemory: 1Gi + maxmemoryPolicy: allkeys-lru + documentSelector: + path: kind + value: RedisFailover + asserts: + - equal: + path: spec.redis.customConfig[3] + value: maxmemory-policy allkeys-lru + - equal: + path: spec.redis.customConfig[4] + value: maxmemory 1073741824 diff --git a/packages/apps/redis/tests/min_replicas_test.yaml b/packages/apps/redis/tests/min_replicas_test.yaml index f0de83e86..13217884d 100644 --- a/packages/apps/redis/tests/min_replicas_test.yaml +++ b/packages/apps/redis/tests/min_replicas_test.yaml @@ -1,7 +1,7 @@ suite: Redis min-replicas write safety -# Both settings are opt-in: unset keeps the Redis defaults, so existing -# instances render the same customConfig as before. +# Both settings are opt-in: unset keeps the Redis defaults, so neither +# directive is rendered. release: name: my-redis @@ -30,7 +30,7 @@ tests: pattern: ^tcp-keepalive 0$ - lengthEqual: path: spec.redis.customConfig - count: 2 + count: 5 - it: "renders min-replicas-to-write when set" set: diff --git a/packages/apps/redis/values.schema.json b/packages/apps/redis/values.schema.json index 1a301615f..183fb6a53 100644 --- a/packages/apps/redis/values.schema.json +++ b/packages/apps/redis/values.schema.json @@ -171,6 +171,39 @@ "type": "integer", "minimum": 0 }, + "maxmemory": { + "description": "Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers.", + "pattern": "^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$", + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + } + ], + "x-kubernetes-int-or-string": true + }, + "maxmemoryPolicy": { + "description": "Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.", + "type": "string", + "default": "noeviction", + "enum": [ + "noeviction", + "allkeys-lru", + "allkeys-lfu", + "allkeys-random", + "volatile-lru", + "volatile-lfu", + "volatile-random", + "volatile-ttl" + ] + }, + "snapshots": { + "description": "Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty.", + "type": "boolean", + "default": true + }, "sentinel": { "description": "Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.", "type": "object", diff --git a/packages/apps/redis/values.yaml b/packages/apps/redis/values.yaml index 7917ca34d..ed14c4f46 100644 --- a/packages/apps/redis/values.yaml +++ b/packages/apps/redis/values.yaml @@ -119,6 +119,24 @@ authEnabled: true ## @param {*int} [minReplicasMaxLag] - Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it. ## @minimum 0 +## @param {quantity} [maxmemory] - Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers. + +## @enum {string} MaxmemoryPolicy - Values accepted by the Redis `maxmemory-policy` directive. +## @value noeviction +## @value allkeys-lru +## @value allkeys-lfu +## @value allkeys-random +## @value volatile-lru +## @value volatile-lfu +## @value volatile-random +## @value volatile-ttl + +## @param {MaxmemoryPolicy} maxmemoryPolicy="noeviction" - Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset. +maxmemoryPolicy: "noeviction" + +## @param {bool} snapshots - Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty. +snapshots: true + ## ## @section Sentinel parameters ## diff --git a/packages/apps/tcp-balancer/templates/service.yaml b/packages/apps/tcp-balancer/templates/service.yaml index 8384cc85b..689db7e16 100644 --- a/packages/apps/tcp-balancer/templates/service.yaml +++ b/packages/apps/tcp-balancer/templates/service.yaml @@ -27,7 +27,7 @@ spec: port: 443 targetPort: https {{- end }} - {{- with .Values.httpAndHttps }} + {{- with .Values.kubernetesAndTalos }} - name: kubernetes protocol: TCP port: 6443 diff --git a/packages/apps/tcp-balancer/tests/service_test.yaml b/packages/apps/tcp-balancer/tests/service_test.yaml new file mode 100644 index 000000000..57191115c --- /dev/null +++ b/packages/apps/tcp-balancer/tests/service_test.yaml @@ -0,0 +1,97 @@ +suite: haproxy service + +templates: + - templates/service.yaml + - templates/deployment.yaml + - templates/configmap.yaml + +tests: + - it: publishes only http and https by default, both on container ports the Deployment declares + asserts: + - equal: + path: spec.ports + value: + - name: http + protocol: TCP + port: 80 + targetPort: http + - name: https + protocol: TCP + port: 443 + targetPort: https + template: templates/service.yaml + - equal: + path: spec.template.spec.containers[0].ports + value: + - containerPort: 8080 + name: http + - containerPort: 8443 + name: https + template: templates/deployment.yaml + + - it: publishes kubernetes and talos for kubernetesAndTalos alone, both on container ports the Deployment declares + set: + httpAndHttps: null + kubernetesAndTalos: + endpoints: + - 10.1.1.1 + asserts: + - equal: + path: spec.ports + value: + - name: kubernetes + protocol: TCP + port: 6443 + targetPort: kubernetes + - name: talos + protocol: TCP + port: 50000 + targetPort: talos + template: templates/service.yaml + - equal: + path: spec.template.spec.containers[0].ports + value: + - containerPort: 6443 + name: kubernetes + - containerPort: 50000 + name: talos + template: templates/deployment.yaml + + - it: publishes all four ports when both are set, each on a container port the Deployment declares + set: + kubernetesAndTalos: + endpoints: + - 10.1.1.1 + asserts: + - equal: + path: spec.ports + value: + - name: http + protocol: TCP + port: 80 + targetPort: http + - name: https + protocol: TCP + port: 443 + targetPort: https + - name: kubernetes + protocol: TCP + port: 6443 + targetPort: kubernetes + - name: talos + protocol: TCP + port: 50000 + targetPort: talos + template: templates/service.yaml + - equal: + path: spec.template.spec.containers[0].ports + value: + - containerPort: 8080 + name: http + - containerPort: 8443 + name: https + - containerPort: 6443 + name: kubernetes + - containerPort: 50000 + name: talos + template: templates/deployment.yaml diff --git a/packages/apps/tenant/templates/cleanup-job.yaml b/packages/apps/tenant/templates/cleanup-job.yaml index ffa07d0fe..7285a4d79 100644 --- a/packages/apps/tenant/templates/cleanup-job.yaml +++ b/packages/apps/tenant/templates/cleanup-job.yaml @@ -40,11 +40,22 @@ subjects: - kind: ServiceAccount name: {{ include "tenant.name" . }}-cleanup namespace: cozy-system +{{- /* The Job name becomes the job-name label of its pod, which takes 63 + characters at most, and a nested tenant namespace can use all 63 by + itself. Cutting the name to fit would let a parent and one of its + children land on the same Job in cozy-system, where the child's + before-hook-creation would delete the parent's running cleanup, so a + name that does not fit keeps a hash of the whole namespace instead. */}} +{{- $tenantName := include "tenant.name" . }} +{{- $jobName := printf "%s-cleanup" $tenantName }} +{{- if gt (len $jobName) 63 }} +{{- $jobName = printf "%s-%s-cleanup" ($tenantName | trunc 48 | trimSuffix "-") ($tenantName | sha256sum | trunc 6) }} +{{- end }} --- apiVersion: batch/v1 kind: Job metadata: - name: {{ include "tenant.name" . }}-cleanup + name: {{ $jobName }} namespace: cozy-system annotations: helm.sh/hook: pre-delete @@ -98,7 +109,7 @@ spec: ttlSecondsAfterFinished: 300 template: metadata: - name: {{ include "tenant.name" . }}-cleanup + name: {{ $jobName }} labels: policy.cozystack.io/allow-to-apiserver: "true" spec: diff --git a/packages/apps/tenant/templates/networkpolicy.yaml b/packages/apps/tenant/templates/networkpolicy.yaml index 5285b495a..bc6904e0a 100644 --- a/packages/apps/tenant/templates/networkpolicy.yaml +++ b/packages/apps/tenant/templates/networkpolicy.yaml @@ -259,6 +259,20 @@ spec: - ports: - port: "8443" protocol: TCP +--- +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + # The whole namespace, as for linstor-gui: the ingress controller also + # has to reach the ACME HTTP-01 solver pods cert-manager starts there. + name: allow-to-observability-proxy + namespace: {{ include "tenant.name" . }} +spec: + endpointSelector: {} + egress: + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": cozy-observability-proxy {{- if .Values.monitoring }} --- apiVersion: cilium.io/v2 diff --git a/packages/apps/tenant/tests/job_name_budget_test.yaml b/packages/apps/tenant/tests/job_name_budget_test.yaml new file mode 100644 index 000000000..5cf4b93f6 --- /dev/null +++ b/packages/apps/tenant/tests/job_name_budget_test.yaml @@ -0,0 +1,127 @@ +suite: tenant cleanup Job name budget + +# The API server copies a Job's name into the job-name label of its pods, and a +# label value takes 63 characters at most, while a nested tenant namespace may +# take all 63 by itself. Every tenant's cleanup Job runs in cozy-system, where +# cutting the namespace alone could give a parent and its child one Job, and the +# child's before-hook-creation would delete the parent's while it runs. So a +# name that does not fit is the namespace cut to 48 characters, a hash of the +# whole namespace and -cleanup. The ServiceAccount, Role and RoleBinding have no +# such limit and keep the full name. + +templates: + - templates/cleanup-job.yaml + +tests: + - it: keeps -cleanup for a tenant under tenant-root + release: + name: tenant-test + namespace: tenant-root + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-test-cleanup + + - it: keeps -cleanup for a short nested tenant + release: + name: tenant-child + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-test-child-cleanup + + - it: keeps the name when it lands exactly on 63 characters + release: + name: tenant-0123456789abcdefghijk + namespace: tenant-abcdefghijklmnopqrstuvwxyz + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijk-cleanup + + - it: shortens the name for a 63-character namespace + release: + name: tenant-0123456789abcdefghijklmnopqrs + namespace: tenant-abcdefghijklmnopqrstuvwxyz + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcd-7b3e3e-cleanup + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ + - equal: + path: spec.template.metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcd-7b3e3e-cleanup + - equal: + path: spec.template.spec.serviceAccountName + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmnopqrs-cleanup + + # Cut to 63 characters, the parent below and its child named clea would both + # be tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmn-clea. + - it: gives a parent its own Job name + release: + name: tenant-0123456789abcdefghijklmn + namespace: tenant-abcdefghijklmnopqrstuvwxyz + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcd-571020-cleanup + + - it: gives the child that would collide with that parent another Job name + release: + name: tenant-clea + namespace: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmn + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcd-ec5e9b-cleanup + + - it: does not end the cut namespace with a dash + release: + name: tenant-abcdefghijklmno + namespace: tenant-abcdefghijklmnopqrstuvwxyz0123456789abcd + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz0123456789abcd-477f49-cleanup + + - it: leaves the RBAC objects on the full name for a 63-character namespace + release: + name: tenant-0123456789abcdefghijklmnopqrs + namespace: tenant-abcdefghijklmnopqrstuvwxyz + asserts: + - equal: + path: metadata.name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmnopqrs-cleanup + documentSelector: + path: kind + value: ServiceAccount + - equal: + path: subjects[0].name + value: tenant-abcdefghijklmnopqrstuvwxyz-0123456789abcdefghijklmnopqrs-cleanup + documentSelector: + path: kind + value: RoleBinding diff --git a/packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml b/packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml new file mode 100644 index 000000000..3f3e027fd --- /dev/null +++ b/packages/apps/tenant/tests/networkpolicy_observability_proxy_test.yaml @@ -0,0 +1,64 @@ +suite: tenant egress to the observability proxy +# The observability-proxy Ingress in cozy-observability-proxy is served by +# the ingress controller running in tenant-root. Tenant egress is +# default-deny plus explicit allow-to-* policies, so without this one the +# controller reaches neither the proxy nor the ACME HTTP-01 solver pod that +# cert-manager starts in the same namespace; hence the whole namespace, as +# allow-to-linstor-gui does. +templates: + - templates/networkpolicy.yaml +tests: + - it: lets every pod of the root tenant reach the proxy namespace + release: + name: tenant-root + namespace: tenant-root + asserts: + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + isKind: + of: CiliumNetworkPolicy + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + equal: + path: metadata.namespace + value: tenant-root + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + equal: + path: spec + value: + endpointSelector: {} + egress: + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": cozy-observability-proxy + + - it: renders in a nested tenant with every module off + release: + name: tenant-bar + namespace: tenant-foo + set: + monitoring: false + ingress: false + etcd: false + asserts: + - documentSelector: + path: metadata.name + value: allow-to-observability-proxy + equal: + path: metadata.namespace + value: tenant-foo-bar + + - it: renders nothing where the tenant Cilium policies are off + release: + name: tenant-root + namespace: tenant-root + set: + _cluster: + bundle-name: isp-hosted + asserts: + - hasDocuments: + count: 0 diff --git a/packages/apps/vm-disk/templates/NOTES.txt b/packages/apps/vm-disk/templates/NOTES.txt index 73d07cc59..0940e75ba 100644 --- a/packages/apps/vm-disk/templates/NOTES.txt +++ b/packages/apps/vm-disk/templates/NOTES.txt @@ -1,5 +1,5 @@ {{- if and .Values.source (hasKey .Values.source "upload") }} To upload your disk run: -virtctl image-upload dv -n {{ .Release.Namespace }} {{ .Release.Name }} --image-path +virtctl image-upload dv {{ .Release.Name }} --no-create --namespace {{ .Release.Namespace }} --image-path {{- end }} diff --git a/packages/apps/vm-disk/templates/dv.yaml b/packages/apps/vm-disk/templates/dv.yaml index 7a48b9100..c8750afe1 100644 --- a/packages/apps/vm-disk/templates/dv.yaml +++ b/packages/apps/vm-disk/templates/dv.yaml @@ -146,6 +146,26 @@ {{- fail (printf "source.image %q: golden image %q lives in StorageClass %q, and a clone into StorageClass %q of the same provisioner gets no host-assisted fallback from CDI. LINSTOR creates the clone next to the source data, and StorageClass %q does not allow remote volume access from every node, so the clone may never bind. Set storageClass to %q, or ask the platform administrator to set cloneStrategyOverride to copy in the kubevirt-cdi package." .Values.source.image.name $desiredSource.pvc.name $goldenSC $targetSC $targetSC $goldenSC) }} {{- end }} {{- end }} + {{- /* CDI v1.66.1 refuses to clone a Block source PVC into a smaller + request before it creates the claim (validateClone in + pkg/controller/datavolume/pvc-clone-controller.go), and the + DataVolume then waits forever. It raises the request to the target + StorageProfile's minimumSupportedPvcSize before comparing. A + Filesystem source is not compared, as CDI sizes that clone from + the image inside it or with a filesystem overhead the chart does + not reproduce. */}} + {{- $goldenSize := dig "spec" "resources" "requests" "storage" "" ($goldenPVC | default dict) | toString }} + {{- if and .Values.storage $goldenSize (eq (dig "spec" "volumeMode" "" ($goldenPVC | default dict) | toString) "Block") }} + {{- $goldenBytes := include "cozy-lib.resources.toFloat" $goldenSize | float64 }} + {{- $diskBytes := include "cozy-lib.resources.toFloat" (.Values.storage | toString) | float64 }} + {{- if and (lt $diskBytes $goldenBytes) $targetSC }} + {{- $minimum := dig "metadata" "annotations" "cdi.kubevirt.io/minimumSupportedPvcSize" "" (lookup "cdi.kubevirt.io/v1beta1" "StorageProfile" "" $targetSC | default dict) | toString }} + {{- $diskBytes = maxf $diskBytes (include "cozy-lib.resources.toFloat" $minimum | float64) }} + {{- end }} + {{- if lt $diskBytes $goldenBytes }} + {{- fail (printf "storage %s is smaller than golden image %q (%s), so CDI refuses to clone it and the disk would stay pending forever. Set storage to %s or more." (.Values.storage | toString) $desiredSource.pvc.name $goldenSize $goldenSize) }} + {{- end }} + {{- end }} {{- end }} {{- else if hasKey .Values.source "http" }} {{- $desiredSource = dict "http" (dict "url" (required "A valid .Values.source.http.url entry required!" .Values.source.http.url)) }} diff --git a/packages/apps/vm-disk/templates/pvc-resize-hook.yaml b/packages/apps/vm-disk/templates/pvc-resize-hook.yaml index 5e327c6f5..224da33ba 100644 --- a/packages/apps/vm-disk/templates/pvc-resize-hook.yaml +++ b/packages/apps/vm-disk/templates/pvc-resize-hook.yaml @@ -15,7 +15,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: "{{ .Release.Name }}-volume-resize-hook" + name: {{ printf "%s-volume-resize-hook" .Release.Name | trunc 63 | trimSuffix "-" | quote }} annotations: helm.sh/hook: pre-install,pre-upgrade helm.sh/hook-weight: "0" diff --git a/packages/apps/vm-disk/tests/dv_golden_size_test.yaml b/packages/apps/vm-disk/tests/dv_golden_size_test.yaml new file mode 100644 index 000000000..6a68874c7 --- /dev/null +++ b/packages/apps/vm-disk/tests/dv_golden_size_test.yaml @@ -0,0 +1,252 @@ +suite: vm-disk refuses a disk smaller than the golden image it clones +release: + name: test-vm-disk + namespace: tenant-test +templates: + - templates/dv.yaml +tests: + - it: refuses a new disk at the chart default size from a larger Block golden + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: &scheme + "cdi.kubevirt.io/v1beta1/DataVolume": + gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: datavolumes} + namespaced: true + "cdi.kubevirt.io/v1beta1/StorageProfile": + gvr: {group: cdi.kubevirt.io, version: v1beta1, resource: storageprofiles} + namespaced: false + "v1/Namespace": + gvr: {group: "", version: v1, resource: namespaces} + namespaced: false + "v1/PersistentVolumeClaim": + gvr: {group: "", version: v1, resource: persistentvolumeclaims} + namespaced: true + "storage.k8s.io/v1/StorageClass": + gvr: {group: storage.k8s.io, version: v1, resource: storageclasses} + namespaced: false + objects: + - &namespace + kind: Namespace + apiVersion: v1 + metadata: {name: tenant-test} + - &goldenPVC + kind: PersistentVolumeClaim + apiVersion: v1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + spec: + storageClassName: replicated + volumeMode: Block + resources: {requests: {storage: 20Gi}} + asserts: + - failedTemplate: + errorPattern: 'storage 5Gi is smaller than golden image "vm-default-images-ubuntu" \(20Gi\), so CDI refuses to clone it and the disk would stay pending forever\. Set storage to 20Gi or more\.' + + - it: clones into a disk the size of the golden + set: + source: {image: {name: ubuntu}} + storage: 20Gi + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - equal: + path: spec.storage.resources.requests.storage + value: 20Gi + + - it: clones into a disk larger than the golden + set: + source: {image: {name: ubuntu}} + storage: 25Gi + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - equal: + path: spec.storage.resources.requests.storage + value: 25Gi + + - it: compares bytes, so 21G is smaller than a 20Gi golden + set: + source: {image: {name: ubuntu}} + storage: 21G + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - failedTemplate: + errorPattern: 'storage 21G is smaller than golden image "vm-default-images-ubuntu" \(20Gi\)' + + - it: compares bytes, so 20480Mi is the size of a 20Gi golden + set: + source: {image: {name: ubuntu}} + storage: 20480Mi + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - equal: + path: spec.storage.resources.requests.storage + value: 20480Mi + + - it: clones when the target StorageProfile's minimum PVC size covers the golden + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + - kind: StorageProfile + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: replicated + annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 20Gi} + asserts: + - notFailedTemplate: {} + + - it: refuses when the target StorageProfile's minimum PVC size is still below the golden + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + - kind: StorageProfile + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: replicated + annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 10Gi} + asserts: + - failedTemplate: + errorPattern: 'storage 5Gi is smaller than golden image "vm-default-images-ubuntu" \(20Gi\)' + + - it: reads the minimum PVC size of the default class for an empty storageClass + set: + source: {image: {name: ubuntu}} + storageClass: "" + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + - kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: replicated + annotations: {storageclass.kubernetes.io/is-default-class: "true"} + provisioner: linstor.csi.linbit.com + - kind: StorageProfile + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: replicated + annotations: {cdi.kubevirt.io/minimumSupportedPvcSize: 20Gi} + asserts: + - notFailedTemplate: {} + + - it: does not compare sizes for a Filesystem golden + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + spec: + storageClassName: replicated + volumeMode: Filesystem + resources: {requests: {storage: 20Gi}} + asserts: + - notFailedTemplate: {} + + - it: renders without a cluster to look the golden up in + set: + source: {image: {name: ubuntu}} + asserts: + - equal: + path: spec.storage.resources.requests.storage + value: 5Gi + + - it: clones a golden whose PVC does not exist yet + set: + source: {image: {name: ubuntu}} + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: {name: vm-default-images-ubuntu, namespace: cozy-public} + status: {phase: ImportScheduled} + asserts: + - notFailedTemplate: {} + + - it: leaves a disk without a size to CDI, which takes the golden's + set: + source: {image: {name: ubuntu}} + storage: null + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - notFailedTemplate: {} + + - it: leaves an existing disk alone + set: + source: {image: {name: ubuntu}} + storage: 25Gi + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + - kind: DataVolume + apiVersion: cdi.kubevirt.io/v1beta1 + metadata: + name: test-vm-disk + namespace: tenant-test + annotations: + vm-disk.cozystack.io/source: '{"image":{"name":"ubuntu"}}' + vm-disk.cozystack.io/storage-class: replicated + spec: + source: {pvc: {name: vm-default-images-ubuntu, namespace: cozy-public}} + storage: {resources: {requests: {storage: 10Gi}}, storageClassName: replicated} + asserts: + - notFailedTemplate: {} + + - it: does not compare sizes for an http source + set: + source: {http: {url: "https://example.com/disk.img"}} + storage: 1Gi + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - notFailedTemplate: {} + + - it: does not compare sizes for an upload source + set: + source: {upload: {}} + storage: 1Gi + kubernetesProvider: + scheme: *scheme + objects: + - *namespace + - *goldenPVC + asserts: + - notFailedTemplate: {} diff --git a/packages/apps/vm-disk/tests/job_name_budget_test.yaml b/packages/apps/vm-disk/tests/job_name_budget_test.yaml new file mode 100644 index 000000000..5922dcc7b --- /dev/null +++ b/packages/apps/vm-disk/tests/job_name_budget_test.yaml @@ -0,0 +1,127 @@ +suite: vm-disk resize Job name budget + +# The API server copies a Job's name into the job-name label of its pods, and a +# label value takes 63 characters at most, while a release name may take 53. +# The hook renders only when an existing PVC grows, so each release below gets +# a smaller PVC to look up. The ServiceAccount, Role and RoleBinding have no +# such limit and keep the full name, so the ones an earlier resize left behind +# are replaced, not orphaned. + +templates: + - templates/pvc-resize-hook.yaml + +set: + storage: 2Gi + +kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + objects: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: test-vm-disk + namespace: tenant-test + spec: + resources: + requests: + storage: 1Gi + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789 + namespace: tenant-test + spec: + resources: + requests: + storage: 1Gi + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcd + namespace: tenant-test + spec: + resources: + requests: + storage: 1Gi + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi + namespace: tenant-test + spec: + resources: + requests: + storage: 1Gi + +tests: + - it: keeps -volume-resize-hook for a short release name + release: + name: test-vm-disk + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: test-vm-disk-volume-resize-hook + + - it: keeps the name when it lands exactly on 63 characters + release: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789 + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789-volume-resize-hook + + - it: does not end the cut name with a dash + release: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcd + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcd-volume-resize + + - it: cuts the name to 63 characters for a 53-character release name + release: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi-volume-re + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ + - equal: + path: spec.template.spec.serviceAccountName + value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi-volume-resize-hook + + - it: leaves the ServiceAccount on the full name for a 53-character release name + release: + name: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi + namespace: tenant-test + documentSelector: + path: kind + value: ServiceAccount + asserts: + - equal: + path: metadata.name + value: vm-disk-abcdefghijklmnopqrstuvwxyz0123456789abcdefghi-volume-resize-hook diff --git a/packages/apps/vm-disk/tests/upload_notes_test.yaml b/packages/apps/vm-disk/tests/upload_notes_test.yaml new file mode 100644 index 000000000..16a6432a3 --- /dev/null +++ b/packages/apps/vm-disk/tests/upload_notes_test.yaml @@ -0,0 +1,18 @@ +suite: VM disk upload notes +templates: + - templates/NOTES.txt + +release: + name: vm-disk-demo + namespace: tenant-root + +tests: + - it: reuses the chart-created DataVolume for local image upload + set: + source: + upload: {} + asserts: + - matchRegexRaw: + pattern: 'virtctl image-upload dv vm-disk-demo --no-create --namespace tenant-root --image-path ' + - notMatchRegexRaw: + pattern: 'virtctl image-upload dv -n' diff --git a/packages/apps/vm-instance/README.md b/packages/apps/vm-instance/README.md index 2365ca9c1..f2a0d001d 100644 --- a/packages/apps/vm-instance/README.md +++ b/packages/apps/vm-instance/README.md @@ -48,6 +48,7 @@ virtctl ssh @ | `disks` | List of disks to attach. | `[]object` | `[]` | | `disks[i].name` | Disk name. | `string` | `""` | | `disks[i].bus` | Disk bus type (e.g. "sata"). | `string` | `""` | +| `disks[i].bootOrder` | Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both. | `*int` | `null` | | `networks` | Networks to attach the VM to. | `[]object` | `[]` | | `networks[i].name` | Network attachment name. | `string` | `""` | | `subnets` | Deprecated: use networks instead. | `[]object` | `[]` | diff --git a/packages/apps/vm-instance/templates/service.yaml b/packages/apps/vm-instance/templates/service.yaml index cfeffa810..b757ab96d 100644 --- a/packages/apps/vm-instance/templates/service.yaml +++ b/packages/apps/vm-instance/templates/service.yaml @@ -30,6 +30,13 @@ spec: {{- if .Values.external }} {{- if and (eq .Values.externalMethod "WholeIP") (not .Values.externalPorts) }} - port: 65535 + {{- else if not .Values.externalPorts }} + {{- /* A Service needs at least one port. cozy-proxy only admits TCP and + UDP ports into its PortList filter, so an SCTP placeholder keeps the + external IP while every inbound TCP and UDP packet is dropped. */}} + - name: no-ports + port: 65535 + protocol: SCTP {{- else }} {{- range .Values.externalPorts }} - name: port-{{ . }} diff --git a/packages/apps/vm-instance/templates/vm-update-hook.yaml b/packages/apps/vm-instance/templates/vm-update-hook.yaml index 72bc617e2..ab77f21b1 100644 --- a/packages/apps/vm-instance/templates/vm-update-hook.yaml +++ b/packages/apps/vm-instance/templates/vm-update-hook.yaml @@ -1,5 +1,6 @@ {{- $vmName := include "virtual-machine.fullname" . -}} {{- $namespace := .Release.Namespace -}} +{{- $jobName := printf "%s-update-hook" .Release.Name | trunc 63 | trimSuffix "-" -}} {{- $existingVM := lookup "kubevirt.io/v1" "VirtualMachine" $namespace $vmName -}} {{- $existingService := lookup "v1" "Service" $namespace $vmName -}} @@ -48,7 +49,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: "{{ $.Release.Name }}-update-hook" + name: "{{ $jobName }}" annotations: helm.sh/hook: pre-install,pre-upgrade helm.sh/hook-weight: "0" @@ -57,7 +58,7 @@ spec: template: metadata: labels: - app: "{{ $.Release.Name }}-update-hook" + app: "{{ $jobName }}" policy.cozystack.io/allow-to-apiserver: "true" spec: serviceAccountName: {{ $.Release.Name }}-update-hook diff --git a/packages/apps/vm-instance/templates/vm.yaml b/packages/apps/vm-instance/templates/vm.yaml index 8153adf8f..0d7845621 100644 --- a/packages/apps/vm-instance/templates/vm.yaml +++ b/packages/apps/vm-instance/templates/vm.yaml @@ -92,7 +92,16 @@ spec: {{- end }} {{- end }} disks: + {{- /* KubeVirt rejects two devices sharing a bootOrder, so a clash is + reported here with the disk names rather than surfacing as a + webhook denial on the VirtualMachine. */}} + {{- $bootOrderOwners := dict }} {{- range $i, $disk := .Values.disks }} + {{- $bootOrder := int ($disk.bootOrder | default (add $i 1)) }} + {{- with get $bootOrderOwners (toString $bootOrder) }} + {{- fail (printf "disks %q and %q both have bootOrder %d; boot orders must be unique" . $disk.name $bootOrder) }} + {{- end }} + {{- $_ := set $bootOrderOwners (toString $bootOrder) $disk.name }} - name: disk-{{ $disk.name }} {{- $dv := lookup "cdi.kubevirt.io/v1beta1" "DataVolume" $.Release.Namespace (printf "vm-disk-%s" $disk.name) }} {{- if $dv }} @@ -104,7 +113,7 @@ spec: {{- else }} {{ $deviceKey }}: {} {{- end }} - bootOrder: {{ add $i 1 }} + bootOrder: {{ $bootOrder }} {{- else }} {{- fail (printf "Specified disk not exists in cluster: %s" .name) }} {{- end }} diff --git a/packages/apps/vm-instance/tests/disk_boot_order_test.yaml b/packages/apps/vm-instance/tests/disk_boot_order_test.yaml new file mode 100644 index 000000000..146cbaf53 --- /dev/null +++ b/packages/apps/vm-instance/tests/disk_boot_order_test.yaml @@ -0,0 +1,128 @@ +suite: vm-instance disk boot order + +# The first entry of `disks` is the one callers treat as the system disk, so +# booting from installer or rescue media must not require moving that media to +# the front of the list. These cases pin that an explicit bootOrder decides +# which device the VM boots from while list order stays as given, and that +# leaving it unset keeps the positional order existing VMs were rendered with. +release: + name: test-vm + namespace: tenant-test +templates: + - templates/vm.yaml +set: + fullnameOverride: test-vm + instanceType: "" + instanceProfile: "" + resources: + cpu: 1 + sockets: 1 + memory: 1Gi + _cluster: + scheduling: + dedicatedNodesForWindowsVMs: "false" +kubernetesProvider: + scheme: + "cdi.kubevirt.io/v1beta1/DataVolume": + gvr: + group: cdi.kubevirt.io + version: v1beta1 + resource: datavolumes + namespaced: true + objects: + - apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: vm-disk-system + namespace: tenant-test + annotations: {} + - apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: vm-disk-iso + namespace: tenant-test + annotations: + vm-disk.cozystack.io/optical: "true" + - apiVersion: cdi.kubevirt.io/v1beta1 + kind: DataVolume + metadata: + name: vm-disk-data + namespace: tenant-test + annotations: {} +tests: + - it: keeps the positional boot order when no disk sets bootOrder + set: + disks: + - name: system + - name: iso + - name: data + asserts: + - equal: + path: spec.template.spec.domain.devices.disks + value: + - name: disk-system + disk: {} + bootOrder: 1 + - name: disk-iso + cdrom: {} + bootOrder: 2 + - name: disk-data + disk: {} + bootOrder: 3 + + - it: boots from a CD-ROM in second position without reordering the disks + set: + disks: + - name: system + bootOrder: 2 + - name: iso + bus: sata + bootOrder: 1 + - name: data + asserts: + - equal: + path: spec.template.spec.domain.devices.disks + value: + - name: disk-system + disk: {} + bootOrder: 2 + - name: disk-iso + cdrom: + bus: sata + bootOrder: 1 + - name: disk-data + disk: {} + bootOrder: 3 + - equal: + path: spec.template.spec.volumes[0].dataVolume.name + value: vm-disk-system + + - it: refuses an explicit bootOrder that collides with another disk's positional one + set: + disks: + - name: system + - name: iso + bootOrder: 1 + asserts: + - failedTemplate: + errorMessage: 'disks "system" and "iso" both have bootOrder 1; boot orders must be unique' + + - it: refuses two disks given the same explicit bootOrder + set: + disks: + - name: system + bootOrder: 3 + - name: iso + bootOrder: 3 + asserts: + - failedTemplate: + errorMessage: 'disks "system" and "iso" both have bootOrder 3; boot orders must be unique' + + - it: rejects a bootOrder below 1 through the values schema + set: + disks: + - name: system + bootOrder: 0 + asserts: + - failedTemplate: + errorPattern: "minimum" diff --git a/packages/apps/vm-instance/tests/job_name_budget_test.yaml b/packages/apps/vm-instance/tests/job_name_budget_test.yaml new file mode 100644 index 000000000..fb4190930 --- /dev/null +++ b/packages/apps/vm-instance/tests/job_name_budget_test.yaml @@ -0,0 +1,107 @@ +suite: vm-instance update Job name budget + +# The API server copies a Job's name into the job-name label of its pods, and a +# label value takes 63 characters at most, while a release name may take 53. +# The pod's own app label carries the same name and is held to the same limit. +# The hook renders only when an existing VM or Service needs reconciling, so +# each release below gets a Service of the wrong type to look up. The +# ServiceAccount, Role and RoleBinding have no such limit and keep the full +# name, so the ones an earlier update left behind are replaced, not orphaned. + +templates: + - templates/vm-update-hook.yaml + +set: + external: false + +kubernetesProvider: + scheme: + "v1/Service": + gvr: + group: "" + version: "v1" + resource: "services" + namespaced: true + objects: + - kind: Service + apiVersion: v1 + metadata: + name: vm-instance-test + namespace: tenant-test + spec: + type: LoadBalancer + - kind: Service + apiVersion: v1 + metadata: + name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abc + namespace: tenant-test + spec: + type: LoadBalancer + - kind: Service + apiVersion: v1 + metadata: + name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde + namespace: tenant-test + spec: + type: LoadBalancer + +tests: + - it: keeps -update-hook for a short release name + release: + name: vm-instance-test + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: vm-instance-test-update-hook + - equal: + path: spec.template.metadata.labels.app + value: vm-instance-test-update-hook + + - it: keeps the name when it lands exactly on 63 characters + release: + name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abc + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abc-update-hook + + - it: cuts the Job name and the app label to 63 characters for a 53-character release name + release: + name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde + namespace: tenant-test + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.name + value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde-update-ho + - matchRegex: + path: metadata.name + pattern: ^[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?$ + - equal: + path: spec.template.metadata.labels.app + value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde-update-ho + - equal: + path: spec.template.spec.serviceAccountName + value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde-update-hook + + - it: leaves the ServiceAccount on the full name for a 53-character release name + release: + name: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde + namespace: tenant-test + documentSelector: + path: kind + value: ServiceAccount + asserts: + - equal: + path: metadata.name + value: vm-instance-abcdefghijklmnopqrstuvwxyz0123456789abcde-update-hook diff --git a/packages/apps/vm-instance/tests/service_ports_test.yaml b/packages/apps/vm-instance/tests/service_ports_test.yaml new file mode 100644 index 000000000..d7689d957 --- /dev/null +++ b/packages/apps/vm-instance/tests/service_ports_test.yaml @@ -0,0 +1,70 @@ +suite: vm-instance external Service ports + +# The API server rejects a Service without ports, so an external PortList VM +# whose port list was emptied could not be upgraded at all. The placeholder +# must be SCTP: cozy-proxy only admits TCP and UDP ports into its filter, so +# any TCP or UDP placeholder would open that port on the VM. +release: + name: test-vm + namespace: tenant-test +templates: + - templates/service.yaml +set: + fullnameOverride: test-vm + external: true + +tests: + - it: renders an SCTP placeholder for PortList with no ports + set: + externalMethod: PortList + externalPorts: [] + asserts: + - equal: + path: spec.type + value: LoadBalancer + - equal: + path: metadata.annotations["networking.cozystack.io/wholeIP"] + value: "false" + - lengthEqual: + path: spec.ports + count: 1 + - equal: + path: spec.ports[0] + value: + name: no-ports + port: 65535 + protocol: SCTP + + - it: renders the listed ports for PortList + set: + externalMethod: PortList + externalPorts: [22, 443] + asserts: + - lengthEqual: + path: spec.ports + count: 2 + - equal: + path: spec.ports[0] + value: + name: port-22 + port: 22 + targetPort: 22 + - notContains: + path: spec.ports + content: + name: no-ports + port: 65535 + protocol: SCTP + + - it: keeps the WholeIP placeholder when no ports are listed + set: + externalMethod: WholeIP + externalPorts: [] + asserts: + - equal: + path: metadata.annotations["networking.cozystack.io/wholeIP"] + value: "true" + - equal: + path: spec.ports + value: + - port: 65535 diff --git a/packages/apps/vm-instance/values.schema.json b/packages/apps/vm-instance/values.schema.json index 7c24a9e52..45adaac72 100644 --- a/packages/apps/vm-instance/values.schema.json +++ b/packages/apps/vm-instance/values.schema.json @@ -69,6 +69,11 @@ "name" ], "properties": { + "bootOrder": { + "description": "Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both.", + "type": "integer", + "minimum": 1 + }, "bus": { "description": "Disk bus type (e.g. \"sata\").", "type": "string" diff --git a/packages/apps/vm-instance/values.yaml b/packages/apps/vm-instance/values.yaml index 88fc6f8dd..2730812a0 100644 --- a/packages/apps/vm-instance/values.yaml +++ b/packages/apps/vm-instance/values.yaml @@ -10,6 +10,8 @@ ## @field {string} name - Disk name. ## @x-cozystack-options {source: vmdisk} ## @field {string} [bus] - Disk bus type (e.g. "sata"). +## @field {*int} [bootOrder] - Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both. +## @minimum 1 ## @typedef {struct} Network - Network to attach the VM to. ## @field {string} [name] - Network attachment name. @@ -67,6 +69,12 @@ disks: [] ## - name: example-system ## - name: example-data ## bus: sata +## Boot from an attached ISO while keeping the system disk first in the list: +## disks: +## - name: example-system +## bootOrder: 2 +## - name: example-iso +## bootOrder: 1 ## @param {[]Network} networks - Networks to attach the VM to. networks: [] diff --git a/packages/apps/vpc/Makefile b/packages/apps/vpc/Makefile index fce015f3e..6cc55dbed 100644 --- a/packages/apps/vpc/Makefile +++ b/packages/apps/vpc/Makefile @@ -1,7 +1,7 @@ include ../../../hack/package.mk generate: - cozyvalues-gen -m 'vpc' -v values.yaml -s values.schema.json -g ../../../api/apps/v1alpha1/vpc/types.go + cozyvalues-gen -m 'vpc' -v values.yaml -s values.schema.json -r README.md -g ../../../api/apps/v1alpha1/vpc/types.go ../../../hack/update-crd.sh update: diff --git a/packages/apps/vpc/README.md b/packages/apps/vpc/README.md index 890f12d89..f2dd2a5f9 100644 --- a/packages/apps/vpc/README.md +++ b/packages/apps/vpc/README.md @@ -27,10 +27,18 @@ A VM or a pod may be connected to multiple secondary Subnets at once. Each secon ### Common parameters -| Name | Description | Type | Value | -| -------------------- | -------------------------------- | ------------------- | ------- | -| `subnets` | Subnets of a VPC | `map[string]object` | `{...}` | -| `subnets[name].cidr` | Subnet CIDR, e.g. 192.168.0.0/24 | `cidr` | `{}` | +| Name | Description | Type | Value | +| -------------------------- | ----------------------------------------------------------------------------------------------------------------- | ---------- | ----- | +| `subnets` | Subnets of a VPC | `[]object` | `[]` | +| `subnets[i].name` | Subnet name | `string` | `""` | +| `subnets[i].cidr` | IP address range | `string` | `""` | +| `subnets[i].allowSubnets` | Additional source CIDRs admitted into this private subnet, such as remote networks routed in through a gateway VM | `[]string` | `[]` | +| `peers` | VPC peering connections (bidirectional declaration required) | `[]object` | `[]` | +| `peers[i].vpcName` | Logical name of the remote VPC (without "virtualprivatecloud-" prefix) | `string` | `""` | +| `peers[i].tenantNamespace` | Namespace of the remote tenant | `string` | `""` | +| `routes` | Static routes for the VPC | `[]object` | `[]` | +| `routes[i].cidr` | Destination CIDR | `string` | `""` | +| `routes[i].nextHopIP` | Next hop IP address | `string` | `""` | ## Examples @@ -41,10 +49,10 @@ metadata: name: vpc00 spec: subnets: - sub00: + - name: sub00 cidr: 172.16.0.0/24 - sub01: + - name: sub01 cidr: 172.16.1.0/24 - sub02: + - name: sub02 cidr: 172.16.2.0/24 ``` diff --git a/packages/core/platform/sources/foundationdb-operator.yaml b/packages/core/platform/sources/foundationdb-operator.yaml index 0af881f60..28328c3d1 100644 --- a/packages/core/platform/sources/foundationdb-operator.yaml +++ b/packages/core/platform/sources/foundationdb-operator.yaml @@ -20,3 +20,9 @@ spec: install: namespace: cozy-foundationdb-operator releaseName: foundationdb-operator + # helm-controller leaves existing CRDs alone on upgrade, so a cluster + # installed before v1.6.0 keeps the v2.13.0 schema under the v2.30.0 + # operator, and the apiserver prunes the fields added since, + # status.backupDetails.restorable among them. The schema diff adds + # fields and drops none, so replace is safe. + upgradeCRDs: CreateReplace diff --git a/packages/core/platform/sources/keycloak-operator.yaml b/packages/core/platform/sources/keycloak-operator.yaml index ce3a5f7e0..5b116e5e6 100644 --- a/packages/core/platform/sources/keycloak-operator.yaml +++ b/packages/core/platform/sources/keycloak-operator.yaml @@ -20,3 +20,10 @@ spec: install: namespace: cozy-keycloak releaseName: keycloak-operator + # helm-controller leaves existing CRDs alone on upgrade, so a cluster + # installed before v1.2.0 keeps the v1.25.0 schema and the apiserver + # prunes the realm session settings keycloak-configure sets. v1.32.0 + # drops the realm, targetRealm and keycloakOwner fields deprecated in + # favour of realmRef and keycloakRef. The platform renders only the + # refs, so replace is safe. + upgradeCRDs: CreateReplace diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index 906c8d95f..814bf201e 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -178,9 +178,19 @@ {{- $controllerValues := dict "cozystackController" (dict "disableTelemetry" true) -}} {{- $_ := set $cozystackEngineComponents "cozystack-controller" (dict "values" $controllerValues) -}} {{- end -}} +{{- $keycloakTruststores := .Values.authentication.oidc.keycloakTruststores -}} +{{- $hasKeycloakTruststores := and (not (kindIs "invalid" $keycloakTruststores)) (or (not (kindIs "slice" $keycloakTruststores)) (not (empty $keycloakTruststores))) -}} +{{- $keycloakComponents := dict -}} +{{- if $hasKeycloakTruststores }} +{{- $_ := set $keycloakComponents "keycloak" (dict "values" (dict "truststores" $keycloakTruststores)) -}} +{{- end }} {{- if .Values.authentication.oidc.enabled }} {{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "oidc" $ $cozystackEngineComponents) }} +{{- if $hasKeycloakTruststores }} +{{include "cozystack.platform.package" (list "cozystack.keycloak" "default" $ $keycloakComponents) }} +{{- else }} {{include "cozystack.platform.package.default" (list "cozystack.keycloak" $) }} +{{- end }} {{include "cozystack.platform.package.default" (list "cozystack.keycloak-operator" $) }} {{- else }} {{include "cozystack.platform.package" (list "cozystack.cozystack-engine" "default" $ $cozystackEngineComponents) }} diff --git a/packages/core/platform/tests/bundles_keycloak_truststores_test.yaml b/packages/core/platform/tests/bundles_keycloak_truststores_test.yaml new file mode 100644 index 000000000..6319c6dd9 --- /dev/null +++ b/packages/core/platform/tests/bundles_keycloak_truststores_test.yaml @@ -0,0 +1,203 @@ +suite: bundles.system Keycloak truststores wiring +templates: + - templates/bundles/system.yaml +release: + name: cozystack + namespace: cozy-system +tests: + - it: emits cozystack.keycloak without a components block when keycloakTruststores is unset + set: + bundles: + system: + enabled: true + variant: isp-full + authentication: + oidc: + enabled: true + documentSelector: + path: metadata.name + value: cozystack.keycloak + asserts: + - equal: + path: kind + value: Package + - equal: + path: spec.variant + value: default + - notExists: + path: spec.components + + - it: omits the components block when keycloakTruststores is explicitly empty + set: + bundles: + system: + enabled: true + variant: isp-full + authentication: + oidc: + enabled: true + keycloakTruststores: [] + documentSelector: + path: metadata.name + value: cozystack.keycloak + asserts: + - notExists: + path: spec.components + + - it: omits the components block when keycloakTruststores is explicitly null + set: + bundles: + system: + enabled: true + variant: isp-full + authentication: + oidc: + enabled: true + keycloakTruststores: null + documentSelector: + path: metadata.name + value: cozystack.keycloak + asserts: + - notExists: + path: spec.components + + - it: forwards false so the Keycloak chart can reject it instead of silently removing trust + set: + bundles: + system: + enabled: true + variant: isp-full + authentication: + oidc: + enabled: true + keycloakTruststores: false + documentSelector: + path: metadata.name + value: cozystack.keycloak + asserts: + - equal: + path: spec.components.keycloak.values.truststores + value: false + + - it: forwards zero so the Keycloak chart can reject it instead of silently removing trust + set: + bundles: + system: + enabled: true + variant: isp-full + authentication: + oidc: + enabled: true + keycloakTruststores: 0 + documentSelector: + path: metadata.name + value: cozystack.keycloak + asserts: + - equal: + path: spec.components.keycloak.values.truststores + value: 0 + + - it: forwards an empty string so the Keycloak chart can reject it instead of silently removing trust + set: + bundles: + system: + enabled: true + variant: isp-full + authentication: + oidc: + enabled: true + keycloakTruststores: "" + documentSelector: + path: metadata.name + value: cozystack.keycloak + asserts: + - equal: + path: spec.components.keycloak.values.truststores + value: "" + + - it: forwards an empty map so the Keycloak chart can reject it instead of silently removing trust + set: + bundles: + system: + enabled: true + variant: isp-full + authentication: + oidc: + enabled: true + keycloakTruststores: {} + documentSelector: + path: metadata.name + value: cozystack.keycloak + asserts: + - equal: + path: spec.components.keycloak.values.truststores + value: {} + + - it: forwards the complete truststores list without taking over chart defaults + set: + bundles: + system: + enabled: true + variant: isp-full + authentication: + oidc: + enabled: true + keycloakTruststores: + - configMapName: organization-ca + - secretName: partner-ca + key: certificates.pem + - configMapName: secondary-ca + key: chain.pem + documentSelector: + path: metadata.name + value: cozystack.keycloak + asserts: + - equal: + path: spec.components.keycloak.values + value: + truststores: + - configMapName: organization-ca + - secretName: partner-ca + key: certificates.pem + - configMapName: secondary-ca + key: chain.pem + + - it: forwards keycloakTruststores on the isp-slim variant too + set: + bundles: + system: + enabled: true + variant: isp-slim + authentication: + oidc: + enabled: true + keycloakTruststores: + - configMapName: organization-ca + key: ca.crt + documentSelector: + path: metadata.name + value: cozystack.keycloak + asserts: + - equal: + path: spec.components.keycloak.values.truststores + value: + - configMapName: organization-ca + key: ca.crt + + - it: does not emit cozystack.keycloak for a malformed false-y value when OIDC is disabled + set: + bundles: + system: + enabled: true + variant: isp-full + authentication: + oidc: + enabled: false + keycloakTruststores: false + asserts: + - containsDocument: + apiVersion: cozystack.io/v1alpha1 + kind: Package + name: cozystack.keycloak + any: true + not: true diff --git a/packages/core/platform/tests/observability_proxy_wiring_test.yaml b/packages/core/platform/tests/observability_proxy_wiring_test.yaml index 9e6ec583d..84777ca7f 100644 --- a/packages/core/platform/tests/observability_proxy_wiring_test.yaml +++ b/packages/core/platform/tests/observability_proxy_wiring_test.yaml @@ -5,6 +5,7 @@ suite: observability-proxy package wiring templates: - templates/sources.yaml - templates/bundles/system.yaml + - templates/apps.yaml release: name: cozystack namespace: cozy-system @@ -67,3 +68,12 @@ tests: kind: Package name: cozystack.observability-proxy any: true + + # The proxy's HTTPRoute lives in its own namespace, and the tenant Gateway + # admits routes only from the namespaces listed here. + - it: lets the proxy namespace attach a route to the publishing Gateway + template: templates/apps.yaml + asserts: + - matchRegex: + path: stringData["values.yaml"] + pattern: 'gateway-attached-namespaces: "([^"]*,)?cozy-observability-proxy(,[^"]*)?"' diff --git a/packages/core/platform/tests/sources_foundationdb_operator_crds_test.yaml b/packages/core/platform/tests/sources_foundationdb_operator_crds_test.yaml new file mode 100644 index 000000000..a0976ae57 --- /dev/null +++ b/packages/core/platform/tests/sources_foundationdb_operator_crds_test.yaml @@ -0,0 +1,20 @@ +suite: foundationdb-operator applies CRD updates on upgrade +# The fdb-operator v2.30.0 CRDs add fields the v2.13.0 ones lack, among them +# status.backupDetails.restorable on FoundationDBBackup. Helm never upgrades +# CRDs shipped in crds/, so without CreateReplace a cluster installed before +# v1.6.0 keeps the v2.13.0 CRDs and the apiserver prunes those fields on every +# write. Pin the policy so an upgrade cannot silently regress. +templates: + - templates/sources.yaml +release: + name: cozystack + namespace: cozy-system +tests: + - it: foundationdb-operator PackageSource sets upgradeCRDs to CreateReplace + documentSelector: + path: metadata.name + value: cozystack.foundationdb-operator + asserts: + - equal: + path: spec.variants[0].components[0].install.upgradeCRDs + value: CreateReplace diff --git a/packages/core/platform/tests/sources_keycloak_operator_crds_test.yaml b/packages/core/platform/tests/sources_keycloak_operator_crds_test.yaml new file mode 100644 index 000000000..5cccfc757 --- /dev/null +++ b/packages/core/platform/tests/sources_keycloak_operator_crds_test.yaml @@ -0,0 +1,22 @@ +suite: keycloak-operator applies CRD updates on upgrade +# keycloak-configure sets spec.sessions and spec.organizationsEnabled, and +# spec.login when configured, on the cozy ClusterKeycloakRealm. The CRDs of +# keycloak-operator v1.25.0 declare none of them. Helm never upgrades CRDs +# shipped in crds/, so without CreateReplace a cluster installed before v1.2.0 +# keeps those CRDs, the apiserver drops the fields without an error, and the +# operator never applies them to the realm. Pin the policy so an upgrade cannot +# silently regress. +templates: + - templates/sources.yaml +release: + name: cozystack + namespace: cozy-system +tests: + - it: keycloak-operator PackageSource sets upgradeCRDs to CreateReplace + documentSelector: + path: metadata.name + value: cozystack.keycloak-operator + asserts: + - equal: + path: spec.variants[0].components[0].install.upgradeCRDs + value: CreateReplace diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index f2e35b5e5..8ea4a054a 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -549,6 +549,19 @@ authentication: # OIDC discovery; any other value -> used verbatim. An explicit value is not rewritten, so a # custom realm paired with an explicit URL must name the same realm. keycloakInternalUrl: null + # Additional CA trust anchors for the bundled Keycloak server (LDAPS, + # external IdPs, SMTP). Each entry selects one key from an existing Secret + # or ConfigMap in cozy-keycloak; key defaults to ca.crt. Trust applies to + # the entire server across all realms. See packages/system/keycloak/values.yaml + # for supported formats, incoming-client-mTLS scope and rotation behavior. + # Ignored when oidc.enabled is false: no Keycloak Package is emitted. An + # existing Package and its rendered truststores remain on the cluster + # (helm.sh/resource-policy: keep). + keycloakTruststores: [] + # - configMapName: organization-ca + # key: ca.crt + # - secretName: partner-ca + # key: certificates.pem # Gateway API configuration gateway: # Enable Gateway API support across the platform. When true: @@ -668,6 +681,7 @@ gateway: - cozy-kubevirt-cdi - cozy-monitoring - cozy-linstor-gui + - cozy-observability-proxy - default # Platform ingress configuration. ingress: diff --git a/packages/extra/etcd/README.md b/packages/extra/etcd/README.md index e96c63832..91341ff32 100644 --- a/packages/extra/etcd/README.md +++ b/packages/extra/etcd/README.md @@ -34,14 +34,14 @@ coordinates). ### Common parameters -| Name | Description | Type | Value | -| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------- | -| `version` | etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image. | `string` | `3.6.11` | -| `size` | Persistent Volume size. | `quantity` | `4Gi` | -| `storageClass` | StorageClass used to store the data. | `string` | `""` | -| `replicas` | Number of etcd replicas. | `int` | `3` | -| `resources` | Resource configuration for etcd. | `object` | `{}` | -| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` | -| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` | -| `affinity` | Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively. | `object` | `{}` | +| Name | Description | Type | Value | +| ------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------- | -------- | +| `version` | etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image. | `string` | `3.6.11` | +| `size` | Persistent Volume size. | `quantity` | `4Gi` | +| `storageClass` | StorageClass used to store the data. | `string` | `""` | +| `replicas` | Number of etcd replicas. | `int` | `3` | +| `resources` | Resource configuration for etcd. | `object` | `{}` | +| `resources.cpu` | Number of CPU cores allocated. | `quantity` | `1000m` | +| `resources.memory` | Amount of memory allocated. | `quantity` | `512Mi` | +| `affinity` | Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones. | `object` | `{}` | diff --git a/packages/extra/etcd/templates/etcd-cluster.yaml b/packages/extra/etcd/templates/etcd-cluster.yaml index 2824ce371..3676f53cb 100644 --- a/packages/extra/etcd/templates/etcd-cluster.yaml +++ b/packages/extra/etcd/templates/etcd-cluster.yaml @@ -69,7 +69,8 @@ spec: topologySpreadConstraints: - maxSkew: 1 topologyKey: "kubernetes.io/hostname" - whenUnsatisfiable: ScheduleAnyway + whenUnsatisfiable: DoNotSchedule + nodeTaintsPolicy: Ignore labelSelector: matchLabels: app.kubernetes.io/instance: etcd @@ -78,11 +79,15 @@ spec: affinity: {{- toYaml . | nindent 4 }} {{- else }} - # Soft anti-affinity: spread etcd members across nodes on a best-effort - # basis so a single node failure is unlikely to take out more than one - # member. Preferred (not required) because a hard rule would leave a member - # Pending when replicas exceed schedulable nodes, or when node-local storage - # pins a member to a node. Override via .Values.affinity for a hard rule. + # Soft anti-affinity, a preference on top of the spread constraint above, + # which by default is the hard rule that keeps two members off one node + # while another node has none. Preferred, not required, because a required + # rule also forbids two members on one node when every node already has + # one, so an etcd with more members than schedulable nodes could not start + # all of them. With node-local storage a recreated member Pod can only + # return to its own node, and the default spread keeps it Pending there + # while that node holds another member and some other node holds none. + # Override via .Values.affinity for a hard rule. affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: diff --git a/packages/extra/etcd/templates/etcd-defrag.yaml b/packages/extra/etcd/templates/etcd-defrag.yaml index 52cc63518..c1eefa0db 100644 --- a/packages/extra/etcd/templates/etcd-defrag.yaml +++ b/packages/extra/etcd/templates/etcd-defrag.yaml @@ -29,11 +29,11 @@ spec: drop: ["ALL"] resources: requests: - cpu: 200m - memory: 256Mi + cpu: 10m + memory: 32Mi limits: - cpu: 500m - memory: 512Mi + cpu: 100m + memory: 64Mi args: # The v1alpha2 operator names the headless Service after the # cluster ({{ $.Release.Name }}) and does not use StatefulSet diff --git a/packages/extra/etcd/tests/defrag_test.yaml b/packages/extra/etcd/tests/defrag_test.yaml index 8d74e1a6b..b3acc3401 100644 --- a/packages/extra/etcd/tests/defrag_test.yaml +++ b/packages/extra/etcd/tests/defrag_test.yaml @@ -41,3 +41,16 @@ tests: documentSelector: path: kind value: EtcdCluster + + - it: asks the tenant quota only for what the defrag client uses + template: templates/etcd-defrag.yaml + asserts: + - equal: + path: spec.jobTemplate.spec.template.spec.containers[0].resources + value: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi diff --git a/packages/extra/etcd/tests/topology-spread_test.yaml b/packages/extra/etcd/tests/topology-spread_test.yaml index 4b96fc310..11bb92243 100644 --- a/packages/extra/etcd/tests/topology-spread_test.yaml +++ b/packages/extra/etcd/tests/topology-spread_test.yaml @@ -8,7 +8,7 @@ templates: - templates/etcd-cluster.yaml tests: - - it: uses a soft per-host spread by default + - it: uses a strict per-host spread by default set: _cluster: {} documentSelector: @@ -16,17 +16,41 @@ tests: value: EtcdCluster asserts: - equal: - path: spec.topologySpreadConstraints[0].maxSkew - value: 1 + path: spec.topologySpreadConstraints[0] + value: + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + nodeTaintsPolicy: Ignore + labelSelector: + matchLabels: + app.kubernetes.io/instance: etcd + - lengthEqual: + path: spec.topologySpreadConstraints + count: 1 + + - it: renders a soft cluster-wide spread policy as written + set: + _cluster: + scheduling: + globalAppTopologySpreadConstraints: | + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + documentSelector: + path: kind + value: EtcdCluster + asserts: - equal: - path: spec.topologySpreadConstraints[0].topologyKey - value: kubernetes.io/hostname - - equal: - path: spec.topologySpreadConstraints[0].whenUnsatisfiable - value: ScheduleAnyway - - equal: - path: spec.topologySpreadConstraints[0].labelSelector.matchLabels["app.kubernetes.io/instance"] - value: etcd + path: spec.topologySpreadConstraints[0] + value: + maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/instance: etcd - lengthEqual: path: spec.topologySpreadConstraints count: 1 diff --git a/packages/extra/etcd/values.schema.json b/packages/extra/etcd/values.schema.json index b438ca9d7..c33219f4f 100644 --- a/packages/extra/etcd/values.schema.json +++ b/packages/extra/etcd/values.schema.json @@ -67,7 +67,7 @@ } }, "affinity": { - "description": "Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.", + "description": "Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.", "type": "object", "default": {}, "x-kubernetes-preserve-unknown-fields": true diff --git a/packages/extra/etcd/values.yaml b/packages/extra/etcd/values.yaml index 61b994d7c..c5be94909 100644 --- a/packages/extra/etcd/values.yaml +++ b/packages/extra/etcd/values.yaml @@ -23,5 +23,5 @@ resources: cpu: 1000m memory: 512Mi -## @param {object} [affinity] - Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively. +## @param {object} [affinity] - Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones. affinity: {} diff --git a/packages/extra/ingress/templates/nginx-scrape.yaml b/packages/extra/ingress/templates/nginx-scrape.yaml index d04c5b651..4917206de 100644 --- a/packages/extra/ingress/templates/nginx-scrape.yaml +++ b/packages/extra/ingress/templates/nginx-scrape.yaml @@ -9,6 +9,13 @@ spec: matchNames: - {{ .Release.Namespace }} podMetricsEndpoints: + # Most series describe the Ingress objects served, in any namespace, so + # the application identity the vmagent scrape class stamps from the + # controller pod would misattribute them; and with honorLabels an + # exporter-supplied identity would win over the class. The controller's + # own series (nginx process, reload status, go_*, up, scrape_*) lose the + # Ingress application's identity too; that is the accepted cost. The + # target-stage drop covers up and scrape_*, which skip metric relabeling. - port: metrics honorLabels: true interval: 30s @@ -19,6 +26,11 @@ spec: targetLabel: node - targetLabel: tier replacement: cluster + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) + metricRelabelConfigs: + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) selector: matchLabels: app.kubernetes.io/component: controller @@ -46,6 +58,11 @@ spec: targetLabel: controller - targetLabel: tier replacement: cluster + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) + metricRelabelConfigs: + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) selector: matchLabels: app.kubernetes.io/component: controller diff --git a/packages/extra/ingress/tests/application_identity_test.yaml b/packages/extra/ingress/tests/application_identity_test.yaml new file mode 100644 index 000000000..4914049c8 --- /dev/null +++ b/packages/extra/ingress/tests/application_identity_test.yaml @@ -0,0 +1,62 @@ +suite: ingress-nginx series do not carry an application identity +templates: + - templates/nginx-scrape.yaml +release: + name: ingress + namespace: tenant-root + +# The vmagent scrape class stamps the scraped pod's application identity on +# every series. Most ingress-nginx series describe the Ingress objects it +# serves, in whatever namespace they live (honorLabels keeps their +# namespace), so the controller's identity would misattribute them, and with +# honorLabels true an exporter-supplied identity would win over the class. +# The controller's own health series lose that identity too, by design. Both +# endpoints drop the identity instead: from the target, because vmagent +# adds up and scrape_* with the target labels after metric relabeling, and +# from the scraped samples, because an exporter's copy is not renamed. +tests: + - it: drops the identity on every endpoint + asserts: + - lengthEqual: + path: spec.podMetricsEndpoints + count: 1 + - equal: + path: spec.podMetricsEndpoints[0].metricRelabelConfigs + value: + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) + + # Endpoint rules run after the scrape class rules, so the drop takes + # effect wherever it sits in this list; it is pinned last so that no + # later rule can set the identity again. + - it: drops the identity from the basic endpoint's target as its last rule + documentIndex: 0 + asserts: + - equal: + path: spec.podMetricsEndpoints[0].relabelConfigs + value: + - replacement: nginx-ingress-controller + targetLabel: job + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node + - targetLabel: tier + replacement: cluster + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) + + - it: drops the identity from the detailed endpoint's target as its last rule + documentIndex: 1 + asserts: + - equal: + path: spec.podMetricsEndpoints[0].relabelConfigs + value: + - replacement: nginx-ingress-controller + targetLabel: job + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node + - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + targetLabel: controller + - targetLabel: tier + replacement: cluster + - action: labeldrop + regex: apps_cozystack_io_application_(group|kind|name) diff --git a/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag b/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag index bf4084277..b84ea7484 100644 --- a/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag +++ b/packages/extra/seaweedfs/images/seaweedfs-cosi-driver.tag @@ -1 +1 @@ -ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.4.0 +ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.5.0 diff --git a/packages/extra/seaweedfs/templates/client/cosi-bucket-class.yaml b/packages/extra/seaweedfs/templates/client/cosi-bucket-class.yaml index ce049b316..0b262f47d 100644 --- a/packages/extra/seaweedfs/templates/client/cosi-bucket-class.yaml +++ b/packages/extra/seaweedfs/templates/client/cosi-bucket-class.yaml @@ -35,4 +35,13 @@ driverName: {{ .Release.Namespace }}.seaweedfs.objectstorage.k8s.io authenticationType: KEY parameters: accessPolicy: readonly +--- +kind: BucketAccessClass +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: {{ .Release.Namespace }}-public +driverName: {{ .Release.Namespace }}.seaweedfs.objectstorage.k8s.io +authenticationType: KEY +parameters: + accessPolicy: public-read {{- end }} diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index 92b40ef4d..4c4174a78 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -32,8 +32,8 @@ {{- if not (regexMatch "^[a-z0-9]([a-z0-9-]*[a-z0-9])?$" $poolName) }} {{- fail (printf "volume.pools key '%s' must be a valid DNS label (lowercase alphanumeric and hyphens, no dots)." $poolName) }} {{- end }} -{{- if or (hasSuffix "-lock" $poolName) (hasSuffix "-readonly" $poolName) }} -{{- fail (printf "volume.pools key '%s' must not end with '-lock' or '-readonly' (reserved suffixes for COSI resources)." $poolName) }} +{{- if or (hasSuffix "-lock" $poolName) (hasSuffix "-readonly" $poolName) (hasSuffix "-public" $poolName) (eq "public" $poolName) }} +{{- fail (printf "volume.pools key '%s' must not be 'public' or end with '-lock', '-readonly' or '-public' (reserved for COSI resources)." $poolName) }} {{- end }} {{- if not $pool.diskType }} {{- fail (printf "volume.pools.%s.diskType is required." $poolName) }} @@ -52,8 +52,8 @@ {{- if not (regexMatch "^[a-z0-9]([a-z0-9-]*[a-z0-9])?$" $poolName) }} {{- fail (printf "volume.zones.%s.pools key '%s' must be a valid DNS label." $zoneName $poolName) }} {{- end }} -{{- if or (hasSuffix "-lock" $poolName) (hasSuffix "-readonly" $poolName) }} -{{- fail (printf "volume.zones.%s.pools key '%s' must not end with '-lock' or '-readonly' (reserved suffixes for COSI resources)." $zoneName $poolName) }} +{{- if or (hasSuffix "-lock" $poolName) (hasSuffix "-readonly" $poolName) (hasSuffix "-public" $poolName) (eq "public" $poolName) }} +{{- fail (printf "volume.zones.%s.pools key '%s' must not be 'public' or end with '-lock', '-readonly' or '-public' (reserved for COSI resources)." $zoneName $poolName) }} {{- end }} {{- if not $pool.diskType }} {{- fail (printf "volume.zones.%s.pools.%s.diskType is required." $zoneName $poolName) }} @@ -248,13 +248,19 @@ spec: {{- /* An explicit null is dropped before schema validation, so neither bound sees it, and the master exits on the empty flag it renders. */}} volumeSizeLimitMB: {{ required "master.volumeSizeLimitMB must be set" .Values.master.volumeSizeLimitMB }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.master.resourcesPreset .Values.master.resources $) | nindent 10 }} +{{- $masterResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.master.resourcesPreset .Values.master.resources $) | fromYaml }} + resources: {{- toYaml $masterResources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $masterResources | quote }} volume: {{ if eq .Values.topology "MultiZone" }} enabled: false {{- end }} replicas: {{ .Values.volume.replicas }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.volume.resourcesPreset .Values.volume.resources $) | nindent 10 }} +{{- $volumeResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.volume.resourcesPreset .Values.volume.resources $) | fromYaml }} + resources: {{- toYaml $volumeResources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $volumeResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -273,7 +279,13 @@ spec: {{- range $poolName, $pool := .Values.volume.pools }} {{ $poolName }}: replicas: {{ ternary $pool.replicas $.Values.volume.replicas (hasKey $pool "replicas") }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | nindent 12 }} +{{- $poolResources := include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | fromYaml }} + resources: {{- toYaml $poolResources | nindent 12 }} + {{- /* The subchart merges volume into every entry of volumes, so a + pool that does not set its own value runs on the default + volume's, sized for a different limit. */}} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $poolResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -289,7 +301,9 @@ spec: {{- range $zoneName, $zone := .Values.volume.zones }} {{ $zoneName }}: replicas: {{ ternary $zone.replicas $.Values.volume.replicas (hasKey $zone "replicas") }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list $.Values.volume.resourcesPreset $.Values.volume.resources $) | nindent 12 }} + resources: {{- toYaml $volumeResources | nindent 12 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $volumeResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -314,7 +328,13 @@ spec: {{- range $poolName, $pool := (dig "pools" dict $zone) }} {{ $zoneName }}-{{ $poolName }}: replicas: {{ ternary $pool.replicas (ternary $zone.replicas $.Values.volume.replicas (hasKey $zone "replicas")) (hasKey $pool "replicas") }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | nindent 12 }} +{{- $poolResources := include "cozy-lib.resources.defaultingSanitize" (list ($pool.resourcesPreset | default $.Values.volume.resourcesPreset) (default dict $pool.resources) $) | fromYaml }} + resources: {{- toYaml $poolResources | nindent 12 }} + {{- /* The subchart merges volume into every entry of volumes, so a + pool that does not set its own value runs on the default + volume's, sized for a different limit. */}} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $poolResources | quote }} dataDirs: - name: data1 type: "persistentVolumeClaim" @@ -345,7 +365,10 @@ spec: replicas: {{ .Values.filer.replicas }} s3: domainName: {{ .Values.host | default (printf "s3.%s" $host) }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.filer.resourcesPreset .Values.filer.resources $) | nindent 10 }} +{{- $filerResources := include "cozy-lib.resources.defaultingSanitize" (list .Values.filer.resourcesPreset .Values.filer.resources $) | fromYaml }} + resources: {{- toYaml $filerResources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $filerResources | quote }} s3: replicas: {{ .Values.s3.replicas }} ingress: @@ -381,7 +404,10 @@ spec: {{- if not $wildcardSecret }} secretName: {{ .Release.Name }}-s3-ingress-tls {{- end }} - resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.s3.resourcesPreset .Values.s3.resources $) | nindent 10 }} +{{- $s3Resources := include "cozy-lib.resources.defaultingSanitize" (list .Values.s3.resourcesPreset .Values.s3.resources $) | fromYaml }} + resources: {{- toYaml $s3Resources | nindent 10 }} + extraEnvironmentVars: + GOMEMLIMIT: {{ include "cozy-lib.resources.goMemLimit" $s3Resources | quote }} admin: enabled: {{ .Values.lifecycle.enabled }} secret: diff --git a/packages/extra/seaweedfs/templates/storage-pool-bucket-classes.yaml b/packages/extra/seaweedfs/templates/storage-pool-bucket-classes.yaml index bf58340a2..faa99f137 100644 --- a/packages/extra/seaweedfs/templates/storage-pool-bucket-classes.yaml +++ b/packages/extra/seaweedfs/templates/storage-pool-bucket-classes.yaml @@ -51,5 +51,14 @@ driverName: {{ $.Release.Namespace }}.seaweedfs.objectstorage.k8s.io authenticationType: KEY parameters: accessPolicy: readonly +--- +kind: BucketAccessClass +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: {{ $.Release.Namespace }}-{{ $poolName }}-public +driverName: {{ $.Release.Namespace }}.seaweedfs.objectstorage.k8s.io +authenticationType: KEY +parameters: + accessPolicy: public-read {{- end }} {{- end }} diff --git a/packages/extra/seaweedfs/tests/gomemlimit_test.yaml b/packages/extra/seaweedfs/tests/gomemlimit_test.yaml new file mode 100644 index 000000000..2b298b10e --- /dev/null +++ b/packages/extra/seaweedfs/tests/gomemlimit_test.yaml @@ -0,0 +1,162 @@ +suite: seaweedfs components carry a GOMEMLIMIT derived from their memory limit +templates: + - templates/seaweedfs.yaml + +# Every SeaweedFS component is a Go binary. Without GOMEMLIMIT the runtime +# collects on a relative target and keeps no absolute ceiling, so a burst of +# allocation grows the heap past the cgroup limit and the kernel kills the +# container. Each component sizes its own value from its own memory limit, +# so the assertions below track the limit rather than a fixed string. + +tests: + - it: derives GOMEMLIMIT from the default preset on every component + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + asserts: + - equal: + path: spec.values.seaweedfs.master.resources.limits.memory + value: 512Mi + - equal: + path: spec.values.seaweedfs.master.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + - equal: + path: spec.values.seaweedfs.volume.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + - equal: + path: spec.values.seaweedfs.filer.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + - equal: + path: spec.values.seaweedfs.s3.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + + - it: follows the memory limit when a component is resized + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + s3: + resources: + memory: 8Gi + volume: + resources: + memory: 2Gi + asserts: + - equal: + path: spec.values.seaweedfs.s3.resources.limits.memory + value: 8Gi + - equal: + path: spec.values.seaweedfs.s3.extraEnvironmentVars.GOMEMLIMIT + value: 6144MiB + - equal: + path: spec.values.seaweedfs.volume.extraEnvironmentVars.GOMEMLIMIT + value: 1536MiB + - equal: + path: spec.values.seaweedfs.master.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + + - it: keeps GOMEMLIMIT below the cgroup limit it protects + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + s3: + resources: + memory: 1Gi + asserts: + - equal: + path: spec.values.seaweedfs.s3.resources.limits.memory + value: 1Gi + - equal: + path: spec.values.seaweedfs.s3.extraEnvironmentVars.GOMEMLIMIT + value: 768MiB + + # The subchart merges volume into every entry of volumes, so a pool that + # did not carry its own value would run on the default volume's, sized for + # a larger limit than its own. + - it: sizes a Simple pool from its own limit, not the default volume's + documentSelector: + path: kind + value: HelmRelease + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + volume: + pools: + small: + diskType: ssd + resourcesPreset: t1.nano + asserts: + - equal: + path: spec.values.seaweedfs.volumes.small.resources.limits.memory + value: 128Mi + - equal: + path: spec.values.seaweedfs.volumes.small.extraEnvironmentVars.GOMEMLIMIT + value: 96MiB + - equal: + path: spec.values.seaweedfs.volume.extraEnvironmentVars.GOMEMLIMIT + value: 384MiB + + - it: sizes zones and zone pools from their own limits in MultiZone + documentSelector: + path: kind + value: HelmRelease + set: + topology: MultiZone + replicationFactor: 1 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + volume: + resources: + memory: 2Gi + zones: + a: + pools: + small: + diskType: ssd + resourcesPreset: t1.nano + asserts: + - equal: + path: spec.values.seaweedfs.volumes.a.extraEnvironmentVars.GOMEMLIMIT + value: 1536MiB + - equal: + path: spec.values.seaweedfs.volumes.a-small.resources.limits.memory + value: 128Mi + - equal: + path: spec.values.seaweedfs.volumes.a-small.extraEnvironmentVars.GOMEMLIMIT + value: 96MiB diff --git a/packages/extra/seaweedfs/tests/public_access_class_test.yaml b/packages/extra/seaweedfs/tests/public_access_class_test.yaml new file mode 100644 index 000000000..dbbcc84e7 --- /dev/null +++ b/packages/extra/seaweedfs/tests/public_access_class_test.yaml @@ -0,0 +1,113 @@ +suite: seaweedfs public-read BucketAccessClass +release: + name: seaweedfs + namespace: tenant-root + +set: + filer: + grpcHost: seaweedfs-filer.tenant-other.svc + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + +tests: + - it: offers a -public access class to a Client instance + template: templates/client/cosi-bucket-class.yaml + set: + topology: Client + documentSelector: + path: metadata.name + value: tenant-root-public + asserts: + - isKind: + of: BucketAccessClass + - equal: + path: parameters.accessPolicy + value: public-read + - equal: + path: driverName + value: tenant-root.seaweedfs.objectstorage.k8s.io + + - it: offers a -public access class per storage pool + template: templates/storage-pool-bucket-classes.yaml + set: + topology: Simple + volume: + pools: + fast: + diskType: ssd + documentSelector: + path: metadata.name + value: tenant-root-fast-public + asserts: + - isKind: + of: BucketAccessClass + - equal: + path: parameters.accessPolicy + value: public-read + + # A pool named -public would render a readwrite BucketAccessClass + # --public, the same name as pool 's public-read class; a pool named + # public would take -public, the instance's own public-read class. + - it: refuses a pool name ending in -public + template: templates/seaweedfs.yaml + set: + volume: + pools: + fast: + diskType: ssd + fast-public: + diskType: nvme + asserts: + - failedTemplate: + errorPattern: volume.pools key 'fast-public' must not be 'public' or end with + + - it: refuses a pool named public + template: templates/seaweedfs.yaml + set: + volume: + pools: + public: + diskType: ssd + asserts: + - failedTemplate: + errorPattern: volume.pools key 'public' must not be 'public' or end with + + - it: refuses a zone pool name ending in -public + template: templates/seaweedfs.yaml + set: + topology: MultiZone + replicationFactor: 1 + volume: + zones: + a: + pools: + fast-public: + diskType: ssd + asserts: + - failedTemplate: + errorPattern: volume.zones.a.pools key 'fast-public' must not be 'public' or end with + + - it: still refuses a pool name ending in -readonly + template: templates/seaweedfs.yaml + set: + volume: + pools: + fast-readonly: + diskType: ssd + asserts: + - failedTemplate: + errorPattern: volume.pools key 'fast-readonly' must not be 'public' or end with + + - it: accepts a pool name that only contains public + template: templates/seaweedfs.yaml + set: + volume: + pools: + publicfast: + diskType: ssd + asserts: + - notFailedTemplate: {} diff --git a/packages/library/cozy-lib/templates/_barman.tpl b/packages/library/cozy-lib/templates/_barman.tpl index c37b9673f..3b187dda6 100644 --- a/packages/library/cozy-lib/templates/_barman.tpl +++ b/packages/library/cozy-lib/templates/_barman.tpl @@ -13,10 +13,12 @@ LimitRange, which is cozy-keycloak and any tenant that leaves resourceQuotas empty, gave the sidecar no requests and no limit at all, so nothing reserved memory for it and nothing bounded it. The same request and limit settle both: 256Mi holds the measured working set and 1Gi is four times the measurement. -No CPU limit is set, matching the entityOperator precedent in -packages/apps/kafka: a throttled sidecar stalls WAL archiving instead of -failing it. Inside a tenant the limit counts against the ResourceQuota -limits.memory budget, 1Gi per instance pod. +The CPU limit is explicit for the same reason: left unset, that LimitRange +gives the sidecar its 250m default, so gzip compression of full backups and +WAL segments runs on a quarter of a core (a 1.6 GB database took 30 minutes +to back up). One core keeps the sidecar from starving the instance while +removing that ceiling. Inside a tenant both limits count against the +ResourceQuota budget, one core and 1Gi per instance pod. Checksum: since botocore ~1.36 (early 2025) the default RequestChecksumCalculation is when_supported, which attaches a flexible checksum @@ -39,6 +41,7 @@ instanceSidecarConfiguration: cpu: 100m memory: 256Mi limits: + cpu: "1" memory: 1Gi env: - name: AWS_REQUEST_CHECKSUM_CALCULATION diff --git a/packages/library/cozy-lib/templates/_resources.tpl b/packages/library/cozy-lib/templates/_resources.tpl index 48b0718fd..17ffac8c0 100644 --- a/packages/library/cozy-lib/templates/_resources.tpl +++ b/packages/library/cozy-lib/templates/_resources.tpl @@ -68,6 +68,8 @@ number of milli-CPUs (a CPU quantity finer than 1m is invalid for the VPA admission webhook and meaningless for the scheduler). + A memory below 1Mi fails the render, see cozy-lib.resources.checkMemory. + The template now expects **one flat map** as input (no nested `requests:` / `limits:` sections). Each value in that map is taken as the *limit* for the corresponding resource. Usage example: @@ -92,6 +94,31 @@ devices.com/nvidia: "1" # = limit */}} {{- define "cozy-lib.resources.sanitize" }} +{{- include "cozy-lib.resources.checkMemory" (index . 0) }} +{{- include "cozy-lib.resources.requestsAndLimits" . }} +{{- end }} + +{{- /* + A quantity without a unit counts bytes, so a memory of `8` typed for 8Gi + sizes the container at 8 bytes and nothing else refuses it. Every preset is + 128Mi or more, so a floor of 1Mi only stops that kind of slip. +*/}} +{{- define "cozy-lib.resources.checkMemory" }} +{{- if hasKey . "memory" }} +{{- $memory := index . "memory" }} +{{- if not (kindIs "invalid" $memory) }} +{{- if lt (include "cozy-lib.resources.toFloat" $memory | float64) (include "cozy-lib.resources.toFloat" "1Mi" | float64) }} +{{- fail (printf "memory (%v) must be at least 1Mi. A value without a unit is read as bytes, so add the unit, such as 512Mi or 8Gi." $memory) }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} + +{{- /* + sanitize without the memory floor. flatten renders a ResourceQuota with it, + where a memory of 0 is a valid quota that allows none. +*/}} +{{- define "cozy-lib.resources.requestsAndLimits" }} {{- $cpuAllocationRatio := include "cozy-lib.resources.cpuAllocationRatio" . | float64 }} {{- $memoryAllocationRatio := include "cozy-lib.resources.memoryAllocationRatio" . | float64 }} {{- $ephemeralStorageAllocationRatio := include "cozy-lib.resources.ephemeralStorageAllocationRatio" . | float64 }} @@ -202,9 +229,29 @@ {{- printf `-Xms%dm -Xmx%dm` $xmsMi $xmxMi }} {{- end }} +{{- /* + goMemLimit takes a sanitized .Values.resources and returns a GOMEMLIMIT + value set to 75% of the memory limit. Without it the Go runtime collects + on a relative target (GOGC) with no absolute ceiling, so a burst of + allocation grows the heap past the cgroup limit and the kernel kills the + container. The 25% headroom covers what GOMEMLIMIT does not account for: + goroutine stacks and any mmap'd region. Accepts only sanitized resource + maps. +*/}} +{{- define "cozy-lib.resources.goMemLimit" }} +{{- $memoryLimitInt := include "cozy-lib.resources.toFloat" .limits.memory | float64 | int64 }} +{{- /* 4194304 is 4Mi */}} +{{- $limitMiB := div (mul $memoryLimitInt 3) 4194304 }} +{{- /* Go accepts 0, and then collects almost without pause. */}} +{{- if lt $limitMiB 1 }} +{{- fail (printf "ERROR: a memory limit of %s leaves no room for a Go heap; GOMEMLIMIT needs a limit of at least 1398102 bytes (4/3 Mi) to come out at 1MiB." .limits.memory) }} +{{- end }} +{{- printf `%dMiB` $limitMiB }} +{{- end }} + {{- define "cozy-lib.resources.flatten" -}} {{- $out := dict -}} -{{- $res := include "cozy-lib.resources.sanitize" . | fromYaml -}} +{{- $res := include "cozy-lib.resources.requestsAndLimits" . | fromYaml -}} {{- range $section, $values := $res }} {{- range $k, $v := $values }} {{- with include "cozy-lib.resources.flattenResource" (list $section $k) }} diff --git a/packages/system/backupstrategy-controller/Makefile b/packages/system/backupstrategy-controller/Makefile index 94b1931d9..aabc11d17 100644 --- a/packages/system/backupstrategy-controller/Makefile +++ b/packages/system/backupstrategy-controller/Makefile @@ -6,9 +6,9 @@ include ../../../hack/package.mk # PHONY so a stray file/dir named `image` cannot mark the aggregate up to date # and skip both builds. -.PHONY: image image-backupstrategy-controller image-redis-backup image-backup-http-client +.PHONY: image image-backupstrategy-controller image-redis-backup image-backup-http-client image-kubernetes-backup -image: image-backupstrategy-controller image-redis-backup image-backup-http-client +image: image-backupstrategy-controller image-redis-backup image-backup-http-client image-kubernetes-backup image-backupstrategy-controller: docker buildx build -f images/backupstrategy-controller/Dockerfile ../../.. \ @@ -46,5 +46,16 @@ image-backup-http-client: image-redis-backup yq -i '.backupStrategyController.chBackupClientImage = strenv(IMAGE) | .backupStrategyController.rabbitmqBackupClientImage = strenv(IMAGE)' values.yaml rm -f images/backup-http-client.json +# Chained after image-backup-http-client: one `yq -i` writer at a time. +image-kubernetes-backup: image-backup-http-client + docker buildx build -f images/kubernetes-backup/Dockerfile images/kubernetes-backup \ + $(call image-tags,kubernetes-backup,v$(COZYSTACK_VERSION)) \ + $(call cache-args,kubernetes-backup) \ + --metadata-file images/kubernetes-backup.json \ + $(BUILDX_ARGS) + IMAGE="$(REGISTRY)/kubernetes-backup:$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/kubernetes-backup.json -o json -r)" \ + yq -i '.backupStrategyController.kubernetesBackupClientImage = strenv(IMAGE)' values.yaml + rm -f images/kubernetes-backup.json + test: helm unittest . diff --git a/packages/system/backupstrategy-controller/definitions/strategy.backups.cozystack.io_kubernetes.yaml b/packages/system/backupstrategy-controller/definitions/strategy.backups.cozystack.io_kubernetes.yaml new file mode 100644 index 000000000..0b681c5be --- /dev/null +++ b/packages/system/backupstrategy-controller/definitions/strategy.backups.cozystack.io_kubernetes.yaml @@ -0,0 +1,147 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: kubernetes.strategy.backups.cozystack.io +spec: + group: strategy.backups.cozystack.io + names: + kind: Kubernetes + listKind: KubernetesList + plural: kubernetes + singular: kubernetes + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + Kubernetes defines a backup strategy for a tenant Kubernetes application. + A tenant cluster keeps its state in two places the platform can reach: the + cluster's key prefix in the tenant's shared etcd DataStore, and host-side + objects (control-plane PKI Secrets, the application HelmReleases, and the + DataVolumes that back the tenant's PersistentVolumes). The strategy captures + the first with a one-shot Job and the second with Velero. The two are taken + one after another, so a backup is crash-consistent at best. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: KubernetesSpec specifies the desired behavior of a Kubernetes + backup strategy. + properties: + certificateWarningThreshold: + description: |- + CertificateWarningThreshold is how close to expiry a control-plane + certificate may be before a backup records a warning. Defaults to 720h. + type: string + template: + description: |- + Template holds a PodTemplateSpec for the Job that dumps the cluster's + key prefix from its etcd DataStore to object storage, writes it back on + restore, and deletes the object on cleanup. The Job runs in the + application namespace. Helm-like Go templates are supported in every + string field and are rendered against: + - `.Release.Name` and `.Release.Namespace` — the application's name and + namespace; `.Release.Prefix` is `kubernetes-`. + - `.Mode` — `"backup"`, `"restore"` or `"cleanup"`. + - `.ObjectKey` — the object-storage key of this backup's dump. + - `.Datastore.Prefix` — the cluster's key prefix, without slashes. + - `.Datastore.Endpoints` — comma-separated etcd client URLs. + - `.Datastore.CertificateSecret` — the Secret holding the cluster's + etcd client certificate (`ca.crt`, `server.crt`, `server.key`). + - `.Parameters` — the resolved BackupClass strategy parameters. + + Stored schemaless for the same CRD-size reason as redis_types.go. + x-kubernetes-preserve-unknown-fields: true + velero: + description: |- + Velero holds the Velero Backup and Restore specs for the host-side + objects and volumes. Rendered against `.Application` and `.Parameters`, + like the Velero strategy. + x-kubernetes-preserve-unknown-fields: true + required: + - template + - velero + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packages/system/backupstrategy-controller/images/kubernetes-backup/Dockerfile b/packages/system/backupstrategy-controller/images/kubernetes-backup/Dockerfile new file mode 100644 index 000000000..b897e51be --- /dev/null +++ b/packages/system/backupstrategy-controller/images/kubernetes-backup/Dockerfile @@ -0,0 +1,14 @@ +# kubernetes-backup runs the Kubernetes strategy Pod: it dumps a tenant +# cluster's key prefix from its etcd DataStore with etcdctl, moves the dump to +# S3 with curl --aws-sigv4, and writes it back with etcdctl on restore. etcdctl +# matches the etcd the platform deploys. Everything is installed at build time +# so the Pods never run `apk add` and work on air-gapped clusters. +FROM quay.io/coreos/etcd:v3.6.11@sha256:6ae247c7666ceec554c51ba1f9bc8dd2212f975370dbd65710c9ca0e36ae1fff AS etcd + +FROM alpine:3.24@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6 + +RUN apk add --no-cache curl jq ca-certificates coreutils + +COPY --from=etcd /usr/local/bin/etcdctl /usr/local/bin/etcdctl + +USER 65532:65532 diff --git a/packages/system/backupstrategy-controller/templates/backupclass-default.yaml b/packages/system/backupstrategy-controller/templates/backupclass-default.yaml index 3f3b271e3..347c704d1 100644 --- a/packages/system/backupstrategy-controller/templates/backupclass-default.yaml +++ b/packages/system/backupstrategy-controller/templates/backupclass-default.yaml @@ -78,6 +78,13 @@ spec: apiGroup: strategy.backups.cozystack.io kind: Redis name: cozy-default-redis + - application: + apiGroup: apps.cozystack.io + kind: Kubernetes + strategyRef: + apiGroup: strategy.backups.cozystack.io + kind: Kubernetes + name: cozy-default-kubernetes - application: apiGroup: apps.cozystack.io kind: Kafka diff --git a/packages/system/backupstrategy-controller/templates/rbac.yaml b/packages/system/backupstrategy-controller/templates/rbac.yaml index 8d90980a8..883fd1fe9 100644 --- a/packages/system/backupstrategy-controller/templates/rbac.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac.yaml @@ -62,7 +62,25 @@ rules: # through the dynamic client, so no list/watch) - apiGroups: ["cdi.kubevirt.io"] resources: ["datavolumes"] - verbs: ["get", "delete"] + verbs: ["get", "list", "delete"] +# Kubernetes strategy: read where a tenant cluster keeps its etcd prefix, and +# stop and resume its control plane, worker pools and cluster-autoscaler around +# an in-place restore. +- apiGroups: ["kamaji.clastix.io"] + resources: ["tenantcontrolplanes", "datastores"] + verbs: ["get"] +- apiGroups: ["controlplane.cluster.x-k8s.io"] + resources: ["kamajicontrolplanes"] + verbs: ["get", "patch"] +- apiGroups: ["cluster.x-k8s.io"] + resources: ["machinedeployments"] + verbs: ["get", "list", "patch"] +- apiGroups: ["cluster.x-k8s.io"] + resources: ["machines"] + verbs: ["list", "patch"] +- apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "patch"] # HelmReleases: pre-restore suspends HRs; post-restore rename creates new HR # and deletes old. patch additionally lets the DefaultObjectsGate stamp # reconcile.fluxcd.io/forceAt + requestedAt on THIS chart's own release to @@ -146,15 +164,16 @@ rules: # MariaDB strategy: read the operator-side MariaDB CR to surface "not yet # rendered" preconditions, create one-shot k8s.mariadb.com/Backup CRs on # the BackupJob path, and create k8s.mariadb.com/Restore CRs on the -# RestoreJob path. The driver only ever Creates/Lists/Gets these objects - -# it does not patch or delete them; their lifecycle is owned by -# mariadb-operator and (for cleanup) by tenants / explicit teardown flows, -# never by this controller. +# RestoreJob path. A Backup is deleted once the dump it wrote is, when its +# Cozystack Backup is deleted; a Restore is never deleted. - apiGroups: ["k8s.mariadb.com"] resources: ["mariadbs"] verbs: ["get", "list", "watch"] - apiGroups: ["k8s.mariadb.com"] - resources: ["backups", "restores"] + resources: ["backups"] + verbs: ["get", "list", "watch", "create", "delete"] +- apiGroups: ["k8s.mariadb.com"] + resources: ["restores"] verbs: ["get", "list", "watch", "create"] # MongoDB strategy: read the operator-side PerconaServerMongoDB CR to gate on # the backup preconditions, create one-shot PerconaServerMongoDBBackup CRs on the diff --git a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml index 21d05f8a4..3e6747966 100644 --- a/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml +++ b/packages/system/backupstrategy-controller/templates/strategy-altinity-default.yaml @@ -58,32 +58,75 @@ spec: # text/template's missing-key marker, when a controller that # predates archive names renders this template during an upgrade. archive="{{ printf `{{ .ArchiveName | default "" }}` }}" - for _ in $(seq 1 60); do - if curl -fsS ${auth} --max-time 2 "${api}/backup/status" >/dev/null 2>&1; then break; fi - sleep 5 - done + # Every request but the POST is retried on failure, every 5s, + # for $2 attempts in all: once the operation is accepted, a + # dropped poll must not fail the attempt, since the Job retry it + # causes cannot tell a lost poll from a lost operation. + api_get() { + n=0 + while true; do + if out=$(curl -fsS ${auth} --max-time 10 "${api}$1"); then + printf '%s\n' "${out}" + return 0 + fi + n=$((n + 1)) + if [ "${n}" -ge "$2" ]; then + echo "GET ${api}$1 failed ${n} times in a row" >&2 + return 1 + fi + sleep 5 + done + } + # The sidecar answers 423 to a new operation while any other is + # in progress, so starting before it is idle burns the attempt. + # The exception is the operation $1 itself already running, as + # left by an earlier attempt of this Job that died: it is taken + # over rather than started twice. + wait_turn() { + for _ in $(seq 1 60); do + if actions=$(curl -fsS ${auth} --max-time 10 "${api}/backup/actions"); then + turn=$(printf '%s\n' "${actions}" | jq -rs --arg cmd "$1" ' + if any(.[]; .status == "in progress" and .command == $cmd) then "join" + elif any(.[]; .status == "in progress") then "busy" + else "idle" end') || turn= + case "${turn}" in join|idle) return 0 ;; esac + fi + sleep 5 + done + echo "clickhouse-backup sidecar at ${api} did not become idle" >&2 + exit 1 + } if [ "{{ printf "{{ .Mode }}" }}" = "backup" ]; then prefix="clickhouse-{{ printf "{{ .Release.Name }}" }}-" - # Only that older controller leaves the name empty, and it - # records no name on the Backup either. + # Only a controller that predates archive names leaves it + # empty, and that one records no name on the Backup either. target="${archive:-${prefix}$(date -u +%Y%m%dT%H%M%SZ)}" - # A retried Pod gets the same name. Wait on an attempt the - # sidecar is still running, which would answer a new one with - # 423, and ask for resume, so a local or partial remote backup - # an earlier attempt left is resumed rather than refused as - # existing, whatever use_resumable_state the sidecar runs with. - state=$(curl -fsS ${auth} "${api}/backup/actions" \ - | jq -rs --arg suffix "${target}" \ - '[.[] | select((.command // "") | endswith($suffix))] | sort_by(.start) | last | .status // ""') - if [ "${state}" != "in progress" ]; then - echo "create_remote ${target}" + # A retried Pod gets the same name, so it takes over an attempt + # the sidecar is still running, and otherwise asks for resume, + # so a local or partial remote backup an earlier attempt left is + # resumed rather than refused as existing, whatever + # use_resumable_state the sidecar runs with. The sidecar logs + # the request as "create_remote --resume ". The name is + # this BackupJob's alone, so an upload of it that last ended in + # success is an earlier attempt's finished result: resuming + # over it would upload the archive again and leave it broken if + # that upload died halfway. + cmd="create_remote --resume ${target}" + wait_turn "${cmd}" + if [ "${turn}" = "join" ]; then + echo "${cmd} is already running; waiting for it" + elif printf '%s\n' "${actions}" | jq -es --arg cmd "${cmd}" \ + '[.[] | select(.command == $cmd)] | sort_by(.start) | last | .status == "success"' >/dev/null; then + echo "${cmd} already succeeded in an earlier attempt" + else + echo "${cmd}" curl -fsS ${auth} -X POST "${api}/backup/create_remote?name=${target}&resume=true" >/dev/null fi else app="{{ printf "{{ .Backup.ApplicationRef.Name }}" }}" backup="{{ printf "{{ .Backup.Name }}" }}" prefix="clickhouse-${app}-" - remote=$(curl -fsS ${auth} "${api}/backup/list/remote") + remote=$(api_get /backup/list/remote 60) if [ -n "${archive}" ]; then if ! printf '%s\n' "${remote}" | jq -es --arg name "${archive}" 'any(.[]; .name == $name)' >/dev/null; then echo "archive ${archive} recorded on Backup ${backup} is not in remote storage; not restoring any other archive" >&2 @@ -104,19 +147,39 @@ spec: exit 1 fi fi - echo "restore_remote ${target}" - curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null + # restore_remote drops every table it restores before + # recreating it, so a second run discards whatever was written + # after the first one finished. Only a running one is taken + # over: the action log is never trimmed, and a finished row of + # this archive may be left by an earlier RestoreJob. + cmd="restore_remote ${target}" + wait_turn "${cmd}" + if [ "${turn}" = "join" ]; then + echo "${cmd} is already running; waiting for it" + else + echo "${cmd}" + curl -fsS ${auth} -X POST "${api}/backup/restore_remote/${target}" >/dev/null + fi fi while true; do - row=$(curl -fsS ${auth} "${api}/backup/actions" \ - | jq -rs --arg suffix "${target}" \ - '[.[] | select((.command // "") | endswith($suffix))] | sort_by(.start) | last') + actions=$(api_get /backup/actions 6) + row=$(printf '%s\n' "${actions}" \ + | jq -rs --arg cmd "${cmd}" \ + '[.[] | select(.command == $cmd)] | sort_by(.start) | last') + # The sidecar logs the operation before acknowledging the POST + # and keeps that log only in memory, so a missing row means it + # restarted and the operation is gone; nothing will resume it. + if [ "${row}" = "null" ]; then + echo "operation ${target} is no longer in the sidecar action log; the sidecar restarted and dropped it" >&2 + exit 1 + fi status=$(echo "${row}" | jq -r '.status // ""') case "${status}" in success) echo "operation succeeded" ; exit 0 ;; - error) + "in progress") ;; + *) err=$(echo "${row}" | jq -r '.error // "unknown"') - echo "operation failed: ${err}" >&2 + echo "operation ended with status ${status}: ${err}" >&2 exit 1 ;; esac diff --git a/packages/system/backupstrategy-controller/templates/strategy-kubernetes-default.yaml b/packages/system/backupstrategy-controller/templates/strategy-kubernetes-default.yaml new file mode 100644 index 000000000..3fab192ad --- /dev/null +++ b/packages/system/backupstrategy-controller/templates/strategy-kubernetes-default.yaml @@ -0,0 +1,304 @@ +{{/* + Two template layers, as in strategy-redis-default.yaml: Helm renders this + file, then the strategy controller renders the resulting CR per BackupJob. + Literal "{{ .Foo }}" for the second layer goes through printf. + + Gated on both a Velero BSL and a resolved $bucketName: the Velero half of + the backup goes to the BSL and the etcd dump to the bucket. Also gated on + its CRD being established, for the upgrade of an existing install that + first ships this kind: see strategy-mongodb-default.yaml. +*/}} +{{- $bucketName := include "backupstrategy-controller.bucketName" . -}} +{{- $crdEstablished := include "backupstrategy-controller.crdEstablished" (dict "root" . "crd" "kubernetes.strategy.backups.cozystack.io") -}} +{{- if and .Values.velero.bslEnabled $bucketName $crdEstablished -}} +apiVersion: strategy.backups.cozystack.io/v1alpha1 +kind: Kubernetes +metadata: + name: cozy-default-kubernetes + annotations: + backups.cozystack.io/endpoint: {{ include "backupstrategy-controller.endpoint" . | quote }} +spec: + template: + spec: + restartPolicy: Never + activeDeadlineSeconds: {{ int .Values.backupStrategyController.kubernetesBackupActiveDeadlineSeconds }} + volumes: + {{- /* + The cluster's own etcd client certificate, the one its kube-apiserver + uses. Optional so a missing Secret fails the script with a clear cause + instead of wedging the Pod on FailedMount. + */}} + - name: etcd-client + secret: + secretName: {{ printf "{{ .Datastore.CertificateSecret }}" | quote }} + optional: true + {{- if .Values.backupStorage.endpointCASecretName }} + - name: s3-ca + secret: + secretName: {{ .Values.backupStorage.endpointCASecretName | quote }} + optional: true + {{- end }} + containers: + - name: kubernetes-backup + image: "{{ .Values.backupStrategyController.kubernetesBackupClientImage }}" + imagePullPolicy: IfNotPresent + volumeMounts: + - name: etcd-client + mountPath: /etc/etcd-client + readOnly: true + {{- if .Values.backupStorage.endpointCASecretName }} + - name: s3-ca + mountPath: /etc/s3-ca + readOnly: true + {{- end }} + env: + - name: ETCDCTL_ENDPOINTS + value: {{ printf "{{ .Datastore.Endpoints }}" | quote }} + - name: S3_ENDPOINT + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: endpoint + optional: true + - name: S3_BUCKET + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: bucketName + optional: true + - name: S3_REGION + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: region + optional: true + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: AWS_ACCESS_KEY_ID + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: cozy-backups-creds + key: AWS_SECRET_ACCESS_KEY + optional: true + - name: S3_INSECURE_SKIP_VERIFY + value: {{ .Values.backupStorage.insecureSkipTLSVerify | quote }} + command: ["/bin/sh", "-c"] + args: + - | + set -euo pipefail + : "${S3_ENDPOINT:?missing endpoint in cozy-backups-creds}" + : "${S3_BUCKET:?missing bucketName in cozy-backups-creds}" + : "${S3_REGION:?missing region in cozy-backups-creds}" + : "${AWS_ACCESS_KEY_ID:?missing AWS_ACCESS_KEY_ID in cozy-backups-creds}" + : "${AWS_SECRET_ACCESS_KEY:?missing AWS_SECRET_ACCESS_KEY in cozy-backups-creds}" + + MODE="{{ printf "{{ .Mode }}" }}" + KEY="{{ printf "{{ .ObjectKey }}" }}" + SCHEMA="{{ printf "{{ .Datastore.Prefix }}" }}" + SOURCE_SCHEMA="{{ printf "{{ .Datastore.SourcePrefix }}" }}" + + case "${S3_ENDPOINT}" in + http://*|https://*) BASE="${S3_ENDPOINT}" ;; + *) BASE="https://${S3_ENDPOINT}" ;; + esac + TLS="" + case "${BASE}" in + https://*) + if [ -f /etc/s3-ca/ca.crt ]; then TLS="--cacert /etc/s3-ca/ca.crt"; + elif [ -f /etc/s3-ca/tls.crt ]; then TLS="--cacert /etc/s3-ca/tls.crt"; + elif [ "${S3_INSECURE_SKIP_VERIFY:-false}" = "true" ]; then TLS="-k"; + fi + ;; + esac + s3() { + curl -fsS ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" "$@" + } + OBJ="${BASE}/${S3_BUCKET}/${KEY}" + {{- /* + Where a restore keeps the prefix it replaces. One per backup, so + cleanup removes it with the dump. A restore writes it only when it + is absent and deletes it on success, so after a failed restore it + holds the prefix from before the first failed attempt and a retry + cannot overwrite it with a partial one. + */}} + PRE_KEY="${KEY}.pre-restore" + PRE_OBJ="${BASE}/${S3_BUCKET}/${PRE_KEY}" + + if [ "${MODE}" = "cleanup" ]; then + for o in "${OBJ}" "${PRE_OBJ}"; do + echo "deleting ${o#"${BASE}/"}" + code=$(curl -sS -o /dev/null -w '%{http_code}' ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + -X DELETE "${o}") + case "${code}" in + 2??|404) echo "deleted (HTTP ${code})" ;; + *) echo "delete failed: HTTP ${code}" >&2; exit 1 ;; + esac + done + exit 0 + fi + + if [ "${MODE}" = "verify" ]; then + s3 -I "${OBJ}" > /dev/null + echo "s3://${S3_BUCKET}/${KEY} exists" + exit 0 + fi + + {{- /* + The prefix comes from the TenantControlPlane: Kamaji's schema is the + TenantControlPlane's UID, or _ for one created by + an older Kamaji. Refuse anything else: an empty or slash-bearing + value would turn the restore's `del --prefix` into a wipe of every + cluster in the shared etcd. + */}} + for s in "${SCHEMA}" "${SOURCE_SCHEMA}"; do + case "${s}" in + ""|*[!a-z0-9_-]*) echo "refusing unexpected datastore prefix '${s}'" >&2; exit 1 ;; + esac + done + PREFIX="/${SCHEMA}/" + {{- /* + The prefix the dump was taken under. A rebuilt cluster has a new + TenantControlPlane and so a new schema; its keys are written under + the new prefix. + */}} + SOURCE_PREFIX="/${SOURCE_SCHEMA}/" + for f in ca.crt server.crt server.key; do + [ -f "/etc/etcd-client/${f}" ] || { echo "etcd client certificate is missing /etc/etcd-client/${f}" >&2; exit 1; } + done + export ETCDCTL_CACERT=/etc/etcd-client/ca.crt + export ETCDCTL_CERT=/etc/etcd-client/server.crt + export ETCDCTL_KEY=/etc/etcd-client/server.key + + if [ "${MODE}" = "backup" ]; then + etcdctl get "${PREFIX}" --prefix -w json > /tmp/dump.json + count=$(jq '.kvs // [] | length' /tmp/dump.json) + if [ "${count}" -eq 0 ]; then + echo "etcd prefix ${PREFIX} is empty" >&2 + exit 1 + fi + echo "dumped ${count} keys under ${PREFIX}; uploading s3://${S3_BUCKET}/${KEY}" + s3 -X PUT --upload-file /tmp/dump.json "${OBJ}" + elif [ "${MODE}" = "restore" ]; then + {{- /* + Everything that can fail without touching etcd runs before the + prefix is deleted: the download, the check that every key belongs + to this cluster, and a copy of the prefix as it is now, which is + what an operator restores from if a failure comes after the delete. + */}} + echo "downloading s3://${S3_BUCKET}/${KEY}" + s3 -o /tmp/dump.json "${OBJ}" + jq -e '.kvs | length > 0' /tmp/dump.json >/dev/null || { echo "dump holds no keys" >&2; exit 1; } + jq -e --arg p "${SOURCE_PREFIX}" 'all(.kvs[]; .key | @base64d | startswith($p))' /tmp/dump.json >/dev/null \ + || { echo "dump holds keys outside ${SOURCE_PREFIX}" >&2; exit 1; } + {{- /* + etcdctl put cannot write an empty value, so a dump holding one + must stop here rather than halfway through the write. + */}} + jq -e 'all(.kvs[]; (.value // "") != "")' /tmp/dump.json >/dev/null \ + || { echo "dump holds a key with an empty value, which etcdctl cannot write" >&2; exit 1; } + jq -r '.kvs[] | [.key, .value] | @tsv' /tmp/dump.json > /tmp/kvs.tsv + {{- /* + Only a definite answer decides about the copy: an existing one is + kept, a missing one is taken, and any other reply (a 5xx, a + timeout) stops the restore while the prefix is still intact, + rather than overwriting the copy an earlier failed restore left. + */}} + code=$(curl -sS -o /dev/null -w '%{http_code}' -I ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + "${PRE_OBJ}") || code=000 + case "${code}" in + 200) echo "keeping s3://${S3_BUCKET}/${PRE_KEY} from an earlier failed restore" ;; + 404) + etcdctl get "${PREFIX}" --prefix -w json > /tmp/before.json + echo "saving the current prefix to s3://${S3_BUCKET}/${PRE_KEY}" + s3 -X PUT --upload-file /tmp/before.json "${PRE_OBJ}" + ;; + *) echo "cannot tell whether s3://${S3_BUCKET}/${PRE_KEY} exists: HTTP ${code}" >&2; exit 1 ;; + esac + {{- /* + Delete first so keys created after the backup do not survive the + restore. Events and master leases are bound to etcd leases that + did not survive; written back without one they would never expire, + so they are skipped. They are matched as whole top-level resources + under the prefix, not as a path segment anywhere, which would also + drop every object in a namespace named "events". + */}} + {{- /* + The termination message tells the driver that this Pod got as far + as changing the prefix: only then is the control plane left + stopped after a failure. + */}} + echo writing > /dev/termination-log + etcdctl del "${PREFIX}" --prefix + n=0 + while IFS="$(printf '\t')" read -r k v; do + key=$(printf '%s' "${k}" | base64 -d) + rel="${key#"${SOURCE_PREFIX}"}" + case "${rel}" in events/*|masterleases/*) continue ;; esac + printf '%s' "${v}" | base64 -d > /tmp/value + etcdctl put "${PREFIX}${rel}" < /tmp/value > /dev/null + n=$((n + 1)) + done < /tmp/kvs.tsv + echo "restored ${n} keys under ${PREFIX}" + code=$(curl -sS -o /dev/null -w '%{http_code}' ${TLS} \ + --aws-sigv4 "aws:amz:${S3_REGION}:s3" \ + --user "${AWS_ACCESS_KEY_ID}:${AWS_SECRET_ACCESS_KEY}" \ + -X DELETE "${PRE_OBJ}") || code=000 + case "${code}" in + 2??|404) echo "dropped s3://${S3_BUCKET}/${PRE_KEY}" ;; + *) echo "WARNING: could not drop s3://${S3_BUCKET}/${PRE_KEY} (HTTP ${code}); a later restore would keep it as the pre-restore copy, so delete it by hand" >&2 ;; + esac + else + echo "unknown mode: ${MODE}" >&2 + exit 1 + fi + resources: + {{- toYaml .Values.backupStrategyController.kubernetesBackupResources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + velero: + {{- /* + Secrets (control-plane and Talos PKI, by the Kamaji and Helm instance + labels) and the DataVolumes and PVCs backing the cluster's + PersistentVolumes (the CSI driver stamps the cluster-name label on them; + worker system disks do not carry it). The HelmReleases are recorded by the + driver on the Backup instead: selecting them by application label would + also select worker disks the lineage webhook labels the same way. The + driver sets the restore's resources and selectors per phase. + */}} + spec: + includedNamespaces: + - {{ printf "{{ .Application.metadata.namespace }}" | quote }} + orLabelSelectors: + - matchLabels: + kamaji.clastix.io/name: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }} + - matchLabels: + app.kubernetes.io/instance: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }} + - matchLabels: + cluster.x-k8s.io/cluster-name: {{ printf "kubernetes-{{ .Application.metadata.name }}" | quote }} + includedResources: + - secrets + - persistentvolumeclaims + - datavolumes.cdi.kubevirt.io + includeClusterResources: false + storageLocation: cozy-default + snapshotVolumes: true + snapshotMoveData: true + ttl: 720h0m0s + itemOperationTimeout: 24h0m0s + restoreSpec: + includeClusterResources: false +{{- end -}} diff --git a/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml b/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml new file mode 100644 index 000000000..21ac6e807 --- /dev/null +++ b/packages/system/backupstrategy-controller/tests/mariadb_cleanup_rbac_test.yaml @@ -0,0 +1,28 @@ +suite: RBAC of the MariaDB dump cleanup + +# Deleting a MariaDB Backup deletes its operator Backup once the dump is gone. +# The Go unit tests run with no authorization, so a missing verb only shows up +# on a cluster, where every such Backup would stay Terminating. + +templates: + - templates/rbac.yaml + +release: + name: backupstrategy-controller + namespace: cozy-backup-controller + +tests: + - it: "ClusterRole can delete operator Backups but not Restores" + asserts: + - contains: + path: rules + content: + apiGroups: ["k8s.mariadb.com"] + resources: ["backups"] + verbs: ["get", "list", "watch", "create", "delete"] + - contains: + path: rules + content: + apiGroups: ["k8s.mariadb.com"] + resources: ["restores"] + verbs: ["get", "list", "watch", "create"] diff --git a/packages/system/backupstrategy-controller/tests/rendering_test.yaml b/packages/system/backupstrategy-controller/tests/rendering_test.yaml index 3db6adb14..941444070 100644 --- a/packages/system/backupstrategy-controller/tests/rendering_test.yaml +++ b/packages/system/backupstrategy-controller/tests/rendering_test.yaml @@ -22,6 +22,7 @@ templates: - templates/strategy-foundationdb-default.yaml - templates/strategy-velero-vminstance-default.yaml - templates/strategy-velero-vmdisk-default.yaml + - templates/strategy-kubernetes-default.yaml release: name: t @@ -88,11 +89,14 @@ tests: - hasDocuments: count: 0 template: templates/strategy-velero-vmdisk-default.yaml + - hasDocuments: + count: 0 + template: templates/strategy-kubernetes-default.yaml - hasDocuments: count: 0 template: templates/velero-bsl.yaml - - it: "cozy-default BackupClass is the Day-1 tenant interface: renders unconditionally with all eleven routes (FoundationDB intentionally unbound)" + - it: "cozy-default BackupClass is the Day-1 tenant interface: renders unconditionally with all twelve routes (FoundationDB intentionally unbound)" asserts: - hasDocuments: count: 1 @@ -103,7 +107,18 @@ tests: template: templates/backupclass-default.yaml - lengthEqual: path: spec.strategies - count: 11 + count: 12 + template: templates/backupclass-default.yaml + - contains: + path: spec.strategies + content: + application: + apiGroup: apps.cozystack.io + kind: Kubernetes + strategyRef: + apiGroup: strategy.backups.cozystack.io + kind: Kubernetes + name: cozy-default-kubernetes template: templates/backupclass-default.yaml - contains: path: spec.strategies @@ -240,6 +255,18 @@ tests: - hasDocuments: count: 1 template: templates/strategy-velero-vmdisk-default.yaml + - hasDocuments: + count: 1 + template: templates/strategy-kubernetes-default.yaml + # The driver renders these per BackupJob; Helm must leave them literal. + - equal: + path: spec.template.spec.volumes[0].secret.secretName + value: "{{ .Datastore.CertificateSecret }}" + template: templates/strategy-kubernetes-default.yaml + - equal: + path: spec.velero.spec.orLabelSelectors[2].matchLabels["cluster.x-k8s.io/cluster-name"] + value: "kubernetes-{{ .Application.metadata.name }}" + template: templates/strategy-kubernetes-default.yaml - hasDocuments: count: 1 template: templates/velero-bsl.yaml @@ -347,6 +374,7 @@ tests: - {hasDocuments: {count: 0}, template: templates/strategy-foundationdb-default.yaml} - {hasDocuments: {count: 0}, template: templates/strategy-velero-vminstance-default.yaml} - {hasDocuments: {count: 0}, template: templates/strategy-velero-vmdisk-default.yaml} + - {hasDocuments: {count: 0}, template: templates/strategy-kubernetes-default.yaml} # The ungated BackupClass keeps rendering, which is what makes the counts # above an assertion about the CRD gate rather than about a chart-wide # render failure. @@ -432,6 +460,7 @@ tests: - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: etcds.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: foundationdbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kafkas.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} + - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kubernetes.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mariadbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mongodbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: rabbitmqs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} @@ -449,6 +478,7 @@ tests: - {hasDocuments: {count: 0}, template: templates/strategy-redis-default.yaml} - {hasDocuments: {count: 0}, template: templates/strategy-velero-vminstance-default.yaml} - {hasDocuments: {count: 0}, template: templates/strategy-velero-vmdisk-default.yaml} + - {hasDocuments: {count: 0}, template: templates/strategy-kubernetes-default.yaml} - it: "with every CRD Established and the BucketClaim resolved, every Strategy CR renders against the COSI bucket name" # The positive half of the case above, through the same live lookups and no @@ -471,6 +501,7 @@ tests: - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: etcds.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: foundationdbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kafkas.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} + - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: kubernetes.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mariadbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: mongodbs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} - {apiVersion: apiextensions.k8s.io/v1, kind: CustomResourceDefinition, metadata: {name: rabbitmqs.strategy.backups.cozystack.io}, status: {conditions: [{type: Established, status: "True"}]}} @@ -489,6 +520,7 @@ tests: - {hasDocuments: {count: 1}, template: templates/strategy-redis-default.yaml} - {hasDocuments: {count: 1}, template: templates/strategy-velero-vminstance-default.yaml} - {hasDocuments: {count: 1}, template: templates/strategy-velero-vmdisk-default.yaml} + - {hasDocuments: {count: 1}, template: templates/strategy-kubernetes-default.yaml} - it: "velero.bslEnabled=false: Velero strategies and BSL drop out while other strategies keep rendering" set: diff --git a/packages/system/backupstrategy-controller/tests/restore_rbac_test.yaml b/packages/system/backupstrategy-controller/tests/restore_rbac_test.yaml index b13b88173..fa3406d58 100644 --- a/packages/system/backupstrategy-controller/tests/restore_rbac_test.yaml +++ b/packages/system/backupstrategy-controller/tests/restore_rbac_test.yaml @@ -13,11 +13,11 @@ release: namespace: cozy-backup-controller tests: - - it: "ClusterRole can read and delete DataVolumes" + - it: "ClusterRole can read, list and delete DataVolumes" asserts: - contains: path: rules content: apiGroups: ["cdi.kubevirt.io"] resources: ["datavolumes"] - verbs: ["get", "delete"] + verbs: ["get", "list", "delete"] diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index 39f45c741..8de863a27 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -37,6 +37,24 @@ backupStrategyController: cpu: "1" memory: 1Gi ephemeral-storage: 4Gi + # kubernetesBackupClientImage is rendered into the Kubernetes strategy Pod, + # which dumps a tenant cluster's etcd prefix with etcdctl and moves it to S3 + # with curl. Built from images/kubernetes-backup/; `make image` re-digests it. + kubernetesBackupClientImage: "ghcr.io/cozystack/cozystack/kubernetes-backup:v1.6.0" + # kubernetesBackupActiveDeadlineSeconds bounds one etcd dump/restore Job. + kubernetesBackupActiveDeadlineSeconds: 3600 + # kubernetesBackupResources sizes the Kubernetes strategy Pod. The dump is held + # in ephemeral storage while it moves and jq reads it into memory, so a + # cluster with a large etcd prefix needs more of both. + kubernetesBackupResources: + requests: + cpu: 10m + memory: 64Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 1Gi + ephemeral-storage: 2Gi # The Kafka metadata strategy carries no client image here: unlike the Altinity # and RabbitMQ drivers, which need only curl (plus jq for Altinity), its driver # runs the Kafka Admin-API CLI (kafka-topics / kafka-configs), which only the diff --git a/packages/system/bucket-rd/cozyrds/bucket.yaml b/packages/system/bucket-rd/cozyrds/bucket.yaml index f75c736bd..0f05f4dfe 100644 --- a/packages/system/bucket-rd/cozyrds/bucket.yaml +++ b/packages/system/bucket-rd/cozyrds/bucket.yaml @@ -8,7 +8,7 @@ spec: plural: buckets singular: bucket openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"locking":{"description":"Provisions bucket from the `-lock` BucketClass (with object lock enabled).","type":"boolean","default":false},"storagePool":{"description":"Selects a specific BucketClass by storage pool name.","type":"string","default":"","x-cozystack-options":{"source":"storagepool"}},"ui":{"description":"Deploys the web UI (s3manager) with its Service and its Ingress or HTTPRoute at `.`. Set to `false` to run the bucket without them.","type":"boolean","default":true},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"readonly":{"description":"Whether the user has read-only access.","type":"boolean"}}}}}} + {"title":"Chart Values","type":"object","properties":{"locking":{"description":"Provisions bucket from the `-lock` BucketClass (with object lock enabled).","type":"boolean","default":false},"storagePool":{"description":"Selects a specific BucketClass by storage pool name.","type":"string","default":"","x-cozystack-options":{"source":"storagepool"}},"ui":{"description":"Deploys the web UI (s3manager) with its Service and its Ingress or HTTPRoute at `.`. Set to `false` to run the bucket without them.","type":"boolean","default":true},"publicRead":{"description":"Serves the bucket's objects to anyone without credentials at `//`. Anonymous callers can read objects but cannot list the bucket or write to it.","type":"boolean","default":false},"users":{"description":"Users configuration map.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"readonly":{"description":"Whether the user has read-only access.","type":"boolean"}}}}}} release: prefix: bucket- labels: @@ -26,7 +26,7 @@ spec: tags: - storage icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODNfMzA5MSkiLz4KPHBhdGggZmlsbC1ydWxlPSJldmVub2RkIiBjbGlwLXJ1bGU9ImV2ZW5vZGQiIGQ9Ik03MiAzMC4xNjQxTDExNy45ODMgMzYuNzc4OVY0MC42NzM5QzExNy45ODMgNDYuNDY1MyA5Ny4zODYyIDUxLjEzMzIgNzEuOTgyNyA1MS4xMzMyQzQ2LjU3OTIgNTEuMTMzMiAyNiA0Ni40NjUzIDI2IDQwLjY3MzlWMzYuNDQzMUw3MiAzMC4xNjQxWk03MiA1OC4yNjc4QzkxLjIwODQgNTguMjY3OCAxMDcuNjU4IDU1LjU5ODYgMTE0LjU0NyA1MS44MDQ4TDExNi44MDMgNDguMTExTDExNy43MjMgNDQuNzUzVjQ4LjkxNzFMMTAyLjY3OSAxMTEuMDMzQzEwMi42NzkgMTE0Ljg5NSA4OC45NTMzIDExOCA3Mi4wMTcyIDExOEM1NS4wODEyIDExOCA0MS4zNzQzIDExNC44OTUgNDEuMzc0MyAxMTEuMDMzTDI2LjMzIDQ4LjkxNzFWNDQuODM2OUwyOS44MDA3IDUxLjkzODJDMzYuNzA2NSA1NS42NjUzIDUyLjk5OTcgNTguMjY3OCA3MiA1OC4yNjc4WiIgZmlsbD0iIzhDMzEyMyIvPgo8cGF0aCBmaWxsLXJ1bGU9ImV2ZW5vZGQiIGNsaXAtcnVsZT0iZXZlbm9kZCIgZD0iTTcyLjAwMDMgMjZDOTcuNDAzOCAyNiAxMTggMzAuNjgzOSAxMTggMzYuNDQyQzExOCA0Mi4yIDk3LjM4NjYgNDYuODUwNyA3Mi4wMDAzIDQ2Ljg1MDdDNDYuNjE0MSA0Ni44NTA3IDI2LjAxNzYgNDIuMjM0NSAyNi4wMTc2IDM2LjQ0MkMyNi4wMTc2IDMwLjY0OTQgNDYuNTk2OCAyNiA3Mi4wMDAzIDI2Wk03Mi4wMDAzIDU0LjEwMzdDOTUuNjg1NyA1NC4xMDM3IDExNS4xNzIgNTAuMDU4IDExNy43MDYgNDQuODE5N0wxMDIuNjYyIDEwNi45MzdDMTAyLjY2MiAxMTAuNzk5IDg4LjkzNjQgMTEzLjkwNSA3Mi4wMDAzIDExMy45MDVDNTUuMDY0MyAxMTMuOTA1IDQxLjMzOSAxMTAuODE2IDQxLjMzOSAxMDYuOTU0TDI2LjI5NTkgNDQuODM3QzI4Ljg0NjYgNTAuMDU4IDQ4LjMzMzMgNTQuMTAzNyA3Mi4wMDAzIDU0LjEwMzdaIiBmaWxsPSIjRTA1MjQzIi8+CjxwYXRoIGZpbGwtcnVsZT0iZXZlbm9kZCIgY2xpcC1ydWxlPSJldmVub2RkIiBkPSJNNjEuMTcyNSA2MC4wMjkzSDgxLjA5MjhWNzkuMTY3Nkg2MS4xNzI1VjYwLjAyOTNaTTQ1LjMzMDEgOTUuMzY4OEM0NS4zMzAxIDkwLjE0MiA0OS43MTA0IDg1LjkzNDIgNTUuMTUxMSA4NS45MzQyQzYwLjU5MTcgODUuOTM0MiA2NC45NzIxIDkwLjE0MiA2NC45NzIxIDk1LjM2ODhDNjQuOTcyMSAxMDAuNTk2IDYwLjU5MTcgMTA0LjgwMyA1NS4xNTExIDEwNC44MDNDNDkuNzEwNCAxMDQuODAzIDQ1LjMzMDEgMTAwLjU5NiA0NS4zMzAxIDk1LjM2ODhaTTk2LjQ0ODcgMTA0LjM2OEg3Ni43NzIyTDg2LjYxMDUgODYuNzczN0w5Ni40NDg3IDEwNC4zNjhaIiBmaWxsPSJ3aGl0ZSIvPgo8ZGVmcz4KPGxpbmVhckdyYWRpZW50IGlkPSJwYWludDBfbGluZWFyXzY4M18zMDkxIiB4MT0iMCIgeTE9IjAiIHgyPSIxNTEiIHkyPSIxODAiIGdyYWRpZW50VW5pdHM9InVzZXJTcGFjZU9uVXNlIj4KPHN0b3Agc3RvcC1jb2xvcj0iI0ZGRjBFRSIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiNFQzg4N0QiLz4KPC9saW5lYXJHcmFkaWVudD4KPC9kZWZzPgo8L3N2Zz4K - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "locking"], ["spec", "storagePool"], ["spec", "ui"], ["spec", "users"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "locking"], ["spec", "storagePool"], ["spec", "ui"], ["spec", "publicRead"], ["spec", "users"]] secrets: exclude: [] include: diff --git a/packages/system/cozystack-basics/templates/clusterroles.yaml b/packages/system/cozystack-basics/templates/clusterroles.yaml index 9807d32f1..5df762886 100644 --- a/packages/system/cozystack-basics/templates/clusterroles.yaml +++ b/packages/system/cozystack-basics/templates/clusterroles.yaml @@ -235,6 +235,17 @@ metadata: labels: rbac.cozystack.io/aggregate-to-tenant-admin: "true" rules: +# virtctl reads both the disk PVC and CDI's dynamically named PVC Prime. +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: + - get +- apiGroups: ["upload.cdi.kubevirt.io"] + resources: + - uploadtokenrequests + verbs: + - create - apiGroups: [""] resources: - pods diff --git a/packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml b/packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml new file mode 100644 index 000000000..cbc928dc4 --- /dev/null +++ b/packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml @@ -0,0 +1,59 @@ +{{- if .Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: cozystack-tenant-namespace-delete-policy + labels: + internal.cozystack.io/managed-by-cozystack: "" +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["DELETE"] + resources: ["namespaces"] + variables: + - name: platformCaller + expression: >- + has(request.userInfo.groups) && request.userInfo.groups.exists(g, + g == "system:serviceaccounts:cozy-fluxcd" || + g == "system:serviceaccounts:kube-system") + - name: name + expression: oldObject.metadata.name + - name: cut + expression: variables.name.lastIndexOf("-") + - name: tenant + expression: variables.name.substring(variables.cut + 1) + - name: parentNamespace + expression: >- + variables.cut > 0 && variables.name.substring(0, variables.cut) != "tenant" + ? variables.name.substring(0, variables.cut) + : "tenant-root" + validations: + - expression: variables.platformCaller + messageExpression: >- + "Namespace " + variables.name + " belongs to the Tenant " + variables.tenant + + " in " + variables.parentNamespace + ". Delete the Tenant instead with kubectl delete tenants.apps.cozystack.io " + + variables.tenant + " --namespace " + variables.parentNamespace + + ". Deleting the namespace directly can remove the tenant etcd before the Kubernetes clusters that use it and leave the namespace stuck in Terminating." + + " If the Tenant is already gone, remove the label first with kubectl label namespace " + variables.name + + " tenant.cozystack.io/tenant-root-" + reason: Forbidden +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: cozystack-tenant-namespace-delete-policy + labels: + internal.cozystack.io/managed-by-cozystack: "" +spec: + policyName: cozystack-tenant-namespace-delete-policy + validationActions: [Deny] + matchResources: + objectSelector: + matchExpressions: + - key: tenant.cozystack.io/tenant-root + operator: Exists +{{- end }} diff --git a/packages/system/cozystack-basics/tests/cdi_upload_rbac_test.yaml b/packages/system/cozystack-basics/tests/cdi_upload_rbac_test.yaml new file mode 100644 index 000000000..b99df44bf --- /dev/null +++ b/packages/system/cozystack-basics/tests/cdi_upload_rbac_test.yaml @@ -0,0 +1,97 @@ +suite: tenant admin CDI upload RBAC + +templates: + - templates/clusterroles.yaml + +release: + name: cozystack-basics + namespace: cozy-system + +tests: + - it: lets tenant admins inspect upload PVCs and create CDI upload tokens + documentSelector: + path: metadata.name + value: cozy:tenant:admin:base + asserts: + - equal: + path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant-admin"] + value: "true" + - contains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - contains: + path: rules + content: + apiGroups: ["upload.cdi.kubevirt.io"] + resources: ["uploadtokenrequests"] + verbs: ["create"] + + - it: keeps the admin grant out of the use, view and service-account aggregation + documentSelector: + path: metadata.name + value: cozy:tenant:admin:base + asserts: + - notExists: + path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant-use"] + - notExists: + path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant-view"] + - notExists: + path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant"] + + - it: does not grant CDI upload permissions to the use tier + documentSelector: + path: metadata.name + value: cozy:tenant:use:base + asserts: + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get"] + - notContains: + path: rules + content: + apiGroups: ["upload.cdi.kubevirt.io"] + resources: ["uploadtokenrequests"] + verbs: ["create"] + + - it: aggregates upload permissions through admin into super-admin + documentSelector: + path: metadata.name + value: cozy:tenant:admin + asserts: + - equal: + path: metadata.labels["rbac.cozystack.io/aggregate-to-tenant-super-admin"] + value: "true" + - contains: + path: aggregationRule.clusterRoleSelectors + content: + matchLabels: + rbac.cozystack.io/aggregate-to-tenant-admin: "true" + + - it: selects admin permissions for super-admin + documentSelector: + path: metadata.name + value: cozy:tenant:super-admin + asserts: + - contains: + path: aggregationRule.clusterRoleSelectors + content: + matchLabels: + rbac.cozystack.io/aggregate-to-tenant-super-admin: "true" + + - it: does not grant CDI upload token creation to the view tier + documentSelector: + path: metadata.name + value: cozy:tenant:view:base + asserts: + - notContains: + path: rules + content: + apiGroups: ["upload.cdi.kubevirt.io"] + resources: ["uploadtokenrequests"] + verbs: ["create"] diff --git a/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml index 61b8be507..96cfdfaa3 100644 --- a/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml +++ b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml @@ -1,5 +1,6 @@ suite: cozystack-basics VAP policies are gated on the ValidatingAdmissionPolicy API templates: + - templates/tenant-namespace-delete-policy.yaml - templates/route-hostname-policy.yaml - templates/gateway-hostname-policy.yaml - templates/ingress-hostname-policy.yaml @@ -20,6 +21,9 @@ capabilities: tests: - it: none of the VAP policies render when the API is unavailable asserts: + - hasDocuments: + count: 0 + template: templates/tenant-namespace-delete-policy.yaml - hasDocuments: count: 0 template: templates/route-hostname-policy.yaml diff --git a/packages/system/cozystack-basics/tests/tenant-namespace-delete-policy_test.yaml b/packages/system/cozystack-basics/tests/tenant-namespace-delete-policy_test.yaml new file mode 100644 index 000000000..a0e53c77f --- /dev/null +++ b/packages/system/cozystack-basics/tests/tenant-namespace-delete-policy_test.yaml @@ -0,0 +1,51 @@ +suite: tenant namespace delete policy renders with its binding +templates: + - templates/tenant-namespace-delete-policy.yaml +release: + name: cozystack-basics + namespace: cozy-system +capabilities: + apiVersions: + - admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy +tests: + - it: renders the policy and a Deny binding to it + asserts: + - hasDocuments: + count: 2 + - documentIndex: 0 + equal: + path: kind + value: ValidatingAdmissionPolicy + - documentIndex: 0 + equal: + path: metadata.name + value: cozystack-tenant-namespace-delete-policy + - documentIndex: 1 + equal: + path: spec.policyName + value: cozystack-tenant-namespace-delete-policy + - documentIndex: 1 + equal: + path: spec.validationActions + value: [Deny] + + - it: matches only DELETE of core namespaces + asserts: + - documentIndex: 0 + equal: + path: spec.matchConstraints.resourceRules + value: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["DELETE"] + resources: ["namespaces"] + + - it: binds to the namespaces the tenant chart labels as descendants of tenant-root + asserts: + - documentIndex: 1 + equal: + path: spec.matchResources.objectSelector + value: + matchExpressions: + - key: tenant.cozystack.io/tenant-root + operator: Exists diff --git a/packages/system/cozystack-basics/tests/tenant-root_test.yaml b/packages/system/cozystack-basics/tests/tenant-root_test.yaml index e22dbb649..54210174c 100644 --- a/packages/system/cozystack-basics/tests/tenant-root_test.yaml +++ b/packages/system/cozystack-basics/tests/tenant-root_test.yaml @@ -79,3 +79,9 @@ tests: equal: path: spec.chartRef.name value: cozystack-tenant-application-default-tenant + + - it: tenant-root namespace carries no tenant.cozystack.io/tenant-root label, which the tenant namespace delete policy selects on + asserts: + - documentIndex: 0 + notExists: + path: metadata.labels["tenant.cozystack.io/tenant-root"] diff --git a/packages/system/dashboard/images/console/CLAUDE.md b/packages/system/dashboard/images/console/CLAUDE.md index 2acf80504..036e527d6 100644 --- a/packages/system/dashboard/images/console/CLAUDE.md +++ b/packages/system/dashboard/images/console/CLAUDE.md @@ -32,6 +32,8 @@ Implications: - **RJSF** (`@rjsf/core` + `validator-ajv8`) for application config forms. - **Vitest + jsdom + @testing-library/react** for tests. - **Monaco** for YAML view, **noVNC** for the VM console tab. +- **uPlot** for the Metrics tab charts, lazy-loaded with that tab. +- Metrics and logs come from the observability proxy, not kube-apiserver: `lib/observability/proxy.ts` reads its native Prometheus JSON and VictoriaLogs NDJSON through `K8sClient.raw`, under the paths nginx routes to it ahead of `/apis`. ## Dev loop @@ -79,6 +81,7 @@ the apiserver checks that user's RBAC. to a chunked-encoding WATCH against the same `resourceVersion`. Don't poll. Don't add `refetchInterval`. If you need a one-shot, pass `{ watch: false }`. + The one exception is the metrics panels in `components/observability`: the observability proxy serves the Prometheus query API, which has no watch, so they refresh on a fixed interval instead. Log lines stream through the tail endpoint and are not polled. 4. **Tenant scoping.** Most resources live in `tenant-` namespaces. Pull the active tenant from `useTenantContext()` — never read the namespace from a URL param or guess it. diff --git a/packages/system/dashboard/images/console/apps/console/package.json b/packages/system/dashboard/images/console/apps/console/package.json index fcddb7023..254195f28 100644 --- a/packages/system/dashboard/images/console/apps/console/package.json +++ b/packages/system/dashboard/images/console/apps/console/package.json @@ -31,7 +31,8 @@ "react": "^19.2.4", "react-dom": "^19.2.4", "react-router": "^7.6.1", - "simple-icons": "^16.22.0" + "simple-icons": "^16.22.0", + "uplot": "^1.6.32" }, "devDependencies": { "@tailwindcss/vite": "^4.2.2", diff --git a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts new file mode 100644 index 000000000..37985fca5 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.headers.test.ts @@ -0,0 +1,31 @@ +import { describe, it, expect, vi, afterEach } from "vitest" +import { K8sClient } from "@cozystack/k8s-client" + +afterEach(() => { + vi.unstubAllGlobals() +}) + +describe("K8sClient request headers", () => { + it("lets a method's own content type replace the JSON default", async () => { + const fetchMock = vi.fn(async () => new Response("{}")) + vi.stubGlobal("fetch", fetchMock) + const client = new K8sClient({ getToken: async () => "t0k" }) + + await client.patch("apps", "v1", "deployments", "web", { spec: {} }, "default", "merge") + + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + const headers = new Headers(init.headers) + expect(headers.get("Content-Type")).toBe("application/merge-patch+json") + expect(headers.get("Authorization")).toBe("Bearer t0k") + }) + + it("sends JSON by default", async () => { + const fetchMock = vi.fn(async () => new Response("{}")) + vi.stubGlobal("fetch", fetchMock) + + await new K8sClient().getApiGroups() + + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(new Headers(init.headers).get("Content-Type")).toBe("application/json") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts new file mode 100644 index 000000000..0ac0d75ae --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/__tests__/k8s-client/client.raw.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect, vi, afterEach } from "vitest" +import { K8sClient } from "@cozystack/k8s-client" + +afterEach(() => { + vi.unstubAllGlobals() + vi.restoreAllMocks() +}) + +describe("K8sClient.raw", () => { + it("returns the response untouched, including a non-2xx one", async () => { + const res = new Response("not found", { status: 404 }) + vi.stubGlobal("fetch", vi.fn(async () => res)) + const client = new K8sClient() + + await expect(client.raw("/x")).resolves.toBe(res) + }) + + it("prefixes the base URL and attaches the bearer token", async () => { + const fetchMock = vi.fn(async () => new Response("{}")) + vi.stubGlobal("fetch", fetchMock) + const client = new K8sClient({ baseUrl: "/base", getToken: async () => "t0k" }) + + await client.raw("/apis/x", { headers: { Accept: "application/json" } }) + + const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + expect(url).toBe("/base/apis/x") + const headers = new Headers(init.headers) + expect(headers.get("Accept")).toBe("application/json") + expect(headers.get("Authorization")).toBe("Bearer t0k") + }) + + it.each([ + ["a Headers instance", new Headers({ Accept: "application/x-ndjson" })], + ["a tuple list", [["Accept", "application/x-ndjson"]] as [string, string][]], + ])("keeps caller headers passed as %s", async (_, given) => { + const fetchMock = vi.fn(async () => new Response("{}")) + vi.stubGlobal("fetch", fetchMock) + const client = new K8sClient({ getToken: async () => "t0k" }) + + await client.raw("/x", { headers: given }) + + const [, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit] + const headers = new Headers(init.headers) + expect(headers.get("Accept")).toBe("application/x-ndjson") + expect(headers.get("Authorization")).toBe("Bearer t0k") + }) + + it("routes a 401 through the client's unauthorized handler", async () => { + const onUnauthorized = vi.fn() + vi.stubGlobal("fetch", vi.fn(async () => new Response("", { status: 401 }))) + const client = new K8sClient({ onUnauthorized }) + + const res = await client.raw("/x") + + expect(res.status).toBe(401) + expect(onUnauthorized).toHaveBeenCalledTimes(1) + }) + + it("does not swallow a network failure", async () => { + vi.stubGlobal("fetch", vi.fn(async () => { throw new TypeError("offline") })) + const client = new K8sClient() + + await expect(client.raw("/x")).rejects.toThrow("offline") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx new file mode 100644 index 000000000..8872f4d43 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.test.tsx @@ -0,0 +1,17 @@ +import { describe, it, expect } from "vitest" +import { render, screen } from "@testing-library/react" +import { EmptyState } from "./EmptyState.tsx" + +describe("EmptyState", () => { + it("announces its message as a status", () => { + render() + + expect(screen.getByRole("status")).toHaveTextContent("No data for this range.Pick a longer range.") + }) + + it("marks itself busy while something is loading", () => { + render() + + expect(screen.getByRole("status")).toHaveAttribute("aria-busy", "true") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx new file mode 100644 index 000000000..5b41ef13d --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/EmptyState.tsx @@ -0,0 +1,27 @@ +import type { ReactNode } from "react" +import { cn, Spinner } from "@cozystack/ui" + +interface EmptyStateProps { + title: ReactNode + hint?: ReactNode + action?: ReactNode + busy?: boolean + tone?: "light" | "dark" + className?: string +} + +export function EmptyState({ title, hint, action, busy, tone = "light", className }: EmptyStateProps) { + const dark = tone === "dark" + return ( +
+ {busy && } +

{title}

+ {hint &&

{hint}

} + {action &&
{action}
} +
+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx new file mode 100644 index 000000000..a0f519852 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.test.tsx @@ -0,0 +1,81 @@ +import { describe, it, expect } from "vitest" +import { fireEvent, render, screen } from "@testing-library/react" +import { LogLines } from "./LogLines.tsx" + +const entry = (n: number) => ({ _time: `2026-11-04T10:00:0${n}Z`, _msg: `line ${n}`, _stream_id: "s" }) + +function sized(list: HTMLElement, scrollHeight: number) { + Object.defineProperty(list, "scrollHeight", { configurable: true, value: scrollHeight }) + Object.defineProperty(list, "clientHeight", { configurable: true, value: 100 }) +} + +describe("LogLines", () => { + it("keeps the console frame and shows the placeholder inside it when there are no lines", () => { + render() + + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(frame).toHaveTextContent("Waiting for new log lines…") + expect(screen.queryByRole("list")).toBeNull() + }) + + it("puts the lines in the same frame once they arrive", () => { + const { rerender } = render() + const frame = screen.getByRole("region", { name: "Log lines" }) + + rerender() + + expect(screen.getByRole("region", { name: "Log lines" })).toBe(frame) + expect(frame).toContainElement(screen.getByRole("list")) + expect(frame).not.toHaveTextContent("Waiting") + }) + + it("shows a line without _time with no timestamp and no empty source", () => { + render() + + expect(screen.getByRole("listitem")).toHaveTextContent(/^visible message$/) + }) + + it("shows the remaining fields of a row without _msg", () => { + render() + + expect(screen.getByRole("listitem")).toHaveTextContent(/^db-0\/ rows=42$/) + }) + + it("keeps following the newest line while the reader is at the bottom", () => { + const { rerender } = render() + const list = screen.getByRole("list") + sized(list, 500) + + rerender() + + expect(list.scrollTop).toBe(500) + }) + + it("leaves the position alone once the reader scrolls up", () => { + const { rerender } = render() + const list = screen.getByRole("list") + sized(list, 500) + list.scrollTop = 120 + fireEvent.scroll(list) + + rerender() + + expect(list.scrollTop).toBe(120) + }) + + it("follows again once the list is cleared for a new run", () => { + const { rerender } = render() + const first = screen.getByRole("list") + sized(first, 500) + first.scrollTop = 120 + fireEvent.scroll(first) + + rerender() + rerender() + const second = screen.getByRole("list") + sized(second, 300) + rerender() + + expect(second.scrollTop).toBe(300) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx new file mode 100644 index 000000000..f29af017c --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogLines.tsx @@ -0,0 +1,62 @@ +import { useEffect, useRef, type ReactNode } from "react" +import { CONTAINER_FIELD, POD_FIELD } from "../../lib/observability/logs.ts" +import type { LogEntry } from "../../lib/observability/proxy.ts" + +const BOTTOM_SLACK_PX = 24 +const SHOWN_APART = new Set(["_time", "_msg", "_stream_id", POD_FIELD, CONTAINER_FIELD]) + +function source(e: LogEntry): string | null { + const pod = e[POD_FIELD] + const container = e[CONTAINER_FIELD] + return pod || container ? `${pod ?? ""}/${container ?? ""}` : null +} + +// A `fields` or `stats` pipe can leave a row without _msg; its other fields are then the content. +function message(e: LogEntry): string { + if (e._msg !== undefined) return e._msg + return Object.entries(e) + .filter(([k, v]) => v !== undefined && !SHOWN_APART.has(k)) + .map(([k, v]) => `${k}=${v}`) + .join(" ") +} + +export function LogLines({ entries, placeholder }: { entries: LogEntry[]; placeholder: ReactNode }) { + const ref = useRef(null) + const pinned = useRef(true) + + useEffect(() => { + if (entries.length === 0) pinned.current = true + else if (ref.current && pinned.current) ref.current.scrollTop = ref.current.scrollHeight + }, [entries]) + + return ( +
+ {entries.length === 0 ? ( + placeholder + ) : ( +
    { + const el = e.currentTarget + pinned.current = el.scrollHeight - el.scrollTop - el.clientHeight <= BOTTOM_SLACK_PX + }} + className="h-full overflow-auto p-3" + > + {entries.map((e, i) => { + const from = source(e) + return ( +
  1. + {e._time !== undefined && <>{new Date(e._time).toLocaleTimeString()}{" "}} + {from !== null && <>{from}{" "}} + {message(e)} +
  2. + ) + })} +
+ )} +
+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx new file mode 100644 index 000000000..faba85a4e --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.test.tsx @@ -0,0 +1,416 @@ +import { describe, it, expect, vi } from "vitest" +import { render, screen, waitFor, within } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { K8sClient, K8sProvider } from "@cozystack/k8s-client" +import { renderWithK8sProvider } from "../../test-utils/render.tsx" +import { LogsView } from "./LogsView.tsx" + +const app = { namespace: "tenant-a", plural: "postgreses", name: "db" } +const ROOT = "/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/" + +const line = (time: string, msg: string, pod = "db-0") => + JSON.stringify({ _time: time, _msg: msg, _stream_id: "s", kubernetes_pod_name: pod, kubernetes_container_name: "postgres" }) + +function clientWith(handler: (endpoint: string, params: URLSearchParams) => Response | Promise) { + const client = new K8sClient() + const raw = vi.spyOn(client, "raw").mockImplementation(async (path) => { + const url = new URL(path, "http://x") + return handler(url.pathname, url.searchParams) + }) + return { client, raw } +} + +const values = (...v: string[]) => Response.json({ values: v.map((value) => ({ value, hits: 1 })) }) + +function defaultHandler(endpoint: string, params?: URLSearchParams): Response | Promise { + if (endpoint.endsWith("stream_field_values")) { + return params?.get("field") === "kubernetes_pod_name" ? values("db-0", "db-1") : values("postgres") + } + if (endpoint.endsWith("query")) return new Response(`${line("2026-11-04T10:00:01Z", "ready to accept connections")}\n`) + return new Promise(() => {}) +} + +// The provider's own QueryClient keeps the previous result while a key changes; it is +// shared across tests, so each test below uses namespaces no other test reads. +function renderWithProductionDefaults(client: K8sClient, namespace: string) { + const view = (ns: string) => ( + + + + ) + const result = render(view(namespace)) + return { switchTo: (ns: string) => result.rerender(view(ns)) } +} + +describe("LogsView with the production query defaults", () => { + it("drops the previous tenant's lines and pods while the next tenant loads", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.includes("/tenant-switch-b/") ? new Promise(() => {}) : defaultHandler(endpoint, params), + ) + const { switchTo } = renderWithProductionDefaults(client, "tenant-switch-a") + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + await userEvent.click(screen.getByRole("combobox", { name: "Pod" })) + await screen.findByRole("option", { name: "db-1" }) + await userEvent.keyboard("{Escape}") + + switchTo("tenant-switch-b") + + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(within(frame).getByText("Loading log lines…")).toBeInTheDocument() + expect(screen.queryByText("ready to accept connections")).toBeNull() + await userEvent.click(screen.getByRole("combobox", { name: "Pod" })) + expect((await screen.findAllByRole("option")).map((o) => o.textContent)).toEqual(["All"]) + }) + + it("starts the next tenant with no filter, pod or container of the previous one", async () => { + const { client, raw } = clientWith(defaultHandler) + const { switchTo } = renderWithProductionDefaults(client, "tenant-reset-a") + await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}") + await userEvent.click(screen.getByRole("combobox", { name: "Pod" })) + await userEvent.click(await screen.findByRole("option", { name: "db-1" })) + await waitFor(() => + expect(raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("query") === '{kubernetes_pod_name="db-1"} (error)')).toBe(true), + ) + + switchTo("tenant-reset-b") + + await waitFor(() => expect(raw.mock.calls.some(([p]) => p.includes("/tenant-reset-b/") && p.includes("select/logsql/query?"))).toBe(true)) + const first = raw.mock.calls.find(([p]) => p.includes("/tenant-reset-b/") && p.includes("select/logsql/query?"))! + expect(new URL(first[0], "http://x").searchParams.get("query")).toBe("*") + expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("All") + expect(screen.getByRole("combobox", { name: "Container" })).toHaveTextContent("All") + expect(screen.getByLabelText("LogsQL filter")).toHaveValue("") + }) + + it("keeps the current lines while a new filter in the same tenant loads", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") && params.get("query") === "slow" ? new Promise(() => {}) : defaultHandler(endpoint, params), + ) + renderWithProductionDefaults(client, "tenant-filter-a") + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + + await userEvent.type(screen.getByLabelText("LogsQL filter"), "slow{Enter}") + + expect(screen.getByText("ready to accept connections")).toBeInTheDocument() + }) + + it("says it is loading, not that nothing matched, while a longer range replaces an empty one", async () => { + const { client } = clientWith((endpoint, params) => { + if (!endpoint.endsWith("query")) return defaultHandler(endpoint, params) + return Date.parse(params.get("start") ?? "") < Date.now() - 2 * 3600_000 + ? new Promise(() => {}) + : new Response("") + }) + renderWithProductionDefaults(client, "tenant-range-a") + const frame = () => screen.getByRole("region", { name: "Log lines" }) + expect(await within(frame()).findByText("No log lines for this range and filter.")).toBeInTheDocument() + + await userEvent.click(screen.getByRole("button", { name: "24h" })) + + await waitFor(() => expect(within(frame()).getByRole("status")).toHaveAttribute("aria-busy", "true")) + expect(within(frame()).queryByText("No log lines for this range and filter.")).toBeNull() + }) + + it("marks the lines of the previous range as standing in while a longer range loads", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") && Date.parse(params.get("start") ?? "") < Date.now() - 2 * 3600_000 + ? new Promise(() => {}) + : defaultHandler(endpoint, params), + ) + renderWithProductionDefaults(client, "tenant-range-b") + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + expect(screen.queryByText(/previous lines/)).toBeNull() + + await userEvent.click(screen.getByRole("button", { name: "24h" })) + + expect(await screen.findByText(/previous lines/)).toHaveAttribute("aria-busy", "true") + expect(screen.getByText("ready to accept connections")).toBeInTheDocument() + }) + + it("drops the standing-in note once the longer range answers", async () => { + let answer: (r: Response) => void = () => {} + const longer = new Promise((resolve) => { answer = resolve }) + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") && Date.parse(params.get("start") ?? "") < Date.now() - 2 * 3600_000 + ? longer + : defaultHandler(endpoint, params), + ) + renderWithProductionDefaults(client, "tenant-range-c") + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + await userEvent.click(screen.getByRole("button", { name: "24h" })) + expect(await screen.findByText(/previous lines/)).toBeInTheDocument() + + answer(new Response(`${line("2026-11-03T12:00:00Z", "checkpoint complete")}\n`)) + + expect(await screen.findByText("checkpoint complete")).toBeInTheDocument() + expect(screen.queryByText(/previous lines/)).toBeNull() + expect(screen.queryByText("ready to accept connections")).toBeNull() + }) + + it("shows no standing-in note while following", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("tail") + ? new Response(`${line("2026-11-04T10:00:02Z", "live line")}\n`) + : endpoint.endsWith("query") && params.get("query")?.includes("error") + ? new Promise(() => {}) + : defaultHandler(endpoint, params), + ) + renderWithProductionDefaults(client, "tenant-follow-a") + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + + await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}") + expect(await screen.findByText(/previous lines/)).toBeInTheDocument() + await userEvent.click(screen.getByRole("button", { name: "Follow" })) + + expect(await screen.findByText("live line")).toBeInTheDocument() + expect(screen.queryByText(/previous lines/)).toBeNull() + }) +}) + +describe("LogsView", () => { + it("shows recent lines from the application root", async () => { + const { client, raw } = clientWith(defaultHandler) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + const call = raw.mock.calls.find(([p]) => p.includes("select/logsql/query?"))! + expect(call[0].startsWith(`${ROOT}select/logsql/query?`)).toBe(true) + const params = new URL(call[0], "http://x").searchParams + expect(params.get("query")).toBe("*") + expect(params.get("limit")).toBe("500") + }) + + it("narrows to the picked pod with a stream filter", async () => { + const { client, raw } = clientWith(defaultHandler) + renderWithK8sProvider(, { client }) + + await userEvent.click(screen.getByRole("combobox", { name: "Pod" })) + await userEvent.click(await screen.findByRole("option", { name: "db-1" })) + + await waitFor(() => + expect( + raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("query") === '{kubernetes_pod_name="db-1"} (*)'), + ).toBe(true), + ) + }) + + it("offers All next to every pod and starts on it", async () => { + const { client } = clientWith(defaultHandler) + renderWithK8sProvider(, { client }) + const pod = screen.getByRole("combobox", { name: "Pod" }) + expect(pod).toHaveTextContent("All") + + await userEvent.click(pod) + + expect(await screen.findByRole("option", { name: "db-1" })).toBeInTheDocument() + expect(screen.getAllByRole("option").map((o) => o.textContent)).toEqual(["All", "db-0", "db-1"]) + }) + + it("keeps the log console and says so inside it when nothing matches", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") ? new Response("") : defaultHandler(endpoint, params), + ) + + renderWithK8sProvider(, { client }) + + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(await within(frame).findByText("No log lines for this range and filter.")).toBeInTheDocument() + }) + + it("suggests a longer range only while one exists", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") ? new Response("") : defaultHandler(endpoint, params), + ) + renderWithK8sProvider(, { client }) + const frame = () => screen.getByRole("region", { name: "Log lines" }) + expect(await within(frame()).findByText("Try a longer range or a broader filter.")).toBeInTheDocument() + + await userEvent.click(screen.getByRole("button", { name: "24h" })) + + await waitFor(() => expect(within(frame()).getByText("Try a broader filter.")).toBeInTheDocument()) + expect(within(frame()).queryByText(/longer range/)).toBeNull() + }) + + it("shows loading inside the log console", () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") ? new Promise(() => {}) : defaultHandler(endpoint, params), + ) + + renderWithK8sProvider(, { client }) + + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(within(frame).getByRole("status")).toHaveAttribute("aria-busy", "true") + }) + + it("says it is waiting while Follow has no line yet", async () => { + const { client } = clientWith(defaultHandler) + renderWithK8sProvider(, { client }) + + await userEvent.click(screen.getByRole("button", { name: "Follow" })) + + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(within(frame).getByText("Waiting for new log lines…")).toBeInTheDocument() + }) + + it("applies the LogsQL filter on submit", async () => { + const { client, raw } = clientWith(defaultHandler) + renderWithK8sProvider(, { client }) + + await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}") + + await waitFor(() => + expect(raw.mock.calls.some(([p]) => p.includes("select/logsql/query?") && p.includes("query=error"))).toBe(true), + ) + }) + + it("follows the live tail when Follow is pressed", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("tail") ? new Response(`${line("2026-11-04T10:00:05Z", "checkpoint complete")}\n`) : defaultHandler(endpoint, params), + ) + renderWithK8sProvider(, { client }) + + await userEvent.click(screen.getByRole("button", { name: "Follow" })) + + expect(await screen.findByText("checkpoint complete")).toBeInTheDocument() + expect(screen.getByRole("button", { name: "Follow" })).toHaveAttribute("aria-pressed", "true") + }) + + it("says the tail stopped when the proxy refuses the query", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("tail") ? new Response("pipe not allowed in tail", { status: 400 }) : defaultHandler(endpoint, params), + ) + renderWithK8sProvider(, { client }) + + await userEvent.click(screen.getByRole("button", { name: "Follow" })) + + expect(await screen.findByText(/Stopped/)).toBeInTheDocument() + expect(screen.getByText("pipe not allowed in tail")).toBeInTheDocument() + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(within(frame).getByText("Follow stopped before any line arrived.")).toBeInTheDocument() + }) + + it("renders a cluster without the proxy as not available", async () => { + const { client } = clientWith(() => new Response("404 page not found", { status: 404 })) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("Logs are not available on this cluster.")).toBeInTheDocument() + expect(screen.queryByLabelText("LogsQL filter")).toBeNull() + }) + + it("renders the tenant logs page as not available when kube-apiserver answers instead of the proxy", async () => { + const { client } = clientWith(() => + Response.json({ kind: "Status", apiVersion: "v1", status: "Failure", reason: "Forbidden", code: 403 }, { status: 403 }), + ) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("Logs are not available on this cluster.")).toBeInTheDocument() + }) + + it("keeps the controls and explains a rejected query", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") ? new Response("cannot parse query", { status: 400 }) : defaultHandler(endpoint, params), + ) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("cannot parse query")).toBeInTheDocument() + expect(screen.getByLabelText("LogsQL filter")).toBeInTheDocument() + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(within(frame).getByText("No log lines to show.")).toBeInTheDocument() + }) + + it("offers All once when some lines carry no pod name", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("stream_field_values") && params.get("field") === "kubernetes_pod_name" + ? values("", "db-0") + : defaultHandler(endpoint, params), + ) + renderWithK8sProvider(, { client }) + + await userEvent.click(screen.getByRole("combobox", { name: "Pod" })) + + await screen.findByRole("option", { name: "db-0" }) + expect(screen.getAllByRole("option").map((o) => o.textContent)).toEqual(["All", "db-0"]) + }) + + it("shows rows of a query whose pipe removed _time", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") ? new Response('{"_msg":"visible message"}\n') : defaultHandler(endpoint, params), + ) + renderWithK8sProvider(, { client }) + + await userEvent.type(screen.getByLabelText("LogsQL filter"), "* | fields _msg{Enter}") + + const frame = screen.getByRole("region", { name: "Log lines" }) + expect(await within(frame).findByText("visible message")).toBeInTheDocument() + expect(within(frame).queryByText(/No log lines/)).toBeNull() + }) + + it("keeps the view when only the pod list is refused", async () => { + const status = JSON.stringify({ kind: "Status", apiVersion: "v1", status: "Failure", code: 404 }) + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("stream_field_values") ? new Response(status, { status: 404 }) : defaultHandler(endpoint, params), + ) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + expect(screen.queryByText("Logs are not available on this cluster.")).toBeNull() + }) + + it("retries a failed history query on request", async () => { + let calls = 0 + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") && calls++ === 0 ? new Response("upstream down", { status: 502 }) : defaultHandler(endpoint, params), + ) + renderWithK8sProvider(, { client }) + + await userEvent.click(await screen.findByRole("button", { name: "Retry" })) + + expect(await screen.findByText("ready to accept connections")).toBeInTheDocument() + expect(screen.queryByRole("button", { name: "Retry" })).toBeNull() + }) + + it("says it is loading while a retry runs", async () => { + let calls = 0 + const { client } = clientWith((endpoint, params) => { + if (!endpoint.endsWith("query")) return defaultHandler(endpoint, params) + return calls++ === 0 ? new Response("upstream down", { status: 502 }) : new Promise(() => {}) + }) + renderWithK8sProvider(, { client }) + + await userEvent.click(await screen.findByRole("button", { name: "Retry" })) + + const frame = screen.getByRole("region", { name: "Log lines" }) + await waitFor(() => expect(within(frame).getByRole("status")).toHaveAttribute("aria-busy", "true")) + expect(within(frame).queryByRole("button", { name: "Retry" })).toBeNull() + }) + + it("opens a new history result at its newest line, not the old scroll position", async () => { + const { client } = clientWith((endpoint, params) => + endpoint.endsWith("query") + ? new Response(`${line("2026-11-04T10:00:01Z", `result for ${params.get("query")}`)}\n`) + : defaultHandler(endpoint, params), + ) + const { queryClient } = renderWithK8sProvider(, { client }) + queryClient.setDefaultOptions({ queries: { ...queryClient.getDefaultOptions().queries, placeholderData: (prev: unknown) => prev } }) + await screen.findByText("result for *") + const before = screen.getByRole("list") + + await userEvent.type(screen.getByLabelText("LogsQL filter"), "error{Enter}") + + await screen.findByText("result for error") + expect(screen.getByRole("list")).not.toBe(before) + }) + + it("names the source pod and container of each line", async () => { + const { client } = clientWith(defaultHandler) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("db-0/postgres")).toBeInTheDocument() + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx new file mode 100644 index 000000000..a4376df47 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/LogsView.tsx @@ -0,0 +1,180 @@ +import { useState } from "react" +import { useQuery } from "@tanstack/react-query" +import { useK8sClient } from "@cozystack/k8s-client" +import { Button } from "@cozystack/ui" +import { Input } from "../ui/input.tsx" +import { SelectItem, SelectLabel, SelectPopup, SelectRoot, SelectTrigger, SelectValue } from "../ui/select.tsx" +import { + CONTAINER_FIELD, + LOG_RANGES, + POD_FIELD, + buildLogsQuery, + type LogRangeKey, +} from "../../lib/observability/logs.ts" +import { + fieldValues, + isScopeWide, + logsRoot, + queryLogs, + type ObservabilityScope, +} from "../../lib/observability/proxy.ts" +import { useLogTail } from "../../lib/observability/use-log-tail.ts" +import { ProxyNotice } from "./ProxyNotice.tsx" +import { RangePicker } from "./RangePicker.tsx" +import { LogLines } from "./LogLines.tsx" +import { EmptyState } from "./EmptyState.tsx" + +const HISTORY_LIMIT = 500 +const FOLLOW_CAP = 1000 +const TAIL_STATE_LABEL = { connecting: "Connecting…", live: "Live", reconnecting: "Reconnecting…", stopped: "Stopped" } + +function windowFor(range: LogRangeKey) { + const end = Date.now() + return { start: new Date(end - LOG_RANGES[range] * 1000).toISOString(), end: new Date(end).toISOString() } +} + +const ALL = "" + +function Picker({ label, value, options, onChange }: { label: string; value: string; options: string[]; onChange: (v: string) => void }) { + const items = [{ value: ALL, label: "All" }, ...options.filter((o) => o !== ALL).map((o) => ({ value: o, label: o }))] + return ( + onChange(v ?? ALL)}> +
+ {label} + + + +
+ + {items.map((item) => ( + + {item.label} + + ))} + +
+ ) +} + +const RANGE_KEYS = Object.keys(LOG_RANGES) as LogRangeKey[] +const LONGEST_RANGE = RANGE_KEYS[RANGE_KEYS.length - 1] + +function placeholderFor({ follow, stopped, loading, failed, longestRange, retry }: { follow: boolean; stopped: boolean; loading: boolean; failed: boolean; longestRange: boolean; retry: () => void }) { + if (follow) { + return stopped ? ( + + ) : ( + + ) + } + if (loading) return + if (failed) { + return ( + Retry} + /> + ) + } + return +} + +// Filter, pod, container and the lines already read belong to one scope; a new tenant or application starts clean. +export function LogsView({ scope }: { scope: ObservabilityScope }) { + const root = logsRoot(scope) + return +} + +function ScopedLogsView({ root }: { root: string }) { + const client = useK8sClient() + const [range, setRange] = useState("1h") + const [draft, setDraft] = useState("") + const [filter, setFilter] = useState("") + const [pod, setPod] = useState("") + const [container, setContainer] = useState("") + const [followRun, setFollowRun] = useState(null) + const follow = followRun !== null + const query = buildLogsQuery({ filter, pod, container }) + + const pickerQuery = (field: string, narrowTo?: string) => ({ + queryKey: ["observability", "logs-fields", root, field, range, narrowTo ?? ""], + queryFn: ({ signal }: { signal: AbortSignal }) => + fieldValues(client, root, { + field, + stream: true, + query: buildLogsQuery({ filter: "", pod: narrowTo }), + ...windowFor(range), + }, signal), + retry: false, + }) + const pods = useQuery(pickerQuery(POD_FIELD)) + const containers = useQuery(pickerQuery(CONTAINER_FIELD, pod || undefined)) + + const history = useQuery({ + queryKey: ["observability", "logs", root, query, range], + queryFn: ({ signal }) => queryLogs(client, root, { query, limit: HISTORY_LIMIT, ...windowFor(range) }, signal), + enabled: !follow, + retry: false, + }) + const tail = useLogTail(root, query, followRun, { cap: FOLLOW_CAP }) + + // The pod list is optional: refusing it alone must not hide lines the query can still read. + const wide = [history.error, tail.error].find(isScopeWide) + if (wide) { + return ( +
+ +
+ ) + } + + const error = follow ? tail.error : history.error + const entries = follow ? tail.entries : (history.data ?? []) + + return ( +
+
+
{ + e.preventDefault() + setFilter(draft) + }} + > + +
+ { setPod(v); setContainer("") }} /> + + {!follow && } + +
+ {follow && ( +

+ {TAIL_STATE_LABEL[tail.state]} + {` · newest ${FOLLOW_CAP} lines kept`} +

+ )} + {!follow && history.isPlaceholderData && entries.length > 0 && ( +

+ Showing the previous lines until the new range or filter loads… +

+ )} + {error ? : null} + void history.refetch() })} + /> +
+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx new file mode 100644 index 000000000..fccb6f43d --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.test.tsx @@ -0,0 +1,169 @@ +import { describe, it, expect, vi } from "vitest" +import { render, screen, waitFor, within } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { K8sClient, K8sProvider } from "@cozystack/k8s-client" +import { renderWithK8sProvider } from "../../test-utils/render.tsx" +import type { ChartData } from "../../lib/observability/metrics.ts" + +vi.mock("./TimeSeriesChart.tsx", () => ({ + TimeSeriesChart: ({ data }: { data: ChartData }) => ( +
{data.series.map((s) => s.label).join(",")}
+ ), +})) + +const { MetricsView } = await import("./MetricsView.tsx") + +const scope = { namespace: "tenant-a", plural: "postgreses", name: "db" } + +function matrix(pod: string) { + return Response.json({ + status: "success", + data: { resultType: "matrix", result: [{ metric: { pod }, values: [[1, "1"]] }] }, + }) +} + +function clientWith(handler: (url: URL) => Response | Promise) { + const client = new K8sClient() + const raw = vi.spyOn(client, "raw").mockImplementation(async (path) => handler(new URL(path, "http://x"))) + return { client, raw } +} + +describe("MetricsView", () => { + it("draws CPU, memory and network from the application root", async () => { + const { client, raw } = clientWith(() => matrix("db-0")) + + renderWithK8sProvider(, { client }) + + await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3)) + expect(screen.getByText("CPU (cores)")).toBeInTheDocument() + expect(screen.getByText("Memory (working set)")).toBeInTheDocument() + expect(screen.getByText("Network")).toBeInTheDocument() + for (const [path] of raw.mock.calls) { + expect(path.startsWith("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/api/v1/query_range?")).toBe(true) + } + }) + + it("requeries with a coarser step when a longer range is picked", async () => { + const { client, raw } = clientWith(() => matrix("db-0")) + renderWithK8sProvider(, { client }) + await waitFor(() => expect(raw).toHaveBeenCalled()) + + await userEvent.click(screen.getByRole("button", { name: "7d" })) + + await waitFor(() => + expect(raw.mock.calls.some(([p]) => new URL(p, "http://x").searchParams.get("step") === "2520")).toBe(true), + ) + }) + + it("shows one quiet notice instead of three errors when the proxy is absent", async () => { + const { client } = clientWith(() => new Response("404 page not found", { status: 404 })) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("Metrics are not available on this cluster.")).toBeInTheDocument() + expect(screen.getAllByRole("status")).toHaveLength(1) + expect(screen.queryByText("CPU (cores)")).toBeNull() + }) + + it("keeps the other panels when one query hits a limit", async () => { + const { client } = clientWith((url) => + url.searchParams.get("query")?.includes("memory") + ? Response.json({ status: "error", error: "too many series" }, { status: 422 }) + : matrix("db-0"), + ) + + renderWithK8sProvider(, { client }) + + expect(await screen.findByText("too many series. Choose a shorter range.")).toBeInTheDocument() + await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(2)) + }) + + it("says so when a panel has no data", async () => { + const { client } = clientWith(() => Response.json({ status: "success", data: { resultType: "matrix", result: [] } })) + + renderWithK8sProvider(, { client }) + + await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(3)) + for (const status of screen.getAllByRole("status")) expect(status).toHaveClass("h-[230px]") + }) + + it("suggests a longer range only while one exists", async () => { + const { client } = clientWith(() => Response.json({ status: "success", data: { resultType: "matrix", result: [] } })) + renderWithK8sProvider(, { client }) + await waitFor(() => expect(screen.getAllByText("Try a longer range.")).toHaveLength(3)) + + await userEvent.click(screen.getByRole("button", { name: "7d" })) + + await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(3)) + expect(screen.queryByText("Try a longer range.")).toBeNull() + }) + + it("shows a panel's samples as a table on request", async () => { + const { client } = clientWith(() => matrix("db-0")) + renderWithK8sProvider(, { client }) + await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3)) + const toggle = screen.getByRole("button", { name: "Show data for CPU (cores)" }) + expect(toggle).toHaveAttribute("aria-expanded", "false") + + await userEvent.click(toggle) + + const table = screen.getByRole("region", { name: "CPU (cores) data" }) + expect(toggle).toHaveAttribute("aria-expanded", "true") + expect(toggle).toHaveAttribute("aria-controls", table.id) + expect(toggle).toHaveTextContent("Hide data") + expect(within(table).getByRole("columnheader", { name: "db-0" })).toBeInTheDocument() + expect(screen.queryByRole("region", { name: "Memory (working set) data" })).toBeNull() + }) + + it("offers no table while a panel has no samples", () => { + const { client } = clientWith(() => new Promise(() => {})) + + renderWithK8sProvider(, { client }) + + expect(screen.getByRole("button", { name: "Show data for Network" })).toBeDisabled() + }) + + it("offers no table for a panel with no series or an error", async () => { + const { client } = clientWith((url) => + url.searchParams.get("query")?.includes("memory") + ? Response.json({ status: "error", error: "too many series" }, { status: 422 }) + : Response.json({ status: "success", data: { resultType: "matrix", result: [] } }), + ) + renderWithK8sProvider(, { client }) + + await screen.findByText("too many series. Choose a shorter range.") + await waitFor(() => expect(screen.getAllByText("No data for this range.")).toHaveLength(2)) + for (const title of ["CPU (cores)", "Memory (working set)", "Network"]) { + expect(screen.getByRole("button", { name: `Show data for ${title}` })).toBeDisabled() + } + }) + + it("drops the previous application's charts while the next one loads, with the production query defaults", async () => { + const { client } = clientWith((url) => (url.pathname.includes("/switch-b/") ? new Promise(() => {}) : matrix("db-0"))) + const view = (name: string) => ( + + + + ) + const { rerender } = render(view("switch-a")) + await waitFor(() => expect(screen.getAllByTestId("chart")).toHaveLength(3)) + + rerender(view("switch-b")) + + expect(screen.queryAllByTestId("chart")).toHaveLength(0) + expect(screen.getAllByText("Loading…")).toHaveLength(3) + }) + + it("holds each chart's place while it loads", () => { + const { client } = clientWith(() => new Promise(() => {})) + + renderWithK8sProvider(, { client }) + + const statuses = screen.getAllByRole("status") + expect(statuses).toHaveLength(3) + for (const status of statuses) { + expect(status).toHaveAttribute("aria-busy", "true") + expect(status).toHaveClass("h-[230px]") + } + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx new file mode 100644 index 000000000..4663b2ae3 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/MetricsView.tsx @@ -0,0 +1,105 @@ +import { useId, useState } from "react" +import { useQueries } from "@tanstack/react-query" +import { useK8sClient } from "@cozystack/k8s-client" +import { Button, Section } from "@cozystack/ui" +import { + METRIC_PANELS, + RANGES, + rangeWindow, + toChartData, + type RangeKey, +} from "../../lib/observability/metrics.ts" +import { isScopeWide, metricsRoot, queryRange, type ObservabilityScope } from "../../lib/observability/proxy.ts" +import { ProxyNotice } from "./ProxyNotice.tsx" +import { TimeSeriesChart } from "./TimeSeriesChart.tsx" +import { RangePicker } from "./RangePicker.tsx" +import { EmptyState } from "./EmptyState.tsx" +import { SeriesTable } from "./SeriesTable.tsx" + +const REFRESH_MS = 60_000 +// The uPlot canvas plus its one-row legend, so a placeholder takes the chart's place without a jump. +const CHART_SLOT = "h-[230px] rounded-md border border-dashed border-slate-200 bg-slate-50" + +export function MetricsView({ scope }: { scope: ObservabilityScope }) { + const client = useK8sClient() + const root = metricsRoot(scope) + const [range, setRange] = useState("1h") + const [tables, setTables] = useState>(new Set()) + const idBase = useId() + const rangeKeys = Object.keys(RANGES) as RangeKey[] + const longestRange = range === rangeKeys[rangeKeys.length - 1] + + const results = useQueries({ + queries: METRIC_PANELS.map((panel) => ({ + queryKey: ["observability", "metrics", root, panel.id, range], + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const window = rangeWindow(range, Math.floor(Date.now() / 1000)) + const series = await Promise.all( + panel.queries(window).map(async (q) => ({ + label: q.label, + series: await queryRange(client, root, { query: q.expr, ...window }, signal), + })), + ) + return toChartData(series) + }, + retry: false, + refetchInterval: REFRESH_MS, + })), + }) + + const wide = results.find((r) => isScopeWide(r.error))?.error + + return ( +
+
+

Resource consumption

+ +
+ {wide ? ( + + ) : ( + METRIC_PANELS.map((panel, i) => { + const { data, error, isLoading } = results[i] + const hasSeries = !error && !isLoading && !!data && data.series.length > 0 + const tableOpen = hasSeries && tables.has(panel.id) + const tableId = `${idBase}-${panel.id}-data` + const toggle = ( + + ) + return ( +
+ {error ? ( + + ) : isLoading || !data ? ( + + ) : data.series.length === 0 ? ( + + ) : ( + <> + + {tableOpen && } + + )} +
+ ) + }) + )} +
+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx new file mode 100644 index 000000000..c4b4e89c1 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.test.tsx @@ -0,0 +1,59 @@ +import { describe, it, expect } from "vitest" +import { render, screen } from "@testing-library/react" +import { ProxyError } from "../../lib/observability/proxy.ts" +import { ProxyNotice } from "./ProxyNotice.tsx" + +describe("ProxyNotice", () => { + it("renders a 404 as not available on this cluster, not as an error", () => { + render() + + expect(screen.getByRole("status")).toHaveTextContent("Metrics are not available on this cluster.") + expect(screen.queryByRole("alert")).toBeNull() + }) + + it("renders a 502 as a quiet notice about the store", () => { + render() + + expect(screen.getByRole("status")).toHaveTextContent("The logs store is unreachable right now.") + }) + + it.each([ + ["forbidden", 403, "You do not have permission to read metrics here."], + ["limit", 422, "too many points. Choose a shorter range."], + ["rate-limited", 429, "Too many requests. Try again in a moment."], + ["timeout", 504, "The store did not answer in time. Choose a shorter range or a narrower filter."], + ["unavailable", 503, "The service is temporarily unavailable. Try again in a moment."], + ] as const)("explains %s", (kind, status, text) => { + render() + + expect(screen.getByRole("alert")).toHaveTextContent(text) + }) + + it.each([ + "the application has more than 50 pods in this window; use a shorter range", + "1200 points per series exceed the limit of 1000; raise step", + ])("does not add advice to a limit message that already carries its own: %s", (message) => { + render() + + expect(screen.getByRole("alert").textContent).toBe(`${message}.`) + }) + + it("still advises a shorter range for a store limit whose own advice is for operators", () => { + const message = "the number of matching timeseries exceeds 300000; either narrow down the search or increase -search.max* command-line flag values" + render() + + expect(screen.getByRole("alert").textContent).toBe(`${message}. Choose a shorter range.`) + }) + + it("shows the server message for a rejected query", () => { + render() + + expect(screen.getByRole("alert")).toHaveTextContent("cannot parse query") + }) + + it("falls back to the message of an unexpected error", () => { + render() + + expect(screen.getByRole("alert")).toHaveTextContent("boom") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx new file mode 100644 index 000000000..1fb30c8ff --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/ProxyNotice.tsx @@ -0,0 +1,56 @@ +import { ProxyError } from "../../lib/observability/proxy.ts" + +interface ProxyNoticeProps { + what: "Metrics" | "Logs" + error: unknown +} + +// Remedies the observability proxy itself appends; vmselect's 422 text carries +// advice for operators (raise -search.max* flags), so it still gets ours. +const PROXY_REMEDIES = ["; use a shorter range", "; raise step"] + +function describe(what: ProxyNoticeProps["what"], error: unknown): { quiet: boolean; text: string } { + const lower = what.toLowerCase() + if (!(error instanceof ProxyError)) { + return { quiet: false, text: error instanceof Error ? error.message : String(error) } + } + switch (error.kind) { + case "not-available": + return { quiet: true, text: `${what} are not available on this cluster.` } + case "store-unreachable": + return { quiet: true, text: `The ${lower} store is unreachable right now.` } + case "unauthorized": + return { quiet: false, text: "Your session has expired. Sign in again." } + case "forbidden": + return { quiet: false, text: `You do not have permission to read ${lower} here.` } + case "limit": + return { + quiet: false, + text: PROXY_REMEDIES.some((r) => error.message.endsWith(r)) ? `${error.message}.` : `${error.message}. Choose a shorter range.`, + } + case "rate-limited": + return { quiet: false, text: "Too many requests. Try again in a moment." } + case "timeout": + return { quiet: false, text: "The store did not answer in time. Choose a shorter range or a narrower filter." } + case "unavailable": + return { quiet: false, text: "The service is temporarily unavailable. Try again in a moment." } + default: + return { quiet: false, text: error.message } + } +} + +export function ProxyNotice({ what, error }: ProxyNoticeProps) { + const { quiet, text } = describe(what, error) + if (quiet) { + return ( +

+ {text} +

+ ) + } + return ( +

+ {text} +

+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx new file mode 100644 index 000000000..9787963b2 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.test.tsx @@ -0,0 +1,23 @@ +import { describe, it, expect, vi } from "vitest" +import { render, screen, within } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { RangePicker } from "./RangePicker.tsx" + +describe("RangePicker", () => { + it("groups the ranges under one name and marks the chosen one pressed", () => { + render( {}} />) + + const group = screen.getByRole("group", { name: "Time range" }) + expect(within(group).getByRole("button", { name: "6h" })).toHaveAttribute("aria-pressed", "true") + expect(within(group).getByRole("button", { name: "1h" })).toHaveAttribute("aria-pressed", "false") + }) + + it("reports the range clicked", async () => { + const onChange = vi.fn() + render() + + await userEvent.click(screen.getByRole("button", { name: "1h" })) + + expect(onChange).toHaveBeenCalledWith("1h") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx new file mode 100644 index 000000000..49abd8b1d --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/RangePicker.tsx @@ -0,0 +1,30 @@ +import { cn } from "@cozystack/ui" + +export function RangePicker({ + options, + value, + onChange, +}: { + options: readonly K[] + value: K + onChange: (value: K) => void +}) { + return ( +
+ {options.map((key) => ( + + ))} +
+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx new file mode 100644 index 000000000..e85041c0a --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.test.tsx @@ -0,0 +1,37 @@ +import { describe, it, expect } from "vitest" +import { render, screen, within } from "@testing-library/react" +import { SeriesTable } from "./SeriesTable.tsx" + +const data = { + timestamps: [1_700_000_000, 1_700_000_060], + series: [ + { label: "db-0", values: [1, 2] }, + { label: "db-1", values: [null, 3] }, + ], +} + +describe("SeriesTable", () => { + it("is a named region the keyboard can reach", () => { + render() + + const region = screen.getByRole("region", { name: "Memory data" }) + expect(region).toHaveAttribute("id", "t") + expect(region).toHaveAttribute("tabindex", "0") + }) + + it("has a time column and one column per series", () => { + render() + + expect(screen.getAllByRole("columnheader").map((h) => h.textContent)).toEqual(["Time", "db-0", "db-1"]) + }) + + it("lists every sample newest first in the chart's units", () => { + render( `${v} B`} />) + + const rows = screen.getAllByRole("row").slice(1) + expect(rows).toHaveLength(2) + expect(within(rows[0]).getAllByRole("cell").map((c) => c.textContent)).toEqual(["2 B", "3 B"]) + expect(within(rows[1]).getAllByRole("cell").map((c) => c.textContent)).toEqual(["1 B", "–"]) + expect(within(rows[0]).getByRole("rowheader")).toHaveTextContent(new Date(1_700_000_060_000).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit" })) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx new file mode 100644 index 000000000..5e9d88ba1 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/SeriesTable.tsx @@ -0,0 +1,44 @@ +import { formatSampleTime, type ChartData } from "../../lib/observability/metrics.ts" + +interface SeriesTableProps { + id: string + title: string + data: ChartData + format: (value: number) => string +} + +export function SeriesTable({ id, title, data, format }: SeriesTableProps) { + const newestFirst = data.timestamps.map((t, i) => ({ t, i })).reverse() + return ( +
+ + + + + + {data.series.map((s, j) => ( + + ))} + + + + {newestFirst.map(({ t, i }) => ( + + + {data.series.map((s, j) => { + const v = s.values[i] + return + })} + + ))} + +
{`${title}, newest first`}
Time{s.label}
{formatSampleTime(t)}{v === null ? "–" : format(v)}
+
+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx new file mode 100644 index 000000000..bb76edb5a --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.test.tsx @@ -0,0 +1,69 @@ +import { describe, it, expect, vi, beforeEach } from "vitest" +import { render } from "@testing-library/react" + +type LegendValue = (u: unknown, v: number | null, sidx: number, idx: number | null) => string +type Opts = { series: { label?: string; value?: LegendValue }[] } + +const created: { opts: Opts; data: unknown[]; destroy: ReturnType }[] = [] + +vi.mock("uplot", () => ({ + default: class { + destroy = vi.fn() + constructor(opts: Opts, data: unknown[]) { + created.push({ opts, data, destroy: this.destroy }) + } + setSize() {} + }, +})) + +const { TimeSeriesChart } = await import("./TimeSeriesChart.tsx") +const { formatSampleTime } = await import("../../lib/observability/metrics.ts") + +beforeEach(() => { + created.length = 0 +}) + +describe("TimeSeriesChart", () => { + const data = { + timestamps: [1, 2], + series: [ + { label: "db-0", values: [1, 2] }, + { label: "db-1", values: [null, 3] }, + ], + } + + it("hands uPlot the timestamps first and one labelled series per input", () => { + render() + + expect(created).toHaveLength(1) + expect(created[0].data).toEqual([[1, 2], [1, 2], [null, 3]]) + expect(created[0].opts.series.slice(1).map((s) => s.label)).toEqual(["db-0", "db-1"]) + }) + + it("shows each series' latest value in the legend while nothing is hovered", () => { + render( `${v} B`} />) + + const legend = created[0].opts.series[1].value! + + expect(legend(null, null, 1, null)).toBe("2 B") + expect(legend(null, 1, 1, 0)).toBe("1 B") + expect(legend(null, null, 1, 2)).toBe("–") + }) + + it("labels the legend's time as latest while nothing is hovered", () => { + render() + + const time = created[0].opts.series[0].value! + + expect(time(null, null, 0, null)).toBe("latest") + expect(time(null, 2, 0, 1)).toBe(formatSampleTime(2)) + }) + + it("destroys the chart on unmount", () => { + const { unmount } = render() + + unmount() + + expect(created[0].destroy).toHaveBeenCalled() + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx new file mode 100644 index 000000000..607d44593 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/observability/TimeSeriesChart.tsx @@ -0,0 +1,53 @@ +import { useEffect, useRef } from "react" +import uPlot from "uplot" +import "uplot/dist/uPlot.min.css" +import { formatSampleTime, type ChartData } from "../../lib/observability/metrics.ts" + +const COLORS = ["#2563eb", "#16a34a", "#d97706", "#9333ea", "#dc2626", "#0891b2", "#4b5563", "#db2777"] +const HEIGHT = 200 + +interface TimeSeriesChartProps { + data: ChartData + format: (value: number) => string +} + +export function TimeSeriesChart({ data, format }: TimeSeriesChartProps) { + const ref = useRef(null) + + useEffect(() => { + const el = ref.current + if (!el) return + const fmt = (v: number | null) => (v === null ? "–" : format(v)) + const latest = data.series.map((s) => s.values.findLast((v) => v !== null) ?? null) + const chart = new uPlot( + { + width: el.clientWidth || 600, + height: HEIGHT, + scales: { x: { time: true } }, + legend: { live: true }, + series: [ + { value: (_: uPlot, v: number | null, _sidx: number, idx: number | null) => (idx === null || v === null ? "latest" : formatSampleTime(v)) }, + ...data.series.map((s, i) => ({ + label: s.label, + stroke: COLORS[i % COLORS.length], + width: 1.5, + // Without a hovered point the legend shows the latest sample instead of a dash. + value: (_: uPlot, v: number | null, sidx: number, idx: number | null) => fmt(idx === null ? latest[sidx - 1] : v), + })), + ], + axes: [{}, { size: 84, values: (_: uPlot, splits: number[]) => splits.map(format) }], + }, + [data.timestamps, ...data.series.map((s) => s.values)], + el, + ) + if (typeof ResizeObserver === "undefined") return () => chart.destroy() + const observer = new ResizeObserver(() => chart.setSize({ width: el.clientWidth, height: HEIGHT })) + observer.observe(el) + return () => { + observer.disconnect() + chart.destroy() + } + }, [data, format]) + + return
+} diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx new file mode 100644 index 000000000..e65a70a3a --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.test.tsx @@ -0,0 +1,21 @@ +import { describe, it, expect, vi } from "vitest" +import { render, screen } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { Input } from "./input.tsx" + +describe("Input", () => { + it("is a labelled text box that reports what is typed", async () => { + const onValueChange = vi.fn() + render() + + await userEvent.type(screen.getByRole("textbox", { name: "Filter" }), "x") + + expect(onValueChange).toHaveBeenCalledWith("x", expect.anything()) + }) + + it("keeps the caller's classes next to its own", () => { + render() + + expect(screen.getByRole("textbox", { name: "Filter" })).toHaveClass("font-mono", "rounded-lg") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx new file mode 100644 index 000000000..b079e6b35 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/input.tsx @@ -0,0 +1,17 @@ +import { Input as InputPrimitive } from "@base-ui/react/input" +import { cn } from "@cozystack/ui" + +function Input({ className, ...props }: InputPrimitive.Props) { + return ( + + ) +} + +export { Input } diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx new file mode 100644 index 000000000..fb5b84e6f --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.test.tsx @@ -0,0 +1,64 @@ +import { describe, it, expect, vi } from "vitest" +import { render, screen } from "@testing-library/react" +import userEvent from "@testing-library/user-event" +import { SelectItem, SelectLabel, SelectPopup, SelectRoot, SelectTrigger, SelectValue } from "./select.tsx" + +const items = [ + { value: "", label: "All" }, + { value: "db-0", label: "db-0" }, + { value: "db-1", label: "db-1" }, +] + +function PodSelect({ value = "", onValueChange = () => {} }: { value?: string; onValueChange?: (v: string) => void }) { + return ( + onValueChange(v ?? "")}> + Pod + + + + + {items.map((i) => ( + + {i.label} + + ))} + + + ) +} + +describe("Select", () => { + it("names the trigger by its label and shows the chosen item", () => { + render() + + expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("db-1") + }) + + it("shows the label of an empty-string value", () => { + render() + + expect(screen.getByRole("combobox", { name: "Pod" })).toHaveTextContent("All") + }) + + it("lists the items and reports the one clicked", async () => { + const onValueChange = vi.fn() + render() + + await userEvent.click(screen.getByRole("combobox", { name: "Pod" })) + await userEvent.click(await screen.findByRole("option", { name: "db-1" })) + + expect(onValueChange).toHaveBeenCalledWith("db-1") + }) + + it("can be driven from the keyboard", async () => { + const onValueChange = vi.fn() + render() + + screen.getByRole("combobox", { name: "Pod" }).focus() + await userEvent.keyboard("{ArrowDown}") + expect(await screen.findByRole("listbox")).toBeInTheDocument() + await userEvent.keyboard("{ArrowDown}{Enter}") + + expect(onValueChange).toHaveBeenCalledWith("db-0") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx new file mode 100644 index 000000000..f256dba45 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/components/ui/select.tsx @@ -0,0 +1,107 @@ +import { Select as SelectPrimitive } from "@base-ui/react/select" +import { Check, ChevronDown } from "lucide-react" +import { cn } from "@cozystack/ui" + +const SelectRoot = SelectPrimitive.Root + +function SelectLabel({ + className, + ...props +}: SelectPrimitive.Label.Props) { + return ( + + ) +} + +function SelectTrigger({ + className, + children, + ...props +}: SelectPrimitive.Trigger.Props) { + return ( + + {children} + + + + + ) +} + +function SelectValue({ + className, + ...props +}: SelectPrimitive.Value.Props) { + return ( + + ) +} + +function SelectPopup({ + className, + children, + ...props +}: SelectPrimitive.Popup.Props) { + return ( + + + + {children} + + + + ) +} + +function SelectItem({ + className, + children, + ...props +}: SelectPrimitive.Item.Props) { + return ( + + + + + {children} + + ) +} + +export { + SelectRoot, + SelectLabel, + SelectTrigger, + SelectValue, + SelectPopup, + SelectItem, +} diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts new file mode 100644 index 000000000..278da6890 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect } from "vitest" +import { buildLogsQuery, compareTime, mergeTail } from "./logs.ts" + +describe("buildLogsQuery", () => { + it("passes the filter through when nothing is picked", () => { + expect(buildLogsQuery({ filter: "error or warn" })).toBe("error or warn") + expect(buildLogsQuery({ filter: " " })).toBe("*") + }) + + it("ANDs the picked pod and container as a stream filter around the whole filter", () => { + expect(buildLogsQuery({ filter: "error or warn", pod: "db-0", container: "postgres" })).toBe( + '{kubernetes_pod_name="db-0",kubernetes_container_name="postgres"} (error or warn)', + ) + }) + + it("keeps the user's pipes after the scoped filter", () => { + expect(buildLogsQuery({ filter: 'a or "x|y" | fields _msg', pod: "db-0" })).toBe( + '{kubernetes_pod_name="db-0"} (a or "x|y") | fields _msg', + ) + }) + + it("escapes quotes in picked values", () => { + expect(buildLogsQuery({ filter: "", pod: 'a"b' })).toBe('{kubernetes_pod_name="a\\"b"} (*)') + }) +}) + +describe("compareTime", () => { + it("orders RFC3339 times with differing fraction lengths", () => { + expect(compareTime("2026-11-04T10:00:01Z", "2026-11-04T10:00:01.1Z")).toBeLessThan(0) + expect(compareTime("2026-11-04T10:00:01.5Z", "2026-11-04T10:00:01.123Z")).toBeGreaterThan(0) + expect(compareTime("2026-11-04T10:00:01.10Z", "2026-11-04T10:00:01.1Z")).toBe(0) + }) +}) + +const e = (time: string, msg: string, stream = "s1") => ({ _time: time, _msg: msg, _stream_id: stream }) + +describe("mergeTail", () => { + it("appends and keeps only the newest lines over the cap", () => { + const out = mergeTail([e("t1", "a"), e("t2", "b")], [e("t3", "c")], 2) + + expect(out.map((x) => x._msg)).toEqual(["b", "c"]) + }) + + it("drops replayed lines at or before the resume point", () => { + const buffer = [e("2026-01-01T00:00:01Z", "a"), e("2026-01-01T00:00:02Z", "b")] + + const out = mergeTail(buffer, [e("2026-01-01T00:00:02Z", "b"), e("2026-01-01T00:00:03Z", "c")], 10, "2026-01-01T00:00:02Z") + + expect(out.map((x) => x._msg)).toEqual(["a", "b", "c"]) + }) + + it("keeps a line at the resume point from another stream", () => { + const buffer = [e("2026-01-01T00:00:02Z", "b", "s1")] + + const out = mergeTail(buffer, [e("2026-01-01T00:00:02Z", "other", "s2")], 10, "2026-01-01T00:00:02Z") + + expect(out).toHaveLength(2) + }) + + it("never deduplicates lines without _time, even on a resume", () => { + const out = mergeTail([{ _msg: "a" }], [{ _msg: "a" }], 10, "2026-01-01T00:00:02Z") + + expect(out).toHaveLength(2) + }) + + it("never deduplicates lines a pipe stripped of _stream_id, even on a resume", () => { + const t = "2026-01-01T00:00:02Z" + const buffer = [{ _time: t, _msg: "from stream 1" }] + + const out = mergeTail(buffer, [{ _time: t, _msg: "from stream 2" }], 10, t) + + expect(out.map((x) => x._msg)).toEqual(["from stream 1", "from stream 2"]) + }) + + it("deduplicates only the complete rows of a mixed resume", () => { + const t = "2026-01-01T00:00:02Z" + const buffer = [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }] + + const out = mergeTail(buffer, [e(t, "seen"), { _time: t, _msg: "no stream" }, { _stream_id: "s1", _msg: "no time" }], 10, t) + + expect(out.map((x) => x._msg)).toEqual(["seen", "no stream", "no time", "no stream", "no time"]) + }) + + it("does not deduplicate outside a resume", () => { + const out = mergeTail([e("t1", "a")], [e("t1", "a")], 10) + + expect(out).toHaveLength(2) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts new file mode 100644 index 000000000..b1005a1cd --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/logs.ts @@ -0,0 +1,79 @@ +import type { LogEntry } from "./proxy.ts" + +export const LOG_RANGES = { + "15m": 15 * 60, + "1h": 3600, + "6h": 6 * 3600, + "24h": 24 * 3600, +} as const + +export type LogRangeKey = keyof typeof LOG_RANGES + +export const POD_FIELD = "kubernetes_pod_name" +export const CONTAINER_FIELD = "kubernetes_container_name" + +function quote(value: string): string { + return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"` +} + +function splitAtFirstPipe(query: string): [string, string] { + let quoteChar = "" + for (let i = 0; i < query.length; i++) { + const c = query[i] + if (quoteChar) { + if (c === "\\") i++ + else if (c === quoteChar) quoteChar = "" + } else if (c === '"' || c === "'" || c === "`") { + quoteChar = c + } else if (c === "|") { + return [query.slice(0, i), query.slice(i + 1)] + } + } + return [query, ""] +} + +// The picker narrows the filter part only: AND binds tighter than OR in +// LogsQL, and the user's pipes must still run after the narrowed filter. +export function buildLogsQuery({ filter, pod, container }: { filter: string; pod?: string; container?: string }): string { + const trimmed = filter.trim() || "*" + const matchers = [ + pod ? `${POD_FIELD}=${quote(pod)}` : "", + container ? `${CONTAINER_FIELD}=${quote(container)}` : "", + ].filter(Boolean) + if (matchers.length === 0) return trimmed + const [head, pipes] = splitAtFirstPipe(trimmed) + const scoped = `{${matchers.join(",")}} (${head.trim() || "*"})` + return pipes.trim() ? `${scoped} | ${pipes.trim()}` : scoped +} + +// VictoriaLogs trims trailing zeros from the fraction, so the strings do not sort lexically. +function padFraction(time: string): string { + const m = /^([^.Z]+)(?:\.(\d+))?Z$/.exec(time) + return m ? `${m[1]}.${(m[2] ?? "").padEnd(9, "0")}Z` : time +} + +export function compareTime(a: string, b: string): number { + const x = padFraction(a) + const y = padFraction(b) + return x < y ? -1 : x > y ? 1 : 0 +} + +// Null when a pipe removed either identity field: such a row cannot be told from another stream's. +const entryKey = (e: LogEntry) => + e._stream_id !== undefined && e._time !== undefined ? `${e._stream_id}\u0000${e._time}` : null + +// (_stream_id, _time) is not unique in VictoriaLogs, so dedup applies only to +// lines a resumed tail replays at or before the resume point, and only to +// lines that still carry both fields. +export function mergeTail(buffer: LogEntry[], incoming: LogEntry[], cap: number, resumedFrom?: string): LogEntry[] { + let fresh = incoming + if (resumedFrom) { + const seen = new Set(buffer.map(entryKey).filter((k) => k !== null)) + fresh = incoming.filter((e) => { + const key = entryKey(e) + return key === null || compareTime(e._time ?? "", resumedFrom) > 0 || !seen.has(key) + }) + } + const merged = buffer.concat(fresh) + return merged.length > cap ? merged.slice(merged.length - cap) : merged +} diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts new file mode 100644 index 000000000..0598c0c7f --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.test.ts @@ -0,0 +1,116 @@ +import { describe, it, expect } from "vitest" +import { + METRIC_PANELS, + formatBytes, + formatCores, + rangeWindow, + toChartData, +} from "./metrics.ts" + +describe("rangeWindow", () => { + it("ends now and spans the range in about 240 points", () => { + const w = rangeWindow("6h", 1_000_000) + + expect(w.end).toBe(1_000_000) + expect(w.start).toBe(1_000_000 - 6 * 3600) + expect(w.step).toBe(90) + }) + + it("never steps finer than 15s", () => { + expect(rangeWindow("1h", 0).step).toBe(15) + }) +}) + +describe("METRIC_PANELS", () => { + const window = { start: 0, end: 3600, step: 15 } + + it("rates CPU over at least five minutes so sparse scrapes still draw", () => { + const [query] = METRIC_PANELS.find((p) => p.id === "cpu")!.queries(window) + + expect(query.expr).toBe( + 'sum by (pod) (rate(container_cpu_usage_seconds_total{container!="",container!="POD"}[300s]))', + ) + }) + + // Long ranges are served by the longterm store, deduplicated at 5m, where a + // narrower window can hold no samples and the rate comes back empty. + it.each([ + [15, "300s"], + [75, "300s"], + [90, "360s"], + [2520, "10080s"], + ])("rates over max(5m, 4 x step) for step %is", (step, window) => { + for (const panel of METRIC_PANELS) { + for (const q of panel.queries({ start: 0, end: 1, step })) { + if (q.expr.includes("rate(")) expect(q.expr).toContain(`[${window}]`) + } + } + }) + + it("reads memory as working set per pod", () => { + const [query] = METRIC_PANELS.find((p) => p.id === "memory")!.queries(window) + + expect(query.expr).toBe('sum by (pod) (container_memory_working_set_bytes{container!="",container!="POD"})') + }) + + it("draws network receive and transmit as two named series", () => { + const queries = METRIC_PANELS.find((p) => p.id === "network")!.queries(window) + + expect(queries.map((q) => q.label)).toEqual(["receive", "transmit"]) + expect(queries[0].expr).toBe("sum(rate(container_network_receive_bytes_total[300s]))") + expect(queries[1].expr).toBe("sum(rate(container_network_transmit_bytes_total[300s]))") + }) + + it("names no application kind in any query", () => { + for (const panel of METRIC_PANELS) { + for (const q of panel.queries(window)) expect(q.expr).not.toMatch(/apps_cozystack_io|namespace=/) + } + }) +}) + +describe("toChartData", () => { + it("aligns series on the union of timestamps and leaves gaps as null", () => { + const data = toChartData([ + { label: "a", series: [{ metric: {}, values: [[10, "1"], [20, "2"]] }] }, + { label: "b", series: [{ metric: {}, values: [[20, "5"], [30, "6"]] }] }, + ]) + + expect(data.timestamps).toEqual([10, 20, 30]) + expect(data.series).toEqual([ + { label: "a", values: [1, 2, null] }, + { label: "b", values: [null, 5, 6] }, + ]) + }) + + it("labels each series by its pod, falling back to the query label", () => { + const data = toChartData([ + { + label: "cpu", + series: [ + { metric: { pod: "db-1" }, values: [[1, "0.5"]] }, + { metric: {}, values: [[1, "0.1"]] }, + ], + }, + ]) + + expect(data.series.map((s) => s.label)).toEqual(["db-1", "cpu"]) + }) + + it("drops NaN samples instead of drawing them as zero", () => { + const data = toChartData([{ label: "x", series: [{ metric: {}, values: [[1, "NaN"]] }] }]) + + expect(data.series[0].values).toEqual([null]) + }) +}) + +describe("formatters", () => { + it("formats CPU in millicores below one core", () => { + expect(formatCores(0.25)).toBe("250m") + expect(formatCores(2)).toBe("2") + }) + + it("formats bytes with binary units", () => { + expect(formatBytes(512)).toBe("512 B") + expect(formatBytes(3 * 1024 * 1024)).toBe("3.0 MiB") + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts new file mode 100644 index 000000000..2f77e251f --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/metrics.ts @@ -0,0 +1,125 @@ +import type { PromSeries } from "./proxy.ts" + +export const RANGES = { + "1h": 3600, + "6h": 6 * 3600, + "24h": 24 * 3600, + "7d": 7 * 24 * 3600, +} as const + +export type RangeKey = keyof typeof RANGES + +export interface RangeWindow { + start: number + end: number + step: number +} + +const POINTS = 240 +const MIN_STEP = 15 + +export function rangeWindow(range: RangeKey, nowSeconds: number): RangeWindow { + const span = RANGES[range] + return { start: nowSeconds - span, end: nowSeconds, step: Math.max(MIN_STEP, Math.round(span / POINTS)) } +} + +// The longterm store keeps one sample per 5m, so a narrower window can be empty; +// 4 x step also keeps samples between points from being skipped. +function rateWindow(step: number): string { + return `${Math.max(300, step * 4)}s` +} + +const CONTAINERS = 'container!="",container!="POD"' + +export interface MetricQuery { + label: string + expr: string +} + +export interface MetricPanel { + id: "cpu" | "memory" | "network" + title: string + format: (value: number) => string + queries: (window: RangeWindow) => MetricQuery[] +} + +export function formatCores(value: number): string { + if (value === 0) return "0" + if (Math.abs(value) < 1) return `${Math.round(value * 1000)}m` + return `${Number(value.toFixed(2))}` +} + +const BYTE_UNITS = ["B", "KiB", "MiB", "GiB", "TiB"] + +export function formatBytes(value: number): string { + let v = value + let unit = 0 + while (Math.abs(v) >= 1024 && unit < BYTE_UNITS.length - 1) { + v /= 1024 + unit++ + } + return unit === 0 ? `${Math.round(v)} B` : `${v.toFixed(1)} ${BYTE_UNITS[unit]}` +} + +// The proxy injects the namespace or application scope, so these stay kind-agnostic. +export const METRIC_PANELS: MetricPanel[] = [ + { + id: "cpu", + title: "CPU (cores)", + format: formatCores, + queries: ({ step }) => [ + { + label: "cpu", + expr: `sum by (pod) (rate(container_cpu_usage_seconds_total{${CONTAINERS}}[${rateWindow(step)}]))`, + }, + ], + }, + { + id: "memory", + title: "Memory (working set)", + format: formatBytes, + queries: () => [ + { label: "memory", expr: `sum by (pod) (container_memory_working_set_bytes{${CONTAINERS}})` }, + ], + }, + { + id: "network", + title: "Network", + format: (v) => `${formatBytes(v)}/s`, + queries: ({ step }) => [ + { label: "receive", expr: `sum(rate(container_network_receive_bytes_total[${rateWindow(step)}]))` }, + { label: "transmit", expr: `sum(rate(container_network_transmit_bytes_total[${rateWindow(step)}]))` }, + ], + }, +] + +export function formatSampleTime(seconds: number): string { + return new Date(seconds * 1000).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit", second: "2-digit" }) +} + +export interface ChartSeries { + label: string + values: (number | null)[] +} + +export interface ChartData { + timestamps: number[] + series: ChartSeries[] +} + +export function toChartData(results: { label: string; series: PromSeries[] }[]): ChartData { + const flat = results.flatMap(({ label, series }) => + series.map((s) => ({ label: s.metric.pod ?? label, points: new Map(s.values.map(([t, v]) => [t, Number(v)])) })), + ) + const timestamps = [...new Set(flat.flatMap((s) => [...s.points.keys()]))].sort((a, b) => a - b) + return { + timestamps, + series: flat.map(({ label, points }) => ({ + label, + values: timestamps.map((t) => { + const v = points.get(t) + return v === undefined || Number.isNaN(v) ? null : v + }), + })), + } +} diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts new file mode 100644 index 000000000..f2ba8d7c3 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.test.ts @@ -0,0 +1,242 @@ +import { describe, it, expect, vi } from "vitest" +import { K8sClient } from "@cozystack/k8s-client" +import { + ProxyError, + errorKindForStatus, + fieldValues, + isScopeWide, + logsRoot, + metricsRoot, + queryLogs, + queryRange, + tailLogs, +} from "./proxy.ts" + +const app = { namespace: "tenant-a", plural: "postgreses", name: "db" } +const ns = { namespace: "tenant-a" } + +function clientReturning(...responses: Response[]) { + const client = new K8sClient({ onUnauthorized: () => {} }) + const raw = vi.spyOn(client, "raw") + for (const r of responses) raw.mockResolvedValueOnce(r) + return { client, raw } +} + +function calledUrl(raw: ReturnType["raw"], call = 0): URL { + return new URL(raw.mock.calls[call][0], "http://x") +} + +describe("roots", () => { + it("builds the application roots under apps.cozystack.io", () => { + expect(metricsRoot(app)).toBe("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/metrics/") + expect(logsRoot(app)).toBe("/apis/apps.cozystack.io/v1alpha1/namespaces/tenant-a/postgreses/db/log/") + }) + + it("builds the namespace roots under cozystack.io", () => { + expect(metricsRoot(ns)).toBe("/apis/cozystack.io/v1alpha1/namespaces/tenant-a/metrics/") + expect(logsRoot(ns)).toBe("/apis/cozystack.io/v1alpha1/namespaces/tenant-a/logs/") + }) +}) + +describe("errorKindForStatus", () => { + it.each([ + [400, "bad-request"], + [401, "unauthorized"], + [403, "forbidden"], + [404, "not-available"], + [422, "limit"], + [429, "rate-limited"], + [502, "store-unreachable"], + [503, "unavailable"], + [504, "timeout"], + [500, "unknown"], + ])("maps %i to %s", (status, kind) => { + expect(errorKindForStatus(status)).toBe(kind) + }) +}) + +describe("queryRange", () => { + it("GETs query_range under the root and returns the matrix", async () => { + const body = { status: "success", data: { resultType: "matrix", result: [{ metric: { pod: "p" }, values: [[1, "2"]] }] } } + const { client, raw } = clientReturning(Response.json(body)) + + const result = await queryRange(client, metricsRoot(app), { query: "up", start: 10, end: 20, step: 5 }) + + const url = calledUrl(raw) + expect(url.pathname).toBe(`${metricsRoot(app)}api/v1/query_range`) + expect(Object.fromEntries(url.searchParams)).toEqual({ query: "up", start: "10", end: "20", step: "5" }) + expect(result).toEqual(body.data.result) + }) + + it("throws a ProxyError carrying the Prometheus error text", async () => { + const { client } = clientReturning( + Response.json({ status: "error", error: "range too long" }, { status: 422 }), + ) + + const err = await queryRange(client, metricsRoot(app), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e) + + expect(err).toBeInstanceOf(ProxyError) + expect(err).toMatchObject({ kind: "limit", status: 422, message: "range too long" }) + }) + + it("treats a Kubernetes Status 403 as a cluster without the proxy route", async () => { + const status = { kind: "Status", apiVersion: "v1", status: "Failure", reason: "Forbidden", code: 403, message: "logs.cozystack.io \"select\" is forbidden" } + const { client } = clientReturning(Response.json(status, { status: 403 })) + + const err = await queryRange(client, metricsRoot(ns), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e) + + expect(err).toMatchObject({ kind: "not-available", status: 403 }) + }) + + it("keeps the proxy's own plain-text 403 as forbidden", async () => { + const { client } = clientReturning(new Response("forbidden\n", { status: 403 })) + + const err = await queryRange(client, metricsRoot(ns), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e) + + expect(err).toMatchObject({ kind: "forbidden" }) + }) + + it("reports a fetch failure as a network error", async () => { + const client = new K8sClient() + vi.spyOn(client, "raw").mockRejectedValue(new TypeError("offline")) + + const err = await queryRange(client, metricsRoot(app), { query: "up", start: 0, end: 1, step: 1 }).catch((e) => e) + + expect(err).toMatchObject({ kind: "network" }) + }) +}) + +describe("queryLogs", () => { + it("parses NDJSON lines and sorts them by _time ascending", async () => { + const ndjson = [ + JSON.stringify({ _time: "2026-11-04T10:00:02Z", _msg: "b" }), + "", + JSON.stringify({ _time: "2026-11-04T10:00:01Z", _msg: "a" }), + ].join("\n") + const { client, raw } = clientReturning(new Response(ndjson)) + + const entries = await queryLogs(client, logsRoot(ns), { query: "*", start: "s", end: "e", limit: 100 }) + + expect(calledUrl(raw).pathname).toBe(`${logsRoot(ns)}select/logsql/query`) + expect(Object.fromEntries(calledUrl(raw).searchParams)).toEqual({ query: "*", start: "s", end: "e", limit: "100" }) + expect(entries.map((e) => e._msg)).toEqual(["a", "b"]) + }) + + it("keeps rows a pipe stripped of _time, in the order the store sent them", async () => { + const ndjson = ['{"_msg":"visible message"}', '{"_msg":"second"}', "[1]", "null", "{torn"].join("\n") + const { client } = clientReturning(new Response(ndjson)) + + const entries = await queryLogs(client, logsRoot(ns), { query: "* | fields _msg", start: "s", end: "e", limit: 100 }) + + expect(entries).toEqual([{ _msg: "visible message" }, { _msg: "second" }]) + }) + + it("keeps a plain-text error body as the message", async () => { + const { client } = clientReturning(new Response("cannot parse query", { status: 400 })) + + const err = await queryLogs(client, logsRoot(ns), { query: "(", start: "s", end: "e", limit: 1 }).catch((e) => e) + + expect(err).toMatchObject({ kind: "bad-request", message: "cannot parse query" }) + }) +}) + +describe("fieldValues", () => { + it("uses stream_field_values for stream fields and returns the values", async () => { + const { client, raw } = clientReturning( + Response.json({ values: [{ value: "pod-b", hits: 1 }, { value: "pod-a", hits: 9 }] }), + ) + + const values = await fieldValues(client, logsRoot(app), { + field: "kubernetes_pod_name", + stream: true, + query: "*", + start: "s", + end: "e", + }) + + expect(calledUrl(raw).pathname).toBe(`${logsRoot(app)}select/logsql/stream_field_values`) + expect(calledUrl(raw).searchParams.get("field")).toBe("kubernetes_pod_name") + expect(values).toEqual(["pod-a", "pod-b"]) + }) + + it("uses field_values for other fields", async () => { + const { client, raw } = clientReturning(Response.json({ values: [] })) + + await fieldValues(client, logsRoot(app), { field: "level", stream: false, query: "*", start: "s", end: "e" }) + + expect(calledUrl(raw).pathname).toBe(`${logsRoot(app)}select/logsql/field_values`) + }) +}) + +describe("tailLogs", () => { + it("delivers every line as it arrives, including one split across chunks", async () => { + const enc = new TextEncoder() + const chunks = ['{"_time":"t1","_msg":"a"}\n{"_ti', 'me":"t2","_msg":"b"}\n'] + const body = new ReadableStream({ + start(c) { + for (const ch of chunks) c.enqueue(enc.encode(ch)) + c.close() + }, + }) + const { client, raw } = clientReturning(new Response(body)) + const seen: (string | undefined)[] = [] + + await tailLogs(client, logsRoot(ns), { query: "*", since: "t0" }, (e) => seen.push(e._msg)) + + expect(calledUrl(raw).pathname).toBe(`${logsRoot(ns)}select/logsql/tail`) + expect(calledUrl(raw).searchParams.get("since")).toBe("t0") + expect(seen).toEqual(["a", "b"]) + }) + + it("delivers a line without _time", async () => { + const { client } = clientReturning(new Response('{"_msg":"untimed"}\n')) + const seen: (string | undefined)[] = [] + + await tailLogs(client, logsRoot(ns), { query: "* | fields _msg" }, (e) => seen.push(e._msg)) + + expect(seen).toEqual(["untimed"]) + }) + + it("omits since on the first connect", async () => { + const { client, raw } = clientReturning(new Response("")) + + await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}) + + expect(calledUrl(raw).searchParams.has("since")).toBe(false) + }) + + it("throws a ProxyError when the tail is refused", async () => { + const { client } = clientReturning(new Response("", { status: 403 })) + + const err = await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}).catch((e) => e) + + expect(err).toMatchObject({ kind: "forbidden" }) + }) + + it("says the stream opened before any line arrives", async () => { + const { client } = clientReturning(new Response("")) + const onOpen = vi.fn() + + await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}, undefined, onOpen) + + expect(onOpen).toHaveBeenCalledTimes(1) + }) + + it("does not say the stream opened when the tail is refused", async () => { + const { client } = clientReturning(new Response("", { status: 429 })) + const onOpen = vi.fn() + + await tailLogs(client, logsRoot(ns), { query: "*" }, () => {}, undefined, onOpen).catch(() => {}) + + expect(onOpen).not.toHaveBeenCalled() + }) +}) + +describe("isScopeWide", () => { + it("is true for answers that do not depend on the query", () => { + expect(isScopeWide(new ProxyError("not-available", 404, ""))).toBe(true) + expect(isScopeWide(new ProxyError("forbidden", 403, ""))).toBe(true) + expect(isScopeWide(new ProxyError("limit", 422, ""))).toBe(false) + expect(isScopeWide(new Error("x"))).toBe(false) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts new file mode 100644 index 000000000..c28bd12d4 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/proxy.ts @@ -0,0 +1,228 @@ +import type { K8sClient } from "@cozystack/k8s-client" +import { compareTime } from "./logs.ts" + +export type ObservabilityScope = + | { namespace: string } + | { namespace: string; plural: string; name: string } + +function appBase(scope: ObservabilityScope): string | null { + if (!("plural" in scope)) return null + return `/apis/apps.cozystack.io/v1alpha1/namespaces/${scope.namespace}/${scope.plural}/${scope.name}` +} + +export function metricsRoot(scope: ObservabilityScope): string { + const app = appBase(scope) + return app ? `${app}/metrics/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/metrics/` +} + +export function logsRoot(scope: ObservabilityScope): string { + const app = appBase(scope) + return app ? `${app}/log/` : `/apis/cozystack.io/v1alpha1/namespaces/${scope.namespace}/logs/` +} + +export type ProxyErrorKind = + | "bad-request" + | "unauthorized" + | "forbidden" + | "not-available" + | "limit" + | "rate-limited" + | "store-unreachable" + | "unavailable" + | "timeout" + | "network" + | "unknown" + +const KIND_BY_STATUS: Record = { + 400: "bad-request", + 401: "unauthorized", + 403: "forbidden", + 404: "not-available", + 422: "limit", + 429: "rate-limited", + 502: "store-unreachable", + 503: "unavailable", + 504: "timeout", +} + +export function errorKindForStatus(status: number): ProxyErrorKind { + return KIND_BY_STATUS[status] ?? "unknown" +} + +export class ProxyError extends Error { + kind: ProxyErrorKind + status: number + + constructor(kind: ProxyErrorKind, status: number, message: string) { + super(message) + this.name = "ProxyError" + this.kind = kind + this.status = status + } +} + +// These answers hold for every query in the scope, so one notice replaces the whole view. +export function isScopeWide(error: unknown): boolean { + return ( + error instanceof ProxyError && + (error.kind === "not-available" || error.kind === "forbidden" || error.kind === "unauthorized") + ) +} + +// The proxy answers in plain text; a Kubernetes Status body means nginx sent +// the path to kube-apiserver, which RBAC-denies or 404s a route it does not serve. +function isKubernetesStatus(text: string): boolean { + try { + const parsed: unknown = JSON.parse(text) + return !!parsed && typeof parsed === "object" && "kind" in parsed && parsed.kind === "Status" + } catch { + return false + } +} + +// Prometheus answers {"error": "..."}, VictoriaLogs and nginx answer plain text. +function errorMessage(text: string, status: number): string { + try { + const parsed: unknown = JSON.parse(text) + if (parsed && typeof parsed === "object" && "error" in parsed && typeof parsed.error === "string") { + return parsed.error + } + } catch { + // not JSON + } + const trimmed = text.trim() + return trimmed && trimmed.length < 300 && !trimmed.startsWith("<") ? trimmed : `Server returned ${status}` +} + +async function proxyGet( + client: K8sClient, + path: string, + params: Record, + signal?: AbortSignal, +): Promise { + const search = new URLSearchParams() + for (const [k, v] of Object.entries(params)) if (v !== undefined) search.set(k, String(v)) + let res: Response + try { + res = await client.raw(`${path}?${search}`, { signal }) + } catch (err) { + if (signal?.aborted) throw err + throw new ProxyError("network", 0, "Unable to connect to the server") + } + if (!res.ok) { + const text = await res.text().catch(() => "") + const kind = isKubernetesStatus(text) ? "not-available" : errorKindForStatus(res.status) + throw new ProxyError(kind, res.status, errorMessage(text, res.status)) + } + return res +} + +export interface PromSeries { + metric: Record + values: [number, string][] +} + +export interface RangeParams { + query: string + start: number + end: number + step: number +} + +export async function queryRange( + client: K8sClient, + root: string, + params: RangeParams, + signal?: AbortSignal, +): Promise { + const res = await proxyGet(client, `${root}api/v1/query_range`, { ...params }, signal) + const body = (await res.json()) as { data?: { result?: PromSeries[] } } + return body.data?.result ?? [] +} + +// A pipe such as `fields _msg` or `stats` can drop _time and _msg. +export interface LogEntry { + _time?: string + _msg?: string + _stream_id?: string + [field: string]: string | undefined +} + +function parseLine(line: string): LogEntry | null { + if (!line.trim()) return null + try { + const value: unknown = JSON.parse(line) + if (value && typeof value === "object" && !Array.isArray(value)) return value as LogEntry + } catch { + // a torn or non-JSON line carries nothing to show + } + return null +} + +export interface LogsQueryParams { + query: string + start: string + end: string + limit: number +} + +export async function queryLogs( + client: K8sClient, + root: string, + params: LogsQueryParams, + signal?: AbortSignal, +): Promise { + const res = await proxyGet(client, `${root}select/logsql/query`, { ...params }, signal) + const entries = (await res.text()).split("\n").map(parseLine).filter((e) => e !== null) + if (entries.some((e) => e._time === undefined)) return entries + return entries.sort((a, b) => compareTime(a._time ?? "", b._time ?? "")) +} + +export interface FieldValuesParams { + field: string + stream: boolean + query: string + start: string + end: string +} + +export async function fieldValues( + client: K8sClient, + root: string, + { stream, ...params }: FieldValuesParams, + signal?: AbortSignal, +): Promise { + const endpoint = stream ? "stream_field_values" : "field_values" + const res = await proxyGet(client, `${root}select/logsql/${endpoint}`, { ...params }, signal) + const body = (await res.json()) as { values?: { value: string }[] } + return (body.values ?? []).map((v) => v.value).sort() +} + +export async function tailLogs( + client: K8sClient, + root: string, + params: { query: string; since?: string }, + onEntry: (entry: LogEntry) => void, + signal?: AbortSignal, + onOpen?: () => void, +): Promise { + const res = await proxyGet(client, `${root}select/logsql/tail`, params, signal) + onOpen?.() + if (!res.body) return + const reader = res.body.getReader() + const decoder = new TextDecoder() + let buffer = "" + for (;;) { + const { done, value } = await reader.read() + if (done) break + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split("\n") + buffer = lines.pop() ?? "" + for (const line of lines) { + const entry = parseLine(line) + if (entry) onEntry(entry) + } + } + const last = parseLine(buffer) + if (last) onEntry(last) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx new file mode 100644 index 000000000..b5232d18e --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.test.tsx @@ -0,0 +1,133 @@ +import { describe, it, expect, vi } from "vitest" +import { renderHook, waitFor } from "@testing-library/react" +import type { ReactNode } from "react" +import { K8sClient, K8sProvider } from "@cozystack/k8s-client" +import { useLogTail } from "./use-log-tail.ts" + +const line = (time: string, msg: string) => JSON.stringify({ _time: time, _msg: msg, _stream_id: "s" }) + +function ndjson(...lines: string[]): Response { + return new Response(lines.map((l) => `${l}\n`).join("")) +} + +function openStream(): Response { + return new Response(new ReadableStream({ start() {} })) +} + +function setup(...responses: Response[]) { + const client = new K8sClient() + const raw = vi.spyOn(client, "raw") + for (const r of responses) raw.mockResolvedValueOnce(r) + raw.mockImplementation(() => new Promise(() => {})) + const wrapper = ({ children }: { children: ReactNode }) => {children} + return { raw, wrapper } +} + +const T1 = "2026-11-04T10:00:01Z" +const T2 = "2026-11-04T10:00:02Z" +const T3 = "2026-11-04T10:00:03Z" + +describe("useLogTail", () => { + it("streams lines and resumes after a disconnect from the last _time without duplicates", async () => { + const { raw, wrapper } = setup(ndjson(line(T1, "a"), line(T2, "b")), ndjson(line(T2, "b"), line(T3, "c"))) + + const { result } = renderHook(() => useLogTail("/root/", "*", 1, { cap: 100, retryDelayMs: 1 }), { wrapper }) + + await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a", "b", "c"])) + expect(new URL(raw.mock.calls[0][0], "http://x").searchParams.has("since")).toBe(false) + expect(new URL(raw.mock.calls[1][0], "http://x").searchParams.get("since")).toBe(T2) + }) + + it("stops and reports an error the proxy will repeat on every retry", async () => { + const { raw, wrapper } = setup(new Response("", { status: 403 })) + + const { result } = renderHook(() => useLogTail("/root/", "*", 1, { cap: 100, retryDelayMs: 1 }), { wrapper }) + + await waitFor(() => expect(result.current.state).toBe("stopped")) + expect(result.current.error).toMatchObject({ kind: "forbidden" }) + expect(raw).toHaveBeenCalledTimes(1) + }) + + it("starts a new follow session with an empty buffer", async () => { + const { wrapper } = setup(ndjson(line(T1, "first")), ndjson(line(T3, "second"))) + const { result, rerender } = renderHook( + ({ session }: { session: number | null }) => useLogTail("/root/", "*", session, { retryDelayMs: 60_000 }), + { wrapper, initialProps: { session: 1 as number | null } }, + ) + await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["first"])) + + rerender({ session: null }) + rerender({ session: 2 }) + + await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["second"])) + }) + + it("resumes from the newest line seen, not the last one received", async () => { + const { raw, wrapper } = setup(ndjson(line(T3, "c"), line(T1, "a")), ndjson()) + + renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper }) + + await waitFor(() => expect(raw.mock.calls.length).toBeGreaterThan(1)) + expect(new URL(raw.mock.calls[1][0], "http://x").searchParams.get("since")).toBe(T3) + }) + + it("retries a tail refused for load instead of stopping", async () => { + const { wrapper } = setup(new Response("too many concurrent requests", { status: 429 }), ndjson(line(T1, "a"))) + + const { result } = renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper }) + + await waitFor(() => expect(result.current.entries.map((e) => e._msg)).toEqual(["a"])) + }) + + it("goes live and clears a load warning once the stream opens, before any line", async () => { + const { wrapper } = setup(new Response("too many concurrent requests", { status: 429 }), openStream()) + + const { result } = renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 1 }), { wrapper }) + + await waitFor(() => expect(result.current.state).toBe("live")) + expect(result.current.error).toBeNull() + }) + + it("reconnects after an idle close at the base delay, from when the quiet stream opened", async () => { + const delays: number[] = [] + const realSetTimeout = globalThis.setTimeout + const spy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((fn: () => void, ms?: number) => { + if (ms !== undefined && ms % 7 === 0 && ms > 0) delays.push(ms) + return realSetTimeout(fn, ms) + }) as typeof setTimeout) + const refused = () => new Response("", { status: 429 }) + const { raw, wrapper } = setup(refused(), refused(), ndjson()) + + try { + renderHook(() => useLogTail("/root/", "*", 1, { retryDelayMs: 7 }), { wrapper }) + await waitFor(() => expect(raw).toHaveBeenCalledTimes(4)) + } finally { + spy.mockRestore() + } + + expect(delays).toEqual([7, 14, 7]) + expect(new URL(raw.mock.calls[2][0], "http://x").searchParams.has("since")).toBe(false) + const since = new URL(raw.mock.calls[3][0], "http://x").searchParams.get("since") + expect(since).not.toBeNull() + expect(new Date(since!).toISOString()).toBe(since) + }) + + it("does not connect while disabled", () => { + const { raw, wrapper } = setup() + + renderHook(() => useLogTail("/root/", "*", null), { wrapper }) + + expect(raw).not.toHaveBeenCalled() + }) + + it("aborts the stream on unmount", async () => { + const { raw, wrapper } = setup() + const { unmount } = renderHook(() => useLogTail("/root/", "*", 1), { wrapper }) + await waitFor(() => expect(raw).toHaveBeenCalled()) + + unmount() + + const init = raw.mock.calls[0][1] as RequestInit + expect(init.signal?.aborted).toBe(true) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts new file mode 100644 index 000000000..d55bd5708 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/lib/observability/use-log-tail.ts @@ -0,0 +1,111 @@ +import { useEffect, useState } from "react" +import { useK8sClient } from "@cozystack/k8s-client" +import { compareTime, mergeTail } from "./logs.ts" +import { ProxyError, tailLogs, type LogEntry } from "./proxy.ts" + +export type TailState = "connecting" | "live" | "reconnecting" | "stopped" + +interface TailSnapshot { + key: string + entries: LogEntry[] + state: TailState + error: unknown +} + +const FLUSH_MS = 100 +const MAX_RETRY_MS = 30_000 +// Load refusals retry: the proxy caps rate and concurrent tails with 429 (422 in +// its first builds), and a reconnect can race the previous tail's freed slot. +const FATAL: ReadonlySet = new Set(["bad-request", "unauthorized", "forbidden", "not-available"]) + +const sleep = (ms: number, signal: AbortSignal) => + new Promise((resolve) => { + const onAbort = () => { + clearTimeout(t) + resolve() + } + const t = setTimeout(() => { + signal.removeEventListener("abort", onAbort) + resolve() + }, ms) + signal.addEventListener("abort", onAbort, { once: true }) + }) + +// Each non-null session is one follow run with its own buffer; null stops following. +export function useLogTail( + root: string, + query: string, + session: number | null, + { cap = 1000, retryDelayMs = 2000 }: { cap?: number; retryDelayMs?: number } = {}, +): Omit { + const client = useK8sClient() + const key = `${root}\n${query}\n${session}` + const [snap, setSnap] = useState({ key, entries: [], state: "connecting", error: null }) + + useEffect(() => { + if (session === null) return + const controller = new AbortController() + const { signal } = controller + let pending: LogEntry[] = [] + let timer: ReturnType | undefined + let resumedFrom: string | undefined + let lastTime: string | undefined + + const update = (patch: Partial, batch: LogEntry[] = [], from = resumedFrom) => + setSnap((prev) => { + const base = prev.key === key ? prev : { key, entries: [], state: "connecting" as const, error: null } + return { ...base, ...patch, entries: batch.length ? mergeTail(base.entries, batch, cap, from) : base.entries } + }) + const flush = () => { + clearTimeout(timer) + timer = undefined + if (pending.length === 0 || signal.aborted) return + const batch = pending + pending = [] + update({ state: "live", error: null }, batch) + } + + const run = async () => { + let failures = 0 + let openedAt: string | undefined + while (!signal.aborted) { + resumedFrom = lastTime + let delay = retryDelayMs + try { + await tailLogs(client, root, { query, since: lastTime ?? openedAt }, (entry) => { + if (entry._time && (!lastTime || compareTime(entry._time, lastTime) > 0)) lastTime = entry._time + pending.push(entry) + timer ??= setTimeout(flush, FLUSH_MS) + }, signal, () => { + failures = 0 + // With no line yet, the next reconnect resumes from here so the gap is not lost. + openedAt = new Date().toISOString() + update({ state: "live", error: null }) + }) + flush() + } catch (err) { + flush() + if (signal.aborted) return + if (err instanceof ProxyError && FATAL.has(err.kind)) { + update({ state: "stopped", error: err }) + return + } + update({ error: err }) + delay = Math.min(retryDelayMs * 2 ** failures, MAX_RETRY_MS) + failures++ + } + if (signal.aborted) return + update({ state: "reconnecting" }) + await sleep(delay, signal) + } + } + void run() + return () => { + controller.abort() + clearTimeout(timer) + } + }, [client, root, query, session, key, cap, retryDelayMs]) + + const current = snap.key === key ? snap : { key, entries: [], state: "connecting" as const, error: null } + return { entries: current.entries, state: current.state, error: current.error } +} diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx index 7c6d1209c..c3b014f4d 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.routing.test.tsx @@ -70,4 +70,54 @@ describe("ConsolePage routing", () => { await waitFor(() => expect(client.list).toHaveBeenCalled()) expect(screen.queryByText(/Cluster-scoped capacity/i)).toBeNull() }) + + it("serves the tenant logs page under the reserved -/ segment", async () => { + renderWithK8sProvider( + + + , + { client: makeClient(), initialRoute: "/-/observability/logs" }, + ) + + expect(await screen.findByText(/^Every application in/)).toBeInTheDocument() + }) + + it("leaves /logs to an application kind whose plural is logs", async () => { + const client = makeClient() + renderWithK8sProvider( + + + , + { client, initialRoute: "/logs" }, + ) + + await waitFor(() => expect(client.list).toHaveBeenCalled()) + expect(screen.queryByText(/^Every application in/)).toBeNull() + }) + + // No ApplicationDefinition plural or application name can be "-", so only + // routes under it are safe from shadowing an application's own pages. + it("leaves /observability/logs to an application named logs of kind observability", async () => { + const client = makeClient() + vi.mocked(client.list).mockImplementation(async (_g, _v, plural) => ({ + apiVersion: "v1", + kind: `${plural}List`, + metadata: {}, + items: + plural === "tenantnamespaces" + ? [{ apiVersion: "core.cozystack.io/v1alpha1", kind: "TenantNamespace", metadata: { name: "tenant-root" } }] + : [], + }) as K8sList) + const get = vi.spyOn(client, "get").mockRejectedValue(new Error("not found")) + renderWithK8sProvider( + + + , + { client, initialRoute: "/observability/logs" }, + ) + + expect(await screen.findByText(/^Application/)).toHaveTextContent("Application logs not found.") + expect(get).toHaveBeenCalledWith(expect.anything(), expect.anything(), "observability", "logs", expect.anything()) + expect(screen.queryByText(/^Every application in/)).toBeNull() + }) }) diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx index 37caae92f..c78ae619e 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/ConsolePage.tsx @@ -15,6 +15,7 @@ import { MigrationSourceEditPage } from "./MigrationSourceEditPage.tsx" import { MigrationTaskCreatePage } from "./MigrationTaskCreatePage.tsx" import { MigrationTaskDetailPage } from "./MigrationTaskDetailPage.tsx" import { ApplicationOrderPage } from "./ApplicationOrderPage.tsx" +import { LogsPage } from "./LogsPage.tsx" export function ConsolePage() { return ( @@ -96,6 +97,7 @@ export function ConsolePage() { path="migration/vmimporttasks/:name" element={} /> + } /> } /> } /> } /> diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx new file mode 100644 index 000000000..1f4263cf1 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.test.tsx @@ -0,0 +1,20 @@ +import { describe, it, expect, vi } from "vitest" +import { render, screen } from "@testing-library/react" + +vi.mock("../lib/tenant-context.tsx", () => ({ + useTenantContext: () => ({ tenantNamespace: "tenant-a" }), +})) +vi.mock("../components/observability/LogsView.tsx", () => ({ + LogsView: ({ scope }: { scope: object }) =>
{JSON.stringify(scope)}
, +})) + +const { LogsPage } = await import("./LogsPage.tsx") + +describe("LogsPage", () => { + it("reads logs from the selected tenant's namespace root", () => { + render() + + expect(screen.getByRole("heading", { name: "Logs" })).toBeInTheDocument() + expect(JSON.parse(screen.getByTestId("scope").textContent ?? "")).toEqual({ namespace: "tenant-a" }) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx new file mode 100644 index 000000000..7612dfbb2 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/routes/LogsPage.tsx @@ -0,0 +1,15 @@ +import { useTenantContext } from "../lib/tenant-context.tsx" +import { LogsView } from "../components/observability/LogsView.tsx" + +export function LogsPage() { + const { tenantNamespace } = useTenantContext() + return ( +
+
+

Logs

+

Every application in {tenantNamespace ?? "this tenant"}

+
+ {tenantNamespace && } +
+ ) +} diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.observability.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.observability.test.tsx new file mode 100644 index 000000000..e16bf1665 --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.observability.test.tsx @@ -0,0 +1,113 @@ +import { describe, it, expect, vi, beforeEach } from "vitest" +import { act, render, screen, waitFor } from "@testing-library/react" +import { createMemoryRouter, RouterProvider } from "react-router" +import { K8sProvider } from "@cozystack/k8s-client" +import { TenantContext, type TenantContextValue } from "../../lib/tenant-context.tsx" +import type { ChartData } from "../../lib/observability/metrics.ts" + +// Keep the consoles' and the chart's browser-only libraries out of jsdom; the +// Logs and Metrics tabs, the page and the providers stay real. +vi.mock("./VncTab.tsx", () => ({ VncTab: () => null })) +vi.mock("./SerialTab.tsx", () => ({ SerialTab: () => null })) +vi.mock("../../components/observability/TimeSeriesChart.tsx", () => ({ + TimeSeriesChart: ({ data }: { data: ChartData }) => ( +
{data.series.map((s) => s.label).join(",")}
+ ), +})) + +const { ApplicationDetailPage } = await import("./ApplicationDetailPage.tsx") + +const NS = "tenant-test" +const APPS = `/apis/apps.cozystack.io/v1alpha1/namespaces/${NS}/postgreses` + +const tenant: TenantContextValue = { + tenants: [], + selectedTenant: "test", + selectTenant: () => {}, + tenantNamespace: NS, + isLoading: false, + error: null, +} + +const definitions = { + items: [{ metadata: { name: "postgres" }, spec: { application: { plural: "postgreses", kind: "Postgres" } } }], +} + +const instance = (name: string) => ({ + apiVersion: "apps.cozystack.io/v1alpha1", + kind: "Postgres", + metadata: { name, namespace: NS }, +}) + +const logLine = (name: string) => + `${JSON.stringify({ _time: "2026-11-04T10:00:01Z", _msg: `${name} is ready`, _stream_id: "s", kubernetes_pod_name: `${name}-0`, kubernetes_container_name: "postgres" })}\n` + +const matrix = (name: string) => + Response.json({ status: "success", data: { resultType: "matrix", result: [{ metric: { pod: `${name}-0` }, values: [[1, "1"]] }] } }) + +// app-b's own GET never answers, so the page keeps the placeholder of app-a +// that the production query defaults hand it; its proxy reads do answer. +function handle(url: URL): Response | Promise { + const path = url.pathname + if (path.endsWith("/applicationdefinitions")) return Response.json(definitions) + if (path === `${APPS}/app-a`) return Response.json(instance("app-a")) + if (path === `${APPS}/app-b`) return new Promise(() => {}) + const app = path.match(/\/postgreses\/(app-[ab])\//)?.[1] + if (app && path.endsWith("/select/logsql/query")) return new Response(logLine(app)) + if (app && path.endsWith("/stream_field_values")) return Response.json({ values: [] }) + if (app && path.includes("/metrics/api/v1/query_range")) return matrix(app) + return Response.json({ items: [] }) +} + +const fetchMock = vi.fn(async (input: RequestInfo | URL) => handle(new URL(String(input), "http://x"))) + +function renderAt(path: string) { + const router = createMemoryRouter( + [{ path: "/console/:plural/:name/*", element: }], + { initialEntries: [path] }, + ) + // No queryClient: the provider's production defaults keep a query's previous + // result while its key changes, which is what exposes the stale instance. + render( + + + + + , + ) + return { go: (to: string) => act(() => router.navigate(to)) } +} + +const requested = (prefix: string) => + fetchMock.mock.calls.some(([input]) => new URL(String(input), "http://x").pathname.startsWith(prefix)) + +beforeEach(() => { + vi.stubGlobal("fetch", fetchMock) + fetchMock.mockClear() +}) + +describe("ApplicationDetailPage observability while the next application loads", () => { + it("shows the selected application's logs, not the previous one's", async () => { + const { go } = renderAt("/console/postgreses/app-a/logs") + expect(await screen.findByText("app-a is ready")).toBeInTheDocument() + + await go("/console/postgreses/app-b/logs") + + expect(screen.getByRole("heading", { name: "app-b" })).toBeInTheDocument() + expect(screen.queryByText("app-a is ready")).toBeNull() + await waitFor(() => expect(requested(`${APPS}/app-b/log/`)).toBe(true)) + expect(await screen.findByText("app-b is ready")).toBeInTheDocument() + }) + + it("shows the selected application's metrics, not the previous one's", async () => { + const { go } = renderAt("/console/postgreses/app-a/metrics") + await waitFor(() => expect(screen.getAllByTestId("chart")[0]).toHaveTextContent("app-a-0")) + + await go("/console/postgreses/app-b/metrics") + + expect(screen.getByRole("heading", { name: "app-b" })).toBeInTheDocument() + expect(screen.queryAllByText("app-a-0", { exact: false })).toEqual([]) + await waitFor(() => expect(requested(`${APPS}/app-b/metrics/`)).toBe(true)) + await waitFor(() => expect(screen.getAllByTestId("chart")[0]).toHaveTextContent("app-b-0")) + }) +}) diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx index a015a98bf..a0c3a6b84 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.test.tsx @@ -59,6 +59,8 @@ vi.mock("./EventsTab.tsx", () => ({ EventsTab: () => null })) vi.mock("./VncTab.tsx", () => ({ VncTab: () => null })) vi.mock("./SerialTab.tsx", () => ({ SerialTab: () => null })) vi.mock("./VMPowerControls.tsx", () => ({ VMPowerControls: () => null })) +vi.mock("./MetricsTab.tsx", () => ({ MetricsTab: () => null })) +vi.mock("./LogsTab.tsx", () => ({ LogsTab: () => null })) const { ApplicationDetailPage } = await import("./ApplicationDetailPage.tsx") @@ -111,11 +113,40 @@ describe("ApplicationDetailPage tabs for a virtual machine", () => { "VNC", "Workloads", "Services", + "Metrics", + "Logs", "Events", ]) }) }) +describe("ApplicationDetailPage observability tabs", () => { + function showKind(kind: string, plural: string) { + h.appKind = kind + h.appPlural = plural + h.get = { + data: { kind, metadata: { name: "demo", namespace: "tenant-test" } }, + isLoading: false, + error: undefined, + } + h.configMaps = [] + h.configMapsError = undefined + h.tabLabels = [] + renderPage() + } + + it("offers Metrics and Logs on any application kind, ahead of Events", () => { + showKind("Postgres", "postgreses") + expect(h.tabLabels.slice(-4)).toEqual(["Secrets", "Metrics", "Logs", "Events"]) + }) + + it("leaves Metrics and Logs off a VM disk, which runs no containers", () => { + showKind("VMDisk", "vmdisks") + expect(h.tabLabels).not.toContain("Metrics") + expect(h.tabLabels).not.toContain("Logs") + }) +}) + describe("ApplicationDetailPage configuration", () => { it("offers ConfigMaps declared by the application's resource map", () => { h.appKind = "FoundationDB" diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx index 55fa684b8..ab68f7c49 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/ApplicationDetailPage.tsx @@ -37,11 +37,14 @@ import { VncTab } from "./VncTab.tsx" // fetched only when a VM's console is opened. Every dashboard page reaches this // file, and most of them never touch a VM. const SerialTab = lazy(() => import("./SerialTab.tsx").then((m) => ({ default: m.SerialTab }))) +// Same for the charting library behind the Metrics tab. +const MetricsTab = lazy(() => import("./MetricsTab.tsx").then((m) => ({ default: m.MetricsTab }))) import { VMPowerControls } from "./VMPowerControls.tsx" import { useResourceBasePath } from "../../lib/portal.ts" import { useResourcePresence } from "./use-resource-presence.ts" import { useApplicationConfigMaps } from "./use-app-configmaps.ts" import { ConfigMapsTab } from "./ConfigMapsTab.tsx" +import { LogsTab } from "./LogsTab.tsx" export function ApplicationDetailPage() { const { plural, name } = useParams<{ plural: string; name: string }>() @@ -143,6 +146,8 @@ export function ApplicationDetailPage() { { to: `${base}/vnc`, label: "VNC", end: false }, { to: `${base}/workloads`, label: "Workloads", end: false }, { to: `${base}/services`, label: "Services", end: false }, + { to: `${base}/metrics`, label: "Metrics", end: false }, + { to: `${base}/logs`, label: "Logs", end: false }, { to: `${base}/events`, label: "Events", end: false }, ) } else { @@ -163,6 +168,8 @@ export function ApplicationDetailPage() { } tabs.push( { to: `${base}/secrets`, label: "Secrets", end: false }, + { to: `${base}/metrics`, label: "Metrics", end: false }, + { to: `${base}/logs`, label: "Logs", end: false }, { to: `${base}/events`, label: "Events", end: false }, ) } @@ -262,6 +269,15 @@ export function ApplicationDetailPage() { } /> } /> + }> + + + } + /> + } />
diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx new file mode 100644 index 000000000..64cbcc89e --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/LogsTab.tsx @@ -0,0 +1,16 @@ +import type { ApplicationDefinition } from "@cozystack/types" +import { useTenantContext } from "../../lib/tenant-context.tsx" +import { LogsView } from "../../components/observability/LogsView.tsx" + +interface LogsTabProps { + ad: ApplicationDefinition + // The route's name: while the next application loads, the page still holds the previous instance. + name: string +} + +export function LogsTab({ ad, name }: LogsTabProps) { + const { tenantNamespace } = useTenantContext() + const plural = ad.spec?.application.plural + if (!tenantNamespace || !plural) return null + return +} diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx new file mode 100644 index 000000000..992cfc89b --- /dev/null +++ b/packages/system/dashboard/images/console/apps/console/src/routes/detail/MetricsTab.tsx @@ -0,0 +1,16 @@ +import type { ApplicationDefinition } from "@cozystack/types" +import { useTenantContext } from "../../lib/tenant-context.tsx" +import { MetricsView } from "../../components/observability/MetricsView.tsx" + +interface MetricsTabProps { + ad: ApplicationDefinition + // The route's name: while the next application loads, the page still holds the previous instance. + name: string +} + +export function MetricsTab({ ad, name }: MetricsTabProps) { + const { tenantNamespace } = useTenantContext() + const plural = ad.spec?.application.plural + if (!tenantNamespace || !plural) return null + return +} diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx index efb69dc7b..e678c7dd4 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.test.tsx @@ -110,6 +110,16 @@ describe("useConsoleSidebarSections — admin areas moved out", () => { expect(findItem(result.current, "Imports")?.to).toBe("/console/migration/vmimporttasks") expect(result.current.some((s) => s.title === "Migration")).toBe(true) }) + + it("offers the tenant's namespace logs in an Observability group", async () => { + const client = makeClient({ nodes: true, backupclasses: true }) + const { result } = renderHook(() => useConsoleSidebarSections(), { + wrapper: makeWrapper(client), + }) + await waitFor(() => expect(result.current.length).toBeGreaterThan(0)) + expect(findItem(result.current, "Logs")?.to).toBe("/console/-/observability/logs") + expect(result.current.find((s) => s.title === "Observability")?.items).toHaveLength(1) + }) }) function appDef(name: string, kind: string, category: string, module = false) { diff --git a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx index d5d100813..51c9ea00a 100644 --- a/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx +++ b/packages/system/dashboard/images/console/apps/console/src/routes/sidebar-sections.tsx @@ -11,6 +11,7 @@ import { Layers, Network, Package, + ScrollText, Plug, Server, ToyBrick, @@ -118,7 +119,12 @@ export function useConsoleSidebarSections(): SidebarSection[] { ], } - return [...categorySections, backupsSection, migrationSection] + const observabilitySection: SidebarSection = { + title: "Observability", + items: [{ label: "Logs", to: "/console/-/observability/logs", icon: ScrollText }], + } + + return [...categorySections, backupsSection, migrationSection, observabilitySection] }, [grouped]) } diff --git a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts index 12f50e0fd..64af1131d 100644 --- a/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts +++ b/packages/system/dashboard/images/console/packages/k8s-client/src/client.ts @@ -50,14 +50,12 @@ export class K8sClient { } private async request(path: string, init?: RequestInit): Promise { - const headers: Record = { - "Content-Type": "application/json", - ...(init?.headers as Record), - } + const headers = new Headers(init?.headers) + if (!headers.has("Content-Type")) headers.set("Content-Type", "application/json") if (this.getToken) { const token = await this.getToken() - headers["Authorization"] = `Bearer ${token}` + headers.set("Authorization", `Bearer ${token}`) } let res: Response @@ -241,6 +239,22 @@ export class K8sClient { }) } + /** + * Fetch a same-origin path with the client's credentials and 401 handling, + * leaving status and body to the caller. For endpoints that do not speak + * Kubernetes JSON, such as the observability proxy's streamed NDJSON. + */ + async raw(path: string, init?: RequestInit): Promise { + const headers = new Headers(init?.headers) + if (this.getToken) { + headers.set("Authorization", `Bearer ${await this.getToken()}`) + } + const res = await fetch(`${this.baseUrl}${path}`, { ...init, headers }) + if (res.status === 401) this.handleUnauthorized() + else if (res.ok) this.unauthorizedHandled = false + return res + } + getApiGroups(): Promise { return this.request("/apis") } diff --git a/packages/system/dashboard/images/console/pnpm-lock.yaml b/packages/system/dashboard/images/console/pnpm-lock.yaml index 9d90b0c5c..4661c444a 100644 --- a/packages/system/dashboard/images/console/pnpm-lock.yaml +++ b/packages/system/dashboard/images/console/pnpm-lock.yaml @@ -92,6 +92,9 @@ importers: simple-icons: specifier: ^16.22.0 version: 16.22.0 + uplot: + specifier: ^1.6.32 + version: 1.6.32 devDependencies: '@tailwindcss/vite': specifier: ^4.2.2 @@ -1773,6 +1776,9 @@ packages: peerDependencies: browserslist: '>= 4.21.0' + uplot@1.6.32: + resolution: {integrity: sha512-KIMVnG68zvu5XXUbC4LQEPnhwOxBuLyW1AHtpm6IKTXImkbLgkMy+jabjLgSLMasNuGGzQm/ep3tOkyTxpiQIw==} + uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} @@ -3405,6 +3411,8 @@ snapshots: escalade: 3.2.0 picocolors: 1.1.1 + uplot@1.6.32: {} + uri-js@4.4.1: dependencies: punycode: 2.3.1 diff --git a/packages/system/etcd-rd/cozyrds/etcd.yaml b/packages/system/etcd-rd/cozyrds/etcd.yaml index 90ed8a80a..1ce88e8ae 100644 --- a/packages/system/etcd-rd/cozyrds/etcd.yaml +++ b/packages/system/etcd-rd/cozyrds/etcd.yaml @@ -8,7 +8,7 @@ spec: plural: etcds singular: etcd openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"version":{"description":"etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image.","type":"string","default":"3.6.11"},"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"affinity":{"description":"Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity that spreads members across nodes by kubernetes.io/hostname on a best-effort basis, so a single node failure is less likely to take out more than one member. The default is intentionally soft: a hard rule would leave a member Pending when there are fewer schedulable nodes than replicas, or when node-local storage pins a member to a node. Set a hard requiredDuringSchedulingIgnoredDuringExecution rule here to forbid co-location outright (only on clusters with enough nodes and non-node-pinned storage). Note that the operator applies affinity only when it creates or replaces a member Pod and does not reschedule running members, so on a cluster that already has co-located members the rule takes effect only as each member is replaced, not retroactively.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}} + {"title":"Chart Values","type":"object","properties":{"version":{"description":"etcd version (X.Y.Z) to run, set on the EtcdCluster's spec.version; the operator pulls etcd from its own configured image.","type":"string","default":"3.6.11"},"size":{"description":"Persistent Volume size.","default":"4Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":""},"replicas":{"description":"Number of etcd replicas.","type":"integer","default":3},"resources":{"description":"Resource configuration for etcd.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated.","default":"1000m","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated.","default":"512Mi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"affinity":{"description":"Affinity rules set verbatim on the EtcdCluster's spec.affinity, replacing the chart default. When left empty the chart applies a soft (preferred) podAntiAffinity by kubernetes.io/hostname on top of the topologySpreadConstraints, which by default require an even spread of members over nodes (whenUnsatisfiable DoNotSchedule, maxSkew 1) unless the platform sets scheduling.globalAppTopologySpreadConstraints. The anti-affinity stays soft because a required rule forbids two members on one node even when every node already has one, so an etcd with more members than schedulable nodes could not start all of them. Set a requiredDuringSchedulingIgnoredDuringExecution rule here only on clusters with at least as many schedulable nodes as members. With node-local storage a recreated member Pod can only return to its own node, so a required rule keeps it Pending while that node holds another member, and the default spread does the same once some other node holds none. The operator copies affinity into each EtcdMember when it creates the member and builds every Pod of that member from that copy, so a change does not reschedule running members and takes effect only as members are replaced by new ones.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}} release: prefix: "" labels: diff --git a/packages/system/ingress-nginx/tests/client_keepalive_time_test.yaml b/packages/system/ingress-nginx/tests/client_keepalive_time_test.yaml new file mode 100644 index 000000000..bb492e57c --- /dev/null +++ b/packages/system/ingress-nginx/tests/client_keepalive_time_test.yaml @@ -0,0 +1,26 @@ +suite: ingress-nginx client keepalive_time +release: + name: ingress-nginx + namespace: cozy-ingress-nginx +templates: + - charts/ingress-nginx/templates/controller-configmap.yaml +tests: + - it: raises the client keepalive_time above the nginx 1h default + asserts: + - equal: + path: data.http-snippet + value: keepalive_time 12h; + - it: keeps the snippet when a tenant ingress sets its own config keys + # The tenant ingress HelmRelease passes controller.config for whitelist, + # Cloudflare and PROXY protocol; Helm merges maps, so those keys must not + # drop the default snippet. + set: + ingress-nginx.controller.config.use-proxy-protocol: "true" + ingress-nginx.controller.config.whitelist-source-range: 10.0.0.0/8 + asserts: + - equal: + path: data.http-snippet + value: keepalive_time 12h; + - equal: + path: data.use-proxy-protocol + value: "true" diff --git a/packages/system/ingress-nginx/values.yaml b/packages/system/ingress-nginx/values.yaml index fca145843..1cb7b58da 100644 --- a/packages/system/ingress-nginx/values.yaml +++ b/packages/system/ingress-nginx/values.yaml @@ -47,6 +47,13 @@ ingress-nginx: upstream-keepalive-timeout: "60" upstream-keepalive-connections: "320" ssl-session-tickets: "true" + # Client-side keepalive_time has no ConfigMap key, so nginx keeps its + # 1h default and sends GOAWAY to any HTTP/2 connection older than that. + # A long S3 operation (kopia maintenance behind Velero) multiplexes its + # requests over one connection, and the in-flight object listing fails + # instead of being retried. Idle connections still close after + # keep-alive (75s). + http-snippet: "keepalive_time 12h;" resources: limits: cpu: "1" diff --git a/packages/system/keycloak-operator/Makefile b/packages/system/keycloak-operator/Makefile index e2f7f45d5..303f37eeb 100644 --- a/packages/system/keycloak-operator/Makefile +++ b/packages/system/keycloak-operator/Makefile @@ -4,6 +4,9 @@ export NAMESPACE=cozy-keycloak include ../../../hack/common-envs.mk include ../../../hack/package.mk +test: + helm unittest . + image: docker buildx build images/keycloak-operator \ $(call image-tags,keycloak-operator,v$(COZYSTACK_VERSION)) \ diff --git a/packages/system/keycloak-operator/tests/crds_test.yaml b/packages/system/keycloak-operator/tests/crds_test.yaml new file mode 100644 index 000000000..1be3fe611 --- /dev/null +++ b/packages/system/keycloak-operator/tests/crds_test.yaml @@ -0,0 +1,19 @@ +suite: keycloak-operator CRDs declare the realm settings the platform sets +# keycloak-configure renders the cozy ClusterKeycloakRealm with spec.sessions +# and spec.organizationsEnabled, and spec.login when configured. The apiserver +# drops any field the CRD does not declare without an error, so with a chart +# that lacks them the realm never gets those settings. +includeCrds: true +templates: + - charts/keycloak-operator/crds/v1.edp.epam.com_clusterkeycloakrealms.yaml +tests: + - it: declares the session, organization and login settings + asserts: + - exists: + path: spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.sessions.properties.ssoSessionSettings.properties.idleTimeout + - exists: + path: spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.sessions.properties.ssoSessionSettings.properties.maxLifespan + - exists: + path: spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.organizationsEnabled + - exists: + path: spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.login diff --git a/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml b/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml index 00e80ae45..71de2dd63 100644 --- a/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml +++ b/packages/system/kubernetes-nodes-rd/cozyrds/kubernetes-nodes.yaml @@ -21,7 +21,7 @@ spec: singular: kubernetesnodes plural: kubernetesnodeses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool name (the part after `-`) must not collide with a nodeGroup still managed by the parent kubernetes chart (e.g. the default `md0`), or the render fails on ownership conflict.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `\"010\"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `\"010\"` reserves what `\"10\"` does.","type":"object","default":{},"properties":{"cpu":{"description":"Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"podCpuLimit":{"description":"CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"podCpuRequest":{"description":"CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"kernelModules":{"description":"Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls.","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Module name as `modprobe` takes it, e.g. `nvidia_uvm`.","type":"string"},"parameters":{"description":"Module parameters, each as a bare `key=value` string.","type":"array","items":{"type":"string"}}}}},"kubelet":{"description":"Kubelet resource reservations for this pool.","type":"object","default":{},"properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"logSerialConsole":{"description":"Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it.","type":"boolean","default":false},"maxUnhealthy":{"description":"Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is a percentage. Drop to \"0%\" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At \"50%\" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two.","type":"string","default":"50%"},"nodeStartupTimeout":{"description":"Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.","type":"string","default":"20m"},"readyUnknownTimeout":{"description":"How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"5m"},"readyFalseTimeout":{"description":"How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.","type":"string","default":"10m"},"maxNodeProvisionTime":{"description":"How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted. Keep in sync with the parent kubernetes chart.","type":"string","default":"30m"},"version":{"description":"Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool -- and must satisfy the Talos<->Kubernetes support matrix against `talos.version`.","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"osImage":{"description":"Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool.","type":"object","default":{},"properties":{"builtin":{"description":"Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled.","type":"object","properties":{"schematicID":{"description":"Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version of the golden to clone. Defaults to `talos.version`.","type":"string"}}},"factory":{"description":"Import from an Image Factory / mirror (default).","type":"object","properties":{"imageFactoryURL":{"description":"Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. NOTE: this redirects only the worker boot-disk import; the in-guest upgrade installer image is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.","type":"string"},"schematicID":{"description":"Talos schematic ID. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version. Defaults to `talos.version`.","type":"string"}}}}},"talos":{"description":"Talos worker image configuration. Keep in sync with the parent cluster's `talos`.","type":"object","default":{},"required":["imageFactoryURL","installerRepository","registryMirrors","schematicID","version"],"properties":{"imageFactoryCA":{"description":"Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there.","type":"string","default":""},"imageFactoryURL":{"description":"Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal.","type":"string","default":"https://factory.talos.dev"},"installerRepository":{"description":"OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.","type":"string","default":"factory.talos.dev/installer"},"registryMirrors":{"description":"Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true},"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"sysctls":{"description":"Sysctl overrides rendered into Talos `machine.sysctls` on this pool's guest workers only. Values must be strings (quote numeric values); for example, `user.max_user_namespaces: \"11255\"` enables the user namespaces required by the Talos gVisor extension. An empty map preserves Talos defaults and the existing bootstrap hash. Adding, changing or removing an override changes the content-hashed TalosConfigTemplate and rolls the pool's workers. Supported names and values depend on the worker's Talos release and kernel; unsafe settings can weaken guest isolation or prevent workers from booting. This does not install gVisor or configure a RuntimeClass, and does not change the management hosts, control plane or other pools.","type":"object","default":{},"additionalProperties":{"type":"string"}},"version":{"description":"Default Talos release for this pool's worker OS image and in-guest installer. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix rejects a pairing it lists as unsupported; it lists the Talos minors this chart ships and passes any other minor unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.","type":"string","default":"v1.13.6"}}},"substrate":{"description":"Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["network"],"properties":{"dnsServers":{"description":"Nameservers written into the worker machineconfig. On kubevirt the workers use the management cluster's CoreDNS, which they reach over the pod network; an off-cluster Proxmox worker cannot, so it needs reachable resolvers of its own. Required when `substrate` is `proxmox`. Keep in sync with the parent kubernetes chart's `proxmox.dnsServers`.","type":"array","default":[],"items":{"type":"string"}},"full":{"description":"Full clone rather than linked. Default is a linked clone: it costs kilobytes at creation instead of the whole disk, because a ZFS-backed linked clone writes only its own increment (measured: 8K against 10.2G for the same worker full-cloned onto a `sparse 0` pool). The cost is that the clone holds the template's base snapshot, so the template cannot be rotated while any linked clone still references it — which is what the consolidation CronJob resolves 24h after creation. Set true for a pool that must be independent of the template from the first second.","type":"boolean","default":false},"network":{"description":"NIC configuration.","type":"object","default":{},"required":["bridge"],"properties":{"bridge":{"description":"Proxmox bridge the VM NIC attaches to (e.g. `vmbr0`). Required when `substrate` is `proxmox`.","type":"string","default":""},"mtu":{"description":"NIC MTU. Omitted when unset.","type":"integer"},"vlan":{"description":"L2 VLAN tag. Omitted when unset.","type":"integer"}}},"pool":{"description":"Proxmox resource pool the worker VMs are added to. Set the same pool on every node pool of a tenant cluster and grant the tenant's tokens their privileges on `/pool/`: that is the one ACL path that follows every VM capmox creates, replacements included. The pool must exist on the hypervisor; capmox does not create it. Empty adds the workers to no pool.","type":"string","default":""},"storage":{"description":"Proxmox storage for the cloned disk; needs `full: true`. A target storage is a full-clone parameter — a linked clone always lives on the template's storage, and Proxmox refuses the pair — so the render refuses it too. Empty keeps the template's storage.","type":"string","default":""},"templateTags":{"description":"Tags identifying the Talos VM template to clone. capmox matches a template when its tag set is EQUAL to this list, not when it contains it, so every tag on the template must be listed here — a missing one yields `found 0 VM templates with tags ...` while the tags are plainly present. The template has to carry this pool's `talos.schematicID` at `talos.version` in the nocloud flavour, with the Proxmox guest agent enabled for the `qemu-guest-agent` extension of the default schematic; `hack/proxmox-talos-template.sh` builds it from these values and prints the tags to list here. Do not build it from the platform release's `nocloud-amd64.raw.xz`: that is the bare-metal node image, whose `zfs` extension keeps a VM from finishing its boot sequence and reboots it at the Talos boot timeout. Required when `substrate` is `proxmox`.","type":"array","default":[],"items":{"type":"string"}}}}}} + {"title":"Chart Values","type":"object","properties":{"cluster":{"description":"Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-`; the KubernetesNodes CR must be named `-`. The pool's objects are named `kubernetes--`; when a MachineDeployment of that name already exists and belongs to another Helm release, the render fails instead of taking it over.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"cluster is immutable"}]},"storageClass":{"description":"StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"minReplicas":{"description":"Minimum number of replicas in this pool. Used as the cluster-autoscaler floor.","type":"integer","default":0},"maxReplicas":{"description":"Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.","type":"integer","default":10},"instanceType":{"description":"KubeVirt instance type that sizes each worker VM when `resources` is not set. Ignored when `resources.cpu` and `resources.memory` are both set, and therefore on `substrate: proxmox`, which requires them.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"diskSize":{"description":"System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. Has no effect on `substrate: proxmox`, where the worker disk is a clone of the VM template's disk and keeps its size.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"roles":{"description":"List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.","type":"array","default":[],"items":{"type":"string"}},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `\"010\"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `\"010\"` reserves what `\"10\"` does.","type":"object","default":{},"properties":{"cpu":{"description":"Number of vCPU cores for each worker VM, as a whole number such as `2`. On the `kubevirt` substrate the value becomes the VM's integer `domain.cpu.cores`, so millicore, fractional and negative quantities, and counts above 4294967295, are rejected at render time; an unquoted `0` is treated as unset. On the `proxmox` substrate a millicore or fractional quantity is rounded up to whole cores.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"podCpuLimit":{"description":"CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"podCpuRequest":{"description":"CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"kernelModules":{"description":"Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Module name as `modprobe` takes it, e.g. `nvidia_uvm`.","type":"string"},"parameters":{"description":"Module parameters, each as a bare `key=value` string.","type":"array","items":{"type":"string"}}}}},"kubelet":{"description":"Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`.","type":"object","default":{},"properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. When empty, 5% of the worker's CPU, clamped to 50m..500m.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. When empty, 5% of the worker's CPU, clamped to 50m..500m.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.","type":"string"}}},"logSerialConsole":{"description":"Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it.","type":"boolean","default":false},"maxUnhealthy":{"description":"Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer (\"0\", \"1\", ...) or a percentage (\"0%\", \"50%\"); bare numeric strings are rejected, so the safer default is a percentage. Drop to \"0%\" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At \"50%\" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two.","type":"string","default":"50%"},"nodeStartupTimeout":{"description":"Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (the Talos image import from the Image Factory, or a storage class slow to provision or clone the worker disk). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.","type":"string","default":"20m"},"readyUnknownTimeout":{"description":"How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.","type":"string","default":"5m"},"readyFalseTimeout":{"description":"How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.","type":"string","default":"10m"},"maxNodeProvisionTime":{"description":"How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours (\"30m\", \"1h30m\"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to (\"20m0s\", \"1h0m0s\") is accepted.","type":"string","default":"30m"},"version":{"description":"Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool. The render fails when it is ahead. Its pairing with the Talos release the pool resolves (`talos.version`, or the `osImage` override) is checked against the chart's Talos<->Kubernetes support matrix: for a Talos minor the matrix lists, a Kubernetes minor outside that minor's list is refused; a Talos minor the matrix does not list is not checked.","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"osImage":{"description":"Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool.","type":"object","default":{},"properties":{"builtin":{"description":"Clone a golden image from the worker image catalog. Requires a StorageClass whose volumes are reachable from any node, in practice a replicated/DRBD class: the clone is a storage-layer copy, and on a node-pinned class its two ends land on unrelated nodes -- the golden binds where CDI imported it, a worker disk where its VM is scheduled.","type":"object","properties":{"schematicID":{"description":"Talos schematic ID of the golden to clone. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version of the golden to clone. Defaults to `talos.version`.","type":"string"}}},"factory":{"description":"Import from an Image Factory / mirror (default).","type":"object","properties":{"imageFactoryURL":{"description":"Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. This redirects only the worker boot-disk import; the installer image in the worker machine config is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.","type":"string"},"schematicID":{"description":"Talos schematic ID. Defaults to `talos.schematicID`.","type":"string"},"version":{"description":"Talos version. Defaults to `talos.version`.","type":"string"}}}}},"talos":{"description":"Talos worker image configuration for this pool.","type":"object","default":{},"required":["imageFactoryURL","installerRepository","registryMirrors","schematicID","version"],"properties":{"imageFactoryCA":{"description":"Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there. On `substrate: proxmox` it has no effect on the workers either: the ConfigMap is still rendered, but nothing references it.","type":"string","default":""},"imageFactoryURL":{"description":"Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal. On `substrate: proxmox` the workers clone a VM template that already carries the OS, so this field has no effect there beyond that format check.","type":"string","default":"https://factory.talos.dev"},"installerRepository":{"description":"OCI repository prefix for the Talos installer image, written into the worker machine config as `machine.install.image`. Resolved as `/:` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.","type":"string","default":"factory.talos.dev/installer"},"registryMirrors":{"description":"Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true},"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"sysctls":{"description":"Sysctl overrides rendered into Talos `machine.sysctls` on this pool's guest workers only. Values must be strings (quote numeric values); for example, `user.max_user_namespaces: \"11255\"` enables the user namespaces required by the Talos gVisor extension. An empty map preserves Talos defaults and the existing bootstrap hash. Adding, changing or removing an override changes the content-hashed TalosConfigTemplate and rolls the pool's workers. Supported names and values depend on the worker's Talos release and kernel; unsafe settings can weaken guest isolation or prevent workers from booting. This does not install gVisor or configure a RuntimeClass, and does not change the management hosts, control plane or other pools.","type":"object","default":{},"additionalProperties":{"type":"string"}},"version":{"description":"Default Talos release for this pool's worker OS image and for the installer image written into the worker machine config. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix lists, for each Talos minor the chart knows, the Kubernetes minors it supports and refuses a pool `version` outside that list; a Talos minor it does not list passes unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.","type":"string","default":"v1.13.6"}}},"substrate":{"description":"Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["network"],"properties":{"dnsServers":{"description":"Nameservers written into the worker machineconfig. On kubevirt the workers use the management cluster's CoreDNS, which they reach over the pod network; an off-cluster Proxmox worker cannot, so it needs reachable resolvers of its own. Required when `substrate` is `proxmox`. Keep in sync with the parent kubernetes chart's `proxmox.dnsServers`.","type":"array","default":[],"items":{"type":"string"}},"full":{"description":"Full clone rather than linked. Default is a linked clone: it costs kilobytes at creation instead of the whole disk, because a ZFS-backed linked clone writes only its own increment (measured: 8K against 10.2G for the same worker full-cloned onto a `sparse 0` pool). The cost is that the clone holds the template's base snapshot, so the template cannot be rotated while any linked clone still references it — which is what the consolidation CronJob resolves 24h after creation. Set true for a pool that must be independent of the template from the first second.","type":"boolean","default":false},"network":{"description":"NIC configuration.","type":"object","default":{},"required":["bridge"],"properties":{"bridge":{"description":"Proxmox bridge the VM NIC attaches to (e.g. `vmbr0`). Required when `substrate` is `proxmox`.","type":"string","default":""},"mtu":{"description":"NIC MTU. Omitted when unset.","type":"integer"},"vlan":{"description":"L2 VLAN tag. Omitted when unset.","type":"integer"}}},"pool":{"description":"Proxmox resource pool the worker VMs are added to. Set the same pool on every node pool of a tenant cluster and grant the tenant's tokens their privileges on `/pool/`: that is the one ACL path that follows every VM capmox creates, replacements included. The pool must exist on the hypervisor; capmox does not create it. Empty adds the workers to no pool.","type":"string","default":""},"storage":{"description":"Proxmox storage for the cloned disk; needs `full: true`. A target storage is a full-clone parameter — a linked clone always lives on the template's storage, and Proxmox refuses the pair — so the render refuses it too. Empty keeps the template's storage.","type":"string","default":""},"templateTags":{"description":"Tags identifying the Talos VM template to clone. capmox matches a template when its tag set is EQUAL to this list, not when it contains it, so every tag on the template must be listed here — a missing one yields `found 0 VM templates with tags ...` while the tags are plainly present. The template has to carry this pool's `talos.schematicID` at `talos.version` in the nocloud flavour, with the Proxmox guest agent enabled for the `qemu-guest-agent` extension of the default schematic; `hack/proxmox-talos-template.sh` builds it from these values and prints the tags to list here. Do not build it from the platform release's `nocloud-amd64.raw.xz`: that is the bare-metal node image, whose `zfs` extension keeps a VM from finishing its boot sequence and reboots it at the Talos boot timeout. Required when `substrate` is `proxmox`.","type":"array","default":[],"items":{"type":"string"}}}}}} release: labels: sharding.fluxcd.io/key: tenants diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index 3be6aa3b9..52644e331 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -32,7 +32,7 @@ spec: singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","x-cozystack-name":{"type":"string","description":"Cluster name. Every worker pool is a separate KubernetesNodes release named `kubernetes-nodes--`, so the name must leave room for the default `md0` pool inside the 53-character Helm release name limit.","maxLength":32},"properties":{"storageClass":{"description":"Default StorageClass inside the tenant cluster. Remote-accessible LINSTOR classes are auto-propagated to the tenant under the same name.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"substrate":{"description":"Which infrastructure provider backs this cluster's worker VMs. `kubevirt` runs them inside this cluster; `proxmox` runs them on an external Proxmox VE cluster through capmox. The control plane is Kamaji either way — this selects the infrastructure half only. Must match the `substrate` of every KubernetesNodes pool attached to this cluster: the pools reference this cluster's infrastructure object by kind, and a mismatch leaves their Machines unreconciled. Switching an existing cluster is not supported; create a new one.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["ipv4Config"],"properties":{"allowedNodes":{"description":"Proxmox nodes capmox may place VMs on. Empty means every node in the Proxmox cluster.","type":"array","default":[],"items":{"type":"string"}},"ccm":{"description":"Proxmox cloud-controller-manager settings.","type":"object","default":{},"required":["credentialsSecretName"],"properties":{"credentialsSecretName":{"description":"Secret in THIS namespace holding the controller's Proxmox API credentials under the keys `url`, `token_id`, `token_secret` and `region`. May name the same Secret as `csi.credentialsSecretName`; it is a separate knob so the two can hold separate tokens, since this one only reads VM inventory while the driver attaches and detaches disks.","type":"string","default":""}}},"csi":{"description":"Proxmox CSI driver settings.","type":"object","default":{},"required":["credentialsSecretName","enabled"],"properties":{"credentialsSecretName":{"description":"Secret in THIS namespace holding the driver's Proxmox API credentials under the keys `url`, `token_id`, `token_secret` and `region`. The parent chart composes them into the controller's config file here, in the management cluster; nothing reaches the tenant, and no token is written into a rendered manifest. Distinct from the capmox credentials on purpose: the driver attaches and detaches disks, which is a different blast radius from creating VMs.","type":"string","default":""},"enabled":{"description":"Install the driver. Without it a Proxmox-backed tenant has no way to provision PersistentVolumes at all.","type":"boolean","default":true},"storageClasses":{"description":"StorageClasses to create in the tenant.","type":"array","default":[],"items":{"type":"object","required":["name","storage"],"properties":{"fstype":{"description":"Filesystem the driver formats volumes with: ext4 or xfs.","type":"string"},"name":{"description":"StorageClass name inside the tenant cluster.","type":"string"},"reclaimPolicy":{"description":"Delete or Retain.","type":"string"},"ssd":{"description":"Mark volumes as SSD-backed, which changes the discard/cache defaults Proxmox applies.","type":"boolean"},"storage":{"description":"Proxmox storage id the volumes are created on (as it appears in `pvesm status`).","type":"string"}}}}}},"dnsServers":{"description":"Nameservers written into each worker's network config. Required by the ProxmoxCluster schema.","type":"array","default":[],"items":{"type":"string"}},"insecure":{"description":"Skip verification of the Proxmox API server certificate in the CCM and the CSI controller, both of which run in the management cluster. Default false: the certificate is verified unless an operator turns that off. A stock Proxmox VE install serves a self-signed one, so set true for it, or install a certificate the management cluster trusts.","type":"boolean","default":false},"ipv4Config":{"description":"Address pool for workers. capmox assigns static addresses and has no DHCP mode, so this is required rather than optional.","type":"object","default":{},"required":["gateway","prefix"],"properties":{"addresses":{"description":"Ranges or CIDRs, e.g. `[\"10.0.0.120-10.0.0.170\"]`. capmox turns these into an InClusterIPPool, so they must not overlap any DHCP range on the same L2 or two workers will answer to one address.","type":"array","default":[],"items":{"type":"string"}},"gateway":{"description":"Default gateway for the workers.","type":"string","default":""},"prefix":{"description":"Netmask prefix length.","type":"integer","default":24}}}}},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","ouroboros","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin. Always installed as the tenant cluster's CNI; this section only overrides its Helm values.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}},"x-cozystack-no-enable-switch":true},"coredns":{"description":"CoreDNS addon. Always installed as the tenant cluster's DNS; this section only overrides its Helm values.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}},"x-cozystack-no-enable-switch":true},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ouroboros":{"description":"Hairpin-NAT fix for ingress-nginx with PROXY-protocol.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable ouroboros. Requires addons.ingressNginx.enabled (chart-render fail otherwise). Only useful when PROXY-protocol is wired on the tenant ingress-nginx via valuesOverride.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides. Operator-key wins over cozystack defaults.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler. Installed and removed together with addons.monitoringAgents.enabled; this section only overrides its Helm values.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}},"x-cozystack-no-enable-switch":true}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"admissionControllers":{"description":"Admission plugins the tenant kube-apiserver enables, passed through to KamajiControlPlane `spec.admissionControllers`, which the control plane renders as `--enable-admission-plugins`. A non-empty list replaces the control plane's default set instead of adding to it. The kube-apiserver enables these plugins in addition to its own default ones, so leaving a plugin out does not turn it off; to turn one off, name it in `--disable-admission-plugins` in `controlPlane.apiServer.extraArgs` and keep it out of this list, since the apiserver refuses to start with a plugin in both, and the control plane's default set names several. Empty by default, which keeps that default set, taken from the TenantControlPlane CRD: CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, LimitRanger, MutatingAdmissionWebhook, NamespaceLifecycle, PersistentVolumeClaimResize, Priority, ResourceQuota, RuntimeClass, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. `NodeRestriction` is not in it.","type":"array","default":[],"items":{"type":"string","enum":["AlwaysAdmit","AlwaysDeny","AlwaysPullImages","CertificateApproval","CertificateSigning","CertificateSubjectRestriction","DefaultIngressClass","DefaultStorageClass","DefaultTolerationSeconds","DenyEscalatingExec","DenyExecOnPrivileged","DenyServiceExternalIPs","EventRateLimit","ExtendedResourceToleration","ImagePolicyWebhook","LimitPodHardAntiAffinityTopology","LimitRanger","MutatingAdmissionWebhook","NamespaceAutoProvision","NamespaceExists","NamespaceLifecycle","NodeRestriction","OwnerReferencesPermissionEnforcement","PersistentVolumeClaimResize","PersistentVolumeLabel","PodNodeSelector","PodSecurity","PodSecurityPolicy","PodTolerationRestriction","Priority","ResourceQuota","RuntimeClass","SecurityContextDeny","ServiceAccount","StorageObjectInUseProtection","TaintNodesByCondition","ValidatingAdmissionWebhook"]}},"extraArgs":{"description":"Extra command-line flags appended to the tenant kube-apiserver, passed through to KamajiControlPlane `spec.apiServer.extraArgs`. For OIDC use `spec.oidc.mode` — this passthrough is the escape hatch for unrelated apiserver flags. The tenant control plane sets each of the following flags itself and drops an entry naming one before the apiserver sees it. The chart moves an `--enable-admission-plugins=` entry into `controlPlane.apiServer.admissionControllers` and a `--kubelet-preferred-address-types=` entry into `controlPlane.kubelet.preferredAddressTypes` while that field is empty, and refuses any other entry naming one of them: `--advertise-address`, `--client-ca-file`, `--enable-admission-plugins`, `--service-cluster-ip-range`, `--kubelet-client-certificate`, `--kubelet-client-key`, `--kubelet-preferred-address-types`, `--proxy-client-cert-file`, `--proxy-client-key-file`, `--requestheader-allowed-names`, `--requestheader-client-ca-file`, `--secure-port`, `--service-account-key-file`, `--service-account-signing-key-file`, `--tls-cert-file`, `--tls-private-key-file`, `--etcd-servers`, `--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, `--etcd-prefix`, `--etcd-compaction-interval`, `--egress-selector-config-file`. When `spec.oidc.mode` is not `None` the chart also renders OIDC flags here, reads what you have written on them, and refuses some combinations outright; the Aggregated API servers section of the Kubernetes OIDC guide on cozystack.io carries which flags, which combinations, and why. Do NOT add legacy `--oidc-*` flags here when `spec.oidc.mode` is not `None`; the chart injects `--authentication-config` and the apiserver refuses to boot with both. Empty by default (no change to current behavior).","type":"array","default":[],"items":{"type":"string"}},"extraVolumeMounts":{"description":"Extra volume mounts added to the tenant kube-apiserver container, passed through to KamajiControlPlane `spec.apiServer.extraVolumeMounts`. Each `name` must reference a volume declared in `extraVolumes`; the chart-managed talos secret volumes cannot be mounted. Each item is a core/v1 VolumeMount. Empty by default.","type":"array","default":[],"items":{"type":"object","x-kubernetes-preserve-unknown-fields":true}},"extraVolumes":{"description":"Extra volumes added to the control-plane Deployment, passed through to KamajiControlPlane `spec.deployment.extraVolumes`. Use to mount a ConfigMap or Secret holding an AuthenticationConfiguration file referenced by `extraArgs`. Each item is a core/v1 Volume, but because the control-plane pod runs on the management cluster only `configMap` and `secret` sources are allowed (host-reaching sources like `hostPath`/`csi` and token-projection via `projected` are rejected); each volume must have a unique, non-empty name and exactly one source. The names `talos-ca` and `talos-tls-cert` are reserved by the chart. Empty by default.","type":"array","default":[],"items":{"type":"object","x-kubernetes-preserve-unknown-fields":true}},"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"c1.medium","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"kubelet":{"description":"Kubelet connection settings.","type":"object","default":{},"properties":{"preferredAddressTypes":{"description":"Node address types the tenant kube-apiserver tries, in order, when it connects to a kubelet, passed through to KamajiControlPlane `spec.kubelet.preferredAddressTypes`, which the control plane renders as `--kubelet-preferred-address-types`. The KamajiControlPlane CRD rejects a repeated name. Empty by default, which renders `InternalIP`, `ExternalIP`, or the list of a `--kubelet-preferred-address-types=` entry in `controlPlane.apiServer.extraArgs` when there is one.","type":"array","default":[],"items":{"type":"string","enum":["Hostname","InternalIP","ExternalIP","InternalDNS","ExternalDNS"]}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"talos":{"description":"Talos worker image configuration.","type":"object","default":{},"required":["imageFactoryURL","installerRepository","registryMirrors","schematicID","version"],"properties":{"imageFactoryURL":{"description":"Base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash.","type":"string","default":"https://factory.talos.dev"},"installerRepository":{"description":"OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `/:`. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.","type":"string","default":"factory.talos.dev/installer"},"registryMirrors":{"description":"Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true},"schematicID":{"description":"Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here.","type":"string","default":"ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"},"version":{"description":"Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`.","type":"string","default":"v1.13.6"}}},"oidc":{"description":"OIDC authentication and per-user RBAC for the tenant kube-apiserver. See https://cozystack.io/docs/v1.6/kubernetes/oidc-authentication/ for the operator guide.","type":"object","default":{},"required":["mode"],"properties":{"customConfig":{"description":"Tenant-supplied AuthenticationConfiguration; consumed only when `mode: CustomConfig`.","type":"object","default":{},"properties":{"config":{"description":"Inline AuthenticationConfiguration YAML (`apiserver.config.k8s.io/v1beta1`). The chart writes the value verbatim into a Secret mounted at `/etc/kubernetes/authentication-config/config.yaml` on the kube-apiserver. Mutually exclusive with `secretRef.name`.","type":"string","default":""},"secretRef":{"description":"Reference to an existing Secret in the tenant namespace carrying the `AuthenticationConfiguration`.","type":"object","default":{},"properties":{"name":{"description":"Name of an existing Secret in the tenant (release) namespace whose `config.yaml` key holds an `apiserver.config.k8s.io/v1beta1` `AuthenticationConfiguration`. Mutually exclusive with `customConfig.config`.","type":"string","default":""}}}}},"mode":{"description":"Identity mode. `None`: no OIDC, only the static admin kubeconfig works. `System`: trust the platform Keycloak realm (`cozy` by default) via a per-cluster public client with audience binding; zero-config default. `CustomConfig`: trust a tenant-supplied issuer directly (BYO); `cozy` is not in the path.","type":"string","default":"None","enum":["None","System","CustomConfig"]},"users":{"description":"Users granted access to the tenant cluster; each entry produces one ClusterRoleBinding inside the tenant cluster. Works for both `System` and `CustomConfig` modes.","type":"array","default":[],"items":{"type":"object","required":["email","role"],"properties":{"email":{"description":"Email address matched against the `email` claim from the issuer. Used verbatim as the `User:` subject in the ClusterRoleBinding inside the tenant cluster. The `email` claim is a built-in OIDC scope requested explicitly by the chart-generated `kubectl oidc-login` kubeconfig; every conformant OIDC provider (cozy realm included) emits it.","type":"string"},"role":{"description":"Role to bind: `admin` maps to `ClusterRole/cluster-admin`, `view` maps to `ClusterRole/view`.","type":"string","enum":["admin","view"]}}}}}}}} + {"title":"Chart Values","type":"object","x-cozystack-name":{"type":"string","description":"Cluster name. Each worker pool is a KubernetesNodes release named `kubernetes-nodes--`, which has to fit the 53-character Helm release name limit, so a cluster name of at most 32 characters leaves at least three characters for the pool name.","maxLength":32},"properties":{"storageClass":{"description":"Default StorageClass inside the tenant cluster. Remote-accessible LINSTOR classes are auto-propagated to the tenant under the same name.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `kubernetes-.` if empty.","type":"string","default":""},"substrate":{"description":"Which infrastructure provider backs this cluster's worker VMs. `kubevirt` runs them inside this cluster; `proxmox` runs them on an external Proxmox VE cluster through capmox. The control plane is Kamaji either way — this selects the infrastructure half only. Every KubernetesNodes pool of this cluster must set the same `substrate`: a pool set to the other one refuses to render once this cluster exists. Switching an existing cluster is not supported; create a new one.","type":"string","default":"kubevirt","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"substrate is immutable"}]},"proxmox":{"description":"Proxmox substrate settings.","type":"object","default":{},"required":["ipv4Config"],"properties":{"allowedNodes":{"description":"Proxmox nodes capmox may place VMs on. Empty means every node in the Proxmox cluster.","type":"array","default":[],"items":{"type":"string"}},"ccm":{"description":"Proxmox cloud-controller-manager settings.","type":"object","default":{},"required":["credentialsSecretName"],"properties":{"credentialsSecretName":{"description":"Secret in THIS namespace holding the controller's Proxmox API credentials under the keys `url`, `token_id`, `token_secret` and `region`. May name the same Secret as `csi.credentialsSecretName`; it is a separate knob so the two can hold separate tokens, since this one only reads VM inventory while the driver attaches and detaches disks.","type":"string","default":""}}},"csi":{"description":"Proxmox CSI driver settings.","type":"object","default":{},"required":["credentialsSecretName","enabled"],"properties":{"credentialsSecretName":{"description":"Secret in THIS namespace holding the driver's Proxmox API credentials under the keys `url`, `token_id`, `token_secret` and `region`. The parent chart composes them into the controller's config file here, in the management cluster; nothing reaches the tenant, and no token is written into a rendered manifest. Distinct from the capmox credentials on purpose: the driver attaches and detaches disks, which is a different blast radius from creating VMs.","type":"string","default":""},"enabled":{"description":"Install the driver. Without it a Proxmox-backed tenant has no way to provision PersistentVolumes at all.","type":"boolean","default":true},"storageClasses":{"description":"StorageClasses to create in the tenant.","type":"array","default":[],"items":{"type":"object","required":["name","storage"],"properties":{"fstype":{"description":"Filesystem the driver formats volumes with: ext4 or xfs.","type":"string"},"name":{"description":"StorageClass name inside the tenant cluster.","type":"string"},"reclaimPolicy":{"description":"Delete or Retain.","type":"string"},"ssd":{"description":"Mark volumes as SSD-backed, which changes the discard/cache defaults Proxmox applies.","type":"boolean"},"storage":{"description":"Proxmox storage id the volumes are created on (as it appears in `pvesm status`).","type":"string"}}}}}},"dnsServers":{"description":"Nameservers written into each worker's network config. Required by the ProxmoxCluster schema.","type":"array","default":[],"items":{"type":"string"}},"insecure":{"description":"Skip verification of the Proxmox API server certificate in the CCM and the CSI controller, both of which run in the management cluster. Default false: the certificate is verified unless an operator turns that off. A stock Proxmox VE install serves a self-signed one, so set true for it, or install a certificate the management cluster trusts.","type":"boolean","default":false},"ipv4Config":{"description":"Address pool for workers. capmox assigns static addresses and has no DHCP mode, so this is required rather than optional.","type":"object","default":{},"required":["gateway","prefix"],"properties":{"addresses":{"description":"Ranges or CIDRs, e.g. `[\"10.0.0.120-10.0.0.170\"]`. capmox turns these into an InClusterIPPool, so they must not overlap any DHCP range on the same L2 or two workers will answer to one address.","type":"array","default":[],"items":{"type":"string"}},"gateway":{"description":"Default gateway for the workers.","type":"string","default":""},"prefix":{"description":"Netmask prefix length.","type":"integer","default":24}}}}},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","ouroboros","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin. Always installed as the tenant cluster's CNI; this section only overrides its Helm values.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}},"x-cozystack-no-enable-switch":true},"coredns":{"description":"CoreDNS addon. Always installed as the tenant cluster's DNS; this section only overrides its Helm values.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}},"x-cozystack-no-enable-switch":true},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ouroboros":{"description":"Hairpin-NAT fix for ingress-nginx with PROXY-protocol.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable ouroboros. Requires addons.ingressNginx.enabled (chart-render fail otherwise). Only useful when PROXY-protocol is wired on the tenant ingress-nginx via valuesOverride.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides. Operator-key wins over cozystack defaults.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler. Installed and removed together with addons.monitoringAgents.enabled; this section only overrides its Helm values.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}},"x-cozystack-no-enable-switch":true}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"admissionControllers":{"description":"Admission plugins the tenant kube-apiserver enables, passed through to KamajiControlPlane `spec.admissionControllers`, which the control plane renders as `--enable-admission-plugins`. A non-empty list replaces the control plane's default set instead of adding to it. The kube-apiserver enables these plugins in addition to its own default ones, so leaving a plugin out does not turn it off; to turn one off, name it in `--disable-admission-plugins` in `controlPlane.apiServer.extraArgs` and keep it out of this list, since the apiserver refuses to start with a plugin in both, and the control plane's default set names several. Empty by default, which keeps that default set, taken from the TenantControlPlane CRD: CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, LimitRanger, MutatingAdmissionWebhook, NamespaceLifecycle, PersistentVolumeClaimResize, Priority, ResourceQuota, RuntimeClass, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. `NodeRestriction` is not in it.","type":"array","default":[],"items":{"type":"string","enum":["AlwaysAdmit","AlwaysDeny","AlwaysPullImages","CertificateApproval","CertificateSigning","CertificateSubjectRestriction","DefaultIngressClass","DefaultStorageClass","DefaultTolerationSeconds","DenyEscalatingExec","DenyExecOnPrivileged","DenyServiceExternalIPs","EventRateLimit","ExtendedResourceToleration","ImagePolicyWebhook","LimitPodHardAntiAffinityTopology","LimitRanger","MutatingAdmissionWebhook","NamespaceAutoProvision","NamespaceExists","NamespaceLifecycle","NodeRestriction","OwnerReferencesPermissionEnforcement","PersistentVolumeClaimResize","PersistentVolumeLabel","PodNodeSelector","PodSecurity","PodSecurityPolicy","PodTolerationRestriction","Priority","ResourceQuota","RuntimeClass","SecurityContextDeny","ServiceAccount","StorageObjectInUseProtection","TaintNodesByCondition","ValidatingAdmissionWebhook"]}},"extraArgs":{"description":"Extra command-line flags appended to the tenant kube-apiserver, passed through to KamajiControlPlane `spec.apiServer.extraArgs`. For OIDC use `spec.oidc.mode` — this passthrough is the escape hatch for unrelated apiserver flags. The tenant control plane sets each of the following flags itself and drops an entry naming one before the apiserver sees it. The chart moves an `--enable-admission-plugins=` entry into `controlPlane.apiServer.admissionControllers` and a `--kubelet-preferred-address-types=` entry into `controlPlane.kubelet.preferredAddressTypes` while that field is empty, and refuses any other entry naming one of them: `--advertise-address`, `--client-ca-file`, `--enable-admission-plugins`, `--service-cluster-ip-range`, `--kubelet-client-certificate`, `--kubelet-client-key`, `--kubelet-preferred-address-types`, `--proxy-client-cert-file`, `--proxy-client-key-file`, `--requestheader-allowed-names`, `--requestheader-client-ca-file`, `--secure-port`, `--service-account-key-file`, `--service-account-signing-key-file`, `--tls-cert-file`, `--tls-private-key-file`, `--etcd-servers`, `--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, `--etcd-prefix`, `--etcd-compaction-interval`, `--egress-selector-config-file`. When `spec.oidc.mode` is not `None` the chart also renders OIDC flags here, reads what you have written on them, and refuses some combinations outright; the Aggregated API servers section of the Kubernetes OIDC guide on cozystack.io carries which flags, which combinations, and why. Do NOT add legacy `--oidc-*` flags here when `spec.oidc.mode` is not `None`; the chart injects `--authentication-config` and the apiserver refuses to boot with both. Empty by default.","type":"array","default":[],"items":{"type":"string"}},"extraVolumeMounts":{"description":"Extra volume mounts added to the tenant kube-apiserver container, passed through to KamajiControlPlane `spec.apiServer.extraVolumeMounts`. Each `name` must reference a volume declared in `extraVolumes`; the chart-managed talos secret volumes cannot be mounted. Each item is a core/v1 VolumeMount. Empty by default.","type":"array","default":[],"items":{"type":"object","x-kubernetes-preserve-unknown-fields":true}},"extraVolumes":{"description":"Extra volumes added to the control-plane Deployment, passed through to KamajiControlPlane `spec.deployment.extraVolumes`. Use to mount a ConfigMap or Secret holding an AuthenticationConfiguration file referenced by `extraArgs`. Each item is a core/v1 Volume, but because the control-plane pod runs on the management cluster only `configMap` and `secret` sources are allowed (host-reaching sources like `hostPath`/`csi` and token-projection via `projected` are rejected); each volume must have a unique, non-empty name and exactly one source. The names `talos-ca` and `talos-tls-cert` are reserved by the chart. Empty by default.","type":"array","default":[],"items":{"type":"object","x-kubernetes-preserve-unknown-fields":true}},"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"c1.medium","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"kubelet":{"description":"Kubelet connection settings.","type":"object","default":{},"properties":{"preferredAddressTypes":{"description":"Node address types the tenant kube-apiserver tries, in order, when it connects to a kubelet, passed through to KamajiControlPlane `spec.kubelet.preferredAddressTypes`, which the control plane renders as `--kubelet-preferred-address-types`. The KamajiControlPlane CRD rejects a repeated name. Empty by default, which renders `InternalIP`, `ExternalIP`, or the list of a `--kubelet-preferred-address-types=` entry in `controlPlane.apiServer.extraArgs` when there is one.","type":"array","default":[],"items":{"type":"string","enum":["Hostname","InternalIP","ExternalIP","InternalDNS","ExternalDNS"]}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"oidc":{"description":"OIDC authentication and per-user RBAC for the tenant kube-apiserver. See https://cozystack.io/docs/v1.6/kubernetes/oidc-authentication/ for the operator guide.","type":"object","default":{},"required":["mode"],"properties":{"customConfig":{"description":"Tenant-supplied AuthenticationConfiguration; consumed only when `mode: CustomConfig`.","type":"object","default":{},"properties":{"config":{"description":"Inline AuthenticationConfiguration YAML (`apiserver.config.k8s.io/v1beta1`). The chart writes the value verbatim into a Secret mounted at `/etc/kubernetes/authentication-config/config.yaml` on the kube-apiserver. Mutually exclusive with `secretRef.name`.","type":"string","default":""},"secretRef":{"description":"Reference to an existing Secret in the tenant namespace carrying the `AuthenticationConfiguration`.","type":"object","default":{},"properties":{"name":{"description":"Name of an existing Secret in the tenant (release) namespace whose `config.yaml` key holds an `apiserver.config.k8s.io/v1beta1` `AuthenticationConfiguration`. Mutually exclusive with `customConfig.config`.","type":"string","default":""}}}}},"mode":{"description":"Identity mode. `None`: no OIDC, only the static admin kubeconfig works. `System`: trust the platform Keycloak realm (`cozy` by default) via a per-cluster public client with audience binding; zero-config default. `CustomConfig`: trust a tenant-supplied issuer directly (BYO); `cozy` is not in the path.","type":"string","default":"None","enum":["None","System","CustomConfig"]},"users":{"description":"Users granted access to the tenant cluster; each entry produces one ClusterRoleBinding inside the tenant cluster. Works for both `System` and `CustomConfig` modes.","type":"array","default":[],"items":{"type":"object","required":["email","role"],"properties":{"email":{"description":"Email address matched against the `email` claim from the issuer. Used verbatim as the `User:` subject in the ClusterRoleBinding inside the tenant cluster. The `email` claim is a built-in OIDC scope requested explicitly by the chart-generated `kubectl oidc-login` kubeconfig; every conformant OIDC provider (cozy realm included) emits it.","type":"string"},"role":{"description":"Role to bind: `admin` maps to `ClusterRole/cluster-admin`, `view` maps to `ClusterRole/view`.","type":"string","enum":["admin","view"]}}}}}}}} release: prefix: kubernetes- labels: @@ -50,7 +50,7 @@ spec: tags: - compute icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODFfMjg0NSkiLz4KPHBhdGggZD0iTTcxLjk5NjggMTlDNzAuMzAzOSAxOS4wMDAyIDY4LjkzMTIgMjAuNTMyMiA2OC45MzE0IDIyLjQyMjFDNjguOTMxNCAyMi40NTExIDY4LjkzNzMgMjIuNDc4OCA2OC45Mzc5IDIyLjUwNzZDNjguOTM1NCAyMi43NjQ0IDY4LjkyMzEgMjMuMDczNyA2OC45MzE0IDIzLjI5NzNDNjguOTcxNyAyNC4zODczIDY5LjIwODIgMjUuMjIxNiA2OS4zNTA2IDI2LjIyNThDNjkuNjA4NCAyOC4zNzUyIDY5LjgyNDUgMzAuMTU2OSA2OS42OTEyIDMxLjgxM0M2OS41NjE1IDMyLjQzNzUgNjkuMTAzNyAzMy4wMDg2IDY4LjY5NTYgMzMuNDA1Nkw2OC42MjM1IDM0LjcwODZDNjYuNzgzOSAzNC44NjE3IDY0LjkzMTkgMzUuMTQyMSA2My4wODIxIDM1LjU2NDFDNTUuMTIyNiAzNy4zNzk4IDQ4LjI2OTUgNDEuNDk5MSA0My4wNTIgNDcuMDYwOUM0Mi43MTM0IDQ2LjgyODggNDIuMTIxMSA0Ni40MDE5IDQxLjk0NSA0Ni4yNzEyQzQxLjM5NzcgNDYuMzQ1NCA0MC44NDQ1IDQ2LjUxNTEgNDAuMTI0MSA0Ni4wOTM1QzM4Ljc1MjIgNDUuMTY1NyAzNy41MDI4IDQzLjg4NTEgMzUuOTkxIDQyLjM0MjRDMzUuMjk4MiA0MS42MDQ0IDM0Ljc5NjYgNDAuOTAxOCAzMy45NzM1IDQwLjE5MDRDMzMuNzg2NiA0MC4wMjg5IDMzLjUwMTQgMzkuODEwNCAzMy4yOTIzIDM5LjY0NDJDMzIuNjQ4OSAzOS4xMjg4IDMxLjg5IDM4Ljg2IDMxLjE1NyAzOC44MzQ4QzMwLjIxNDcgMzguODAyNCAyOS4zMDc1IDM5LjE3MjUgMjguNzEzOCAzOS45MjA2QzI3LjY1ODQgNDEuMjUwNiAyNy45OTYzIDQzLjI4MzMgMjkuNDY3MSA0NC40NjE0QzI5LjQ4MiA0NC40NzM0IDI5LjQ5NzkgNDQuNDgyNyAyOS41MTI5IDQ0LjQ5NDNDMjkuNzE1IDQ0LjY1ODkgMjkuOTYyNSA0NC44Njk4IDMwLjE0ODMgNDUuMDA3NkMzMS4wMjE3IDQ1LjY1NTUgMzEuODE5NSA0NS45ODcyIDMyLjY4OTcgNDYuNTAxNUMzNC41MjMxIDQ3LjYzOTEgMzYuMDQzIDQ4LjU4MjMgMzcuMjQ4NiA0OS43MTk2QzM3LjcxOTQgNTAuMjIzNyAzNy44MDE2IDUxLjExMjIgMzcuODY0MyA1MS40OTY0TDM4Ljg0NjggNTIuMzc4MkMzMy41ODcyIDYwLjMzMDggMzEuMTUzIDcwLjE1MzkgMzIuNTkxNSA4MC4xNjI3TDMxLjMwNzcgODAuNTM3OEMzMC45NjkzIDgwLjk3NjggMzAuNDkxMiA4MS42Njc2IDI5Ljk5MTEgODEuODczOEMyOC40MTM4IDgyLjM3MjkgMjYuNjM4NyA4Mi41NTYyIDI0LjQ5NTYgODIuNzgxOUMyMy40ODk0IDgyLjg2NiAyMi42MjEzIDgyLjgxNTggMjEuNTU0NiA4My4wMTg4QzIxLjMxOTggODMuMDYzNSAyMC45OTI3IDgzLjE0OTEgMjAuNzM1OCA4My4yMDk3QzIwLjcyNjkgODMuMjExNiAyMC43MTg2IDgzLjIxNDIgMjAuNzA5NiA4My4yMTYyQzIwLjY5NTYgODMuMjE5NSAyMC42NzcyIDgzLjIyNjMgMjAuNjYzOCA4My4yMjk0QzE4Ljg1NyA4My42NjggMTcuNjk2MyA4NS4zMzY1IDE4LjA2OTkgODYuOTgwNUMxOC40NDM3IDg4LjYyNDggMjAuMjA4NiA4OS42MjQ4IDIyLjAyNjIgODkuMjMxMkMyMi4wMzkzIDg5LjIyODIgMjIuMDU4NCA4OS4yMjc3IDIyLjA3MiA4OS4yMjQ2QzIyLjA5MjYgODkuMjE5OSAyMi4xMTA2IDg5LjIwOTkgMjIuMTMxIDg5LjIwNDlDMjIuMzg0NCA4OS4xNDkgMjIuNzAxOSA4OS4wODY4IDIyLjkyMzYgODkuMDI3MkMyMy45NzIzIDg4Ljc0NTEgMjQuNzMxOCA4OC4zMzA2IDI1LjY3NDYgODcuOTY3N0MyNy43MDI5IDg3LjIzNjggMjkuMzgyOCA4Ni42MjYyIDMxLjAxOTUgODYuMzg4M0MzMS43MDMgODYuMzM0NSAzMi40MjMyIDg2LjgxMiAzMi43ODE0IDg3LjAxMzRMMzQuMTE3NyA4Ni43ODMxQzM3LjE5MjYgOTYuMzYxMyA0My42MzY2IDEwNC4xMDMgNTEuNzk2MyAxMDguOTYxTDUxLjIzOTYgMTEwLjMwM0M1MS40NDAzIDExMC44MjQgNTEuNjYxNiAxMTEuNTMgNTEuNTEyMSAxMTIuMDQ1QzUwLjkxNzEgMTEzLjU5NSA0OS44OTggMTE1LjIzMSA0OC43Mzc0IDExNy4wNTVDNDguMTc1NSAxMTcuODk4IDQ3LjYwMDQgMTE4LjU1MiA0Ny4wOTM0IDExOS41MTZDNDYuOTcyIDExOS43NDcgNDYuODE3NSAxMjAuMTAyIDQ2LjcwMDQgMTIwLjM0NkM0NS45MTI1IDEyMi4wMzkgNDYuNDkwNCAxMjMuOTkgNDguMDAzOCAxMjQuNzIyQzQ5LjUyNjggMTI1LjQ1OSA1MS40MTcxIDEyNC42ODIgNTIuMjM1MiAxMjIuOTg1QzUyLjIzNjQgMTIyLjk4MiA1Mi4yNDA2IDEyMi45OCA1Mi4yNDE3IDEyMi45NzhDNTIuMjQyNiAxMjIuOTc2IDUyLjI0MDkgMTIyLjk3MyA1Mi4yNDE3IDEyMi45NzFDNTIuMzU4MiAxMjIuNzMxIDUyLjUyMzMgMTIyLjQxNSA1Mi42MjE2IDEyMi4xODhDNTMuMDU2IDEyMS4xODkgNTMuMjAwNSAxMjAuMzMyIDUzLjUwNTkgMTE5LjM2NUM1NC4zMTcgMTE3LjMxOCA1NC43NjI2IDExNS4xNyA1NS44NzkxIDExMy44MzJDNTYuMTg0OSAxMTMuNDY2IDU2LjY4MzMgMTEzLjMyNSA1Ny4yMDAxIDExMy4xODZMNTcuODk0NCAxMTEuOTIyQzY1LjAwOCAxMTQuNjY1IDcyLjk3MDUgMTE1LjQwMiA4MC45MjQ1IDExMy41ODdDODIuNzM5MSAxMTMuMTczIDg0LjQ5MDggMTEyLjYzNyA4Ni4xODQzIDExMS45OTRDODYuMzc5NCAxMTIuMzQyIDg2Ljc0MiAxMTMuMDExIDg2LjgzOTMgMTEzLjE3OUM4Ny4zNjQ0IDExMy4zNTEgODcuOTM3NyAxMTMuNDM5IDg4LjQwNDcgMTE0LjEzM0M4OS4yNDAxIDExNS41NjcgODkuODExNCAxMTcuMjYzIDkwLjUwNzMgMTE5LjMxMkM5MC44MTI4IDEyMC4yNzkgOTAuOTYzOCAxMjEuMTM2IDkxLjM5ODEgMTIyLjEzNkM5MS40OTcxIDEyMi4zNjMgOTEuNjYxNCAxMjIuNjg0IDkxLjc3OCAxMjIuOTI1QzkyLjU5NDQgMTI0LjYyOCA5NC40OTA3IDEyNS40MDcgOTYuMDE1OSAxMjQuNjY5Qzk3LjUyOTIgMTIzLjkzNyA5OC4xMDc3IDEyMS45ODYgOTcuMzE5NCAxMjAuMjkzQzk3LjIwMjMgMTIwLjA0OSA5Ny4wNDEyIDExOS42OTUgOTYuOTE5OCAxMTkuNDY0Qzk2LjQxMjcgMTE4LjQ5OSA5NS44Mzc3IDExNy44NTIgOTUuMjc1OCAxMTcuMDA5Qzk0LjExNTIgMTE1LjE4NSA5My4xNTI2IDExMy42NyA5Mi41NTc1IDExMi4xMkM5Mi4zMDg3IDExMS4zMiA5Mi41OTk1IDExMC44MjMgOTIuNzkzMyAxMTAuMzAzQzkyLjY3NzIgMTEwLjE3IDkyLjQyODggMTA5LjQxNCA5Mi4yODI0IDEwOS4wNTlDMTAwLjc2MiAxMDQuMDI5IDEwNy4wMTcgOTUuOTk4NSAxMDkuOTU1IDg2LjcyMzlDMTEwLjM1MSA4Ni43ODY1IDExMS4wNDEgODYuOTA5MSAxMTEuMjY1IDg2Ljk1NDJDMTExLjcyNiA4Ni42NDg3IDExMi4xNDkgODYuMjUwMSAxMTIuOTgxIDg2LjMxNTlDMTE0LjYxNyA4Ni41NTM3IDExNi4yOTcgODcuMTY0NSAxMTguMzI2IDg3Ljg5NTNDMTE5LjI2OCA4OC4yNTgxIDEyMC4wMjggODguNjc5MyAxMjEuMDc3IDg4Ljk2MTRDMTIxLjI5OCA4OS4wMjEgMTIxLjYxNiA4OS4wNzY2IDEyMS44NjkgODkuMTMyNUMxMjEuODg5IDg5LjEzNzUgMTIxLjkwOCA4OS4xNDc1IDEyMS45MjggODkuMTUyMkMxMjEuOTQyIDg5LjE1NTMgMTIxLjk2MSA4OS4xNTU4IDEyMS45NzQgODkuMTU4OEMxMjMuNzkyIDg5LjU1MiAxMjUuNTU3IDg4LjU1MjYgMTI1LjkzIDg2LjkwODFDMTI2LjMwMyA4NS4yNjQxIDEyNS4xNDMgODMuNTk1MiAxMjMuMzM2IDgzLjE1N0MxMjMuMDc0IDgzLjA5NyAxMjIuNzAxIDgyLjk5NSAxMjIuNDQ2IDgyLjk0NjVDMTIxLjM3OSA4Mi43NDM1IDEyMC41MTEgODIuNzkzNSAxMTkuNTA1IDgyLjcwOTVDMTE3LjM2MSA4Mi40ODM5IDExNS41ODYgODIuMzAwNCAxMTQuMDA5IDgxLjgwMTRDMTEzLjM2NiA4MS41NTA3IDExMi45MDggODAuNzgxOSAxMTIuNjg2IDgwLjQ2NTVMMTExLjQ0OCA4MC4xMDM1QzExMi4wOSA3NS40MzggMTExLjkxNyA3MC41ODI1IDExMC44MDYgNjUuNzI0M0MxMDkuNjg1IDYwLjgyMDggMTA3LjcwNCA1Ni4zMzYxIDEwNS4wNjIgNTIuMzg0OEMxMDUuMzc5IDUyLjA5NDggMTA1Ljk3OSA1MS41NjEyIDEwNi4xNDkgNTEuNDA0M0MxMDYuMTk5IDUwLjg1MTcgMTA2LjE1NiA1MC4yNzIyIDEwNi43MjUgNDkuNjYwM0MxMDcuOTMxIDQ4LjUyMyAxMDkuNDUxIDQ3LjU3OTkgMTExLjI4NCA0Ni40NDIzQzExMi4xNTQgNDUuOTI3OSAxMTIuOTU5IDQ1LjU5NjQgMTEzLjgzMiA0NC45NDg0QzExNC4wMyA0NC44MDE5IDExNC4yOTkgNDQuNTY5OSAxMTQuNTA3IDQ0LjQwMjJDMTE1Ljk3NyA0My4yMjM3IDExNi4zMTYgNDEuMTkxMSAxMTUuMjYgMzkuODYxNEMxMTQuMjA0IDM4LjUzMTcgMTEyLjE1OSAzOC40MDY1IDExMC42ODggMzkuNTg1QzExMC40NzkgMzkuNzUxNiAxMTAuMTk1IDM5Ljk2ODggMTEwLjAwNyA0MC4xMzEyQzEwOS4xODQgNDAuODQyNiAxMDguNjc2IDQxLjU0NTIgMTA3Ljk4MyA0Mi4yODMyQzEwNi40NzEgNDMuODI1OSAxMDUuMjIyIDQ1LjExMyAxMDMuODUgNDYuMDQwOUMxMDMuMjU1IDQ2LjM4ODUgMTAyLjM4NSA0Ni4yNjgyIDEwMS45OSA0Ni4yNDQ5TDEwMC44MjQgNDcuMDgwNkM5NC4xNzUzIDQwLjA3NjMgODUuMTIzNSAzNS41OTgyIDc1LjM3NjYgMzQuNzI4M0M3NS4zNDk0IDM0LjMxNzkgNzUuMzEzNyAzMy41NzYxIDc1LjMwNDYgMzMuMzUyOUM3NC45MDU2IDMyLjk2OTMgNzQuNDIzNSAzMi42NDE4IDc0LjMwMjQgMzEuODEzQzc0LjE2OTEgMzAuMTU2OSA3NC4zOTE3IDI4LjM3NTIgNzQuNjQ5NiAyNi4yMjU4Qzc0Ljc5MTkgMjUuMjIxNiA3NS4wMjg0IDI0LjM4NzMgNzUuMDY4OCAyMy4yOTczQzc1LjA3OCAyMy4wNDk1IDc1LjA2MzIgMjIuNjkgNzUuMDYyMiAyMi40MjIxQzc1LjA2MiAyMC41MzIyIDczLjY4OTggMTguOTk5OCA3MS45OTY4IDE5Wk02OC4xNTg1IDQyLjg4ODZMNjcuMjQ4IDU5LjA0NDdMNjcuMTgyNSA1OS4wNzc2QzY3LjEyMTQgNjAuNTIyOSA2NS45Mzc1IDYxLjY3NyA2NC40ODM5IDYxLjY3N0M2My44ODg0IDYxLjY3NyA2My4zMzg4IDYxLjQ4NDkgNjIuODkyMiA2MS4xNTcxTDYyLjg2NiA2MS4xNzAzTDQ5LjY4MDcgNTEuNzc5NEM1My43MzMxIDQ3Ljc3NTkgNTguOTE2NCA0NC44MTcyIDY0Ljg5IDQzLjQ1NDZDNjUuOTgxMiA0My4yMDU2IDY3LjA3MTkgNDMuMDIwOSA2OC4xNTg1IDQyLjg4ODZaTTc1Ljg0MTcgNDIuODg4NkM4Mi44MTU5IDQzLjc1MDQgODkuMjY1NyA0Ni45MjMyIDk0LjIwODEgNTEuNzg2TDgxLjEwOCA2MS4xMTc2TDgxLjA2MjEgNjEuMDk3OUM3OS44OTk0IDYxLjk1MTIgNzguMjYxMSA2MS43Mzk0IDc3LjM1NDggNjAuNTk3OEM3Ni45ODM1IDYwLjEzMDEgNzYuNzg4NyA1OS41ODAxIDc2Ljc2NTMgNTkuMDI0OUw3Ni43NTIyIDU5LjAxODRMNzUuODQxNyA0Mi44ODg2Wk00NC44OTkxIDU3LjgxNEw1Ni45MzgyIDY4LjYzM0w1Ni45MjUxIDY4LjY5ODhDNTguMDExNyA2OS42NDc5IDU4LjE3MiA3MS4yOTQ5IDU3LjI2NTcgNzIuNDM2OEM1Ni44OTQ0IDcyLjkwNDUgNTYuMzk3NSA3My4yMTgyIDU1Ljg2MzkgNzMuMzY0N0w1NS44NTA4IDczLjQxNzNMNDAuNDE4OCA3Ny44OTIzQzM5LjYzMzQgNzAuNjc2NSA0MS4zMjYxIDYzLjY2MjEgNDQuODk5MSA1Ny44MTRaTTk5LjAwOTQgNTcuODIwNkMxMDAuNzk4IDYwLjczMzYgMTAyLjE1MyA2My45ODcxIDEwMi45NTkgNjcuNTE0M0MxMDMuNzU2IDcwLjk5OTEgMTAzLjk1NiA3NC40Nzc4IDEwMy42MjcgNzcuODM5N0w4OC4xMTY2IDczLjM1MTVMODguMTAzNSA3My4yODU3Qzg2LjcxNDUgNzIuOTA0MyA4NS44NjA5IDcxLjQ4NDggODYuMTg0MyA3MC4wNjExQzg2LjMxNjggNjkuNDc3OCA4Ni42MjQ5IDY4Ljk4NDQgODcuMDQyMyA2OC42MTk4TDg3LjAzNTggNjguNTg2OUw5OS4wMDk0IDU3LjgyMDZaTTY5LjUyNzQgNjkuNDY4OEg3NC40NTk2TDc3LjUyNTEgNzMuMzE4Nkw3Ni40MjQ3IDc4LjEyMjZMNzEuOTk2OCA4MC4yNjE0TDY3LjU1NTggNzguMTE2MUw2Ni40NTU0IDczLjMxMkw2OS41Mjc0IDY5LjQ2ODhaTTg1LjMzOTMgODIuNjQzN0M4NS41NDg5IDgyLjYzMzEgODUuNzU3NiA4Mi42NTIgODUuOTYxNiA4Mi42ODk4TDg1Ljk4NzggODIuNjU2OUwxMDEuOTUgODUuMzY4MkM5OS42MTQyIDkxLjk2MjQgOTUuMTQ0IDk3LjY3NSA4OS4xNzExIDEwMS40OThMODIuOTc0NyA4Ni40NjA2TDgyLjk5NDQgODYuNDM0M0M4Mi40MjUyIDg1LjEwNTUgODIuOTk0OCA4My41NDcyIDg0LjMwNDQgODIuOTEzNUM4NC42Mzk3IDgyLjc1MTMgODQuOTkgODIuNjYxNCA4NS4zMzkzIDgyLjY0MzdaTTU4LjUyOTggODIuNzA5NUM1OS43NDggODIuNzI2NyA2MC44NDA2IDgzLjU3NjEgNjEuMTIzNyA4NC44MjJDNjEuMjU2MiA4NS40MDUyIDYxLjE5MTcgODUuOTgzMSA2MC45NzMgODYuNDkzNUw2MS4wMTg5IDg2LjU1MjdMNTQuODg4IDEwMS40MzlDNDkuMTU1OSA5Ny43NDMyIDQ0LjU5MDQgOTIuMjA5OSA0Mi4xNDgxIDg1LjQyMDhMNTcuOTczMSA4Mi43MjI3TDU3Ljk5OTMgODIuNzU1NkM1OC4xNzYzIDgyLjcyMjkgNTguMzU1OCA4Mi43MDcxIDU4LjUyOTggODIuNzA5NVpNNzEuODk4NiA4OS4yMzEyQzcyLjMyMjkgODkuMjE1NSA3Mi43NTM0IDg5LjMwMyA3My4xNjI3IDg5LjUwMUM3My42OTkyIDg5Ljc2MDYgNzQuMTEzNiA5MC4xNjkyIDc0LjM3NDUgOTAuNjU5Mkg3NC40MzM0TDgyLjIzNDYgMTA0LjgyMUM4MS4yMjIxIDEwNS4xNjIgODAuMTgxMyAxMDUuNDU0IDc5LjExNjcgMTA1LjY5N0M3My4xNTA1IDEwNy4wNTggNjcuMjAzMiAxMDYuNjQ1IDYxLjgxOCAxMDQuODAyTDY5LjU5OTUgOTAuNjY1OEg2OS42MTI2QzcwLjA3OTUgODkuNzg4OCA3MC45NjUgODkuMjY1NiA3MS44OTg2IDg5LjIzMTJaIiBmaWxsPSJ3aGl0ZSIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIwLjI1Ii8+CjxkZWZzPgo8bGluZWFyR3JhZGllbnQgaWQ9InBhaW50MF9saW5lYXJfNjgxXzI4NDUiIHgxPSIxMCIgeTE9IjE1LjUiIHgyPSIxNDQiIHkyPSIxMzEuNSIgZ3JhZGllbnRVbml0cz0idXNlclNwYWNlT25Vc2UiPgo8c3RvcCBzdG9wLWNvbG9yPSIjNEQ4N0ZGIi8+CjxzdG9wIG9mZnNldD0iMSIgc3RvcC1jb2xvcj0iIzA1NDdEMCIvPgo8L2xpbmVhckdyYWRpZW50Pgo8L2RlZnM+Cjwvc3ZnPgo= - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "version"], ["spec", "host"], ["spec", "substrate"], ["spec", "proxmox"], ["spec", "proxmox", "allowedNodes"], ["spec", "proxmox", "dnsServers"], ["spec", "proxmox", "insecure"], ["spec", "proxmox", "ccm"], ["spec", "proxmox", "ccm", "credentialsSecretName"], ["spec", "proxmox", "csi"], ["spec", "proxmox", "csi", "enabled"], ["spec", "proxmox", "csi", "credentialsSecretName"], ["spec", "proxmox", "csi", "storageClasses"], ["spec", "proxmox", "ipv4Config"], ["spec", "proxmox", "ipv4Config", "addresses"], ["spec", "proxmox", "ipv4Config", "gateway"], ["spec", "proxmox", "ipv4Config", "prefix"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "addons", "ouroboros"], ["spec", "addons", "ouroboros", "enabled"], ["spec", "addons", "ouroboros", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "apiServer", "extraArgs"], ["spec", "controlPlane", "apiServer", "admissionControllers"], ["spec", "controlPlane", "apiServer", "extraVolumes"], ["spec", "controlPlane", "apiServer", "extraVolumeMounts"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "controlPlane", "kubelet"], ["spec", "controlPlane", "kubelet", "preferredAddressTypes"], ["spec", "talos"], ["spec", "talos", "version"], ["spec", "talos", "schematicID"], ["spec", "talos", "imageFactoryURL"], ["spec", "talos", "installerRepository"], ["spec", "talos", "registryMirrors"], ["spec", "oidc"], ["spec", "oidc", "mode"], ["spec", "oidc", "customConfig"], ["spec", "oidc", "customConfig", "config"], ["spec", "oidc", "customConfig", "secretRef"], ["spec", "oidc", "customConfig", "secretRef", "name"], ["spec", "oidc", "users"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "storageClass"], ["spec", "version"], ["spec", "host"], ["spec", "substrate"], ["spec", "proxmox"], ["spec", "proxmox", "allowedNodes"], ["spec", "proxmox", "dnsServers"], ["spec", "proxmox", "insecure"], ["spec", "proxmox", "ccm"], ["spec", "proxmox", "ccm", "credentialsSecretName"], ["spec", "proxmox", "csi"], ["spec", "proxmox", "csi", "enabled"], ["spec", "proxmox", "csi", "credentialsSecretName"], ["spec", "proxmox", "csi", "storageClasses"], ["spec", "proxmox", "ipv4Config"], ["spec", "proxmox", "ipv4Config", "addresses"], ["spec", "proxmox", "ipv4Config", "gateway"], ["spec", "proxmox", "ipv4Config", "prefix"], ["spec", "addons"], ["spec", "addons", "certManager"], ["spec", "addons", "certManager", "enabled"], ["spec", "addons", "certManager", "valuesOverride"], ["spec", "addons", "cilium"], ["spec", "addons", "cilium", "valuesOverride"], ["spec", "addons", "gatewayAPI"], ["spec", "addons", "gatewayAPI", "enabled"], ["spec", "addons", "ingressNginx"], ["spec", "addons", "ingressNginx", "enabled"], ["spec", "addons", "ingressNginx", "exposeMethod"], ["spec", "addons", "ingressNginx", "hosts"], ["spec", "addons", "ingressNginx", "valuesOverride"], ["spec", "addons", "gpuOperator"], ["spec", "addons", "gpuOperator", "enabled"], ["spec", "addons", "gpuOperator", "valuesOverride"], ["spec", "addons", "hami"], ["spec", "addons", "hami", "enabled"], ["spec", "addons", "hami", "valuesOverride"], ["spec", "addons", "monitoringAgents"], ["spec", "addons", "monitoringAgents", "enabled"], ["spec", "addons", "monitoringAgents", "valuesOverride"], ["spec", "addons", "verticalPodAutoscaler"], ["spec", "addons", "verticalPodAutoscaler", "valuesOverride"], ["spec", "addons", "velero"], ["spec", "addons", "velero", "enabled"], ["spec", "addons", "velero", "valuesOverride"], ["spec", "addons", "coredns"], ["spec", "addons", "coredns", "valuesOverride"], ["spec", "addons", "ouroboros"], ["spec", "addons", "ouroboros", "enabled"], ["spec", "addons", "ouroboros", "valuesOverride"], ["spec", "controlPlane"], ["spec", "controlPlane", "replicas"], ["spec", "controlPlane", "apiServer"], ["spec", "controlPlane", "apiServer", "resources"], ["spec", "controlPlane", "apiServer", "resourcesPreset"], ["spec", "controlPlane", "apiServer", "extraArgs"], ["spec", "controlPlane", "apiServer", "admissionControllers"], ["spec", "controlPlane", "apiServer", "extraVolumes"], ["spec", "controlPlane", "apiServer", "extraVolumeMounts"], ["spec", "controlPlane", "controllerManager"], ["spec", "controlPlane", "controllerManager", "resources"], ["spec", "controlPlane", "controllerManager", "resourcesPreset"], ["spec", "controlPlane", "scheduler"], ["spec", "controlPlane", "scheduler", "resources"], ["spec", "controlPlane", "scheduler", "resourcesPreset"], ["spec", "controlPlane", "konnectivity"], ["spec", "controlPlane", "konnectivity", "server"], ["spec", "controlPlane", "konnectivity", "server", "resources"], ["spec", "controlPlane", "konnectivity", "server", "resourcesPreset"], ["spec", "controlPlane", "kubelet"], ["spec", "controlPlane", "kubelet", "preferredAddressTypes"], ["spec", "oidc"], ["spec", "oidc", "mode"], ["spec", "oidc", "customConfig"], ["spec", "oidc", "customConfig", "config"], ["spec", "oidc", "customConfig", "secretRef"], ["spec", "oidc", "customConfig", "secretRef", "name"], ["spec", "oidc", "users"]] secrets: exclude: [] include: diff --git a/packages/system/linstor/Makefile b/packages/system/linstor/Makefile index 74e3f5aef..e58fc8578 100644 --- a/packages/system/linstor/Makefile +++ b/packages/system/linstor/Makefile @@ -4,11 +4,11 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/common-envs.mk include ../../../hack/package.mk -LINSTOR_VERSION ?= 1.33.3 +LINSTOR_VERSION ?= 1.35.3 # The golden-image clone guard in packages/apps/vm-disk/templates/dv.yaml # reads allowRemoteVolumeAccess and layerList the way this release does and # has to be re-checked against the new one. -LINSTOR_CSI_VERSION ?= v1.11.2 +LINSTOR_CSI_VERSION ?= v1.13.2 test: helm unittest . diff --git a/packages/system/linstor/images/linstor-csi/Dockerfile b/packages/system/linstor/images/linstor-csi/Dockerfile index 50e755090..3ff1d154d 100644 --- a/packages/system/linstor/images/linstor-csi/Dockerfile +++ b/packages/system/linstor/images/linstor-csi/Dockerfile @@ -1,8 +1,8 @@ FROM --platform=$BUILDPLATFORM golang:1.27@sha256:3680233e3204827fbdc66088528ae6d4b3d034f51d03a99d454f6de034888244 AS builder ARG GOPROXY -ARG VERSION=v1.11.2 -ARG LINSTOR_WAIT_UNTIL_VERSION=v0.3.1 +ARG VERSION=v1.13.2 +ARG LINSTOR_WAIT_UNTIL_VERSION=v1.0.1 ARG TARGETARCH ARG TARGETOS @@ -15,13 +15,6 @@ RUN git apply /patches/*.diff RUN for attempt in 1 2 3; do go mod download && break; [ "$attempt" = 3 ] && exit 1; sleep $((attempt * 10)); done -# Upstream pins grpc v1.81.0, which uses http2.TrailerPrefix; on Go 1.27 -# golang.org/x/net builds http2 on top of net/http and no longer defines it. -# v1.83.2 also carries the fix for GO-2026-6443, a server panic reachable -# through the CSI gRPC endpoint. -RUN go get google.golang.org/grpc@v1.83.2 && \ - go mod tidy - # go build never compiles a _test.go, so nothing else runs the tests of the # packages the patches touch. Each package is tested from its own directory: # that finds the module it belongs to, and an empty list runs nothing. diff --git a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff index 0e2726708..29c209444 100644 --- a/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff +++ b/packages/system/linstor/images/linstor-csi/patches/001-relocate-after-clone-restore.diff @@ -1,38 +1,37 @@ diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go -index ac4651d..46d565d 100644 --- a/pkg/client/linstor.go +++ b/pkg/client/linstor.go -@@ -462,6 +462,14 @@ func (s *Linstor) Clone(ctx context.Context, vol, src *volume.Info, params *volu +@@ -498,6 +498,14 @@ return err } + if params.RelocateAfterClone { + logger.Debug("relocate resources to optimal nodes") + -+ if err := s.relocateResources(ctx, vol.ID, rGroup.Name); err != nil { ++ if err := s.relocateResources(ctx, vol.ResourceName, rGroup.Name); err != nil { + logger.WithError(err).Warn("resource relocation failed, volume is still usable") + } + } + logger.Debug("reconcile extra properties") - err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties}) -@@ -1297,6 +1305,14 @@ func (s *Linstor) VolFromSnap(ctx context.Context, snap *volume.Snapshot, vol *v + err = s.client.ResourceDefinitions.Modify(ctx, vol.ResourceName, lapi.GenericPropsModify{OverrideProps: vol.Properties}) +@@ -1614,6 +1622,14 @@ return err } + if snapParams != nil && snapParams.RelocateAfterRestore { + logger.Debug("relocate resources to optimal nodes") + -+ if err := s.relocateResources(ctx, vol.ID, rGroup.Name); err != nil { ++ if err := s.relocateResources(ctx, vol.ResourceName, rGroup.Name); err != nil { + logger.WithError(err).Warn("resource relocation failed, volume is still usable") + } + } + logger.Debug("reconcile extra properties") - err = s.client.ResourceDefinitions.Modify(ctx, vol.ID, lapi.GenericPropsModify{OverrideProps: vol.Properties}) -@@ -1487,9 +1503,8 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu + err = s.client.ResourceDefinitions.Modify(ctx, vol.ResourceName, lapi.GenericPropsModify{OverrideProps: vol.Properties}) +@@ -1818,9 +1834,8 @@ return fmt.Errorf("snapshot '%s' not deployed on any node", snap.Name) } @@ -44,7 +43,7 @@ index ac4651d..46d565d 100644 for _, snapNode := range snap.Nodes { if err := s.NodeAvailable(ctx, snapNode); err != nil { logger.WithField("selected node candidate", snapNode).WithError(err).Debug("node is not available") -@@ -1497,13 +1512,23 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu +@@ -1828,15 +1843,25 @@ } if slices.Contains(preferredNodes, snapNode) { @@ -58,8 +57,8 @@ index ac4651d..46d565d 100644 } + + available = append(available, snapNode) -+ } -+ + } + + // Pick a random node from preferred candidates, falling back to any available node. + // Randomization distributes restore load across snapshot nodes, preventing all + // clones of the same source from concentrating on a single node. @@ -71,13 +70,20 @@ index ac4651d..46d565d 100644 + var selectedNode string + if len(candidates) > 0 { + selectedNode = candidates[rand.Intn(len(candidates))] - } - ++ } ++ if selectedNode == "" { -@@ -1696,6 +1721,114 @@ func (s *Linstor) reconcileResourcePlacement(ctx context.Context, vol *volume.In - return nil - } - + return fmt.Errorf("no node [%s] available for snapshot '%s'", strings.Join(snap.Nodes, ","), snap.Name) + } +@@ -2030,6 +2055,114 @@ + err = volumeScheduler.Create(ctx, vol.ResourceName, params, topologies) + if err != nil { + return err ++ } ++ ++ return nil ++} ++ +const propertyRelocationTriggered = "Aux/csi-relocation-triggered" + +// relocateResources migrates replicas from their current nodes to the nodes that @@ -181,19 +187,13 @@ index ac4651d..46d565d 100644 + logger.WithError(err).Warn("migrate-disk failed, skipping this pair") + continue + } -+ } -+ -+ return nil -+} -+ - // FindSnapsByID searches the snapshot in the backend - func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) { - snapshotId, err := volume.ParseSnapshotId(id) + } + + return nil diff --git a/pkg/volume/parameter.go b/pkg/volume/parameter.go -index 39acd95..54d1dfb 100644 --- a/pkg/volume/parameter.go +++ b/pkg/volume/parameter.go -@@ -50,6 +50,7 @@ const ( +@@ -51,6 +51,7 @@ nfsservicename nfssquash nfsrecoveryvolumesize @@ -201,24 +201,24 @@ index 39acd95..54d1dfb 100644 ) // Parameters configuration for linstor volumes. -@@ -118,6 +119,8 @@ type Parameters struct { - // NfsRecoveryVolumeSize sets the volume size (in bytes) of the recovery volume used by the NFS server. - // Defaults to 300MiB. - NfsRecoveryVolumeBytes int64 +@@ -122,6 +123,8 @@ + // IOQoSLimits holds the IO QoS limits parsed from the qos.linbit.com/* parameters. They are stored on + // the LINSTOR resource definition and passed to the nri-volume-qos plugin via VolumeAttachment metadata. + IOQoSLimits meta.Limits + // RelocateAfterClone triggers asynchronous relocation of replicas to optimal nodes after cloning. + RelocateAfterClone bool } const DefaultDisklessStoragePoolName = "DfltDisklessStorPool" -@@ -140,6 +143,7 @@ func NewParameters(params map[string]string, topologyPrefix string) (Parameters, +@@ -144,6 +147,7 @@ NfsServiceName: "linstor-csi-nfs", NfsSquash: "no_root_squash", NfsRecoveryVolumeBytes: 300 * 1024 * 1024, + RelocateAfterClone: true, } - for k, v := range params { -@@ -287,6 +291,13 @@ func NewParameters(params map[string]string, topologyPrefix string) (Parameters, + limits, err := meta.ParseLimits(params) +@@ -298,6 +302,13 @@ } p.NfsRecoveryVolumeBytes = s.Value() @@ -233,10 +233,9 @@ index 39acd95..54d1dfb 100644 } diff --git a/pkg/volume/paramkey_enumer.go b/pkg/volume/paramkey_enumer.go -index 5474963..2eb8a7c 100644 --- a/pkg/volume/paramkey_enumer.go +++ b/pkg/volume/paramkey_enumer.go -@@ -7,11 +7,11 @@ import ( +@@ -7,11 +7,11 @@ "strings" ) @@ -251,7 +250,7 @@ index 5474963..2eb8a7c 100644 func (i paramKey) String() string { if i < 0 || i >= paramKey(len(_paramKeyIndex)-1) { -@@ -50,9 +50,10 @@ func _paramKeyNoOp() { +@@ -50,9 +50,10 @@ _ = x[nfsservicename-(23)] _ = x[nfssquash-(24)] _ = x[nfsrecoveryvolumesize-(25)] @@ -262,17 +261,24 @@ index 5474963..2eb8a7c 100644 +var _paramKeyValues = []paramKey{allowremotevolumeaccess, autoplace, clientlist, disklessonremaining, disklessstoragepool, donotplacewithregex, encryption, fsopts, layerlist, mountopts, nodelist, placementcount, placementpolicy, replicasondifferent, replicasonsame, sizekib, storagepool, postmountxfsopts, resourcegroup, usepvcname, overprovision, xreplicasondifferent, nfsconfigtemplate, nfsservicename, nfssquash, nfsrecoveryvolumesize, relocateafterclone} var _paramKeyNameToValueMap = map[string]paramKey{ - _paramKeyName[0:23]: allowremotevolumeaccess, -@@ -107,6 +108,8 @@ var _paramKeyNameToValueMap = map[string]paramKey{ + _paramKeyName[0:23]: allowremotevolumeaccess, +@@ -81,6 +82,7 @@ + _paramKeyName[310:324]: nfsservicename, + _paramKeyName[324:333]: nfssquash, + _paramKeyName[333:354]: nfsrecoveryvolumesize, ++ _paramKeyName[354:372]: relocateafterclone, + } + + var _paramKeyLowerNameToValueMap = map[string]paramKey{ +@@ -110,6 +112,7 @@ + _paramKeyLowerName[310:324]: nfsservicename, _paramKeyLowerName[324:333]: nfssquash, - _paramKeyName[333:354]: nfsrecoveryvolumesize, _paramKeyLowerName[333:354]: nfsrecoveryvolumesize, -+ _paramKeyName[354:372]: relocateafterclone, + _paramKeyLowerName[354:372]: relocateafterclone, } var _paramKeyNames = []string{ -@@ -136,6 +139,7 @@ var _paramKeyNames = []string{ +@@ -139,6 +142,7 @@ _paramKeyName[310:324], _paramKeyName[324:333], _paramKeyName[333:354], @@ -281,18 +287,17 @@ index 5474963..2eb8a7c 100644 // paramKeyString retrieves an enum value from the enum constants string name. diff --git a/pkg/volume/snapshot_params.go b/pkg/volume/snapshot_params.go -index d167cb8..50b70fb 100644 --- a/pkg/volume/snapshot_params.go +++ b/pkg/volume/snapshot_params.go -@@ -35,6 +35,7 @@ type SnapshotParameters struct { - LinstorTargetUrl string `json:"linstor-target-url,omitempty"` - LinstorTargetClusterID string `json:"linstor-target-cluster-id,omitempty"` - LinstorTargetStoragePool string `json:"linstor-target-storage-pool,omitempty"` -+ RelocateAfterRestore bool `json:"relocate-after-restore,omitempty"` +@@ -38,6 +38,7 @@ + LinstorTargetUrl string `json:"linstor-target-url,omitempty"` + LinstorTargetClusterID string `json:"linstor-target-cluster-id,omitempty"` + LinstorTargetStoragePool string `json:"linstor-target-storage-pool,omitempty"` ++ RelocateAfterRestore bool `json:"relocate-after-restore,omitempty"` } func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParameters, error) { -@@ -91,6 +92,13 @@ func NewSnapshotParameters(params, secrets map[string]string) (*SnapshotParamete +@@ -116,6 +117,13 @@ p.LinstorTargetClusterID = v case "/linstor-target-storage-pool": p.LinstorTargetStoragePool = v diff --git a/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff b/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff deleted file mode 100644 index 279c636b1..000000000 --- a/packages/system/linstor/images/linstor-csi/patches/002-protocol-c-override-for-dual-attach.diff +++ /dev/null @@ -1,132 +0,0 @@ -diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go -index ac4651d..9d75c79 100644 ---- a/pkg/client/linstor.go -+++ b/pkg/client/linstor.go -@@ -653,11 +653,35 @@ func (s *Linstor) Attach(ctx context.Context, volId, node string, rwxBlock bool) - } - - if otherResInUse > 0 && rwxBlock { -- rdPropsModify := lapi.GenericPropsModify{OverrideProps: map[string]string{ -+ rdProps := map[string]string{ - linstor.PropertyAllowTwoPrimaries: "yes", -- }} -+ } - -- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify) -+ // DRBD requires Protocol C whenever allow-two-primaries is enabled. -+ // If the resource is configured with Protocol A or B (typically -+ // through its resource-group), drbdadm adjust on the satellites -+ // fails with "Protocol C required" (errno 139) and live migration -+ // cannot proceed. Override Protocol to C on the resource-definition -+ // itself so it applies to every connection (including diskless -+ // peers, e.g. a TieBreaker). The marker lets Detach revert exactly -+ // the override we installed without disturbing operator-set props. -+ proto, protoErr := s.getEffectiveDrbdProtocol(ctx, volId) -+ if protoErr != nil { -+ s.log.WithError(protoErr).WithField("volume", volId). -+ Warn("failed to determine effective DRBD protocol; skipping Protocol=C override for dual-attach") -+ } else if proto == "A" || proto == "B" { -+ s.log.WithFields(logrus.Fields{ -+ "volume": volId, -+ "protocol": proto, -+ }).Info("installing Protocol=C override on resource-definition for dual-attach") -+ -+ rdProps[linstor.PropertyDrbdNetProtocol] = "C" -+ rdProps[linstor.PropertyCsiProtocolOverride] = "yes" -+ } -+ -+ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{ -+ OverrideProps: rdProps, -+ }) - if err != nil { - return "", err - } -@@ -774,11 +798,26 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error { - } - - if resInUse == 1 { -- rdPropsModify := lapi.GenericPropsModify{DeleteProps: []string{ -- linstor.PropertyAllowTwoPrimaries, -- }} -+ deleteProps := []string{linstor.PropertyAllowTwoPrimaries} -+ -+ // Drop the Protocol=C override only if Attach installed it (gated by -+ // our marker), so an operator-set Protocol property on the -+ // resource-definition is preserved. -+ rd, getErr := s.client.ResourceDefinitions.Get(ctx, volId) -+ if getErr != nil { -+ log.WithError(getErr).Warn("failed to fetch resource-definition props; skipping Protocol=C override removal") -+ } else if rd.Props[linstor.PropertyCsiProtocolOverride] != "" { -+ log.Info("removing Protocol=C override from resource-definition") - -- err = s.client.ResourceDefinitions.Modify(ctx, volId, rdPropsModify) -+ deleteProps = append(deleteProps, -+ linstor.PropertyDrbdNetProtocol, -+ linstor.PropertyCsiProtocolOverride, -+ ) -+ } -+ -+ err = s.client.ResourceDefinitions.Modify(ctx, volId, lapi.GenericPropsModify{ -+ DeleteProps: deleteProps, -+ }) - if err != nil { - return nil404(err) - } -@@ -805,6 +844,33 @@ func (s *Linstor) Detach(ctx context.Context, volId, node string) error { - return nil404(s.client.Resources.Delete(ctx, volId, node)) - } - -+// getEffectiveDrbdProtocol returns the DRBD network protocol ("A", "B" or "C") -+// that LINSTOR will hand to drbdadm for the given resource. Resource-definition -+// properties take precedence; otherwise the value is inherited from the -+// resource-group. An empty string is returned when neither level sets the -+// property (LINSTOR's compiled-in default is "C"). -+func (s *Linstor) getEffectiveDrbdProtocol(ctx context.Context, volId string) (string, error) { -+ rd, err := s.client.ResourceDefinitions.Get(ctx, volId) -+ if err != nil { -+ return "", err -+ } -+ -+ if v, ok := rd.Props[linstor.PropertyDrbdNetProtocol]; ok { -+ return v, nil -+ } -+ -+ if rd.ResourceGroupName == "" { -+ return "", nil -+ } -+ -+ rg, err := s.client.ResourceGroups.Get(ctx, rd.ResourceGroupName) -+ if err != nil { -+ return "", err -+ } -+ -+ return rg.Props[linstor.PropertyDrbdNetProtocol], nil -+} -+ - // CapacityBytes returns the amount of free space in the storage pool specified by the params and topology. - func (s *Linstor) CapacityBytes(ctx context.Context, storagePools []string, overProvision *float64, segments map[string]string) (int64, error) { - log := s.log.WithField("storage-pools", storagePools).WithField("segments", segments) -diff --git a/pkg/linstor/const.go b/pkg/linstor/const.go -index 9a5f79c..c8bc9c3 100644 ---- a/pkg/linstor/const.go -+++ b/pkg/linstor/const.go -@@ -44,6 +44,19 @@ const ( - // PropertyAllowTwoPrimaries is DRBD option to allow second primary. Mainly used for live-migration. - PropertyAllowTwoPrimaries = lc.NamespcDrbdNetOptions + "/allow-two-primaries" - -+ // PropertyDrbdNetProtocol is the DRBD network replication protocol option -+ // (A = async, B = semi-sync, C = sync). DRBD requires Protocol C whenever -+ // allow-two-primaries is enabled, otherwise drbdadm adjust fails with -+ // "Protocol C required" (errno 139). -+ PropertyDrbdNetProtocol = lc.NamespcDrbdNetOptions + "/protocol" -+ -+ // PropertyCsiProtocolOverride marks a resource-connection where this driver -+ // has installed a temporary Protocol=C override to make a Protocol-A/B -+ // resource compatible with allow-two-primaries during live migration. The -+ // marker lets us safely remove only the overrides we set, without touching -+ // connection-level overrides installed by the operator. -+ PropertyCsiProtocolOverride = lc.NamespcAuxiliary + "/csi-protocol-override" -+ - // CreatedForTemporaryDisklessAttach marks a resource as temporary, i.e. it should be removed after it is no longer - // needed. - CreatedForTemporaryDisklessAttach = "temporary-diskless-attach" diff --git a/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff b/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff new file mode 100644 index 000000000..7f96ed5f9 --- /dev/null +++ b/packages/system/linstor/images/linstor-csi/patches/003-snapshot-restore-node-selection.diff @@ -0,0 +1,259 @@ +diff --git a/cmd/linstor-csi/linstor-csi.go b/cmd/linstor-csi/linstor-csi.go +index e260d7b..2062787 100644 +--- a/cmd/linstor-csi/linstor-csi.go ++++ b/cmd/linstor-csi/linstor-csi.go +@@ -148,6 +148,7 @@ func main() { + client.LogOut(logOut), + client.PropertyNamespace(*propNs), + client.LabelBySP(*labelBySP), ++ client.ConfigureKubernetesIfAvailable(), + ) + if err != nil { + log.Fatal(err) +diff --git a/pkg/client/linstor.go b/pkg/client/linstor.go +index 5b52dd4..82f10d0 100644 +--- a/pkg/client/linstor.go ++++ b/pkg/client/linstor.go +@@ -46,6 +46,8 @@ import ( + "github.com/pborman/uuid" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ++ apierrors "k8s.io/apimachinery/pkg/api/errors" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/mount-utils" + utilexec "k8s.io/utils/exec" +@@ -73,6 +75,9 @@ type Linstor struct { + mounter mount.Interface + resizer *mount.ResizeFs + labelBySP bool ++ // nodeSchedulable reports whether new pods can be scheduled on a node. ++ // It is nil when not running in Kubernetes: every node then counts as schedulable. ++ nodeSchedulable func(ctx context.Context, node string) (bool, error) + } + + // NewLinstor returns a high-level linstor client for CSI applications to interact with +@@ -126,6 +131,34 @@ func APIClient(c *lc.HighLevelClient) func(*Linstor) error { + } + } + ++// ConfigureKubernetesIfAvailable lets the client read Kubernetes Node objects, so that a snapshot is ++// not restored on a cordoned node when another node holds it. Outside of Kubernetes it does nothing. ++func ConfigureKubernetesIfAvailable() func(*Linstor) error { ++ return func(l *Linstor) error { ++ cl, _, err := utils.KubernetesClient() ++ if err != nil { ++ // Not running in kubernetes ++ return nil ++ } ++ ++ l.nodeSchedulable = func(ctx context.Context, node string) (bool, error) { ++ n, err := cl.CoreV1().Nodes().Get(ctx, node, metav1.GetOptions{}) ++ if err != nil { ++ if apierrors.IsNotFound(err) { ++ // A LINSTOR node without a Kubernetes Node: nothing says it is cordoned. ++ return true, nil ++ } ++ ++ return true, err ++ } ++ ++ return !n.Spec.Unschedulable, nil ++ } ++ ++ return nil ++ } ++} ++ + // LogOut sets the Linstor client to write logs to the provided io.Writer + // instead of discarding logs. + func LogOut(out io.Writer) func(*Linstor) error { +@@ -1834,34 +1867,29 @@ func (s *Linstor) reconcileSnapshotResources(ctx context.Context, snapshot *volu + return fmt.Errorf("snapshot '%s' not deployed on any node", snap.Name) + } + +- // Collect available snapshot nodes, preferring those matching the topology hints. +- var preferred, available []string ++ // Collect the snapshot nodes LINSTOR can restore on, and whether Kubernetes can schedule pods there. ++ var available []string ++ schedulable := make(map[string]bool, len(snap.Nodes)) + for _, snapNode := range snap.Nodes { + if err := s.NodeAvailable(ctx, snapNode); err != nil { + logger.WithField("selected node candidate", snapNode).WithError(err).Debug("node is not available") + continue + } + +- if slices.Contains(preferredNodes, snapNode) { +- preferred = append(preferred, snapNode) +- } +- + available = append(available, snapNode) +- } ++ schedulable[snapNode] = true + +- // Pick a random node from preferred candidates, falling back to any available node. +- // Randomization distributes restore load across snapshot nodes, preventing all +- // clones of the same source from concentrating on a single node. +- candidates := preferred +- if len(candidates) == 0 { +- candidates = available +- } ++ if s.nodeSchedulable != nil { ++ ok, err := s.nodeSchedulable(ctx, snapNode) ++ if err != nil { ++ logger.WithField("selected node candidate", snapNode).WithError(err).Warn("could not check if node is cordoned, assuming it is not") ++ } + +- var selectedNode string +- if len(candidates) > 0 { +- selectedNode = candidates[rand.Intn(len(candidates))] ++ schedulable[snapNode] = ok ++ } + } + ++ selectedNode := selectRestoreNode(available, preferredNodes, func(node string) bool { return schedulable[node] }) + if selectedNode == "" { + return fmt.Errorf("no node [%s] available for snapshot '%s'", strings.Join(snap.Nodes, ","), snap.Name) + } +@@ -2168,6 +2196,43 @@ func (s *Linstor) relocateResources(ctx context.Context, volID, rgName string) e + return nil + } + ++// selectRestoreNode picks the node a snapshot is restored on, among the available nodes holding it. ++// ++// The restored volume is pinned to that node, so the consumer pod has to run there: ++// - preferredNodes[0] is the node the scheduler selected for a WaitForFirstConsumer volume. If it holds the ++// snapshot, restoring there lets the pod start without another scheduling round. ++// - Otherwise a random node is picked, which spreads restores of the same source across its snapshot nodes. ++// Cordoned nodes are skipped as long as another node holds the snapshot: a pod can never be scheduled on ++// them, so a volume restored there stays unusable until the node is uncordoned. ++// ++// Nodes in preferredNodes come first, but a schedulable node outside them wins over a cordoned one inside them. ++func selectRestoreNode(available, preferredNodes []string, schedulable func(node string) bool) string { ++ isPreferred := func(node string) bool { return slices.Contains(preferredNodes, node) } ++ both := func(node string) bool { return isPreferred(node) && schedulable(node) } ++ ++ var candidates []string ++ for _, keep := range []func(string) bool{both, schedulable, isPreferred} { ++ candidates = slices.DeleteFunc(slices.Clone(available), func(node string) bool { return !keep(node) }) ++ if len(candidates) > 0 { ++ break ++ } ++ } ++ ++ if len(candidates) == 0 { ++ candidates = available ++ } ++ ++ if len(candidates) == 0 { ++ return "" ++ } ++ ++ if len(preferredNodes) > 0 && slices.Contains(candidates, preferredNodes[0]) { ++ return preferredNodes[0] ++ } ++ ++ return candidates[rand.Intn(len(candidates))] ++} ++ + // FindSnapsByID searches the snapshot in the backend + func (s *Linstor) FindSnapsByID(ctx context.Context, id string) ([]*volume.Snapshot, error) { + snapshotId, err := volume.ParseSnapshotId(id) +diff --git a/pkg/client/linstor_test.go b/pkg/client/linstor_test.go +index 54c5242..8da6cad 100644 +--- a/pkg/client/linstor_test.go ++++ b/pkg/client/linstor_test.go +@@ -994,3 +994,93 @@ func TestReconcileResourceDefinitionRGConflict(t *testing.T) { + }) + } + } ++ ++func TestLinstor_reconcileSnapshotResourcesSkipsCordonedNodes(t *testing.T) { ++ // A replicated volume has a snapshot on three nodes, one of them cordoned. The scheduler selected a node ++ // without the snapshot. Restoring on the cordoned node pins the new volume to a node no pod can be scheduled ++ // on, so the restore must always go to one of the other two. ++ const ( ++ sourceRD = "pvc-source" ++ snapName = "snapshot-1" ++ targetRD = "pvc-target" ++ cordoned = "node-3" ++ ) ++ ++ snapNodes := []string{"node-1", "node-2", cordoned} ++ preferred := []string{"node-9", "node-1", "node-2", cordoned} ++ ++ restoredOn := map[string]int{} ++ ++ for range 50 { ++ res := mocks.ResourceProvider{} ++ res.On("GetAll", mock.Anything, targetRD).Return([]lapi.Resource{}, nil) ++ res.On("GetSnapshot", mock.Anything, sourceRD, snapName).Return(lapi.Snapshot{Name: snapName, Nodes: snapNodes}, nil) ++ res.On("RestoreSnapshot", mock.Anything, sourceRD, snapName, mock.Anything).Run(func(args mock.Arguments) { ++ restoredOn[args.Get(3).(lapi.SnapshotRestore).Nodes[0]]++ ++ }).Return(nil) ++ ++ nodes := mocks.NodeProvider{} ++ for _, n := range snapNodes { ++ nodes.On("Get", mock.Anything, n).Return(lapi.Node{Name: n, ConnectionStatus: "ONLINE"}, nil) ++ } ++ ++ cl := Linstor{ ++ client: &lc.HighLevelClient{Client: &lapi.Client{Resources: &res, Nodes: &nodes}}, ++ log: logrus.WithField("test", t.Name()), ++ nodeSchedulable: func(_ context.Context, node string) (bool, error) { ++ return node != cordoned, nil ++ }, ++ } ++ ++ snap := &volume.Snapshot{SnapshotId: volume.SnapshotId{SourceName: sourceRD, SnapshotName: snapName}} ++ assert.NoError(t, cl.reconcileSnapshotResources(context.Background(), snap, targetRD, preferred)) ++ } ++ ++ assert.Zero(t, restoredOn[cordoned], "restored on the cordoned node: %v", restoredOn) ++ assert.Equal(t, 50, restoredOn["node-1"]+restoredOn["node-2"]) ++} ++ ++func TestSelectRestoreNode(t *testing.T) { ++ all := func(string) bool { return true } ++ notNode3 := func(n string) bool { return n != "node-3" } ++ ++ t.Run("selected node holds the snapshot", func(t *testing.T) { ++ for range 20 { ++ assert.Equal(t, "node-2", selectRestoreNode([]string{"node-1", "node-2", "node-3"}, []string{"node-2", "node-1", "node-3"}, all)) ++ } ++ }) ++ ++ t.Run("selected node is cordoned", func(t *testing.T) { ++ for range 20 { ++ assert.NotEqual(t, "node-3", selectRestoreNode([]string{"node-1", "node-3"}, []string{"node-3", "node-1"}, notNode3)) ++ } ++ }) ++ ++ t.Run("schedulable node outside the preferred nodes wins over a cordoned preferred node", func(t *testing.T) { ++ for range 20 { ++ assert.Equal(t, "node-2", selectRestoreNode([]string{"node-3", "node-2"}, []string{"node-3"}, notNode3)) ++ } ++ }) ++ ++ t.Run("only cordoned nodes hold the snapshot", func(t *testing.T) { ++ assert.Equal(t, "node-3", selectRestoreNode([]string{"node-3"}, []string{"node-1", "node-3"}, notNode3)) ++ }) ++ ++ t.Run("snapshot nodes outside the preferred nodes are a fallback", func(t *testing.T) { ++ assert.Equal(t, "node-2", selectRestoreNode([]string{"node-1", "node-2"}, []string{"node-2"}, all)) ++ assert.Equal(t, "node-1", selectRestoreNode([]string{"node-1"}, []string{"node-2"}, all)) ++ }) ++ ++ t.Run("no available node", func(t *testing.T) { ++ assert.Empty(t, selectRestoreNode(nil, []string{"node-1"}, all)) ++ }) ++ ++ t.Run("restores are spread when the selected node lacks the snapshot", func(t *testing.T) { ++ seen := map[string]bool{} ++ for range 200 { ++ seen[selectRestoreNode([]string{"node-1", "node-2"}, []string{"node-9", "node-1", "node-2"}, all)] = true ++ } ++ ++ assert.Equal(t, map[string]bool{"node-1": true, "node-2": true}, seen) ++ }) ++} diff --git a/packages/system/linstor/images/piraeus-server/Dockerfile b/packages/system/linstor/images/piraeus-server/Dockerfile index 2c8a5500a..631303b72 100644 --- a/packages/system/linstor/images/piraeus-server/Dockerfile +++ b/packages/system/linstor/images/piraeus-server/Dockerfile @@ -1,9 +1,10 @@ -ARG DISTRO=bookworm@sha256:f37a335e82bca302e955fa39f9dfe28f1be618f016f8a2b56318e5a5111afc26 +# LINSTOR 1.35 builds and runs on Java 21, which bookworm does not ship. +ARG DISTRO=trixie@sha256:9c6e381421d0f48d1b8133e573f33c185ee16cbf505eda9c4a8a4791b25ea8a6 ARG LINSTOR_VERSION # ------------------------------------------------------------------------------ # Build linstor-server from source -FROM --platform=$BUILDPLATFORM debian:bookworm@sha256:f37a335e82bca302e955fa39f9dfe28f1be618f016f8a2b56318e5a5111afc26 AS builder +FROM --platform=$BUILDPLATFORM debian:trixie@sha256:9c6e381421d0f48d1b8133e573f33c185ee16cbf505eda9c4a8a4791b25ea8a6 AS builder ARG LINSTOR_VERSION ARG VERSION=${LINSTOR_VERSION} @@ -13,10 +14,10 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get -y upgrade \ && apt-get -y install build-essential git default-jdk-headless python3-all debhelper wget unzip && \ - wget https://services.gradle.org/distributions/gradle-8.9-bin.zip -O /tmp/gradle.zip && \ + wget https://services.gradle.org/distributions/gradle-8.14.2-bin.zip -O /tmp/gradle.zip && \ unzip -d /opt /tmp/gradle.zip && \ rm /tmp/gradle.zip && \ - ln -s /opt/gradle-8.9/bin/gradle /usr/local/bin/gradle + ln -s /opt/gradle-8.14.2/bin/gradle /usr/local/bin/gradle RUN git clone https://github.com/LINBIT/linstor-server.git /linstor-server WORKDIR /linstor-server diff --git a/packages/system/linstor/images/piraeus-server/patches/README.md b/packages/system/linstor/images/piraeus-server/patches/README.md index d2ded4285..f0b80dc92 100644 --- a/packages/system/linstor/images/piraeus-server/patches/README.md +++ b/packages/system/linstor/images/piraeus-server/patches/README.md @@ -1,36 +1,22 @@ # LINSTOR Server Patches -Custom patches for piraeus-server (linstor-server) v1.33.3. +Custom patches for piraeus-server (linstor-server) v1.35.3. -- **allow-toggle-disk-retry.diff** — Backport maintainer implementation of toggle-disk retry/abort - - Source PR/comment: [#475](https://github.com/LINBIT/linstor-server/pull/475), [maintainer note](https://github.com/LINBIT/linstor-server/pull/475#issuecomment-3949630419) - - Backported from upstream commit: [`3d97f71c9`](https://github.com/LINBIT/linstor-server/commit/3d97f71c95a493588d3d521c63eac4d846935fb3) -- **fix-duplicate-tcp-ports.diff** — Preserve DRBD TCP ports during toggle-disk and avoid redundant `ensureStackDataExists()` - - Source PR/review: [#476](https://github.com/LINBIT/linstor-server/pull/476), [review suggestion](https://github.com/LINBIT/linstor-server/pull/476#discussion_r3007725079) - - Backported from commits: [`79d6375c5`](https://github.com/kvaps/linstor-server/commit/79d6375c55d6181b35a7b7f0fe8dbdfb86e126cd), [`bcc89902f`](https://github.com/kvaps/linstor-server/commit/bcc89902f4f61ac1589dd07ebb7f5aae1935370d) -- **fix-luks-header-size.diff** — Account for LUKS2 `--offset`, metadata/keyslots sizing, and device `optimal_io_size` - - Source PR/comment: [#472](https://github.com/LINBIT/linstor-server/pull/472), [maintainer note](https://github.com/LINBIT/linstor-server/pull/472#issuecomment-3949687603) - - Backported from commits: [`ccc85fbd2`](https://github.com/LINBIT/linstor-server/commit/ccc85fbd2c65f0b97c52403fa80f1efdb886ec4e), [`71b601554`](https://github.com/LINBIT/linstor-server/commit/71b601554f41bcb50cd5bd06989c5b0d3a814acd) - - Note: upstream commit [`3d0402a0c`](https://github.com/LINBIT/linstor-server/commit/3d0402a0c25f0a4b57b380321f10e89982f26e7a) is already included in `v1.33.1` - **fix-k8s-create-already-exists.diff** — Adopt the existing object when a create in the Kubernetes CRD backend returns 409 AlreadyExists. The HTTP client retries a POST after a 5xx or a dropped connection, so a create the apiserver already persisted comes back as 409; a single-object transaction has no rollback entry, the key is dropped from memory while the CR stays, and every later write of that key fails until the controller restarts. Seen as a PV stuck in `Released` because linstor-csi could not set `BalanceResourcesEnabled` on the resource definition. - Source PR: [#531](https://github.com/LINBIT/linstor-server/pull/531) - Backported from commit: [`8badf8f05`](https://github.com/lexfrei/linstor-server/commit/8badf8f0526ff3b1b391857e4a046ed46b95a79c) - **retry-adjust-after-stale-bitmap.diff** — Retry `drbdadm adjust` after detaching a stale local bitmap state - Source PR: [#491](https://github.com/LINBIT/linstor-server/pull/491) - Backported from commit: [`51ae50a84`](https://github.com/kvaps/linstor-server/commit/51ae50a84dcb98093f543b819652c750a94d96c9) -- **retry-secondary-after-mkfs.diff** — Retry `drbdadm secondary` after mkfs when DRBD reports the device as held open by an external probe (Talos block-controller, udev, multipathd, etc.). Without this retry, a transient `Device is held open by someone` aborts resource initialization, leaves the satellite in an intermediate state, and prevents the controller from receiving the final `UpToDate` event — orphan PVs in `Released` then cannot be cleaned up. - - Related upstream issue: [#268](https://github.com/LINBIT/linstor-server/issues/268) - - Related upstream issue: [drbd #74](https://github.com/LINBIT/drbd/issues/74) (same EBUSY pattern with multipathd) - -- **fix-min-io-probe-device-wait.diff** — Wait for temporary probe devices before reading pool I/O properties; retry failures and fall back to a probe when ZFS volumes have no usable device path. - - Source PR: [#528](https://github.com/LINBIT/linstor-server/pull/528), fixes [#527](https://github.com/LINBIT/linstor-server/issues/527) - - Adapted for v1.33.3; inactive LVM volumes retain the existing no-probe behavior. - + - Rebased onto v1.35.3, where `DrbdAdm` gained the `DFLT_WINDOWS_TIMEOUT_MS` constant; `DrbdadmAdjust` still receives the failed `OutputData` through `ExtCmdFailedException`, so its own retry on "Resize not allowed during resync" keeps working on top of this one. - **native-protoc.diff** — Download the protoc build for the build host's architecture instead of always `linux-x86_64`. The builder stage runs on `$BUILDPLATFORM`, so on a native arm64 builder the x86_64 protoc cannot execute and `./gradlew generateJava` fails with `Syntax error: ")" unexpected`. - Not proposed upstream yet. -- **snapshot-restore-drbd-initialized.diff** — Mark the volume definitions of a restored snapshot as DRBD-initialized, unless the restored resource definition carries `DrbdOptions/ForceInitialSyncPermanent`, which keeps the existing path that recreates metadata, as for a backup. Without the flag the initial UpToDate race is still open on a restored volume, and `Linstor/Drbd/InitialUptoDateOn`, copied from the source volume through the snapshot, names whichever node won it for the source. When linstor-csi autoplaces a replica on that node, the empty new volume wins the race, gets mkfs and goes StandAlone against the restored replica, so a pod attached there sees an empty filesystem. With the flag set, nothing wins the race, and an autoplaced replica resyncs from the restored one like any replica added to an existing volume. - - Backported from upstream commit: [`9d8cdf511`](https://github.com/LINBIT/linstor-server/commit/9d8cdf5118338fc72a34bb99799e452e031ea8f6), first released in v1.34.0; the `CHANGELOG.md` hunk is left out - - Drop when LINSTOR is upgraded to v1.34.0 or later - - On ZFS and thin pools the autoplaced replicas still skip the initial sync and rely on the source volume's day0 UUID being in the restored replica's history, upstream included; if it is not, DRBD reports unrelated data and goes StandAlone rather than syncing - - Fixes [cozystack/cozystack#4862](https://github.com/cozystack/cozystack/issues/4862) +Dropped on the move from v1.33.3, because v1.35.3 already carries them: + +- `allow-toggle-disk-retry.diff` — upstream [`3d97f71c9`](https://github.com/LINBIT/linstor-server/commit/3d97f71c95a493588d3d521c63eac4d846935fb3), v1.34.0 +- `fix-luks-header-size.diff` — upstream [`ccc85fbd2`](https://github.com/LINBIT/linstor-server/commit/ccc85fbd2c65f0b97c52403fa80f1efdb886ec4e) and [`71b601554`](https://github.com/LINBIT/linstor-server/commit/71b601554f41bcb50cd5bd06989c5b0d3a814acd), v1.34.0 +- `snapshot-restore-drbd-initialized.diff` — upstream [`9d8cdf511`](https://github.com/LINBIT/linstor-server/commit/9d8cdf5118338fc72a34bb99799e452e031ea8f6), v1.34.0 +- `fix-duplicate-tcp-ports.diff` — upstream [`521c6d211`](https://github.com/LINBIT/linstor-server/commit/521c6d21121dbbcc26cf6eda723c24a5b34889be) (from [#476](https://github.com/LINBIT/linstor-server/pull/476)), v1.34.0 +- `retry-secondary-after-mkfs.diff` — upstream [`677789c02`](https://github.com/LINBIT/linstor-server/commit/677789c028ac), v1.34.0 +- `fix-min-io-probe-device-wait.diff` — upstream [`6e5557839`](https://github.com/LINBIT/linstor-server/commit/6e5557839c911c7a1db240dcb6fc348383a74266) (from [#528](https://github.com/LINBIT/linstor-server/pull/528)), v1.35.1 diff --git a/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff b/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff deleted file mode 100644 index 952ba16c5..000000000 --- a/packages/system/linstor/images/piraeus-server/patches/allow-toggle-disk-retry.diff +++ /dev/null @@ -1,261 +0,0 @@ -diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -index d93a18014..a944cb809 100644 ---- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -+++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -@@ -111,6 +111,14 @@ import reactor.util.function.Tuple2; - @Singleton - public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionListener - { -+ private enum ToggleDiskAction -+ { -+ NOOP, -+ NORMAL, -+ ABORT, -+ RETRY -+ } -+ - private final AccessContext apiCtx; - private final ScopeRunner scopeRunner; - private final BackgroundRunner backgroundRunner; -@@ -386,69 +394,33 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - "Toggle Disk on %s/%s %s", nodeNameStr, rscNameStr, removeDisk ? "removing disk" : "adding disk"); - - Resource rsc = ctrlApiDataLoader.loadRsc(nodeName, rscName, true); -+ ResourceDefinition rscDfn = rsc.getResourceDefinition(); - -- if (hasDiskAddRequested(rsc)) -- { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.FAIL_RSC_BUSY, -- "Addition of disk to resource already requested", -- true -- )); -- } -- if (hasDiskRemoveRequested(rsc)) -- { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.FAIL_RSC_BUSY, -- "Removal of disk from resource already requested", -- true -- )); -- } -+ ToggleDiskAction action = determineToggleDiskAction(rsc, removeDisk); -+ errorReporter.logDebug("Toggle Disk action: %s", action); - -- if (!removeDisk && !ctrlVlmCrtApiHelper.isDiskless(rsc)) -+ switch (action) - { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.WARN_RSC_ALREADY_HAS_DISK, -- "Resource already has disk", -- true -- )); -- } -- if (removeDisk && ctrlVlmCrtApiHelper.isDiskless(rsc)) -- { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.WARN_RSC_ALREADY_DISKLESS, -- "Resource already diskless", -- true -- )); -+ case NOOP: -+ return handleNoopAction(rsc, removeDisk); -+ case RETRY: -+ return handleRetryAction(rsc, removeDisk, toggleIntoTiebreakerRef, context); -+ case ABORT: -+ clearToggleDiskFlags(rsc); -+ errorReporter.logInfo( -+ "Aborting previous toggle disk transition, starting new transition to %s", -+ removeDisk ? "diskless" : "diskful" -+ ); -+ break; -+ case NORMAL: -+ break; -+ default: -+ throw new ImplementationError("Unhandled case: " + action); - } - -- ResourceDefinition rscDfn = rsc.getResourceDefinition(); -- AccessContext peerCtx = peerAccCtx.get(); -- if (removeDisk) -- { -- // Prevent removal of the last disk -- int haveDiskCount = countDisksAndIsOnline(rscDfn); -- if (haveDiskCount <= 1) -- { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.FAIL_INSUFFICIENT_REPLICA_COUNT, -- "Cannot remove the disk from the only online resource with a disk", -- true -- )); -- } -+ validateToggleDiskPreconditions(rsc, removeDisk); - -- if (!LayerUtils.hasLayer(getLayerData(peerCtx, rsc), DeviceLayerKind.DRBD)) -- { -- throw new ApiRcException(ApiCallRcImpl.simpleEntry( -- ApiConsts.FAIL_INVLD_LAYER_STACK, -- "Toggle disk is only supported in combination with DRBD", -- true -- )); -- } -- } -- else -- { -- ensureAllPeersHavePeerSlotLeft(rscDfn); -- } -+ AccessContext peerCtx = peerAccCtx.get(); - - // Save the requested storage pool in the resource properties. - // This does not cause the storage pool to be used automatically. -@@ -628,10 +600,10 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - - ctrlTransactionHelper.commit(); - -- String action = removeDisk ? "Removal of disk from" : "Addition of disk to"; -+ String actionStr = removeDisk ? "Removal of disk from" : "Addition of disk to"; - responses.addEntry(ApiCallRcImpl.simpleEntry( - ApiConsts.MODIFIED, -- action + " resource '" + rscDfn.getName().displayValue + "' " + -+ actionStr + " resource '" + rscDfn.getName().displayValue + "' " + - "on node '" + rsc.getNode().getName().displayValue + "' registered" - )); - -@@ -651,6 +623,40 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - ); - } - -+ private Flux handleNoopAction(Resource rscRef, boolean removeDiskRef) -+ { -+ String state = removeDiskRef ? "diskless" : "diskful"; -+ ApiCallRcImpl responses = new ApiCallRcImpl(); -+ responses.addEntry(ApiCallRcImpl.simpleEntry( -+ ApiConsts.INFO_NOOP, -+ "Resource '" + rscRef.getResourceDefinition().getName().displayValue + "' on node '" + -+ rscRef.getNode().getName().displayValue + "' is already " + state -+ )); -+ return Flux.just(responses); -+ } -+ -+ private Flux handleRetryAction( -+ Resource rscRef, -+ boolean removeDisk, -+ boolean toggleIntoTiebreakerRef, -+ ResponseContext context -+ ) -+ { -+ String direction = removeDisk ? "diskless" : "diskful"; -+ ApiCallRcImpl responses = new ApiCallRcImpl(); -+ NodeName nodeName = rscRef.getNode().getName(); -+ ResourceName rscName = rscRef.getResourceDefinition().getName(); -+ responses.addEntry(ApiCallRcImpl.simpleEntry( -+ ApiConsts.INFO_NOOP, -+ "Retrying toggle disk to " + direction + " for resource '" + rscName.displayValue + -+ "' on node '" + nodeName.displayValue + "'" -+ )); -+ -+ return Flux -+ .just(responses) -+ .concatWith(updateAndAdjustDisk(nodeName, rscName, removeDisk, toggleIntoTiebreakerRef, context)); -+ } -+ - private long getVlmDfnSizePrivileged(VolumeDefinition vlmDfnRef) - { - try -@@ -781,6 +787,96 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - } - } - -+ private ToggleDiskAction determineToggleDiskAction(Resource rsc, boolean removeDisk) -+ { -+ boolean isDiskless = ctrlVlmCrtApiHelper.isDiskless(rsc); -+ boolean diskAddRequested = hasDiskAddRequested(rsc); -+ boolean diskRemoveRequested = hasDiskRemoveRequested(rsc); -+ -+ ToggleDiskAction action; -+ if (isDiskless) -+ { -+ if (diskAddRequested) -+ { -+ action = removeDisk ? ToggleDiskAction.ABORT : ToggleDiskAction.RETRY; -+ } -+ else if (diskRemoveRequested) -+ { -+ action = removeDisk ? ToggleDiskAction.RETRY : ToggleDiskAction.ABORT; -+ } -+ else -+ { -+ action = removeDisk ? ToggleDiskAction.NOOP : ToggleDiskAction.NORMAL; -+ } -+ } -+ else -+ { -+ if (diskRemoveRequested) -+ { -+ action = removeDisk ? ToggleDiskAction.RETRY : ToggleDiskAction.ABORT; -+ } -+ else -+ { -+ action = removeDisk ? ToggleDiskAction.NORMAL : ToggleDiskAction.NOOP; -+ } -+ } -+ -+ return action; -+ } -+ -+ private void validateToggleDiskPreconditions(Resource rsc, boolean removeDisk) -+ { -+ ResourceDefinition rscDfn = rsc.getResourceDefinition(); -+ if (removeDisk) -+ { -+ validateDiskRemovalAllowed(rsc, rscDfn); -+ } -+ else -+ { -+ ensureAllPeersHavePeerSlotLeft(rscDfn); -+ } -+ } -+ -+ private void clearToggleDiskFlags(Resource rsc) -+ { -+ try -+ { -+ rsc.getStateFlags().disableFlags( -+ apiCtx, -+ Resource.Flags.DISK_ADD_REQUESTED, -+ Resource.Flags.DISK_ADDING, -+ Resource.Flags.DISK_REMOVE_REQUESTED, -+ Resource.Flags.DISK_REMOVING -+ ); -+ } -+ catch (AccessDeniedException | DatabaseException exc) -+ { -+ throw new ImplementationError(exc); -+ } -+ } -+ -+ private void validateDiskRemovalAllowed(Resource rsc, ResourceDefinition rscDfn) -+ { -+ int haveDiskCount = countDisksAndIsOnline(rscDfn); -+ if (haveDiskCount <= 1) -+ { -+ throw new ApiRcException(ApiCallRcImpl.simpleEntry( -+ ApiConsts.FAIL_INSUFFICIENT_REPLICA_COUNT, -+ "Cannot remove the disk from the only online resource with a disk", -+ true -+ )); -+ } -+ -+ if (!LayerUtils.hasLayer(getLayerData(peerAccCtx.get(), rsc), DeviceLayerKind.DRBD)) -+ { -+ throw new ApiRcException(ApiCallRcImpl.simpleEntry( -+ ApiConsts.FAIL_INVLD_LAYER_STACK, -+ "Toggle disk is only supported in combination with DRBD", -+ true -+ )); -+ } -+ } -+ - /** - * Although we need to rebuild the layerData as the layerList might have changed, if we do not - * deactivate (i.e. down) the current resource, we need to make sure that deleting DrbdRscData diff --git a/packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff b/packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff deleted file mode 100644 index ed106f98c..000000000 --- a/packages/system/linstor/images/piraeus-server/patches/fix-duplicate-tcp-ports.diff +++ /dev/null @@ -1,137 +0,0 @@ -diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -index d93a18014..01cfbbacf 100644 ---- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -+++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlRscToggleDiskApiCallHandler.java -@@ -37,6 +37,7 @@ import com.linbit.linstor.core.objects.StorPool; - import com.linbit.linstor.core.objects.Volume; - import com.linbit.linstor.core.objects.VolumeDefinition; - import com.linbit.linstor.core.objects.utils.MixedStorPoolHelper; -+import com.linbit.linstor.core.types.TcpPortNumber; - import com.linbit.linstor.dbdrivers.DatabaseException; - import com.linbit.linstor.event.EventWaiter; - import com.linbit.linstor.event.ObjectIdentifier; -@@ -85,6 +86,7 @@ import java.util.List; - import java.util.Map; - import java.util.Map.Entry; - import java.util.Set; -+import java.util.TreeSet; - - import org.reactivestreams.Publisher; - import reactor.core.publisher.Flux; -@@ -575,12 +577,13 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - - /* - * We also have to remove the currently diskless DrbdRscData and free up the node-id as now we must -- * use the shared resource's node-id -+ * use the shared resource's node-id. We still need to preserve TCP ports though. - */ -+ copyDrbdTcpPortsIfExists(rsc, payload); - } - else - { -- copyDrbdNodeIdIfExists(rsc, payload); -+ copyDrbdSettings(rsc, payload); - } - /* - * rebuilds the layerdata in case we just removed it.. -@@ -782,10 +785,20 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - } - - /** -- * Although we need to rebuild the layerData as the layerList might have changed, if we do not -- * deactivate (i.e. down) the current resource, we need to make sure that deleting DrbdRscData -- * and recreating a new DrbdRscData ends up with the same node-id as before. -+ * Copies DRBD settings (node-id and TCP ports) from the existing DrbdRscData into the payload -+ * before removeLayerData() deletes them. This ensures that recreated DrbdRscData ends up with -+ * the same node-id and TCP ports as before. -+ * -+ * TCP ports must be preserved because if the satellite misses the update (e.g. due to controller -+ * restart or connectivity issues), it will keep the old ports while peers receive the new ones, -+ * causing DRBD connections to fail with StandAlone state. - */ -+ private void copyDrbdSettings(Resource rsc, LayerPayload payload) throws ImplementationError -+ { -+ copyDrbdNodeIdIfExists(rsc, payload); -+ copyDrbdTcpPortsIfExists(rsc, payload); -+ } -+ - private void copyDrbdNodeIdIfExists(Resource rsc, LayerPayload payload) throws ImplementationError - { - Set> drbdRscDataSet = LayerRscUtils.getRscDataByLayer( -@@ -804,6 +817,28 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - } - } - -+ private void copyDrbdTcpPortsIfExists(Resource rsc, LayerPayload payload) throws ImplementationError -+ { -+ Set> drbdRscDataSet = LayerRscUtils.getRscDataByLayer( -+ getLayerData(apiCtx, rsc), -+ DeviceLayerKind.DRBD -+ ); -+ if (!drbdRscDataSet.isEmpty()) -+ { -+ DrbdRscData drbdRscData = (DrbdRscData) drbdRscDataSet.iterator().next(); -+ Collection tcpPorts = drbdRscData.getTcpPortList(); -+ if (tcpPorts != null && !tcpPorts.isEmpty()) -+ { -+ Set portInts = new TreeSet<>(); -+ for (TcpPortNumber port : tcpPorts) -+ { -+ portInts.add(port.value); -+ } -+ payload.drbdRsc.tcpPorts = portInts; -+ } -+ } -+ } -+ - private List removeLayerData(Resource rscRef) - { - List layerList; -@@ -1058,15 +1093,15 @@ public class CtrlRscToggleDiskApiCallHandler implements CtrlSatelliteConnectionL - /* - * We also have to remove the possible meta-children of previous StorageRscData. - * LayerData will be recreated with ensureStackDataExists. -- * However, we still need to remember our node-id if we had / have DRBD in the list -+ * However, we still need to remember our DRBD settings if we had / have DRBD in the list - */ -- copyDrbdNodeIdIfExists(rsc, payload); -+ copyDrbdSettings(rsc, payload); - layerList = removeLayerData(rsc); - } - else - { - markDiskAdded(rsc); -- ctrlLayerStackHelper.resetStoragePools(rsc); -+ ctrlLayerStackHelper.resetStoragePools(rsc, false); - } - ctrlLayerStackHelper.ensureStackDataExists(rsc, layerList, payload); - -diff --git a/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java b/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java -index 3538b380c..f9733b6f1 100644 ---- a/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java -+++ b/controller/src/main/java/com/linbit/linstor/layer/resource/CtrlRscLayerDataFactory.java -@@ -263,6 +263,11 @@ public class CtrlRscLayerDataFactory - } - - public void resetStoragePools(Resource rscRef) -+ { -+ resetStoragePools(rscRef, true); -+ } -+ -+ public void resetStoragePools(Resource rscRef, boolean callEnsureStackDataExistsRef) - { - try - { -@@ -276,8 +281,10 @@ public class CtrlRscLayerDataFactory - - rscDataToProcess.addAll(rscData.getChildren()); - } -- -- ensureStackDataExists(rscRef, null, new LayerPayload()); -+ if (callEnsureStackDataExistsRef) -+ { -+ ensureStackDataExists(rscRef, null, new LayerPayload()); -+ } - } - catch (AccessDeniedException exc) - { diff --git a/packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff b/packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff deleted file mode 100644 index 8611825bc..000000000 --- a/packages/system/linstor/images/piraeus-server/patches/fix-luks-header-size.diff +++ /dev/null @@ -1,570 +0,0 @@ -diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java ---- a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java -+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java -@@ -2081,7 +2081,6 @@ public abstract class AbsStorageProvider< - ) - throws IOException, AccessDeniedException - { -- final StorPoolName storPoolObjName = storPoolObj.getName(); - if (storDevicePath != null) - { - errorReporter.logDebug("updateMinIoSize: Have storDevicePath \"%s\"", storDevicePath); -@@ -2094,35 +2093,18 @@ public abstract class AbsStorageProvider< - "updateMinIoSize: Block device path is \"%s\"", - blockDevicePath.toString() - ); -- final long minIoSize = BlockSizeInfo.getBlockSize(blockDevicePath); -- -- boolean updateValue = true; -- -- final String propKey = StorageConstants.NAMESPACE_INTERNAL + '/' + -- StorageConstants.BLK_DEV_MIN_IO_SIZE; -- final Props storPoolProps = storPoolObj.getProps(storDriverAccCtx); -- final @Nullable String currentPropValue = storPoolProps.getProp(propKey); -- if (currentPropValue != null) -- { -- try -- { -- final long currentPropMinIoSize = Long.parseLong(currentPropValue); -- updateValue = currentPropMinIoSize != minIoSize; -- } -- catch (NumberFormatException ignored) -- { -- } -- } -- -- if (updateValue) -- { -- final String propValue = Long.toString(minIoSize); -- errorReporter.logDebug( -- "Storage pool \"%s\": Set property \"%s\" = \"%s\"", -- storPoolObjName.displayValue, propKey, propValue -- ); -- propsChange.changeStorPoolProp(storPoolObj, propKey, propValue); -- } -+ updatePropIfNeeded( -+ propsChange, -+ storPoolObj, -+ StorageConstants.NAMESPACE_INTERNAL + '/' + StorageConstants.BLK_DEV_MIN_IO_SIZE, -+ Long.toString(BlockSizeInfo.getBlockSize(blockDevicePath)) -+ ); -+ updatePropIfNeeded( -+ propsChange, -+ storPoolObj, -+ StorageConstants.NAMESPACE_INTERNAL + '/' + StorageConstants.BLK_DEV_OPT_IO_SIZE, -+ Long.toString(BlockSizeInfo.getOptimalIoSize(blockDevicePath)) -+ ); - } - else - { -@@ -2130,6 +2112,28 @@ public abstract class AbsStorageProvider< - } - } - -+ private void updatePropIfNeeded( -+ LocalPropsChangePojo propsChangeRef, -+ StorPool storPoolObjRef, -+ String propKeyRef, -+ String propValueRef -+ ) -+ throws AccessDeniedException -+ { -+ final Props storPoolProps = storPoolObjRef.getProps(storDriverAccCtx); -+ final @Nullable String currentPropValue = storPoolProps.getProp(propKeyRef); -+ if (currentPropValue == null || !currentPropValue.equals(propValueRef)) -+ { -+ errorReporter.logDebug( -+ "Storage pool \"%s\": Set property \"%s\" = \"%s\"", -+ storPoolObjRef.getName().displayValue, -+ propKeyRef, -+ propValueRef -+ ); -+ propsChangeRef.changeStorPoolProp(storPoolObjRef, propKeyRef, propValueRef); -+ } -+ } -+ - @SuppressWarnings("unchecked") - @Override - public void updateAllocatedSize(VlmProviderObject vlmDataRef) -diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java ---- a/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java -+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/utils/BlockSizeInfo.java -@@ -3,8 +3,11 @@ package com.linbit.linstor.layer.storage.utils; - import com.linbit.utils.MathUtils; - import com.linbit.utils.SymbolicLinkResolver; - -+import static com.linbit.linstor.layer.storage.BlockSizeConsts.DFLT_OPT_IO_SIZE; - import static com.linbit.linstor.layer.storage.BlockSizeConsts.DFLT_IO_SIZE; -+import static com.linbit.linstor.layer.storage.BlockSizeConsts.MAX_OPT_IO_SIZE; - import static com.linbit.linstor.layer.storage.BlockSizeConsts.MAX_IO_SIZE; -+import static com.linbit.linstor.layer.storage.BlockSizeConsts.MIN_OPT_IO_SIZE; - import static com.linbit.linstor.layer.storage.BlockSizeConsts.MIN_IO_SIZE; - - import java.io.FileInputStream; -@@ -13,6 +16,8 @@ import java.nio.file.Path; - - public class BlockSizeInfo - { -+ private static final int NUMBER_BUFFER_SIZE = 32; -+ - /** - * Determines the blocksize, aka minimum I/O size, for the specified backing storage path. - * -@@ -51,7 +56,7 @@ public class BlockSizeInfo - final Path infoSourceName = blockDevice.getFileName(); - final Path infoSource = Path.of("/sys/block", infoSourceName.toString(), "queue/physical_block_size"); - -- final byte[] data = new byte[32]; -+ final byte[] data = new byte[NUMBER_BUFFER_SIZE]; - try (final FileInputStream fileIn = new FileInputStream(infoSource.toString())) - { - final int readCount = fileIn.read(data); -@@ -75,4 +80,38 @@ public class BlockSizeInfo - } - return blockSize; - } -+ -+ public static long getOptimalIoSize(final Path storageObj) -+ { -+ long optIoSize = DFLT_OPT_IO_SIZE; -+ try -+ { -+ final Path blockDevice = SymbolicLinkResolver.resolveSymLink(storageObj); -+ final Path infoSourceName = blockDevice.getFileName(); -+ final Path infoSource = Path.of("/sys/block", infoSourceName.toString(), "queue/optimal_io_size"); -+ -+ final byte[] data = new byte[NUMBER_BUFFER_SIZE]; -+ try (final FileInputStream fileIn = new FileInputStream(infoSource.toString())) -+ { -+ final int readCount = fileIn.read(data); -+ if (readCount > 0) -+ { -+ String numberStr = new String(data, 0, readCount); -+ numberStr = numberStr.trim(); -+ try -+ { -+ final long unboundedOptIoSize = Long.parseLong(numberStr); -+ optIoSize = MathUtils.bounds(MIN_OPT_IO_SIZE, unboundedOptIoSize, MAX_OPT_IO_SIZE); -+ } -+ catch (NumberFormatException ignored) -+ { -+ } -+ } -+ } -+ } -+ catch (IOException ignored) -+ { -+ } -+ return optIoSize; -+ } - } -diff --git a/server/src/main/java/com/linbit/SizeConv.java b/server/src/main/java/com/linbit/SizeConv.java ---- a/server/src/main/java/com/linbit/SizeConv.java -+++ b/server/src/main/java/com/linbit/SizeConv.java -@@ -16,6 +16,7 @@ public class SizeConv - public enum SizeUnit - { - UNIT_B, -+ UNIT_SECTORS, - UNIT_KiB, - UNIT_MiB, - UNIT_GiB, -@@ -41,6 +42,9 @@ public class SizeConv - case UNIT_B: - factor = FACTOR_B; - break; -+ case UNIT_SECTORS: -+ factor = FACTOR_SECTORS; -+ break; - case UNIT_KiB: - factor = FACTOR_KiB; - break; -@@ -111,6 +115,9 @@ public class SizeConv - case "b": - unit = SizeUnit.UNIT_B; - break; -+ case "s": -+ unit = SizeUnit.UNIT_SECTORS; -+ break; - case "k": - // fall-through - case "kb": -@@ -217,6 +224,9 @@ public class SizeConv - // Factor 1 - public static final BigInteger FACTOR_B = BigInteger.valueOf(1L); - -+ // Factor 512 -+ public static final BigInteger FACTOR_SECTORS = BigInteger.valueOf(512L); -+ - // Factor 1,024 - // Naming convention exception: SI unit capitalization rules - @SuppressWarnings("checkstyle:constantname") -diff --git a/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java b/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java ---- a/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java -+++ b/server/src/main/java/com/linbit/linstor/layer/luks/LuksLayerSizeCalculator.java -@@ -1,27 +1,62 @@ - package com.linbit.linstor.layer.luks; - -+import com.linbit.SizeConv; -+import com.linbit.SizeConv.SizeUnit; - import com.linbit.exceptions.InvalidSizeException; -+import com.linbit.linstor.PriorityProps; -+import com.linbit.linstor.annotation.Nullable; -+import com.linbit.linstor.api.ApiConsts; -+import com.linbit.linstor.core.objects.AbsResource; -+import com.linbit.linstor.core.objects.AbsVolume; -+import com.linbit.linstor.core.objects.Resource; -+import com.linbit.linstor.core.objects.ResourceDefinition; -+import com.linbit.linstor.core.objects.ResourceGroup; -+import com.linbit.linstor.core.objects.Snapshot; -+import com.linbit.linstor.core.objects.SnapshotVolume; -+import com.linbit.linstor.core.objects.StorPool; -+import com.linbit.linstor.core.objects.Volume; -+import com.linbit.linstor.core.objects.VolumeDefinition; - import com.linbit.linstor.dbdrivers.DatabaseException; - import com.linbit.linstor.layer.AbsLayerSizeCalculator; -+import com.linbit.linstor.netcom.Peer; -+import com.linbit.linstor.propscon.InvalidKeyException; -+import com.linbit.linstor.propscon.ReadOnlyProps; - import com.linbit.linstor.security.AccessDeniedException; -+import com.linbit.linstor.storage.StorageConstants; - import com.linbit.linstor.storage.data.adapter.luks.LuksVlmData; - import com.linbit.linstor.storage.interfaces.categories.resource.VlmProviderObject; - import com.linbit.linstor.storage.kinds.DeviceLayerKind; - import com.linbit.linstor.storage.kinds.ExtTools; - import com.linbit.linstor.storage.kinds.ExtToolsInfo; - import com.linbit.linstor.storage.kinds.ExtToolsInfo.Version; -+import com.linbit.linstor.utils.layer.LayerVlmUtils; -+import com.linbit.utils.ShellUtils; -+import com.linbit.utils.SignedAlign; - - import javax.inject.Inject; - import javax.inject.Singleton; - -+import java.util.Iterator; -+import java.util.LinkedList; -+import java.util.List; -+import java.util.Set; -+import java.util.regex.Matcher; -+import java.util.regex.Pattern; -+ - @Singleton - public class LuksLayerSizeCalculator extends AbsLayerSizeCalculator> - { -+ public static final String LUKS2_OPT_METADATA_SIZE = "--luks2-metadata-size"; -+ public static final String LUKS2_OPT_KEYSLOTS_SIZE = "--luks2-keyslots-size"; -+ public static final String LUKS2_OPT_OFFSET = "--offset"; -+ private static final String LUKS2_OPT_ALIGN_PAYLOAD = "--align-payload"; -+ private static final long DFLT_LUKS2_METADATA_SIZE_IN_BYTES = 16L << 10; -+ private static final long DFLT_ALIGNMENT_1MIB_IN_BYTES = 1L << 20; -+ private static final long LUKS1_HEADER_SIZE_IN_KIB = 2L << 10; -+ private static final long LUKS2_HEADER_SIZE_IN_KIB = 16L << 10; -+ private static final long LUKS2_HEADER_SIZE_IN_BYTES = LUKS2_HEADER_SIZE_IN_KIB << 10; - -- // linstor calculates in KiB -- private static final int MIB = 1024; -- private static final int LUKS1_HEADER_SIZE = 2 * MIB; -- private static final int LUKS2_HEADER_SIZE = 16 * MIB; -+ private static final Pattern PATTERN_SIZE = Pattern.compile("(\\d+)([kKmMgGtT]i?[bB]?|[sS]|)"); - - @Inject - public LuksLayerSizeCalculator(AbsLayerSizeCalculatorInit initRef) -@@ -74,30 +109,266 @@ public class LuksLayerSizeCalculator extends AbsLayerSizeCalculator vlmDataRef) -- throws AccessDeniedException -+ throws AccessDeniedException, InvalidSizeException - { -- ExtToolsInfo cryptSetupInfo = vlmDataRef.getRscLayerObject() -+ @Nullable Peer peer = vlmDataRef.getRscLayerObject() - .getAbsResource() - .getNode() -- .getPeer(sysCtx) -- .getExtToolsManager() -+ .getPeer(sysCtx); -+ if (peer == null) -+ { -+ throw new InvalidSizeException( -+ "Could not calculate size of LUKS volume, since cryptsetup's version could not be determined", -+ null -+ ); -+ } -+ ExtToolsInfo cryptSetupInfo = peer.getExtToolsManager() - .getExtToolInfo(ExtTools.CRYPT_SETUP); -- long luksHeaderSize; -+ -+ final long luksHeaderSize; - if (cryptSetupInfo != null && cryptSetupInfo.isSupported()) - { - if (cryptSetupInfo.hasVersionOrHigher(new Version(2, 1))) - { -- luksHeaderSize = LUKS2_HEADER_SIZE; -+ luksHeaderSize = calcLuks2HeaderSize(vlmDataRef); - } - else - { -- luksHeaderSize = LUKS1_HEADER_SIZE; -+ luksHeaderSize = LUKS1_HEADER_SIZE_IN_KIB; - } - } - else - { -- luksHeaderSize = -1; -+ throw new InvalidSizeException( -+ "Could not calculate size of LUKS volume, since cryptsetup's version could not be determined", -+ null -+ ); - } - return luksHeaderSize; - } -+ -+ private long calcLuks2HeaderSize(VlmProviderObject vlmDataRef) throws AccessDeniedException -+ { -+ PriorityProps prioProps = getPrioProps(vlmDataRef); -+ -+ final @Nullable String userOptProp = prioProps.getProp( -+ ApiConsts.KEY_STOR_DRIVER_LUKS_FORMAT_OPTIONS, -+ ApiConsts.NAMESPC_STORAGE_DRIVER -+ ); -+ List userOptions = userOptProp != null ? -+ ShellUtils.shellSplit(userOptProp) : -+ new LinkedList<>(); -+ -+ final long alignedHeaderSizeInBytes; -+ final long unalignedHeaderSizeInBytes; -+ @Nullable Long cryptsetupOffsetInBytes = getLongOptionValue( -+ userOptions, -+ LUKS2_OPT_OFFSET, -+ SizeUnit.UNIT_SECTORS -+ ); -+ if (cryptsetupOffsetInBytes != null) -+ { -+ alignedHeaderSizeInBytes = cryptsetupOffsetInBytes; -+ } -+ else -+ { -+ @Nullable Long cryptsetupLuks2KeyslotsSize = getLongOptionValue( -+ userOptions, -+ LUKS2_OPT_KEYSLOTS_SIZE -+ ); -+ if (cryptsetupLuks2KeyslotsSize == null) -+ { -+ unalignedHeaderSizeInBytes = LUKS2_HEADER_SIZE_IN_BYTES; -+ } -+ else -+ { -+ @Nullable Long cryptsetupLuks2MetadataSize = getLongOptionValue( -+ userOptions, -+ LUKS2_OPT_METADATA_SIZE -+ ); -+ cryptsetupLuks2MetadataSize = cryptsetupLuks2MetadataSize == null ? -+ DFLT_LUKS2_METADATA_SIZE_IN_BYTES : -+ cryptsetupLuks2MetadataSize; -+ -+ unalignedHeaderSizeInBytes = 2 * cryptsetupLuks2MetadataSize + cryptsetupLuks2KeyslotsSize; -+ } -+ -+ long alignment = getAlignment(vlmDataRef, userOptions); -+ alignedHeaderSizeInBytes = new SignedAlign(alignment).ceiling(unalignedHeaderSizeInBytes); -+ } -+ return SizeConv.convert( -+ alignedHeaderSizeInBytes, -+ SizeUnit.UNIT_B, -+ SizeUnit.UNIT_KiB -+ ); -+ } -+ -+ private long getAlignment(VlmProviderObject vlmDataRef, List userOptions) throws AccessDeniedException -+ { -+ @Nullable Long cryptsetupAlignPayloadInBytes = getLongOptionValue( -+ userOptions, -+ LUKS2_OPT_ALIGN_PAYLOAD, -+ SizeUnit.UNIT_SECTORS -+ ); -+ -+ long alignment = DFLT_ALIGNMENT_1MIB_IN_BYTES; -+ if (cryptsetupAlignPayloadInBytes != null) -+ { -+ alignment = cryptsetupAlignPayloadInBytes; -+ } -+ else -+ { -+ final long maxOptIoSize = getMaxOptIoSize(vlmDataRef); -+ alignment = Math.max(alignment, maxOptIoSize); -+ } -+ return alignment; -+ } -+ -+ private long getMaxOptIoSize(VlmProviderObject vlmDataRef) throws InvalidKeyException, AccessDeniedException -+ { -+ long ret = 0; -+ Set storPoolSet = LayerVlmUtils.getStorPoolSet(vlmDataRef, sysCtx); -+ for (StorPool sp : storPoolSet) -+ { -+ @Nullable String strValue = sp.getProps(sysCtx) -+ .getProp( -+ StorageConstants.BLK_DEV_OPT_IO_SIZE, -+ StorageConstants.NAMESPACE_INTERNAL -+ ); -+ if (strValue != null) -+ { -+ try -+ { -+ long parsed = Long.parseLong(strValue); -+ ret = Math.max(parsed, ret); -+ } -+ catch (NumberFormatException ignored) -+ { -+ errorReporter.logWarning( -+ "LuksHeaderSize: Failed to parse '%s' from prop %s. Defaulting to 0 opt_io_size " + -+ "(no recommendation/hint)", -+ strValue, -+ StorageConstants.NAMESPACE_INTERNAL + "/" + StorageConstants.BLK_DEV_OPT_IO_SIZE -+ ); -+ } -+ } -+ } -+ return ret; -+ } -+ -+ private @Nullable Long getLongOptionValue(List userOptionsRef, String optRef) -+ { -+ return getLongOptionValue(userOptionsRef, optRef, SizeUnit.UNIT_B); -+ } -+ -+ @SuppressWarnings("checkstyle:magicnumber") -+ private @Nullable Long getLongOptionValue(List userOptionsRef, String optRef, SizeUnit dfltSizeUnit) -+ { -+ @Nullable Long ret = null; -+ @Nullable String val = findLastValue(userOptionsRef, optRef); -+ if (val != null && !val.isBlank()) -+ { -+ Matcher matcher = PATTERN_SIZE.matcher(val); -+ if (matcher.matches()) -+ { -+ try -+ { -+ ret = Long.parseLong(matcher.group(1)); -+ String unit = matcher.group(2); -+ SizeUnit sizeUnit; -+ if (unit.isBlank()) -+ { -+ sizeUnit = dfltSizeUnit; -+ } -+ else -+ { -+ boolean forcePowerOfTwo = unit.length() == 1 || unit.length() == 3; -+ sizeUnit = SizeUnit.parse(unit, forcePowerOfTwo); -+ } -+ -+ ret = SizeConv.convert(ret, sizeUnit, SizeUnit.UNIT_B); -+ } -+ catch (NumberFormatException ignored) -+ { -+ errorReporter.logWarning( -+ "LuksHeaderSize: Failed to parse '%s' from option '%s %s'.", -+ matcher.group(1), -+ optRef, -+ val -+ ); -+ } -+ } -+ } -+ return ret; -+ } -+ -+ private @Nullable String findLastValue(List userOptionsRef, String optRef) -+ { -+ Iterator it = userOptionsRef.iterator(); -+ @Nullable String val = null; -+ while (it.hasNext()) -+ { -+ String opt = it.next(); -+ if (opt.equals(optRef)) -+ { -+ if (it.hasNext()) -+ { -+ val = it.next(); -+ } -+ else -+ { -+ val = null; -+ } -+ } -+ else if (opt.startsWith(optRef + "=")) -+ { -+ val = opt.substring(optRef.length() + 1); -+ if (val.isBlank()) -+ { -+ val = null; -+ } -+ } -+ } -+ return val; -+ } -+ -+ private PriorityProps getPrioProps(VlmProviderObject vlmDataRef) throws AccessDeniedException -+ { -+ final AbsVolume vlm = vlmDataRef.getVolume(); -+ final AbsResource rsc = vlm.getAbsResource(); -+ final ResourceDefinition rscDfn = vlm.getResourceDefinition(); -+ final VolumeDefinition vlmDfn = vlm.getVolumeDefinition(); -+ final ResourceGroup rscGrp = rscDfn.getResourceGroup(); -+ -+ final ReadOnlyProps vlmProps; -+ final ReadOnlyProps rscProps; -+ if (vlm instanceof Volume) -+ { -+ vlmProps = ((Volume) vlm).getProps(sysCtx); -+ rscProps = ((Resource) rsc).getProps(sysCtx); -+ } -+ else -+ { -+ vlmProps = ((SnapshotVolume) vlm).getVlmProps(sysCtx); -+ rscProps = ((Snapshot) rsc).getRscProps(sysCtx); -+ } -+ -+ final PriorityProps prioProps = new PriorityProps( -+ vlmProps, -+ rscProps -+ ); -+ for (StorPool storPool : LayerVlmUtils.getStorPoolSet(vlmDataRef, sysCtx)) -+ { -+ prioProps.addProps(storPool.getProps(sysCtx)); -+ } -+ prioProps.addProps( -+ rsc.getNode().getProps(sysCtx), -+ vlmDfn.getProps(sysCtx), -+ rscDfn.getProps(sysCtx), -+ rscGrp.getVolumeGroupProps(sysCtx, vlmDfn.getVolumeNumber()), -+ rscGrp.getProps(sysCtx), -+ stltProps -+ ); -+ return prioProps; -+ } - } -diff --git a/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java b/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java ---- a/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java -+++ b/server/src/main/java/com/linbit/linstor/layer/storage/BlockSizeConsts.java -@@ -13,4 +13,9 @@ public class BlockSizeConsts - - // Default value for the minimum_io_size value of non-storage layers - public static final long DFLT_SPECIAL_IO_SIZE = (1L << 12); -+ -+ // optimal_io_size may be 0 to indicate no recommendation. -+ public static final long MIN_OPT_IO_SIZE = 0; -+ public static final long DFLT_OPT_IO_SIZE = 0; -+ public static final long MAX_OPT_IO_SIZE = Long.MAX_VALUE; - } -diff --git a/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java b/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java ---- a/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java -+++ b/server/src/main/java/com/linbit/linstor/storage/StorageConstants.java -@@ -11,6 +11,7 @@ public class StorageConstants - public static final String NAMESPACE_NVME = ApiConsts.NAMESPC_STORAGE_DRIVER + "/NVME"; - public static final String NAMESPACE_INTERNAL = NAMESPACE_STOR_DRIVER + "/internal/"; - -+ public static final String BLK_DEV_OPT_IO_SIZE = "optIoSize"; - public static final String BLK_DEV_MIN_IO_SIZE = "minIoSize"; - public static final String BLK_DEV_MIN_IO_SIZE_AUTO = "minIoSizeAuto"; - public static final String BLK_DEV_MAX_BIO_SIZE = "maxBioSize"; diff --git a/packages/system/linstor/images/piraeus-server/patches/fix-min-io-probe-device-wait.diff b/packages/system/linstor/images/piraeus-server/patches/fix-min-io-probe-device-wait.diff deleted file mode 100644 index bb6185b88..000000000 --- a/packages/system/linstor/images/piraeus-server/patches/fix-min-io-probe-device-wait.diff +++ /dev/null @@ -1,660 +0,0 @@ -diff --git a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java ---- a/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java -+++ b/satellite/src/main/java/com/linbit/linstor/layer/storage/AbsStorageProvider.java -@@ -163,6 +163,8 @@ - } - - private static final long DFLT_WAIT_UNTIL_DEVICE_CREATED_TIMEOUT_IN_MS = 5000; -+ private static final int PROBE_VLM_ATTEMPTS = 3; -+ private static final long PROBE_VLM_RETRY_DELAY_IN_MS = 1000; - protected static final int DFLT_STRIPES = 1; - public static final long SIZE_OF_NOT_FOUND_STOR_POOL = ApiConsts.VAL_STOR_POOL_SPACE_NOT_FOUND; - -@@ -184,6 +186,7 @@ - protected final Set changedStoragePoolStrings = new HashSet<>(); - private final String typeDescr; - private final FileSystemWatch fsWatch; -+ private final Object probeVlmLock = new Object(); - protected final DeviceProviderKind kind; - private final DrbdInvalidateUtils drbdInvalidateUtils; - -@@ -2022,61 +2025,126 @@ - final StorPoolName storPoolObjName = storPoolObj.getName(); - errorReporter.logDebug("ENTER updateMinIoSize method: Storage pool \"%s\"", storPoolObjName.displayValue); - Collection> vlmProviderList = storPoolObj.getVolumes(storDriverAccCtx); -+ boolean haveInfo = false; - if (vlmProviderList != null && vlmProviderList.size() >= 1) - { - errorReporter.logDebug("updateMinIoSize: Have vlmProviderList with %d items", vlmProviderList.size()); - Iterator> vlmProviderIter = vlmProviderList.iterator(); -- boolean haveInfo = false; - while (vlmProviderIter.hasNext() && !haveInfo) - { - VlmProviderObject vlmProvider = vlmProviderIter.next(); - errorReporter.logDebug("updateMinIoSize: Have vlmProvider"); - final String storDevicePath = vlmProvider.getDevicePath(); -+ if (storDevicePath != null) -+ { -+ try -+ { -+ updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange); -+ haveInfo = true; -+ } -+ catch (IOException ignored) -+ { -+ } -+ } -+ } -+ } -+ // An inactive shared LVM volume can be owned by another node; do not probe it by creating a volume. -+ if (!haveInfo && this instanceof ProbeVlmStorageProvider && -+ (vlmProviderList == null || vlmProviderList.isEmpty() || -+ kind == DeviceProviderKind.ZFS || kind == DeviceProviderKind.ZFS_THIN)) -+ { -+ updateMinIoSizeByProbeVlm((ProbeVlmStorageProvider) this, storPoolObj, propsChange); -+ } -+ errorReporter.logDebug("EXIT updateMinIoSize method"); -+ } -+ -+ private void updateMinIoSizeByProbeVlm( -+ ProbeVlmStorageProvider storPrv, -+ StorPool storPoolObj, -+ LocalPropsChangePojo propsChange -+ ) -+ throws AccessDeniedException -+ { -+ // Providers use a fixed probe name. Keep creation, probing and cleanup in one critical section. -+ synchronized (probeVlmLock) -+ { -+ boolean haveInfo = false; -+ boolean interrupted = false; -+ String lastFailure = ""; -+ int attempts = 0; -+ while (attempts < PROBE_VLM_ATTEMPTS && !haveInfo && !interrupted) -+ { -+ if (attempts > 0) -+ { -+ try -+ { -+ Thread.sleep(PROBE_VLM_RETRY_DELAY_IN_MS); -+ } -+ catch (InterruptedException exc) -+ { -+ Thread.currentThread().interrupt(); -+ lastFailure = "interrupted before the next probe attempt"; -+ break; -+ } -+ } -+ attempts++; -+ boolean cleaned = false; - try - { -- updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange); -- haveInfo = true; -- } -- catch (IOException ignored) -- { -- } -- } -- } -- else -- { -- errorReporter.logDebug("updateMinIoSize: Don't have vlmProviderList, using temporary volumes"); -- if (this instanceof ProbeVlmStorageProvider) -- { -- final ProbeVlmStorageProvider storPrv = (ProbeVlmStorageProvider) this; -- try -- { -- final String storDevicePath = storPrv.createTmpProbeVlm(storPoolObj); -+ @Nullable String storDevicePath = storPrv.createTmpProbeVlm(storPoolObj); -+ if (storDevicePath == null) -+ { -+ lastFailure = "temporary volume has no device path"; -+ } -+ else -+ { -+ waitUntilDeviceCreated(storDevicePath, getWaitTimeoutAfterCreate(storPoolObj)); -+ updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange); -+ haveInfo = true; -+ } -+ } -+ catch (StorageException | IOException exc) -+ { -+ lastFailure = exc.getMessage(); -+ interrupted = exc.getCause() instanceof InterruptedException; -+ errorReporter.logDebug("Temporary volume probe attempt %d failed: %s", attempts, lastFailure); -+ } -+ finally -+ { - try - { -- updateMinIoSizeByDevice(storPoolObj, storDevicePath, propsChange); -- } -- catch (IOException ignored) -- { -- } -- } -- catch (StorageException ignored) -- { -- errorReporter.logDebug("updateMinIoSize: Temporary volume creation failed"); -- } -- finally -- { -- try -- { - storPrv.deleteTmpProbeVlm(storPoolObj); -- } -- catch (StorageException ignored) -- { -- errorReporter.logDebug("updateMinIoSize: Temporary volume deletion failed"); -- } -- } -- } -- } -- errorReporter.logDebug("EXIT updateMinIoSize method"); -+ cleaned = true; -+ } -+ catch (StorageException exc) -+ { -+ interrupted |= exc.getCause() instanceof InterruptedException; -+ errorReporter.logWarning( -+ "Failed to delete temporary probe volume for storage pool '%s': %s", -+ storPoolObj.getName().displayValue, -+ exc.getMessage() -+ ); -+ } -+ } -+ if (interrupted) -+ { -+ Thread.currentThread().interrupt(); -+ } -+ if (!cleaned) -+ { -+ break; -+ } -+ } -+ if (!haveInfo) -+ { -+ errorReporter.logWarning( -+ "Could not determine block device information for storage pool '%s' after %d probe attempts: %s", -+ storPoolObj.getName().displayValue, -+ attempts, -+ lastFailure -+ ); -+ } -+ } - } - - private void updateMinIoSizeByDevice( -diff --git a/src/test/java/com/linbit/fsevent/ProbeVolumeDeviceWaitTest.java b/src/test/java/com/linbit/fsevent/ProbeVolumeDeviceWaitTest.java -new file mode 100644 ---- /dev/null -+++ b/src/test/java/com/linbit/fsevent/ProbeVolumeDeviceWaitTest.java -@@ -0,0 +1,467 @@ -+package com.linbit.fsevent; -+ -+import com.linbit.Platform; -+import com.linbit.linstor.core.identifier.StorPoolName; -+import com.linbit.linstor.core.objects.Resource; -+import com.linbit.linstor.core.objects.StorPool; -+import com.linbit.linstor.core.pojos.LocalPropsChangePojo; -+import com.linbit.linstor.layer.storage.AbsStorageProvider.AbsStorageProviderInit; -+import com.linbit.linstor.layer.storage.BlockSizeConsts; -+import com.linbit.linstor.layer.storage.lvm.LvmProvider; -+import com.linbit.linstor.layer.storage.zfs.ZfsThinProvider; -+import com.linbit.linstor.logging.ErrorReporter; -+import com.linbit.linstor.propscon.Props; -+import com.linbit.linstor.storage.StorageConstants; -+import com.linbit.linstor.storage.StorageException; -+import com.linbit.linstor.storage.interfaces.categories.resource.VlmProviderObject; -+ -+import java.io.IOException; -+import java.nio.file.Files; -+import java.nio.file.Path; -+import java.util.Collections; -+import java.util.List; -+import java.util.Map; -+import java.util.concurrent.Callable; -+import java.util.concurrent.CountDownLatch; -+import java.util.concurrent.ExecutorService; -+import java.util.concurrent.Executors; -+import java.util.concurrent.Future; -+import java.util.concurrent.ScheduledExecutorService; -+import java.util.concurrent.ScheduledFuture; -+import java.util.concurrent.TimeUnit; -+import java.util.concurrent.atomic.AtomicInteger; -+ -+import org.junit.After; -+import org.junit.Before; -+import org.junit.Rule; -+import org.junit.Test; -+import org.junit.rules.TemporaryFolder; -+import org.junit.rules.Timeout; -+ -+import static org.junit.Assert.assertEquals; -+import static org.junit.Assert.assertFalse; -+import static org.junit.Assert.assertTrue; -+import static org.junit.Assume.assumeTrue; -+import static org.mockito.Mockito.doReturn; -+import static org.mockito.Mockito.mock; -+import static org.mockito.Mockito.mockingDetails; -+import static org.mockito.Mockito.when; -+ -+public class ProbeVolumeDeviceWaitTest -+{ -+ @Rule -+ public final TemporaryFolder directory = new TemporaryFolder(); -+ -+ @Rule -+ public final Timeout timeout = Timeout.seconds(15); -+ -+ private FileSystemWatch watch; -+ private Path device; -+ private Path link; -+ private StorPool pool; -+ private ProbeProvider provider; -+ private ErrorReporter reporter; -+ private ScheduledExecutorService executor; -+ -+ @Before -+ public void setUp() throws Exception -+ { -+ assumeTrue(Platform.isLinux()); -+ device = directory.newFile("zd-test").toPath(); -+ link = directory.getRoot().toPath().resolve("probe"); -+ executor = Executors.newSingleThreadScheduledExecutor(); -+ reporter = mock(ErrorReporter.class); -+ watch = new FileSystemWatch(reporter); -+ watch.start(); -+ pool = mock(StorPool.class); -+ when(pool.getName()).thenReturn(new StorPoolName("data")); -+ when(pool.getVolumes(null)).thenReturn(Collections.emptyList()); -+ when(pool.getProps(null)).thenReturn(mock(Props.class)); -+ provider = new ProbeProvider(providerInit()); -+ } -+ -+ @After -+ public void tearDown() throws Exception -+ { -+ if (executor != null) -+ { -+ executor.shutdownNow(); -+ assertTrue(executor.awaitTermination(2, TimeUnit.SECONDS)); -+ } -+ if (watch != null) -+ { -+ watch.shutdown(false); -+ watch.awaitShutdown(2000); -+ watch.cancelAllWatchKeys(); -+ } -+ } -+ -+ @Test -+ public void delayedDevicePublishesPropertiesBeforeCleanup() throws Exception -+ { -+ provider.delayMillis = 50; -+ assertPublished(probe()); -+ assertCleaned(1); -+ assertEquals(0, warningCount()); -+ } -+ -+ @Test -+ public void existingProbeDevicePublishesProperties() throws Exception -+ { -+ assertPublished(probe()); -+ assertCleaned(1); -+ } -+ -+ @Test -+ public void missingDeviceTimesOutAndCleansEveryAttempt() throws Exception -+ { -+ provider.delayMillis = -1; -+ provider.timeoutMillis = 20; -+ assertTrue(probe().isEmpty()); -+ assertCleaned(3); -+ assertEquals(1, warningCount()); -+ } -+ -+ @Test -+ public void failedCreationStillCleansEveryAttempt() throws Exception -+ { -+ provider.failCreateTimes = Integer.MAX_VALUE; -+ assertTrue(probe().isEmpty()); -+ assertCleaned(3); -+ assertEquals(1, warningCount()); -+ } -+ -+ @Test -+ public void transientCreationFailureIsRetried() throws Exception -+ { -+ provider.failCreateTimes = 1; -+ provider.delayMillis = 50; -+ assertPublished(probe()); -+ assertCleaned(2); -+ assertEquals(0, warningCount()); -+ } -+ -+ @Test -+ public void nullProbePathDoesNotPublishProperties() throws Exception -+ { -+ provider.nullPath = true; -+ assertTrue(probe().isEmpty()); -+ assertCleaned(3); -+ assertEquals(1, warningCount()); -+ } -+ -+ @Test -+ public void nullFirstPathDoesNotSkipUsableVolume() throws Exception -+ { -+ Files.createSymbolicLink(link, device); -+ doReturn(List.of(volume(null), volume(link))).when(pool).getVolumes(null); -+ assertPublished(probe()); -+ assertEquals(0, provider.createCalls); -+ assertEquals(0, provider.deleteCalls); -+ } -+ -+ @Test -+ public void missingFirstPathDoesNotSkipUsableVolume() throws Exception -+ { -+ Files.createSymbolicLink(link, device); -+ doReturn(List.of(volume(link.resolveSibling("missing")), volume(link))).when(pool).getVolumes(null); -+ assertPublished(probe()); -+ assertEquals(0, provider.createCalls); -+ assertEquals(0, provider.deleteCalls); -+ } -+ -+ @Test -+ public void allNullPathsFallBackToProbe() throws Exception -+ { -+ doReturn(List.of(volume(null))).when(pool).getVolumes(null); -+ provider.delayMillis = 50; -+ assertPublished(probe()); -+ assertCleaned(1); -+ } -+ -+ @Test -+ public void allMissingPathsFallBackToProbe() throws Exception -+ { -+ doReturn(List.of( -+ volume(link.resolveSibling("missing-a")), volume(link.resolveSibling("missing-b")) -+ )).when(pool).getVolumes(null); -+ provider.delayMillis = 50; -+ assertPublished(probe()); -+ assertCleaned(1); -+ } -+ -+ @Test -+ public void cleanupFailureAfterSuccessIsReported() throws Exception -+ { -+ provider.failDelete = true; -+ assertPublished(probe()); -+ assertEquals(1, provider.createCalls); -+ assertEquals(1, provider.deleteCalls); -+ assertEquals(1, warningCount()); -+ } -+ -+ @Test -+ public void cleanupFailurePreventsAnotherCreationAttempt() throws Exception -+ { -+ provider.delayMillis = -1; -+ provider.timeoutMillis = 20; -+ provider.failDelete = true; -+ assertTrue(probe().isEmpty()); -+ assertEquals(1, provider.createCalls); -+ assertEquals(1, provider.deleteCalls); -+ assertEquals(2, warningCount()); -+ } -+ -+ @Test -+ public void interruptedDeviceWaitCleansUpAndPreservesInterrupt() throws Exception -+ { -+ provider.delayMillis = -1; -+ Thread callingThread = Thread.currentThread(); -+ executor.schedule(callingThread::interrupt, 50, TimeUnit.MILLISECONDS); -+ try -+ { -+ assertTrue(probe().isEmpty()); -+ assertTrue(callingThread.isInterrupted()); -+ assertCleaned(1); -+ assertEquals(1, warningCount()); -+ } -+ finally -+ { -+ Thread.interrupted(); -+ } -+ } -+ -+ @Test -+ public void interruptedRetryDelayDoesNotCreateAnotherProbe() throws Exception -+ { -+ provider.failCreateTimes = Integer.MAX_VALUE; -+ Thread callingThread = Thread.currentThread(); -+ executor.schedule(callingThread::interrupt, 100, TimeUnit.MILLISECONDS); -+ try -+ { -+ assertTrue(probe().isEmpty()); -+ assertTrue(callingThread.isInterrupted()); -+ assertCleaned(1); -+ assertEquals(1, warningCount()); -+ } -+ finally -+ { -+ Thread.interrupted(); -+ } -+ } -+ -+ @Test -+ public void interruptedCleanupPreservesInterruptAndStopsRetries() throws Exception -+ { -+ provider.failCreateTimes = Integer.MAX_VALUE; -+ provider.interruptDelete = true; -+ try -+ { -+ assertTrue(probe().isEmpty()); -+ assertTrue(Thread.currentThread().isInterrupted()); -+ assertEquals(1, provider.createCalls); -+ assertEquals(1, provider.deleteCalls); -+ } -+ finally -+ { -+ Thread.interrupted(); -+ } -+ } -+ -+ @Test -+ public void concurrentProbesDoNotShareTheTemporaryVolume() throws Exception -+ { -+ provider.creationDelayMillis = 200; -+ ExecutorService callers = Executors.newFixedThreadPool(2); -+ CountDownLatch ready = new CountDownLatch(2); -+ CountDownLatch start = new CountDownLatch(1); -+ Callable call = () -> -+ { -+ ready.countDown(); -+ start.await(); -+ return probe(); -+ }; -+ try -+ { -+ Future first = callers.submit(call); -+ Future second = callers.submit(call); -+ assertTrue(ready.await(2, TimeUnit.SECONDS)); -+ start.countDown(); -+ assertFalse(first.get(10, TimeUnit.SECONDS).isEmpty()); -+ assertFalse(second.get(10, TimeUnit.SECONDS).isEmpty()); -+ assertEquals(1, provider.peakActiveProbes.get()); -+ assertCleaned(2); -+ } -+ finally -+ { -+ callers.shutdownNow(); -+ assertTrue(callers.awaitTermination(2, TimeUnit.SECONDS)); -+ } -+ } -+ -+ @Test -+ public void inactiveLvmVolumesDoNotCreateProbe() throws Exception -+ { -+ doReturn(List.of(volume(null))).when(pool).getVolumes(null); -+ LvmProvider lvm = new LvmProvider(providerInit()) -+ { -+ @Override -+ public String createTmpProbeVlm(StorPool ignored) -+ { -+ throw new AssertionError("Inactive shared LVM volumes must not trigger a probe volume"); -+ } -+ -+ @Override -+ public void deleteTmpProbeVlm(StorPool ignored) -+ { -+ throw new AssertionError("No probe volume was created"); -+ } -+ }; -+ LocalPropsChangePojo changes = new LocalPropsChangePojo(); -+ lvm.updateMinIoSize(pool, changes); -+ assertTrue(changes.isEmpty()); -+ } -+ -+ private AbsStorageProviderInit providerInit() -+ { -+ return new AbsStorageProviderInit( -+ reporter, null, null, null, null, null, null, null, null, null, watch, null, null, null -+ ); -+ } -+ -+ private LocalPropsChangePojo probe() throws Exception -+ { -+ LocalPropsChangePojo changes = new LocalPropsChangePojo(); -+ provider.updateMinIoSize(pool, changes); -+ return changes; -+ } -+ -+ private void assertPublished(LocalPropsChangePojo changesRef) -+ { -+ String namespace = StorageConstants.NAMESPACE_INTERNAL + '/'; -+ assertEquals(Map.of( -+ namespace + StorageConstants.BLK_DEV_MIN_IO_SIZE, Long.toString(BlockSizeConsts.DFLT_IO_SIZE), -+ namespace + StorageConstants.BLK_DEV_OPT_IO_SIZE, Long.toString(BlockSizeConsts.DFLT_OPT_IO_SIZE) -+ ), changesRef.changedStorPoolProps.get(pool.getName())); -+ } -+ -+ private void assertCleaned(int attemptsRef) -+ { -+ assertEquals(attemptsRef, provider.createCalls); -+ assertEquals(attemptsRef, provider.deleteCalls); -+ assertFalse(Files.exists(link)); -+ } -+ -+ @SuppressWarnings("unchecked") -+ private VlmProviderObject volume(Path pathRef) -+ { -+ VlmProviderObject volume = mock(VlmProviderObject.class); -+ when(volume.getDevicePath()).thenReturn(pathRef == null ? null : pathRef.toString()); -+ return volume; -+ } -+ -+ private long warningCount() -+ { -+ return mockingDetails(reporter).getInvocations().stream() -+ .filter(invocation -> invocation.getMethod().getName().equals("logWarning")) -+ .count(); -+ } -+ -+ private class ProbeProvider extends ZfsThinProvider -+ { -+ private long delayMillis; -+ private long creationDelayMillis; -+ private long timeoutMillis = 4000; -+ private int failCreateTimes; -+ private boolean nullPath; -+ private boolean failDelete; -+ private boolean interruptDelete; -+ private int createCalls; -+ private int deleteCalls; -+ private ScheduledFuture create; -+ private final AtomicInteger activeProbes = new AtomicInteger(); -+ private final AtomicInteger peakActiveProbes = new AtomicInteger(); -+ -+ ProbeProvider(AbsStorageProviderInit initRef) -+ { -+ super(initRef); -+ } -+ -+ @Override -+ protected long getWaitTimeoutAfterCreate(StorPool ignored) -+ { -+ return timeoutMillis; -+ } -+ -+ @Override -+ public String createTmpProbeVlm(StorPool ignored) throws StorageException -+ { -+ peakActiveProbes.accumulateAndGet(activeProbes.incrementAndGet(), Math::max); -+ createCalls++; -+ if (creationDelayMillis > 0) -+ { -+ try -+ { -+ Thread.sleep(creationDelayMillis); -+ } -+ catch (InterruptedException exc) -+ { -+ throw new StorageException("Test probe creation interrupted", exc); -+ } -+ } -+ if (failCreateTimes > 0) -+ { -+ failCreateTimes--; -+ throw new StorageException("Probe creation failed"); -+ } -+ if (nullPath) -+ { -+ return null; -+ } -+ if (delayMillis == 0) -+ { -+ try -+ { -+ Files.createSymbolicLink(link, device); -+ } -+ catch (IOException exc) -+ { -+ throw new StorageException("Test device creation failed", exc); -+ } -+ } -+ else if (delayMillis > 0) -+ { -+ create = executor.schedule(() -> Files.createSymbolicLink(link, device), -+ delayMillis, TimeUnit.MILLISECONDS); -+ } -+ return link.toString(); -+ } -+ -+ @Override -+ public void deleteTmpProbeVlm(StorPool ignored) throws StorageException -+ { -+ deleteCalls++; -+ if (interruptDelete) -+ { -+ throw new StorageException("Probe cleanup interrupted", new InterruptedException()); -+ } -+ if (failDelete) -+ { -+ throw new StorageException("Probe cleanup failed"); -+ } -+ try -+ { -+ if (create != null && !create.cancel(false)) -+ { -+ create.get(); -+ } -+ Files.deleteIfExists(link); -+ activeProbes.decrementAndGet(); -+ } -+ catch (Exception exc) -+ { -+ throw new StorageException("Test device cleanup failed", exc); -+ } -+ } -+ } -+} diff --git a/packages/system/linstor/images/piraeus-server/patches/native-protoc.diff b/packages/system/linstor/images/piraeus-server/patches/native-protoc.diff index f3e601698..fa8596177 100644 --- a/packages/system/linstor/images/piraeus-server/patches/native-protoc.diff +++ b/packages/system/linstor/images/piraeus-server/patches/native-protoc.diff @@ -1,8 +1,8 @@ diff --git a/build.gradle b/build.gradle -index 75b7f64..d2dfec9 100644 +index 9df6c840cdc0..9894a436712c 100644 --- a/build.gradle +++ b/build.gradle -@@ -37,6 +37,9 @@ configurations { +@@ -34,6 +34,9 @@ configurations { } def protobufVersion = '31.1' @@ -12,7 +12,7 @@ index 75b7f64..d2dfec9 100644 def protoc = file("${projectDir}/tools/protoc-" + protobufVersion + '/bin/protoc') def logbackConfig = "logback.xml" def jvmOpts = ["-Xms32M"] -@@ -577,7 +580,7 @@ tasks.register('buildTestCommands', Exec) { +@@ -650,7 +653,7 @@ tasks.register('buildTestCommands', Exec) { tasks.register('unzipProtoc', Copy) { dependsOn(downloadProtoc) @@ -21,7 +21,7 @@ index 75b7f64..d2dfec9 100644 def outputDir = file("tools/protoc-" + protobufVersion) from zipTree(zipFile) -@@ -591,7 +594,7 @@ tasks.register('unzipProtoc', Copy) { +@@ -664,7 +667,7 @@ tasks.register('unzipProtoc', Copy) { tasks.register('downloadProtoc') { outputs.dir "tools" @@ -30,7 +30,7 @@ index 75b7f64..d2dfec9 100644 outputs.file protozip doLast { -@@ -599,7 +602,7 @@ tasks.register('downloadProtoc') { +@@ -672,7 +675,7 @@ tasks.register('downloadProtoc') { mkdir "tools" println "downloading protoc..." new URL('https://github.com/google/protobuf/releases/download/v' diff --git a/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff b/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff index 009591674..e5e0ec013 100644 --- a/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff +++ b/packages/system/linstor/images/piraeus-server/patches/retry-adjust-after-stale-bitmap.diff @@ -1,28 +1,27 @@ diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java -index 5627d1be8..ece191292 100644 +index 0a19f875b702..d44e19b70d88 100644 --- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java +++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java -@@ -42,6 +42,9 @@ import java.util.Arrays; - import java.util.List; +@@ -45,6 +45,8 @@ import java.util.List; + import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.TimeUnit; -+import java.nio.charset.StandardCharsets; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; @Singleton -@@ -56,6 +58,9 @@ public class DrbdAdm - - public static final int WAIT_CONNECT_RES_TIME = 10; - private static final long DOWN_WAIT_TIMEOUT_SEC = 5; +@@ -68,6 +70,9 @@ public class DrbdAdm + private static final long SECONDARY_RETRY_INITIAL_BACKOFF_MS = 250L; + private static final long SECONDARY_RETRY_MAX_BACKOFF_MS = 2_000L; + private static final String DEVICE_HELD_OPEN_ERR_MSG = "Device is held open by someone"; + private static final long FORCE_DETACH_RETRY_WAIT_MS = 250; + private static final String BITMAP_LEAK_ERR_MSG = "already has a bitmap, this should not happen"; + private static final Pattern BITMAP_LEAK_MINOR_PATTERN = Pattern.compile("\\bminor\\s+(\\d+)\\b"); private final ExtCmdFactory extCmdFactory; - private final AccessContext sysCtx; -@@ -131,8 +136,38 @@ public class DrbdAdm + private final StltConfigAccessor stltCfgAccessor; +@@ -143,8 +148,38 @@ public class DrbdAdm // command.add(resName); command.add(drbdRscData.getSuffixedResourceName()); // execute(Arrays.asList("drbdsetup", "show", drbdRscData.getSuffixedResourceName())); @@ -35,7 +34,7 @@ index 5627d1be8..ece191292 100644 + ExtCmd extCmd = extCmdFactory.create(); + if (Platform.isWindows()) + { -+ extCmd.setTimeout(TimeoutType.WAIT, 5 * 60 * 1000); ++ extCmd.setTimeout(TimeoutType.WAIT, DFLT_WINDOWS_TIMEOUT_MS); + } + + OutputData outputData = extCmd.pipeExec(ProcessBuilder.Redirect.from(nullDevice), commandArr); @@ -63,8 +62,8 @@ index 5627d1be8..ece191292 100644 drbdRscData.setAdjustRequired(false); } -@@ -805,6 +840,75 @@ public class DrbdAdm - } +@@ -955,6 +990,75 @@ public class DrbdAdm + return ret; } + static boolean isBitmapLeakOnAttach(OutputData outputData) diff --git a/packages/system/linstor/images/piraeus-server/patches/retry-secondary-after-mkfs.diff b/packages/system/linstor/images/piraeus-server/patches/retry-secondary-after-mkfs.diff deleted file mode 100644 index f213e9037..000000000 --- a/packages/system/linstor/images/piraeus-server/patches/retry-secondary-after-mkfs.diff +++ /dev/null @@ -1,104 +0,0 @@ -diff --git a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java -index 71c1df6..f91903b 100644 ---- a/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java -+++ b/satellite/src/main/java/com/linbit/linstor/layer/drbd/utils/DrbdAdm.java -@@ -62,6 +62,10 @@ public class DrbdAdm - private static final long FORCE_DETACH_RETRY_WAIT_MS = 250; - private static final String BITMAP_LEAK_ERR_MSG = "already has a bitmap, this should not happen"; - private static final Pattern BITMAP_LEAK_MINOR_PATTERN = Pattern.compile("\\bminor\\s+(\\d+)\\b"); -+ private static final int SECONDARY_RETRY_MAX_ATTEMPTS = 7; -+ private static final long SECONDARY_RETRY_INITIAL_BACKOFF_MS = 250L; -+ private static final long SECONDARY_RETRY_MAX_BACKOFF_MS = 2000L; -+ private static final String DEVICE_HELD_OPEN_ERR_MSG = "Device is held open by someone"; - - private final ExtCmdFactory extCmdFactory; - private final AccessContext sysCtx; -@@ -945,6 +949,79 @@ public class DrbdAdm - return true; - } - -+ /** -+ * Switches a resource to secondary mode after mkfs, retrying when DRBD reports the device as held -+ * open by another process. On Talos and other systems with active block-device probing -+ * (machined/udev), the kernel uevent for a freshly-promoted device may cause an external probe -+ * to open the device just before LINSTOR demotes it back to Secondary, producing -+ * "Device is held open by someone" (errno -12) and aborting resource initialization. -+ * Probes typically release the fd within milliseconds, so a short retry with exponential backoff -+ * is sufficient to win the race without changing the contract of {@link #secondary}. -+ */ -+ public void secondaryAfterMkfs(DrbdRscData drbdRscData) throws ExtCmdFailedException -+ { -+ String[] commandArr = new String[] { -+ DRBDADM_UTIL, "-vvv", "secondary", drbdRscData.getSuffixedResourceName() -+ }; -+ -+ File nullDevice = new File(Platform.nullDevice()); -+ -+ OutputData lastOutput = null; -+ for (int attempt = 1; attempt <= SECONDARY_RETRY_MAX_ATTEMPTS; attempt++) -+ { -+ try -+ { -+ ExtCmd extCmd = extCmdFactory.create(); -+ if (Platform.isWindows()) -+ { -+ extCmd.setTimeout(TimeoutType.WAIT, 5 * 60 * 1000); -+ } -+ -+ lastOutput = extCmd.pipeExec(ProcessBuilder.Redirect.from(nullDevice), commandArr); -+ if (lastOutput.exitCode == 0) -+ { -+ return; -+ } -+ if (!isDeviceHeldOpenError(lastOutput) || attempt == SECONDARY_RETRY_MAX_ATTEMPTS) -+ { -+ throw new ExtCmdFailedException(commandArr, lastOutput); -+ } -+ } -+ catch (ChildProcessTimeoutException timeoutExc) -+ { -+ throw new ExtCmdFailedException(commandArr, timeoutExc); -+ } -+ catch (IOException ioExc) -+ { -+ throw new ExtCmdFailedException(commandArr, ioExc); -+ } -+ -+ long sleepMs = Math.min( -+ SECONDARY_RETRY_INITIAL_BACKOFF_MS * (1L << (attempt - 1)), -+ SECONDARY_RETRY_MAX_BACKOFF_MS -+ ); -+ try -+ { -+ Thread.sleep(sleepMs); -+ } -+ catch (InterruptedException ie) -+ { -+ Thread.currentThread().interrupt(); -+ throw new ExtCmdFailedException(commandArr, lastOutput); -+ } -+ } -+ } -+ -+ static boolean isDeviceHeldOpenError(OutputData outputData) -+ { -+ if (outputData == null || outputData.stderrData == null) -+ { -+ return false; -+ } -+ String stderr = new String(outputData.stderrData, StandardCharsets.UTF_8); -+ return stderr.contains(DEVICE_HELD_OPEN_ERR_MSG); -+ } -+ - public static class DrbdPrimary implements AutoCloseable - { - private final DrbdAdm drbdAdm; -@@ -963,7 +1040,7 @@ public class DrbdAdm - @Override - public void close() throws ExtCmdFailedException - { -- drbdAdm.secondary(drbdRscData); -+ drbdAdm.secondaryAfterMkfs(drbdRscData); - } - } - } diff --git a/packages/system/linstor/images/piraeus-server/patches/snapshot-restore-drbd-initialized.diff b/packages/system/linstor/images/piraeus-server/patches/snapshot-restore-drbd-initialized.diff deleted file mode 100644 index 7c72aab69..000000000 --- a/packages/system/linstor/images/piraeus-server/patches/snapshot-restore-drbd-initialized.diff +++ /dev/null @@ -1,67 +0,0 @@ -diff --git a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java -index a639c27cc..4dabc1a83 100644 ---- a/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java -+++ b/controller/src/main/java/com/linbit/linstor/core/apicallhandler/controller/CtrlSnapshotRestoreApiCallHandler.java -@@ -434,26 +434,39 @@ public class CtrlSnapshotRestoreApiCallHandler - } - } - -- /* -- * When a snapshot was received as a backup, it is marked to re-create its local metadata. Before the -- * snapshot was sent the new DRBD_INITIALIZED flag and the VlmDfn property InitialUpToDateOn were -+ /** -+ *

Sets the {@value InternalApiConsts#KEY_LINSTOR_DRBD_INITIAL_UPTODATE_ON} property to the nodeName -+ * of the first Snapshot (which has to have a disk) of the given SnapshotDefinition.

-+ * -+ *

If this SnapshotDefinition is a backup, we also unset the {@link VolumeDefinition.Flags#DRBD_INITIALIZED} -+ * flag. Otherwise we ensure the same flag is set.

-+ * -+ *

Backup path: When a snapshot was received as a backup, it is marked to re-create its local metadata. -+ * Before the snapshot was sent the new DRBD_INITIALIZED flag and the VlmDfn property InitialUpToDateOn were - * cleared. If DRBD_INITIALZED would be set, no peer would even read the property (on purpose). - * Without the property, no peer would think of itself as a winner, thus the DRBD resource would never - * become UpToDate. - * To avoid this, this method sets the InitialUpToDateOn property to let the first resource win the -- * race, so it can become UpToDate and therefore also pull the other peers eventually into UpToDate state. -+ * race, so it can become UpToDate and therefore also pull the other peers eventually into UpToDate state.

- * -- * If the given SnapDfn had also non-backup snapshots, those should be able to keep their metadata. -+ *

If the given SnapDfn had also non-backup snapshots, those should be able to keep their metadata. - * In that case we do not have to do anything. If however all snapshots are descendants from backups - * we must pick one and decide the UpToDate race upfront so that the winner can set its local DRBD -- * as UpToDate, which also triggers the controller to set the DRBD_INITIALZED flag eventually. -+ * as UpToDate, which also triggers the controller to set the DRBD_INITIALZED flag eventually.

-+ * -+ *

Non-Backup path: When a regular snapshot (i.e. not from a backup) is restored, we can or even should -+ * assume that the backing snapshot LVs have correct metadata which do not need be be initialized. When -+ * a SnapshotVolumeDefinition is created it unfortunately does not properly store the VolumeDefinition's flags, -+ * therefore the restored VolumeDefinition will also not have the DRBD_INITIALIZED flag set, although it -+ * refers to a device with healthy and UpToDate metadata. To fix this issue we simply set the flag in this case.

- */ - private void checkDrbdInitializedState(ResourceDefinition rscDfnRef) throws AccessDeniedException - { - AccessContext accCtx = peerAccCtx.get(); -- if (DrbdLayerUtils.isForceInitialSyncSet(accCtx, rscDfnRef) && rscDfnRef.getDiskfulCount(accCtx) > 0) -+ if (rscDfnRef.getDiskfulCount(accCtx) > 0) - { - String winnerNodeName = rscDfnRef.getDiskfulResources(accCtx).get(0).getNode().getName().value; -+ boolean uninitializeDrbd = DrbdLayerUtils.isForceInitialSyncSet(accCtx, rscDfnRef); - Iterator vlmDfnIt = rscDfnRef.iterateVolumeDfn(accCtx); - while (vlmDfnIt.hasNext()) - { -@@ -462,7 +475,14 @@ public class CtrlSnapshotRestoreApiCallHandler - { - vlmDfn.getProps(accCtx) - .setProp(InternalApiConsts.KEY_LINSTOR_DRBD_INITIAL_UPTODATE_ON, winnerNodeName); -- vlmDfn.getFlags().disableFlags(accCtx, VolumeDefinition.Flags.DRBD_INITIALIZED); -+ if (uninitializeDrbd) -+ { -+ vlmDfn.getFlags().disableFlags(accCtx, VolumeDefinition.Flags.DRBD_INITIALIZED); -+ } -+ else -+ { -+ vlmDfn.getFlags().enableFlags(accCtx, VolumeDefinition.Flags.DRBD_INITIALIZED); -+ } - } - catch (InvalidKeyException | InvalidValueException exc) - { diff --git a/packages/system/linstor/templates/cluster.yaml b/packages/system/linstor/templates/cluster.yaml index ba5154375..c648c0a8c 100644 --- a/packages/system/linstor/templates/cluster.yaml +++ b/packages/system/linstor/templates/cluster.yaml @@ -36,6 +36,12 @@ spec: enabled: true podTemplate: spec: + {{- /* run-migration upgrades the database in place: left on the + operator's default image it migrates ahead of the controller, + irreversibly. */}} + initContainers: + - name: run-migration + image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} containers: - name: linstor-controller image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} diff --git a/packages/system/linstor/templates/satellites-cozy.yaml b/packages/system/linstor/templates/satellites-cozy.yaml index 7897189f4..bb6240128 100644 --- a/packages/system/linstor/templates/satellites-cozy.yaml +++ b/packages/system/linstor/templates/satellites-cozy.yaml @@ -11,6 +11,9 @@ spec: spec: # host-network is recommended by Piraeus while it is not default in the upstream hostNetwork: true + initContainers: + - name: setup-lvm-configuration + image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} containers: - name: linstor-satellite image: {{ .Values.piraeusServer.image.repository }}:{{ .Values.piraeusServer.image.tag }} diff --git a/packages/system/linstor/tests/csi-topology_test.yaml b/packages/system/linstor/tests/csi-topology_test.yaml index fbde97a2d..6dd9bb3ea 100644 --- a/packages/system/linstor/tests/csi-topology_test.yaml +++ b/packages/system/linstor/tests/csi-topology_test.yaml @@ -34,7 +34,7 @@ tests: # Until there is, the re-check on a bump is a human obligation, stated in # templates/cluster.yaml beside the list. # - # The list was last verified complete against piraeus-operator v2.10.2: + # The list was last verified complete against piraeus-operator v2.12.1: # eleven upstream flags plus --strict-topology. - it: the restated argument list is complete template: templates/cluster.yaml diff --git a/packages/system/linstor/tests/server-image_test.yaml b/packages/system/linstor/tests/server-image_test.yaml new file mode 100644 index 000000000..d63d19ae4 --- /dev/null +++ b/packages/system/linstor/tests/server-image_test.yaml @@ -0,0 +1,40 @@ +suite: every LINSTOR server container runs the platform image + +# piraeus-operator ships its own default for the LINSTOR server image and +# applies it to every container this package leaves unset. The init +# containers matter most: run-migration upgrades the LINSTOR database in +# place, so an operator default newer than the platform build migrates the +# schema ahead of the controller, which then refuses to start, and the +# migration cannot be undone. Pinning them to the same image as the main +# containers makes the database move only together with the controller. + +templates: + - templates/cluster.yaml + - templates/satellites-cozy.yaml + +set: + piraeusServer: + image: + repository: registry.example/piraeus-server + tag: v0.0.0-test + +tests: + - it: the controller migration runs the platform image + template: templates/cluster.yaml + asserts: + - equal: + path: spec.controller.podTemplate.spec.initContainers[?(@.name=="run-migration")].image + value: registry.example/piraeus-server:v0.0.0-test + - equal: + path: spec.controller.podTemplate.spec.containers[?(@.name=="linstor-controller")].image + value: registry.example/piraeus-server:v0.0.0-test + + - it: the satellite LVM setup runs the platform image + template: templates/satellites-cozy.yaml + asserts: + - equal: + path: spec.podTemplate.spec.initContainers[?(@.name=="setup-lvm-configuration")].image + value: registry.example/piraeus-server:v0.0.0-test + - equal: + path: spec.podTemplate.spec.containers[?(@.name=="linstor-satellite")].image + value: registry.example/piraeus-server:v0.0.0-test diff --git a/packages/system/monitoring-agents/alerts/general.rules.yaml b/packages/system/monitoring-agents/alerts/general.rules.yaml index 924870c40..e713a4970 100644 --- a/packages/system/monitoring-agents/alerts/general.rules.yaml +++ b/packages/system/monitoring-agents/alerts/general.rules.yaml @@ -33,7 +33,8 @@ spec: is working properly. expr: vector(1) labels: - severity: ok + severity: major + timeout: "300" exported_instance: global service: general.rules event: Heartbeat @@ -52,6 +53,6 @@ spec: != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1 labels: - severity: major + severity: none exported_instance: global service: general.rules diff --git a/packages/system/monitoring-agents/alerts/workload-placement.yaml b/packages/system/monitoring-agents/alerts/workload-placement.yaml new file mode 100644 index 000000000..3ae9c8f50 --- /dev/null +++ b/packages/system/monitoring-agents/alerts/workload-placement.yaml @@ -0,0 +1,81 @@ +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: alerts-workload-placement +spec: + groups: + - name: workload-placement + params: {} + rules: + - alert: PodReplicasColocated + expr: |- + count by (cluster, namespace, owner_kind, owner_name, node) ( + label_replace( + label_replace( + ( + max by (cluster, namespace, pod, node, created_by_kind, created_by_name) ( + kube_pod_info{job="kube-state-metrics", created_by_kind=~"StatefulSet|ReplicaSet|Cluster|StrimziPodSet"} + ) + or + label_replace( + label_replace( + max by (cluster, namespace, pod, node, created_by_kind, created_by_name) ( + kube_pod_info{job="kube-state-metrics", created_by_kind="EtcdMember"} + ), + "created_by_name", "$1", "created_by_name", "(.+)-[^-]+" + ), + "created_by_kind", "EtcdCluster", "", "" + ) + ) + and on (cluster, namespace, pod) + kube_pod_status_phase{job="kube-state-metrics", phase="Running"} == 1, + "owner_kind", "$1", "created_by_kind", "(.+)" + ), + "owner_name", "$1", "created_by_name", "(.+)" + ) + ) > 1 + and on (cluster, namespace, owner_kind, owner_name) + count by (cluster, namespace, owner_kind, owner_name) ( + label_replace( + label_replace( + ( + max by (cluster, namespace, pod, node, created_by_kind, created_by_name) ( + kube_pod_info{job="kube-state-metrics", created_by_kind=~"StatefulSet|ReplicaSet|Cluster|StrimziPodSet"} + ) + or + label_replace( + label_replace( + max by (cluster, namespace, pod, node, created_by_kind, created_by_name) ( + kube_pod_info{job="kube-state-metrics", created_by_kind="EtcdMember"} + ), + "created_by_name", "$1", "created_by_name", "(.+)-[^-]+" + ), + "created_by_kind", "EtcdCluster", "", "" + ) + ) + and on (cluster, namespace, pod) + kube_pod_status_phase{job="kube-state-metrics", phase="Running"} == 1, + "owner_kind", "$1", "created_by_kind", "(.+)" + ), + "owner_name", "$1", "created_by_name", "(.+)" + ) + ) + <= on (cluster) group_left () + count by (cluster) ( + max by (cluster, node) (kube_node_info{job="kube-state-metrics"}) + unless on (cluster, node) + kube_node_spec_taint{job="kube-state-metrics", effect=~"NoSchedule|NoExecute"} + ) + for: 1h + labels: + severity: warning + exported_instance: '{{ $labels.namespace }}/{{ $labels.owner_name }}' + service: workload-placement + annotations: + summary: Pods of one workload share a node. + description: >- + {{ $value }} running pods of {{ $labels.owner_kind }} {{ $labels.namespace }}/{{ $labels.owner_name }} + are on node {{ $labels.node }} in cluster {{ $labels.cluster }}, although the cluster has a + schedulable node for each of its pods. Losing this node stops these pods together, which can cost + a quorum-based workload its quorum. The scheduler does not move running pods, so they stay + together until they are recreated. diff --git a/packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml b/packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml new file mode 100644 index 000000000..efdbd1158 --- /dev/null +++ b/packages/system/monitoring-agents/templates/vm-cleanup-hook.yaml @@ -0,0 +1,167 @@ +{{- /* + The VictoriaMetrics operator puts its finalizer on the VMAgent and is the only + thing that ever removes it. The operator ships as a separate release that is + gated on the same addon flag, so disabling the addon removes both at once and + Flux does not order deletions: when the operator goes first, the VMAgent stays + Terminating and the uninstall never finishes. The other kinds are handled the + same way in case an operator version finalizes them too; for those the patch + is a no-op. + + So delete them before Helm does, give a live operator a bounded chance to + finalize them itself, and only then clear the finalizers by hand. The VMAgent's + Deployment, Service and Secrets carry owner references and are garbage + collected once it is gone. + + Cluster-scoped RBAC because the objects span two namespaces, and a Role in the + operator's namespace could not be created if that namespace were already + gone. The hook always exits 0: a cleanup that cannot complete must not be what + keeps the release from uninstalling. +*/ -}} +{{- $name := "vm-objects-cleanup" }} +{{- $img := printf "%s:%s" .Values.kubectlImage.repository .Values.kubectlImage.tag }} +{{- with .Values.kubectlImage.digest }}{{- $img = printf "%s@%s" $img . }}{{- end }} +{{- $kinds := list "vmagents" "vmrules" "vmservicescrapes" "vmpodscrapes" "vmnodescrapes" }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "-10" + helm.sh/hook-delete-policy: hook-succeeded,hook-failed,before-hook-creation +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Namespace }}-{{ $name }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "-10" + helm.sh/hook-delete-policy: hook-succeeded,hook-failed,before-hook-creation +rules: + - apiGroups: ["operator.victoriametrics.com"] + resources: + {{- toYaml $kinds | nindent 6 }} + verbs: ["get", "list", "delete", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Namespace }}-{{ $name }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "-10" + helm.sh/hook-delete-policy: hook-succeeded,hook-failed,before-hook-creation +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Namespace }}-{{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + annotations: + helm.sh/hook: pre-delete + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation +spec: + # Bounds every kubectl call as well: --request-timeout would make kubectl + # skip the in-cluster credentials (hack/check-hook-request-timeout.bats). + # Well inside the HelmRelease's 10m uninstall budget, and above the 60s + # wait plus one call per object on either side of it. + activeDeadlineSeconds: 180 + backoffLimit: 0 + ttlSecondsAfterFinished: 600 + template: + spec: + serviceAccountName: {{ $name }} + restartPolicy: Never + securityContext: + runAsNonRoot: true + # The image user is the non-numeric `nonroot`, which the kubelet cannot + # check against runAsNonRoot. + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + volumes: + - name: home + emptyDir: {} + containers: + - name: cleanup + image: {{ $img | quote }} + imagePullPolicy: {{ .Values.kubectlImage.pullPolicy }} + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + # kubectl writes its discovery cache under $HOME. + env: + - name: HOME + value: /tmp + volumeMounts: + - name: home + mountPath: /tmp + command: + - /bin/sh + - -c + - | + rel={{ .Release.Name | quote }} + relns={{ .Release.Namespace | quote }} + kinds={{ join " " $kinds | quote }} + + # Selected by Helm's ownership annotations rather than by name, so + # the alert rules added under alerts/ are covered without a list + # to keep in step. A kind whose CRD is gone just prints an error. + owned() { + for k in $kinds; do + kubectl get "$k.operator.victoriametrics.com" -A \ + -o jsonpath='{range .items[*]}{.metadata.annotations.meta\.helm\.sh/release-name}{"|"}{.metadata.annotations.meta\.helm\.sh/release-namespace}{"|"}{.metadata.namespace}{"|"}{.metadata.name}{"\n"}{end}' | + while IFS='|' read -r r rns ns n; do + if [ "$r" = "$rel" ] && [ "$rns" = "$relns" ]; then + echo "$k.operator.victoriametrics.com $ns $n" + fi + done + done + } + + objs=$(owned) + if [ -z "$objs" ]; then + echo "no VictoriaMetrics objects left for $relns/$rel" + exit 0 + fi + + echo "$objs" | while read -r k ns n; do + kubectl -n "$ns" delete "$k" "$n" --wait=false + done + + deadline=$(( $(date +%s) + 60 )) + while [ "$(date +%s)" -lt "$deadline" ]; do + sleep 2 + objs=$(owned) + if [ -z "$objs" ]; then + echo "the operator finalized every object" + exit 0 + fi + done + + echo "finalizers still held after 60s; clearing them by hand" + echo "$objs" | while read -r k ns n; do + kubectl -n "$ns" patch "$k" "$n" --type=merge \ + -p '{"metadata":{"finalizers":null}}' + done + exit 0 diff --git a/packages/system/monitoring-agents/templates/vmagent.yaml b/packages/system/monitoring-agents/templates/vmagent.yaml index 7a87fa5b5..0eb6d9d80 100644 --- a/packages/system/monitoring-agents/templates/vmagent.yaml +++ b/packages/system/monitoring-agents/templates/vmagent.yaml @@ -20,6 +20,39 @@ spec: scrapeInterval: 30s selectAllByDefault: true + # The class sets all three identity labels on every target it covers, "-" + # when the pod has no lineage ("-" is not a valid Kubernetes label value, + # so it never collides with a real one). Under honorLabels false every + # exporter-supplied copy then collides, is renamed to exported_* and + # dropped. Under honorLabels true the exporter's copy wins, so such a + # scrape must drop these labels itself. A scrape object's own relabel + # rules run after these: they may drop the labels but must never set them. + # Not covered: VMProbe (operator v0.68.4 takes only auth from a class) and + # raw scrape configs. The placeholder is removed by the remote-write + # relabeling below. + scrapeClasses: + - name: application-identity + default: true + relabelConfigs: + {{- range list "group" "kind" "name" }} + - targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_{{ . }}] + regex: (.+) + targetLabel: apps_cozystack_io_application_{{ . }} + {{- end }} + metricRelabelConfigs: + - action: labeldrop + regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + # Not in metricRelabelConfigs: vmagent adds up and scrape_* after metric + # relabeling, so the placeholder would stay on those series. + inlineRelabelConfig: + {{- range list "group" "kind" "name" }} + - sourceLabels: [apps_cozystack_io_application_{{ . }}] + regex: "-" + targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "" + {{- end }} additionalScrapeConfigs: name: additional-scrape-configs key: prometheus-additional.yaml diff --git a/packages/system/monitoring-agents/templates/vpa.yaml b/packages/system/monitoring-agents/templates/vpa.yaml index 9a31fab94..1ac949f30 100644 --- a/packages/system/monitoring-agents/templates/vpa.yaml +++ b/packages/system/monitoring-agents/templates/vpa.yaml @@ -6,6 +6,11 @@ Initial sets requests at creation without evicting, removing the trigger. Do not switch back to Auto without fixing the underlying Cilium IP-reuse race. */ -}} +{{- /* + Rendered only when the CRD is served: a release manifest that records a VPA + the cluster cannot map is one Helm can no longer uninstall. +*/ -}} +{{- if .Capabilities.APIVersions.Has "autoscaling.k8s.io/v1/VerticalPodAutoscaler" }} apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: @@ -34,3 +39,4 @@ spec: maxAllowed: cpu: 4000m memory: 6G +{{- end }} diff --git a/packages/system/monitoring-agents/tests/uninstall_test.yaml b/packages/system/monitoring-agents/tests/uninstall_test.yaml new file mode 100644 index 000000000..df8a3eaa6 --- /dev/null +++ b/packages/system/monitoring-agents/tests/uninstall_test.yaml @@ -0,0 +1,141 @@ +suite: the release uninstalls whatever order its dependencies go in + +# Disabling the addon removes this release, the VictoriaMetrics operator and the +# VPA CRDs together, and Flux deletes them in parallel. When the operator goes +# first, nothing clears its finalizer on the objects this chart created and the +# uninstall hangs; when the VPA CRD goes first, a release manifest recording a +# VPA cannot be built and the uninstall fails. +release: + name: cozy-monitoring-agents + namespace: cozy-monitoring +templates: + - templates/vm-cleanup-hook.yaml + - templates/vpa.yaml +tests: + - it: runs before Helm deletes anything, bounded and with one attempt + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: metadata.annotations["helm.sh/hook"] + value: pre-delete + - equal: + path: metadata.namespace + value: cozy-monitoring + - equal: + path: spec.activeDeadlineSeconds + value: 180 + - equal: + path: spec.backoffLimit + value: 0 + - matchRegex: + path: spec.template.spec.containers[0].image + pattern: ^docker\.io/clastix/kubectl:v[0-9.]+@sha256:[0-9a-f]{64}$ + + - it: its RBAC exists only for the hook and is removed with it + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: ClusterRole + asserts: + - equal: + path: metadata.annotations["helm.sh/hook"] + value: pre-delete + - equal: + path: metadata.annotations["helm.sh/hook-weight"] + value: "-10" + - matchRegex: + path: metadata.annotations["helm.sh/hook-delete-policy"] + pattern: hook-succeeded + + - it: may delete and clear finalizers on every VictoriaMetrics kind the chart renders + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: ClusterRole + asserts: + - equal: + path: rules[0].apiGroups + value: ["operator.victoriametrics.com"] + - contains: + path: rules[0].resources + content: vmagents + - contains: + path: rules[0].resources + content: vmrules + - contains: + path: rules[0].resources + content: vmservicescrapes + - contains: + path: rules[0].resources + content: vmpodscrapes + - contains: + path: rules[0].resources + content: vmnodescrapes + - equal: + path: rules[0].verbs + value: ["get", "list", "delete", "patch"] + - lengthEqual: + path: rules + count: 1 + + - it: grants that role to the hook's own service account + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: ClusterRoleBinding + asserts: + - equal: + path: roleRef.name + value: cozy-monitoring-vm-objects-cleanup + - equal: + path: subjects[0] + value: + kind: ServiceAccount + name: vm-objects-cleanup + namespace: cozy-monitoring + + - it: touches only objects this release owns, and clears their finalizers + template: templates/vm-cleanup-hook.yaml + documentSelector: + path: kind + value: Job + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: rel="cozy-monitoring-agents" + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: relns="cozy-monitoring" + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: meta\\\.helm\\\.sh/release-name + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'delete "\$k" "\$n" --wait=false' + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: '"finalizers":null' + # A pre-delete hook that fails aborts the uninstall it exists to unblock. + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: exit 0\s*$ + + - it: the VPA is rendered when its CRD is served + template: templates/vpa.yaml + capabilities: + apiVersions: + - autoscaling.k8s.io/v1/VerticalPodAutoscaler + asserts: + - hasDocuments: + count: 1 + - isKind: + of: VerticalPodAutoscaler + + - it: the VPA stays out of the release manifest when its CRD is not served + template: templates/vpa.yaml + asserts: + - hasDocuments: + count: 0 diff --git a/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml new file mode 100644 index 000000000..4e4b93385 --- /dev/null +++ b/packages/system/monitoring-agents/tests/vmagent_application_identity_test.yaml @@ -0,0 +1,77 @@ +suite: cluster-wide vmagent stamps application identity on exporter series +templates: + - templates/vmagent.yaml +release: + name: monitoring-agents + namespace: cozy-monitoring + +# The tenant-resource-metrics proxy scopes a query to one application by +# apps_cozystack_io_application_{group,kind,name}. The labels are only a +# boundary if the scraped pod cannot choose them: the class sets all three +# on every target it covers (a placeholder when the pod has no lineage), so with +# honorLabels false an exporter's own copy is renamed to exported_* and +# dropped. The same rules live in the tenant vmagent of the monitoring +# package and are pinned there by the same assertions. +tests: + - it: every scrape gets the application identity class unless it names another + asserts: + - equal: + path: spec.scrapeClasses[0].name + value: application-identity + - equal: + path: spec.scrapeClasses[0].default + value: true + - lengthEqual: + path: spec.scrapeClasses + count: 1 + + - it: the identity comes from the scraped pod's lineage labels, with a placeholder when the pod has none + asserts: + - equal: + path: spec.scrapeClasses[0].relabelConfigs + value: + - targetLabel: apps_cozystack_io_application_group + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_group] + regex: (.+) + targetLabel: apps_cozystack_io_application_group + - targetLabel: apps_cozystack_io_application_kind + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_kind] + regex: (.+) + targetLabel: apps_cozystack_io_application_kind + - targetLabel: apps_cozystack_io_application_name + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_name] + regex: (.+) + targetLabel: apps_cozystack_io_application_name + + - it: an exporter-supplied identity is dropped + asserts: + - equal: + path: spec.scrapeClasses[0].metricRelabelConfigs + value: + - action: labeldrop + regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + + # vmagent adds up and scrape_* after metric relabeling (VictoriaMetrics + # lib/promscrape addAutoMetrics calls addRows with needRelabel false), so a + # placeholder removed there would stay on those series of every target + # without lineage. The remote-write relabeling sees every series. + - it: the placeholder is removed from every series, including up and scrape_* + asserts: + - equal: + path: spec.inlineRelabelConfig + value: + - sourceLabels: [apps_cozystack_io_application_group] + regex: "-" + targetLabel: apps_cozystack_io_application_group + replacement: "" + - sourceLabels: [apps_cozystack_io_application_kind] + regex: "-" + targetLabel: apps_cozystack_io_application_kind + replacement: "" + - sourceLabels: [apps_cozystack_io_application_name] + regex: "-" + targetLabel: apps_cozystack_io_application_name + replacement: "" diff --git a/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml b/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml new file mode 100644 index 000000000..fe630a80c --- /dev/null +++ b/packages/system/monitoring-agents/tests/watchdog_heartbeat_test.yaml @@ -0,0 +1,29 @@ +suite: Watchdog heartbeat raises an alarm in Alerta when it stops +templates: + - templates/alerts.yaml + +release: + name: monitoring-agents + namespace: cozy-monitoring + +tests: + - it: gives the heartbeat a severity Alerta reports as an alarm + documentSelector: + path: metadata.name + value: alerts-general.rules + asserts: + - equal: + path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.severity + value: major + + - it: lets the heartbeat go stale within minutes, not a day + documentSelector: + path: metadata.name + value: alerts-general.rules + asserts: + - equal: + path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.timeout + value: "300" + - equal: + path: spec.groups[0].rules[?(@.alert == "Watchdog")].labels.event + value: Heartbeat diff --git a/packages/system/monitoring-agents/values.yaml b/packages/system/monitoring-agents/values.yaml index d7dd870b0..b6c046ea5 100644 --- a/packages/system/monitoring-agents/values.yaml +++ b/packages/system/monitoring-agents/values.yaml @@ -575,3 +575,15 @@ prometheus-node-exporter: memory: 32Mi limits: memory: 256Mi + +# Image for the pre-delete hook that releases the VictoriaMetrics objects (see +# templates/vm-cleanup-hook.yaml). Only needs a kubectl binary. Digest-pinned so +# an upstream retag does not change what runs across the fleet; refresh with +# `docker manifest inspect docker.io/clastix/kubectl:v1.32`. Same pin as the +# etcd-operator hook. +# renovate: datasource=docker depName=docker.io/clastix/kubectl +kubectlImage: + repository: docker.io/clastix/kubectl + tag: v1.32 + digest: sha256:b9ef7d8dbe65bcc81a46c09b8dc7543103055021c4f43287bf59e92a8f4fe05c + pullPolicy: IfNotPresent diff --git a/packages/system/monitoring/templates/vm/vmagent.yaml b/packages/system/monitoring/templates/vm/vmagent.yaml index 98bff1e14..919fdef63 100644 --- a/packages/system/monitoring/templates/vm/vmagent.yaml +++ b/packages/system/monitoring/templates/vm/vmagent.yaml @@ -24,6 +24,39 @@ spec: scrapeInterval: 30s selectAllByDefault: false + # The class sets all three identity labels on every target it covers, "-" + # when the pod has no lineage ("-" is not a valid Kubernetes label value, + # so it never collides with a real one). Under honorLabels false every + # exporter-supplied copy then collides, is renamed to exported_* and + # dropped. Under honorLabels true the exporter's copy wins, so such a + # scrape must drop these labels itself. A scrape object's own relabel + # rules run after these: they may drop the labels but must never set them. + # Not covered: VMProbe (operator v0.68.4 takes only auth from a class) and + # raw scrape configs. The placeholder is removed by the remote-write + # relabeling below. + scrapeClasses: + - name: application-identity + default: true + relabelConfigs: + {{- range list "group" "kind" "name" }} + - targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_{{ . }}] + regex: (.+) + targetLabel: apps_cozystack_io_application_{{ . }} + {{- end }} + metricRelabelConfigs: + - action: labeldrop + regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + # Not in metricRelabelConfigs: vmagent adds up and scrape_* after metric + # relabeling, so the placeholder would stay on those series. + inlineRelabelConfig: + {{- range list "group" "kind" "name" }} + - sourceLabels: [apps_cozystack_io_application_{{ . }}] + regex: "-" + targetLabel: apps_cozystack_io_application_{{ . }} + replacement: "" + {{- end }} {{- with .Values.vmagent.inlineScrapeConfig }} inlineScrapeConfig: | {{- . | nindent 4 }} diff --git a/packages/system/monitoring/templates/vm/vmalert-scrape.yaml b/packages/system/monitoring/templates/vm/vmalert-scrape.yaml index ee406a275..b467d5564 100644 --- a/packages/system/monitoring/templates/vm/vmalert-scrape.yaml +++ b/packages/system/monitoring/templates/vm/vmalert-scrape.yaml @@ -1,3 +1,4 @@ +{{- with first .Values.metricsStorages }} --- apiVersion: operator.victoriametrics.com/v1beta1 kind: VMServiceScrape @@ -13,5 +14,6 @@ spec: selector: matchLabels: app.kubernetes.io/component: monitoring - app.kubernetes.io/instance: vmalert + app.kubernetes.io/instance: vmalert-{{ .name }} app.kubernetes.io/name: vmalert +{{- end }} diff --git a/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml new file mode 100644 index 000000000..d4e366180 --- /dev/null +++ b/packages/system/monitoring/tests/alertmanager_blackhole_test.yaml @@ -0,0 +1,51 @@ +suite: Alertmanager blackhole route +templates: + - templates/alerta/alerta.yaml + +release: + name: monitoring + namespace: tenant-root + +set: + _cluster: + root-host: example.org + _namespace: + host: tenant-root.example.org + ingress: tenant-root + +# Rules labelled severity="none", InfoInhibitor among them, exist to be +# dropped, and Alerta answers that severity with a 500. Only the heartbeat +# route may come ahead of the blackhole, so no delivering route claims them. +tests: + - it: drops severity none ahead of every delivering route + asserts: + - documentSelector: + path: metadata.labels.app + value: alertmanager + matchRegex: + path: stringData["alertmanager.yaml"] + pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n' + + - it: keeps severity none out of email even when its alert name is listed + set: + alerta: + alerts: + email: + to: "oncall@example.org" + from: "alerts@example.org" + smarthost: "smtp.example.org:465" + alertnames: + - InfoInhibitor + asserts: + - documentSelector: + path: metadata.labels.app + value: alertmanager + matchRegex: + path: stringData["alertmanager.yaml"] + pattern: '\n routes:\n - receiver: alerta_no_resolve\n repeat_interval: 1m\n matchers:\n - .event="Heartbeat".\n - receiver: blackhole\n matchers:\n - .severity="none".\n' + - documentSelector: + path: metadata.labels.app + value: alertmanager + matchRegex: + path: stringData["alertmanager.yaml"] + pattern: '(?s)\n - receiver: blackhole\n matchers:\n - .severity="none".\n.*\n - receiver: email\n group_wait: 1m\n group_interval: 5m\n repeat_interval: .12h.\n matchers:\n - .alertname=~"\^\(InfoInhibitor\)\$".\n' diff --git a/packages/system/monitoring/tests/vmagent_application_identity_test.yaml b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml new file mode 100644 index 000000000..3a77fe6ee --- /dev/null +++ b/packages/system/monitoring/tests/vmagent_application_identity_test.yaml @@ -0,0 +1,73 @@ +suite: tenant vmagent stamps application identity on exporter series +templates: + - templates/vm/vmagent.yaml +release: + name: monitoring + namespace: tenant-foo + +# Same rules as the cluster-wide vmagent in monitoring-agents, pinned there +# by the same assertions; see that suite for why the placeholder and the +# drop exist. +tests: + - it: every scrape gets the application identity class unless it names another + asserts: + - equal: + path: spec.scrapeClasses[0].name + value: application-identity + - equal: + path: spec.scrapeClasses[0].default + value: true + - lengthEqual: + path: spec.scrapeClasses + count: 1 + + - it: the identity comes from the scraped pod's lineage labels, with a placeholder when the pod has none + asserts: + - equal: + path: spec.scrapeClasses[0].relabelConfigs + value: + - targetLabel: apps_cozystack_io_application_group + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_group] + regex: (.+) + targetLabel: apps_cozystack_io_application_group + - targetLabel: apps_cozystack_io_application_kind + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_kind] + regex: (.+) + targetLabel: apps_cozystack_io_application_kind + - targetLabel: apps_cozystack_io_application_name + replacement: "-" + - sourceLabels: [__meta_kubernetes_pod_label_apps_cozystack_io_application_name] + regex: (.+) + targetLabel: apps_cozystack_io_application_name + + - it: an exporter-supplied identity is dropped + asserts: + - equal: + path: spec.scrapeClasses[0].metricRelabelConfigs + value: + - action: labeldrop + regex: (exported_)+apps_cozystack_io_application_(group|kind|name) + + # vmagent adds up and scrape_* after metric relabeling (VictoriaMetrics + # lib/promscrape addAutoMetrics calls addRows with needRelabel false), so a + # placeholder removed there would stay on those series of every target + # without lineage. The remote-write relabeling sees every series. + - it: the placeholder is removed from every series, including up and scrape_* + asserts: + - equal: + path: spec.inlineRelabelConfig + value: + - sourceLabels: [apps_cozystack_io_application_group] + regex: "-" + targetLabel: apps_cozystack_io_application_group + replacement: "" + - sourceLabels: [apps_cozystack_io_application_kind] + regex: "-" + targetLabel: apps_cozystack_io_application_kind + replacement: "" + - sourceLabels: [apps_cozystack_io_application_name] + regex: "-" + targetLabel: apps_cozystack_io_application_name + replacement: "" diff --git a/packages/system/monitoring/tests/vmalert_scrape_test.yaml b/packages/system/monitoring/tests/vmalert_scrape_test.yaml new file mode 100644 index 000000000..1dfb9ec3e --- /dev/null +++ b/packages/system/monitoring/tests/vmalert_scrape_test.yaml @@ -0,0 +1,38 @@ +suite: vmalert metrics are scraped +templates: + - templates/vm/vmalert-scrape.yaml + +release: + name: monitoring + namespace: tenant-root + +set: + _cluster: + root-host: example.org + _namespace: + host: tenant-root.example.org + ingress: tenant-root + metricsStorages: + - name: shortterm + retentionPeriod: "3d" + deduplicationInterval: "15s" + - name: longterm + retentionPeriod: "14d" + deduplicationInterval: "5m" + +tests: + - it: selects the Service of the VMAlert the chart renders + asserts: + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/component: monitoring + app.kubernetes.io/instance: vmalert-shortterm + app.kubernetes.io/name: vmalert + + - it: renders nothing without metrics storages, like the VMAlert itself + set: + metricsStorages: [] + asserts: + - hasDocuments: + count: 0 diff --git a/packages/system/observability-proxy/templates/deployment.yaml b/packages/system/observability-proxy/templates/deployment.yaml index 581c15fde..3f5cb6d26 100644 --- a/packages/system/observability-proxy/templates/deployment.yaml +++ b/packages/system/observability-proxy/templates/deployment.yaml @@ -38,6 +38,14 @@ spec: - --cluster-domain={{ $clusterDomain }} - --root-tenant={{ .Values.observabilityProxy.rootTenant }} - --auth-cache-ttl={{ .Values.observabilityProxy.authCacheTTL }} + {{- /* Below the 300s stream idle timeout of a Cilium Gateway, so a + published tail that stops sending lines is ended by the proxy, + not reset by Envoy. */}} + - --tail-idle-timeout=4m30s + - --metrics-shortterm-store={{ .Values.observabilityProxy.metrics.shorttermStore }} + - --metrics-longterm-store={{ .Values.observabilityProxy.metrics.longtermStore }} + - --metrics-shortterm-retention={{ .Values.observabilityProxy.metrics.shorttermRetention }} + - --metrics-app-lookback={{ .Values.observabilityProxy.metrics.appLookback }} ports: - name: http containerPort: 8080 diff --git a/packages/system/observability-proxy/templates/httproute.yaml b/packages/system/observability-proxy/templates/httproute.yaml new file mode 100644 index 000000000..1ddd2f720 --- /dev/null +++ b/packages/system/observability-proxy/templates/httproute.yaml @@ -0,0 +1,30 @@ +{{- $cluster := .Values._cluster | default dict }} +{{- $host := index $cluster "root-host" }} +{{- $exposeServices := splitList "," ((index $cluster "expose-services") | default "") }} +{{- $exposeIngress := (index $cluster "expose-ingress") | default "tenant-root" }} +{{- $gatewayEnabled := (index $cluster "gateway-enabled") | default "false" }} +{{- if and (has "observability-proxy" $exposeServices) (eq $gatewayEnabled "true") }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: observability-proxy +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: cozystack + namespace: {{ $exposeIngress }} + hostnames: + - {{ printf "observability-proxy.%s" $host | quote }} + rules: + - backendRefs: + - name: observability-proxy + port: 80 + {{- /* Without a request timeout Cilium leaves Envoy's 15s route default + in place, which would cut every tail. The proxy bounds each + request itself, so the route disables it. Envoy's 300s stream + idle timeout still applies; the proxy's tail idle limit stays + below it (see deployment.yaml). */}} + timeouts: + request: 0s +{{- end }} diff --git a/packages/system/observability-proxy/templates/ingress.yaml b/packages/system/observability-proxy/templates/ingress.yaml new file mode 100644 index 000000000..7c6b1a029 --- /dev/null +++ b/packages/system/observability-proxy/templates/ingress.yaml @@ -0,0 +1,53 @@ +{{- $cluster := .Values._cluster | default dict }} +{{- $solver := (index $cluster "solver") | default "http01" }} +{{- $clusterIssuer := (index $cluster "issuer-name") | default "letsencrypt-prod" }} +{{- $host := index $cluster "root-host" }} +{{- $exposeServices := splitList "," ((index $cluster "expose-services") | default "") }} +{{- $exposeIngress := (index $cluster "expose-ingress") | default "tenant-root" }} +{{- $gatewayEnabled := (index $cluster "gateway-enabled") | default "false" }} +{{- $wildcardSecret := (index $cluster "wildcard-secret-name") | default "" }} +{{- if and (has "observability-proxy" $exposeServices) (ne $gatewayEnabled "true") }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: observability-proxy + annotations: + {{- if not $wildcardSecret }} + cert-manager.io/cluster-issuer: {{ $clusterIssuer }} + {{- if eq $solver "http01" }} + acme.cert-manager.io/http01-ingress-ingressclassname: {{ $exposeIngress }} + {{- end }} + {{- end }} + {{- /* The proxy refuses a body above 1 MiB itself; nginx's 1m is the + same bound, so an oversized body stops at the edge. */}} + nginx.ingress.kubernetes.io/proxy-body-size: 1m + {{- /* A tail flushes every line; buffered, lines would reach the client + in batches. */}} + nginx.ingress.kubernetes.io/proxy-buffering: "off" + {{- /* The controller default is 10s. The longest silence the proxy + allows is a tail with no new line for --tail-idle-timeout (4m30s), + after which it ends the response itself; before the first byte + the token and access reviews may add up to --request-timeout (1m). */}} + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" +spec: + ingressClassName: {{ $exposeIngress }} + rules: + - host: observability-proxy.{{ $host }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: observability-proxy + port: + name: http + tls: + - hosts: + - observability-proxy.{{ $host }} + {{- /* Wildcard mode: ingress-nginx serves its default certificate, and + the tls section stays to keep the HTTP-to-HTTPS redirect. */}} + {{- if not $wildcardSecret }} + secretName: observability-proxy-tls + {{- end }} +{{- end }} diff --git a/packages/system/observability-proxy/templates/rbac.yaml b/packages/system/observability-proxy/templates/rbac.yaml index 3c5392903..f3f67127c 100644 --- a/packages/system/observability-proxy/templates/rbac.yaml +++ b/packages/system/observability-proxy/templates/rbac.yaml @@ -15,6 +15,11 @@ rules: - apiGroups: [""] resources: ["namespaces"] verbs: ["list", "watch"] +# Application roots map the plural in the path to the kind the lineage +# labels carry, from an informer cache as well. +- apiGroups: ["cozystack.io"] + resources: ["applicationdefinitions"] + verbs: ["list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/packages/system/observability-proxy/templates/tenant-clusterroles.yaml b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml index 23460352f..ff5f3b987 100644 --- a/packages/system/observability-proxy/templates/tenant-clusterroles.yaml +++ b/packages/system/observability-proxy/templates/tenant-clusterroles.yaml @@ -1,7 +1,11 @@ # Aggregated into cozy-tenant-view (and consequently use, admin, super-admin). -# No API server serves cozystack.io logs: the rule exists only for the -# SubjectAccessReview the observability proxy asks before it answers a read -# under /apis/cozystack.io/v1alpha1/namespaces//logs/. +# No API server serves cozystack.io logs or metrics: the rule exists only for +# the SubjectAccessReview the observability proxy asks before it answers a +# read under /apis/cozystack.io/v1alpha1/namespaces//{logs,metrics}/, +# and before a read under an application root, +# /apis/apps.cozystack.io/v1alpha1/namespaces////{log,metrics}/, +# which needs the namespace root too. Narrowing this role also cuts off the +# application roots. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -10,5 +14,5 @@ metadata: rbac.cozystack.io/aggregate-to-tenant-view: "true" rules: - apiGroups: ["cozystack.io"] - resources: ["logs"] + resources: ["logs", "metrics"] verbs: ["list"] diff --git a/packages/system/observability-proxy/tests/deployment_test.yaml b/packages/system/observability-proxy/tests/deployment_test.yaml index ddf9d34b8..5705f69d0 100644 --- a/packages/system/observability-proxy/tests/deployment_test.yaml +++ b/packages/system/observability-proxy/tests/deployment_test.yaml @@ -40,6 +40,51 @@ tests: path: spec.template.spec.containers[0].args content: --auth-cache-ttl=10s + # Envoy behind a Cilium Gateway resets a stream after 300s without data + # (envoy.streamIdleTimeoutDurationSeconds), so a silent tail has to be + # ended by the proxy first, or the client sees a reset instead of an end. + - it: ends a silent tail before the Gateway stream idle timeout + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --tail-idle-timeout=4m30s + + - it: passes the metrics store names and short-term retention + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-shortterm-store=shortterm + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-longterm-store=longterm + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-shortterm-retention=72h + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-app-lookback=720h + + - it: follows renamed metrics stores + set: + observabilityProxy.metrics: + shorttermStore: fast + longtermStore: slow + shorttermRetention: 24h + appLookback: 48h + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-shortterm-store=fast + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-longterm-store=slow + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-shortterm-retention=24h + - contains: + path: spec.template.spec.containers[0].args + content: --metrics-app-lookback=48h + - it: falls back to cozy.local when the platform passes no cluster domain asserts: - contains: diff --git a/packages/system/observability-proxy/tests/publishing_test.yaml b/packages/system/observability-proxy/tests/publishing_test.yaml new file mode 100644 index 000000000..676c38e08 --- /dev/null +++ b/packages/system/observability-proxy/tests/publishing_test.yaml @@ -0,0 +1,196 @@ +suite: observability-proxy public endpoint +# Clients outside the cluster reach the proxy on its own host with a +# Kubernetes bearer token, so the route must pass Authorization and the +# /apis/... path through untouched, and must not end a request before the +# proxy does: a non-tail request may take up to --request-timeout (1m), and +# a tail may stay silent up to --tail-idle-timeout (4m30s). Whole objects are +# compared rather than single keys, so an annotation or a field that should +# not be there fails the test too. +templates: + - templates/ingress.yaml + - templates/httproute.yaml + +release: + name: observability-proxy + namespace: cozy-observability-proxy + +tests: + - it: publishes nothing without the cluster values + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: publishes nothing while observability-proxy is not in expose-services + set: + _cluster: + root-host: example.org + expose-services: "api,dashboard" + gateway-enabled: "false" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: publishes nothing in Gateway mode while observability-proxy is not in expose-services + set: + _cluster: + root-host: example.org + expose-services: "api,dashboard" + gateway-enabled: "true" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + hasDocuments: + count: 0 + + - it: renders an Ingress with a per-host certificate when gateway is off + set: + _cluster: + root-host: example.org + expose-services: "dashboard,observability-proxy" + expose-ingress: tenant-public + gateway-enabled: "false" + asserts: + - template: templates/httproute.yaml + hasDocuments: + count: 0 + - template: templates/ingress.yaml + isKind: + of: Ingress + - template: templates/ingress.yaml + equal: + path: metadata.name + value: observability-proxy + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-prod + acme.cert-manager.io/http01-ingress-ingressclassname: tenant-public + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec + value: + ingressClassName: tenant-public + rules: + - host: observability-proxy.example.org + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: observability-proxy + port: + name: http + tls: + - hosts: + - observability-proxy.example.org + secretName: observability-proxy-tls + + - it: leaves the http01 ingress class out under the dns01 solver + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "false" + solver: dns01 + issuer-name: letsencrypt-stage + asserts: + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + cert-manager.io/cluster-issuer: letsencrypt-stage + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec.ingressClassName + value: tenant-root + + - it: relies on the wildcard certificate and keeps the TLS section for the redirect + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "false" + wildcard-secret-name: wildcard-tls + asserts: + - template: templates/ingress.yaml + equal: + path: metadata.annotations + value: + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "360" + - template: templates/ingress.yaml + equal: + path: spec.tls + value: + - hosts: + - observability-proxy.example.org + + - it: renders an HTTPRoute without a route timeout when gateway is on + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + expose-ingress: tenant-public + gateway-enabled: "true" + asserts: + - template: templates/ingress.yaml + hasDocuments: + count: 0 + - template: templates/httproute.yaml + isKind: + of: HTTPRoute + - template: templates/httproute.yaml + isAPIVersion: + of: gateway.networking.k8s.io/v1 + - template: templates/httproute.yaml + equal: + path: metadata.name + value: observability-proxy + - template: templates/httproute.yaml + equal: + path: spec + value: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: cozystack + namespace: tenant-public + hostnames: + - observability-proxy.example.org + rules: + - backendRefs: + - name: observability-proxy + port: 80 + timeouts: + request: 0s + + - it: attaches to the tenant-root Gateway when expose-ingress is unset + set: + _cluster: + root-host: example.org + expose-services: "observability-proxy" + gateway-enabled: "true" + asserts: + - template: templates/httproute.yaml + equal: + path: spec.parentRefs[?(@.name == "cozystack")].namespace + value: tenant-root diff --git a/packages/system/observability-proxy/tests/rbac_test.yaml b/packages/system/observability-proxy/tests/rbac_test.yaml index af339b5d0..5382c2ab6 100644 --- a/packages/system/observability-proxy/tests/rbac_test.yaml +++ b/packages/system/observability-proxy/tests/rbac_test.yaml @@ -2,7 +2,7 @@ suite: observability-proxy RBAC # The proxy authenticates and authorizes every request through the API server, # so a dropped verb here turns every request into an error at runtime rather # than a render failure. The tenant role is what lets a namespace viewer pass -# the SubjectAccessReview the proxy asks for the namespace logs root. +# the SubjectAccessReview the proxy asks for the namespace logs and metrics roots. templates: - templates/rbac.yaml - templates/tenant-clusterroles.yaml @@ -12,7 +12,7 @@ release: namespace: cozy-observability-proxy tests: - - it: lets the proxy review tokens and access, and list and watch namespaces + - it: lets the proxy review tokens and access, and list and watch namespaces and application definitions template: templates/rbac.yaml documentSelector: path: kind @@ -30,6 +30,9 @@ tests: - apiGroups: [""] resources: ["namespaces"] verbs: ["list", "watch"] + - apiGroups: ["cozystack.io"] + resources: ["applicationdefinitions"] + verbs: ["list", "watch"] - it: binds the proxy role to the proxy ServiceAccount in the release namespace template: templates/rbac.yaml @@ -50,7 +53,7 @@ tests: name: observability-proxy namespace: cozy-observability-proxy - - it: aggregates list on cozystack.io logs into the tenant view role + - it: aggregates list on cozystack.io logs and metrics into the tenant view role template: templates/tenant-clusterroles.yaml asserts: - isKind: @@ -63,5 +66,5 @@ tests: path: rules value: - apiGroups: ["cozystack.io"] - resources: ["logs"] + resources: ["logs", "metrics"] verbs: ["list"] diff --git a/packages/system/observability-proxy/values.yaml b/packages/system/observability-proxy/values.yaml index 919e450a9..3e3ac70fe 100644 --- a/packages/system/observability-proxy/values.yaml +++ b/packages/system/observability-proxy/values.yaml @@ -7,6 +7,16 @@ observabilityProxy: replicas: 2 rootTenant: tenant-root authCacheTTL: 10s + # Names and retention of the root tenant's metricsStorages + # (packages/system/monitoring/values.yaml). The retention is a Go + # duration: 3d is written 72h. + metrics: + shorttermStore: shortterm + longtermStore: longterm + shorttermRetention: 72h + # How far before its window an application metrics query may read; + # one reading further back answers 422 on the application root. + appLookback: 720h resources: requests: cpu: 10m diff --git a/packages/system/piraeus-operator-crds/templates/crds.yaml b/packages/system/piraeus-operator-crds/templates/crds.yaml index 5bd9212c6..d190bc2b9 100644 --- a/packages/system/piraeus-operator-crds/templates/crds.yaml +++ b/packages/system/piraeus-operator-crds/templates/crds.yaml @@ -151,13 +151,17 @@ spec: *SecretName using the issuer configured here. properties: group: - description: Group of the resource being referred to. + description: |- + Group of the issuer being referred to. + Defaults to 'cert-manager.io'. type: string kind: - description: Kind of the resource being referred to. + description: |- + Kind of the issuer being referred to. + Defaults to 'Issuer'. type: string name: - description: Name of the resource being referred to. + description: Name of the issuer being referred to. type: string required: - name @@ -201,6 +205,11 @@ spec: type: object x-kubernetes-map-type: atomic x-kubernetes-preserve-unknown-fields: true + replicas: + description: Number of desired pods. Defaults to 1. + format: int32 + minimum: 0 + type: integer type: object csiController: description: CSIController controls the deployment of the CSI Controller @@ -320,13 +329,17 @@ spec: issuer configured here. properties: group: - description: Group of the resource being referred to. + description: |- + Group of the issuer being referred to. + Defaults to 'cert-manager.io'. type: string kind: - description: Kind of the resource being referred to. + description: |- + Kind of the issuer being referred to. + Defaults to 'Issuer'. type: string name: - description: Name of the resource being referred to. + description: Name of the issuer being referred to. type: string required: - name @@ -345,6 +358,18 @@ spec: * Store credentials for accessing remotes for backups. See https://linbit.com/drbd-user-guide/linstor-guide-1_0-en/#s-encrypt_commands for more information. type: string + maxConcurrentEvacuations: + description: |- + MaxConcurrentEvacuations limits how many Satellites are evacuated at the same time when using the + "Evacuate" deletion policy. + + This applies both to Satellites removed because their node no longer matches the cluster, and to nodes + removed through ClusterAPI. While the limit is reached, additional Satellites wait for a free slot before + their volumes are moved and, for ClusterAPI managed nodes, before the node is allowed to drain. + A value of 0 (the default) places no limit on the number of concurrent evacuations. + format: int32 + minimum: 0 + type: integer nfsServer: description: NFSServer controls the deployment of the LINSTOR CSI NFS Server DaemonSet. @@ -568,9 +593,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -1054,13 +1080,17 @@ spec: issuer configured here. properties: group: - description: Group of the resource being referred to. + description: |- + Group of the issuer being referred to. + Defaults to 'cert-manager.io'. type: string kind: - description: Kind of the resource being referred to. + description: |- + Kind of the issuer being referred to. + Defaults to 'Issuer'. type: string name: - description: Name of the resource being referred to. + description: Name of the issuer being referred to. type: string required: - name @@ -1321,6 +1351,15 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + resourceNameSuffixSeparator: + description: |- + ResourceNameSuffixSeparator is inserted between the base resource name and the Satellite name. + + Defaults to ".". + enum: + - . + - '-' + type: string storagePools: description: StoragePools is a list of storage pools to configure on the node. @@ -1347,18 +1386,66 @@ spec: lvmPool: description: Configures a LVM Volume Group as storage pool. properties: - volumeGroup: + externalLocking: + description: |- + ExternalLocking skips LINSTOR's internal locking for the shared VG, relying on an external + lock manager such as lvmlockd on the hosts instead. + type: boolean + physicalVolumeCreateArguments: + description: |- + PhysicalVolumeCreateArguments are arguments to pass to "pvcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array + sharedSpace: + description: |- + SharedSpace marks the VG as shared between multiple nodes, identified by the given name. + All storage pools backed by the same physical storage must use the same shared space name. + A shared VG must be set up manually on the hosts, it cannot be created from source devices. type: string + volumeGroup: + description: VolumeGroup is the name of the Volume Group + (VG) to use. + type: string + volumeGroupCreateArguments: + description: |- + VolumeGroupCreateArguments are arguments to pass to "vgcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array type: object lvmThinPool: description: Configures a LVM Thin Pool as storage pool. properties: + logicalVolumeCreateArguments: + description: |- + LogicalVolumeCreateArguments are arguments to pass to "lvcreate". + This has no effect on an existing thinpool, it only applies when the thinpool initially gets created. + items: + type: string + type: array + physicalVolumeCreateArguments: + description: |- + PhysicalVolumeCreateArguments are arguments to pass to "pvcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array thinPool: description: ThinPool is the name of the thinpool LV (without VG prefix). type: string volumeGroup: type: string + volumeGroupCreateArguments: + description: |- + VolumeGroupCreateArguments are arguments to pass to "vgcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array type: object name: description: Name of the storage pool in linstor. @@ -1447,6 +1534,16 @@ spec: zPool: description: ZPool is the name of the ZFS zpool. type: string + zpoolCreateArguments: + description: |- + ZpoolCreateArguments are arguments to pass to "zpool create". + This has no effect on an existing zpool, it only applies when the zpool initially gets created. + + If not set, "-o failmode=continue" is automatically added to ensure a failed zpool does not stop replication + to and from other nodes in the cluster. + items: + type: string + type: array type: object zfsThinPool: description: Configures a ZFS system based storage pool, allocating @@ -1455,6 +1552,16 @@ spec: zPool: description: ZPool is the name of the ZFS zpool. type: string + zpoolCreateArguments: + description: |- + ZpoolCreateArguments are arguments to pass to "zpool create". + This has no effect on an existing zpool, it only applies when the zpool initially gets created. + + If not set, "-o failmode=continue" is automatically added to ensure a failed zpool does not stop replication + to and from other nodes in the cluster. + items: + type: string + type: array type: object required: - name @@ -1762,13 +1869,17 @@ spec: issuer configured here. properties: group: - description: Group of the resource being referred to. + description: |- + Group of the issuer being referred to. + Defaults to 'cert-manager.io'. type: string kind: - description: Kind of the resource being referred to. + description: |- + Kind of the issuer being referred to. + Defaults to 'Issuer'. type: string name: - description: Name of the resource being referred to. + description: Name of the issuer being referred to. type: string required: - name @@ -1925,6 +2036,15 @@ spec: repository: description: Repository used to pull workload images. type: string + resourceNameSuffixSeparator: + description: |- + ResourceNameSuffixSeparator is inserted between the base resource name and the Satellite name. + + Defaults to ".". + enum: + - . + - '-' + type: string storagePools: description: StoragePools is a list of storage pools to configure on the node. @@ -1951,18 +2071,66 @@ spec: lvmPool: description: Configures a LVM Volume Group as storage pool. properties: - volumeGroup: + externalLocking: + description: |- + ExternalLocking skips LINSTOR's internal locking for the shared VG, relying on an external + lock manager such as lvmlockd on the hosts instead. + type: boolean + physicalVolumeCreateArguments: + description: |- + PhysicalVolumeCreateArguments are arguments to pass to "pvcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array + sharedSpace: + description: |- + SharedSpace marks the VG as shared between multiple nodes, identified by the given name. + All storage pools backed by the same physical storage must use the same shared space name. + A shared VG must be set up manually on the hosts, it cannot be created from source devices. type: string + volumeGroup: + description: VolumeGroup is the name of the Volume Group + (VG) to use. + type: string + volumeGroupCreateArguments: + description: |- + VolumeGroupCreateArguments are arguments to pass to "vgcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array type: object lvmThinPool: description: Configures a LVM Thin Pool as storage pool. properties: + logicalVolumeCreateArguments: + description: |- + LogicalVolumeCreateArguments are arguments to pass to "lvcreate". + This has no effect on an existing thinpool, it only applies when the thinpool initially gets created. + items: + type: string + type: array + physicalVolumeCreateArguments: + description: |- + PhysicalVolumeCreateArguments are arguments to pass to "pvcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array thinPool: description: ThinPool is the name of the thinpool LV (without VG prefix). type: string volumeGroup: type: string + volumeGroupCreateArguments: + description: |- + VolumeGroupCreateArguments are arguments to pass to "vgcreate". + This has no effect on an existing VG, it only applies when the VG initially gets created. + items: + type: string + type: array type: object name: description: Name of the storage pool in linstor. @@ -2051,6 +2219,16 @@ spec: zPool: description: ZPool is the name of the ZFS zpool. type: string + zpoolCreateArguments: + description: |- + ZpoolCreateArguments are arguments to pass to "zpool create". + This has no effect on an existing zpool, it only applies when the zpool initially gets created. + + If not set, "-o failmode=continue" is automatically added to ensure a failed zpool does not stop replication + to and from other nodes in the cluster. + items: + type: string + type: array type: object zfsThinPool: description: Configures a ZFS system based storage pool, allocating @@ -2059,6 +2237,16 @@ spec: zPool: description: ZPool is the name of the ZFS zpool. type: string + zpoolCreateArguments: + description: |- + ZpoolCreateArguments are arguments to pass to "zpool create". + This has no effect on an existing zpool, it only applies when the zpool initially gets created. + + If not set, "-o failmode=continue" is automatically added to ensure a failed zpool does not stop replication + to and from other nodes in the cluster. + items: + type: string + type: array type: object required: - name diff --git a/packages/system/piraeus-operator/Chart.yaml b/packages/system/piraeus-operator/Chart.yaml index 40b3fce28..7a1925e25 100644 --- a/packages/system/piraeus-operator/Chart.yaml +++ b/packages/system/piraeus-operator/Chart.yaml @@ -1,3 +1,7 @@ apiVersion: v2 name: cozy-piraeus-operator version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process +dependencies: + - name: piraeus + version: "*" + repository: file://charts/piraeus diff --git a/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml b/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml index ab679d663..8be364a72 100644 --- a/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml +++ b/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml @@ -29,8 +29,8 @@ spec: LINSTOR Satellite "{{ $labels.hostname }}" reports {{ $value }} errors in the last 15 minutes. Use "linstor error-reports list --nodes {{ $labels.hostname }}" to inspect the reports. expr: | - increase(linstor_error_reports_count{module="SATELLITE"}[15m]) > 0 - and on(instance, job) + increase((max by (hostname) (last_over_time(linstor_error_reports_count{module="SATELLITE"}[1h])))[15m:1m]) > 0 + and on() min_over_time(up{job="linstor-controller"}[15m]) == 1 labels: severity: warning @@ -39,7 +39,7 @@ spec: description: | LINSTOR Controller reports {{ $value }} errors in the last 15 minutes. Use "linstor error-reports list --since 15minutes" to see them. - expr: increase(linstor_error_reports_count{module="CONTROLLER"}[15m]) > 0 + expr: increase((max by (hostname) (last_over_time(linstor_error_reports_count{module="CONTROLLER"}[1h])))[15m:1m]) > 0 labels: severity: warning - alert: linstorSatelliteNotOnline diff --git a/packages/system/piraeus-operator/charts/piraeus/Chart.yaml b/packages/system/piraeus-operator/charts/piraeus/Chart.yaml index 777722f3f..1565e6a27 100644 --- a/packages/system/piraeus-operator/charts/piraeus/Chart.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/Chart.yaml @@ -3,8 +3,8 @@ name: piraeus description: | The Piraeus Operator manages software defined storage clusters using LINSTOR in Kubernetes. type: application -version: 2.10.2 -appVersion: "v2.10.2" +version: 2.12.1 +appVersion: "v2.12.1" maintainers: - name: Piraeus Datastore url: https://piraeus.io diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/config.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/config.yaml index 2e6963f0a..4788ffba0 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/config.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/config.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: {{ include "piraeus-operator.fullname" . }}-image-config + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} data: @@ -17,38 +18,38 @@ data: # quay.io/piraeusdatastore/piraeus-server:v1.24.2 components: linstor-controller: - tag: v1.32.3 + tag: v1.35.3 image: piraeus-server linstor-satellite: - tag: v1.32.3 + tag: v1.35.3 image: piraeus-server linstor-csi: - tag: v1.10.3 + tag: v1.13.2 image: piraeus-csi nfs-server: - tag: v1.10.3 + tag: v1.13.2 image: piraeus-csi-nfs-server drbd-reactor: - tag: v1.10.0 + tag: v1.12.0 image: drbd-reactor ha-controller: - tag: v1.3.1 + tag: v1.3.4 image: piraeus-ha-controller drbd-shutdown-guard: - tag: v1.1.1 + tag: v1.1.2 image: drbd-shutdown-guard ktls-utils: - tag: v1.2.1 + tag: v1.5.0 image: ktls-utils linstor-affinity-controller: - tag: v1.3.0 + tag: v1.5.0 image: linstor-affinity-controller drbd-module-loader: - tag: v9.2.16 + tag: v9.3.4 # The special "match" attribute is used to select an image based on the node's reported OS. # The operator will first check the k8s node's ".status.nodeInfo.osImage" field, and compare it against the list # here. If one matches, that specific image name will be used instead of the fallback image. - image: drbd9-noble # Fallback image: chose a recent kernel, which can hopefully compile whatever config is actually in use + image: drbd9-resolute # Fallback image: chose a recent kernel, which can hopefully compile whatever config is actually in use match: - osImage: Red Hat Enterprise Linux 8\. image: drbd9-almalinux8 @@ -84,6 +85,10 @@ data: image: drbd9-jammy - osImage: Ubuntu 24\.04 image: drbd9-noble + - osImage: Ubuntu 26\.04 + image: drbd9-resolute + - osImage: Debian GNU/Linux 13 + image: drbd9-trixie - osImage: Debian GNU/Linux 12 image: drbd9-bookworm - osImage: Debian GNU/Linux 11 @@ -93,25 +98,22 @@ data: base: registry.k8s.io/sig-storage components: csi-attacher: - tag: v4.10.0 + tag: v4.13.0 image: csi-attacher csi-livenessprobe: - tag: v2.17.0 + tag: v2.20.0 image: livenessprobe csi-provisioner: - tag: v6.1.0 + tag: v6.4.0 image: csi-provisioner csi-snapshotter: - tag: v8.4.0 + tag: v8.7.0 image: csi-snapshotter csi-resizer: - tag: v2.0.0 + tag: v2.3.0 image: csi-resizer - csi-external-health-monitor-controller: - tag: v0.16.0 - image: csi-external-health-monitor-controller csi-node-driver-registrar: - tag: v2.15.0 + tag: v2.18.0 image: csi-node-driver-registrar {{- range $idx, $value := .Values.imageConfigOverride }} {{ add $idx 1 }}_helm_override.yaml: | diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/crds.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/crds.yaml deleted file mode 100644 index 5bd9212c6..000000000 --- a/packages/system/piraeus-operator/charts/piraeus/templates/crds.yaml +++ /dev/null @@ -1,2173 +0,0 @@ - {{ if .Values.installCRDs }} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.18.0 - name: linstorclusters.piraeus.io -spec: - group: piraeus.io - names: - kind: LinstorCluster - listKind: LinstorClusterList - plural: linstorclusters - singular: linstorcluster - scope: Cluster - versions: - - additionalPrinterColumns: - - description: If the LINSTOR Cluster is available - jsonPath: .status.conditions[?(@.type=='Available')].status - name: Available - type: string - - description: If the LINSTOR Cluster is fully configured - jsonPath: .status.conditions[?(@.type=='Configured')].status - name: Configured - type: string - - description: The version of the LINSTOR Cluster - jsonPath: .status.version - name: Version - priority: 10 - type: string - - description: The number of running/expected Satellites - jsonPath: .status.satellites - name: Satellites - type: string - - description: The used capacity in all storage pools - jsonPath: .status.capacity - name: Used Capacity - type: string - - description: The number of volumes in the cluster - jsonPath: .status.numberOfVolumes - name: Volumes - type: integer - - description: The number of snapshots in the cluster - jsonPath: .status.numberOfSnapshots - name: Snapshots - priority: 10 - type: integer - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: LinstorCluster is the Schema for the linstorclusters API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LinstorClusterSpec defines the desired state of LinstorCluster - properties: - affinityController: - description: AffinityController controls the deployment of the Affinity - Controller Deployment. - properties: - enabled: - default: true - description: Enable the component. - type: boolean - podTemplate: - description: |- - Template to apply to Pods of the component. - - The template is applied as a patch to the default deployment, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - replicas: - description: Number of desired pods. Defaults to 1. - format: int32 - minimum: 0 - type: integer - type: object - apiTLS: - description: |- - ApiTLS secures the LINSTOR API. - - This configures the TLS key and certificate used to secure the LINSTOR API. - nullable: true - properties: - affinityControllerSecretName: - description: |- - AffinityControllerSecretName references a secret holding the TLS key and certificate used by the Affinity - Controller to monitor volume state. Defaults to "linstor-affinity-controller-tls". - type: string - apiSecretName: - description: |- - ApiSecretName references a secret holding the TLS key and certificate used to protect the API. - Defaults to "linstor-api-tls". - type: string - caReference: - description: |- - CAReference configures the CA certificate to use when validating TLS certificates. - If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate. - properties: - key: - default: ca.crt - description: |- - Key to select in the resource. - Defaults to ca.crt if not specified. - type: string - kind: - default: Secret - description: Kind of the resource containing the CA Certificate, - either a ConfigMap or Secret. - enum: - - ConfigMap - - Secret - type: string - name: - description: Name of the resource containing the CA Certificate. - type: string - optional: - description: Optional specifies whether the resource and its - key must exist. - type: boolean - required: - - name - type: object - certManager: - description: |- - CertManager references a cert-manager Issuer or ClusterIssuer. - If set, cert-manager.io/Certificate resources will be created, provisioning the secrets referenced in - *SecretName using the issuer configured here. - properties: - group: - description: Group of the resource being referred to. - type: string - kind: - description: Kind of the resource being referred to. - type: string - name: - description: Name of the resource being referred to. - type: string - required: - - name - type: object - clientSecretName: - description: |- - ClientSecretName references a secret holding the TLS key and certificate used by the operator to configure - the cluster. Defaults to "linstor-client-tls". - type: string - csiControllerSecretName: - description: |- - CsiControllerSecretName references a secret holding the TLS key and certificate used by the CSI Controller - to provision volumes. Defaults to "linstor-csi-controller-tls". - type: string - csiNodeSecretName: - description: |- - CsiNodeSecretName references a secret holding the TLS key and certificate used by the CSI Nodes to query - the volume state. Defaults to "linstor-csi-node-tls". - type: string - nfsServerSecretName: - description: |- - NFSServerSecretName references a secret holding the TLS key and certificate used by the NFS Server to query - the cluster state. Defaults to "linstor-csi-nfs-server-tls". - type: string - type: object - controller: - description: Controller controls the deployment of the LINSTOR Controller - Deployment. - properties: - enabled: - default: true - description: Enable the component. - type: boolean - podTemplate: - description: |- - Template to apply to Pods of the component. - - The template is applied as a patch to the default deployment, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - type: object - csiController: - description: CSIController controls the deployment of the CSI Controller - Deployment. - properties: - enabled: - default: true - description: Enable the component. - type: boolean - podTemplate: - description: |- - Template to apply to Pods of the component. - - The template is applied as a patch to the default deployment, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - replicas: - description: Number of desired pods. Defaults to 1. - format: int32 - minimum: 0 - type: integer - type: object - csiNode: - description: CSINode controls the deployment of the CSI Node DaemonSet. - properties: - enabled: - default: true - description: Enable the component. - type: boolean - podTemplate: - description: |- - Template to apply to Pods of the component. - - The template is applied as a patch to the default deployment, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - type: object - externalController: - description: |- - ExternalController references an external controller. - When set, the Operator will skip deploying a LINSTOR Controller and instead use the external cluster - to register satellites. - properties: - url: - description: URL of the external controller. - minLength: 3 - type: string - required: - - url - type: object - highAvailabilityController: - description: HighAvailabilityController controls the deployment of - the High Availability Controller DaemonSet. - properties: - enabled: - default: true - description: Enable the component. - type: boolean - podTemplate: - description: |- - Template to apply to Pods of the component. - - The template is applied as a patch to the default deployment, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - type: object - internalTLS: - description: |- - InternalTLS secures the connection between LINSTOR Controller and Satellite. - - This configures the client certificate used when the Controller connects to a Satellite. This only has an effect - when the Satellite is configured to for secure connections using `LinstorSatellite.spec.internalTLS`. - nullable: true - properties: - caReference: - description: |- - CAReference configures the CA certificate to use when validating TLS certificates. - If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate. - properties: - key: - default: ca.crt - description: |- - Key to select in the resource. - Defaults to ca.crt if not specified. - type: string - kind: - default: Secret - description: Kind of the resource containing the CA Certificate, - either a ConfigMap or Secret. - enum: - - ConfigMap - - Secret - type: string - name: - description: Name of the resource containing the CA Certificate. - type: string - optional: - description: Optional specifies whether the resource and its - key must exist. - type: boolean - required: - - name - type: object - certManager: - description: |- - CertManager references a cert-manager Issuer or ClusterIssuer. - If set, a Certificate resource will be created, provisioning the secret references in SecretName using the - issuer configured here. - properties: - group: - description: Group of the resource being referred to. - type: string - kind: - description: Kind of the resource being referred to. - type: string - name: - description: Name of the resource being referred to. - type: string - required: - - name - type: object - secretName: - description: SecretName references a secret holding the TLS key - and certificates. - type: string - type: object - linstorPassphraseSecret: - description: |- - LinstorPassphraseSecret used to configure the LINSTOR master passphrase. - - The referenced secret must contain a single key "MASTER_PASSPHRASE". The master passphrase is used to - * Derive encryption keys for volumes using the LUKS layer. - * Store credentials for accessing remotes for backups. - See https://linbit.com/drbd-user-guide/linstor-guide-1_0-en/#s-encrypt_commands for more information. - type: string - nfsServer: - description: NFSServer controls the deployment of the LINSTOR CSI - NFS Server DaemonSet. - properties: - enabled: - default: true - description: Enable the component. - type: boolean - podTemplate: - description: |- - Template to apply to Pods of the component. - - The template is applied as a patch to the default deployment, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - type: object - nodeAffinity: - description: |- - NodeAffinity selects the nodes on which LINSTOR Satellites will be deployed. - See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. The terms - are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements by node's - labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies - to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements by node's - fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies - to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - nodeSelector: - additionalProperties: - type: string - description: |- - NodeSelector selects the nodes on which LINSTOR Satellites will be deployed. - See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - type: object - patches: - description: |- - Patches is a list of kustomize patches to apply. - - See https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/ for how to create patches. - items: - description: Patch represent either a Strategic Merge Patch or a - JSON patch and its targets. - properties: - options: - description: Options is a list of options for the patch - properties: - allowKindChange: - description: AllowKindChange allows kind changes to the - resource. - type: boolean - allowNameChange: - description: AllowNameChange allows name changes to the - resource. - type: boolean - type: object - patch: - description: Patch is the content of a patch. - minLength: 1 - type: string - target: - description: Target points to the resources that the patch is - applied to - properties: - annotationSelector: - description: |- - AnnotationSelector is a string that follows the label selection expression - https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api - It matches against the resource annotations. - type: string - group: - type: string - kind: - type: string - labelSelector: - description: |- - LabelSelector is a string that follows the label selection expression - https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api - It matches against the resource labels. - type: string - name: - description: Name of the resource. - type: string - namespace: - description: Namespace the resource belongs to, if it can - belong to a namespace. - type: string - version: - type: string - type: object - required: - - patch - type: object - type: array - properties: - description: |- - Properties to apply on the cluster level. - - Use to create default settings for DRBD that should apply to all resources or to configure some other cluster - wide default. - items: - properties: - name: - description: Name of the property to set. - minLength: 1 - type: string - value: - description: Value to set the property to. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - repository: - description: Repository used to pull workload images. - type: string - tolerations: - description: |- - Tolerations selects the nodes on which LINSTOR Satellites will be deployed. - - The default tolerations for DaemonSets are automatically added. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - x-kubernetes-list-type: atomic - type: object - status: - description: LinstorClusterStatus defines the observed state of LinstorCluster - properties: - availableCapacityBytes: - description: The number of bytes in total in all storage pools in - the LINSTOR Cluster. - format: int64 - type: integer - capacity: - description: Capacity mirrors the information from TotalCapacityBytes - and FreeCapacityBytes in a human-readable string - type: string - conditions: - description: Current LINSTOR Cluster state - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - freeCapacityBytes: - description: The number of bytes free in all storage pools in the - LINSTOR Cluster. - format: int64 - type: integer - numberOfSnapshots: - description: The number of snapshots in the LINSTOR Cluster. - format: int32 - type: integer - numberOfVolumes: - description: The number of volumes in the LINSTOR Cluster. - format: int32 - type: integer - runningSatellites: - description: The number of LINSTOR Satellites currently running. - format: int32 - type: integer - satellites: - description: Satellites mirrors the information from ScheduledSatellites - and RunningSatellites in a human-readable string - type: string - scheduledSatellites: - description: The number of LINSTOR Satellites that are expected to - run. - format: int32 - type: integer - version: - description: The Version of the LINSTOR Cluster. - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.18.0 - name: linstornodeconnections.piraeus.io -spec: - group: piraeus.io - names: - kind: LinstorNodeConnection - listKind: LinstorNodeConnectionList - plural: linstornodeconnections - singular: linstornodeconnection - scope: Cluster - versions: - - additionalPrinterColumns: - - description: If the LINSTOR Node Connection is fully configured - jsonPath: .status.conditions[?(@.type=='Configured')].status - name: Configured - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: LinstorNodeConnection is the Schema for the linstornodeconnections - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LinstorNodeConnectionSpec defines the desired state of LinstorNodeConnection - properties: - paths: - description: Paths configure the network path used when connecting - two nodes. - items: - properties: - interface: - description: Interface to use on both nodes. - type: string - name: - description: Name of the path. - type: string - required: - - interface - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - properties: - description: |- - Properties to apply for the node connection. - - Use to create default settings for DRBD that should apply to all resources connections between a set of - cluster nodes. - items: - properties: - name: - description: Name of the property to set. - minLength: 1 - type: string - value: - description: Value to set the property to. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - selector: - description: |- - Selector selects which pair of Satellites the connection should apply to. - If not given, the connection will be applied to all connections. - items: - description: SelectorTerm matches pairs of nodes by checking that - the nodes match all specified requirements. - properties: - matchLabels: - description: MatchLabels is a list of match expressions that - the node pairs must meet. - items: - properties: - key: - description: Key is the name of a node label. - minLength: 1 - type: string - op: - default: Exists - description: |- - Op to apply to the label. - Exists (default) checks for the presence of the label on both nodes in the pair. - DoesNotExist checks that the label is not present on either node in the pair. - In checks for the presence of the label value given by Values on both nodes in the pair. - NotIn checks that both nodes in the pair do not have any of the label values given by Values. - Same checks that the label value is equal in the node pair. - NotSame checks that the label value is not equal in the node pair. - enum: - - Exists - - DoesNotExist - - In - - NotIn - - Same - - NotSame - type: string - values: - description: Values to match on, using the provided Op. - items: - type: string - type: array - required: - - key - type: object - type: array - required: - - matchLabels - type: object - type: array - type: object - status: - description: LinstorNodeConnectionStatus defines the observed state of - LinstorNodeConnection - properties: - conditions: - description: Current LINSTOR Node Connection state - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.18.0 - name: linstorsatelliteconfigurations.piraeus.io -spec: - group: piraeus.io - names: - kind: LinstorSatelliteConfiguration - listKind: LinstorSatelliteConfigurationList - plural: linstorsatelliteconfigurations - singular: linstorsatelliteconfiguration - scope: Cluster - versions: - - additionalPrinterColumns: - - description: The node selector used - jsonPath: .spec.nodeSelector - name: Selector - type: string - - description: If the Configuration was applied - jsonPath: .status.conditions[?(@.type=='Applied')].status - name: Applied - type: string - - description: Number of Satellites this Configuration has been applied to - jsonPath: .status.matched - name: Matched - type: integer - - description: Satellites this Configuration has been applied to - jsonPath: .status.appliedTo - name: Satellites - priority: 10 - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: LinstorSatelliteConfiguration is the Schema for the linstorsatelliteconfigurations - API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - LinstorSatelliteConfigurationSpec defines a partial, desired state of a LinstorSatelliteSpec. - - All the LinstorSatelliteConfiguration resources with matching NodeSelector will - be merged into a single LinstorSatelliteSpec. - properties: - deletionPolicy: - description: |- - DeletionPolicy configures the way LinstorSatellite resources are deleted. - - A LinstorSatellite may be deleted because: - * It no longer matches the affinity and node selector of the LinstorCluster resource. - * The node it references has been removed from Kubernetes. - * It was manually deleted outside the Operator. - - A LinstorSatellite may store the last copy of a volume, in which case it is not desirable to unconditionally remove - the satellite from the cluster. For this reason, the following deletion policies exist: - - * DeletionPolicyEvacuate will start evacuation of the LINSTOR Satellite and wait until it completes before removing the LinstorSatellite object, comparable to the "linstor node evacuate" command. - * DeletionPolicyRetain will retain the LINSTOR Satellite, keeping it registered in LINSTOR, but removing associated Kubernetes resources. - * DeletionPolicyDelete will remove the LINSTOR Satellite from the LINSTOR Cluster without prior eviction, comparable to the "linstor node lost" command. - enum: - - Evacuate - - Retain - - Delete - type: string - evacuationStrategy: - description: EvacuationStrategy configures the evacuation of volumes - from a Satellite when DeletionPolicy "Evacuate" is used. - nullable: true - properties: - attachedVolumeReattachTimeout: - default: 5m - description: |- - AttachedVolumeReattachTimeout configures how long evacuation waits for attached volumes to reattach on - different nodes. Setting this to 0 disable this evacuation step. - type: string - unattachedVolumeAttachTimeout: - default: 5m - description: |- - UnattachedVolumeAttachTimeout configures how long evacuation waits for unattached volumes to attach on - different nodes. Setting this to 0 disable this evacuation step. - type: string - type: object - internalTLS: - description: |- - InternalTLS configures secure communication for the LINSTOR Satellite. - - If set, the control traffic between LINSTOR Controller and Satellite will be encrypted using mTLS. - nullable: true - properties: - caReference: - description: |- - CAReference configures the CA certificate to use when validating TLS certificates. - If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate. - properties: - key: - default: ca.crt - description: |- - Key to select in the resource. - Defaults to ca.crt if not specified. - type: string - kind: - default: Secret - description: Kind of the resource containing the CA Certificate, - either a ConfigMap or Secret. - enum: - - ConfigMap - - Secret - type: string - name: - description: Name of the resource containing the CA Certificate. - type: string - optional: - description: Optional specifies whether the resource and its - key must exist. - type: boolean - required: - - name - type: object - certManager: - description: |- - CertManager references a cert-manager Issuer or ClusterIssuer. - If set, a Certificate resource will be created, provisioning the secret references in SecretName using the - issuer configured here. - properties: - group: - description: Group of the resource being referred to. - type: string - kind: - description: Kind of the resource being referred to. - type: string - name: - description: Name of the resource being referred to. - type: string - required: - - name - type: object - secretName: - description: SecretName references a secret holding the TLS key - and certificates. - type: string - tlsHandshakeDaemon: - description: |- - TLSHandshakeDaemon enables tlshd for establishing TLS sessions for use by DRBD. - - If enabled, adds a new sidecar to the LINSTOR Satellite that runs the tlshd handshake daemon. - The daemon uses the TLS certificate and key to establish secure connections on behalf of DRBD. - type: boolean - type: object - ipFamilies: - description: |- - IPFamilies configures the IP Family (IPv4 or IPv6) to use to connect to the LINSTOR Satellite. - - If set, the control traffic between LINSTOR Controller and Satellite will use only the given IP Family. - If not set, the Operator will configure all families found in the Satellites Pods' Status. - items: - description: IPFamily represents the IP Family (IPv4 or IPv6). - enum: - - IPv4 - - IPv6 - type: string - type: array - nodeAffinity: - description: |- - NodeAffinity selects which LinstorSatellite resources this spec should be applied to. - See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. The terms - are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements by node's - labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies - to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements by node's - fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector applies - to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - nodeSelector: - additionalProperties: - type: string - description: |- - NodeSelector selects which LinstorSatellite resources this spec should be applied to. - See https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ - type: object - patches: - description: |- - Patches is a list of kustomize patches to apply. - - See https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/ for how to create patches. - items: - description: Patch represent either a Strategic Merge Patch or a - JSON patch and its targets. - properties: - options: - description: Options is a list of options for the patch - properties: - allowKindChange: - description: AllowKindChange allows kind changes to the - resource. - type: boolean - allowNameChange: - description: AllowNameChange allows name changes to the - resource. - type: boolean - type: object - patch: - description: Patch is the content of a patch. - minLength: 1 - type: string - target: - description: Target points to the resources that the patch is - applied to - properties: - annotationSelector: - description: |- - AnnotationSelector is a string that follows the label selection expression - https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api - It matches against the resource annotations. - type: string - group: - type: string - kind: - type: string - labelSelector: - description: |- - LabelSelector is a string that follows the label selection expression - https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api - It matches against the resource labels. - type: string - name: - description: Name of the resource. - type: string - namespace: - description: Namespace the resource belongs to, if it can - belong to a namespace. - type: string - version: - type: string - type: object - required: - - patch - type: object - type: array - podTemplate: - description: |- - Template to apply to Satellite Pods. - - The template is applied as a patch to the default resource, so it can be "sparse", not listing any - containers or volumes that should remain unchanged. - See https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - properties: - description: Properties is a list of properties to set on the node. - items: - properties: - expandFrom: - description: |- - ExpandFrom can reference multiple resource fields at once. - It either sets the property to an aggregate value based on matched resource fields, or expands to multiple - properties. - properties: - delimiter: - description: Delimiter used to join multiple key and value - pairs together. - type: string - nameTemplate: - description: |- - NameTemplate defines how the property key is expanded. - If set, the template is appended to the defined property name, creating multiple properties instead of one - aggregate. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - nodeFieldRef: - description: Select a field of the node. Supports `metadata.name`, - `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - valueTemplate: - description: |- - ValueTemplate defines how the property value is expanded. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - required: - - nodeFieldRef - type: object - name: - description: Name of the property to set. - minLength: 1 - type: string - optional: - description: Optional values are only set if they have a non-empty - value - type: boolean - value: - description: Value to set the property to. - type: string - valueFrom: - description: ValueFrom sets the value from an existing resource. - properties: - nodeFieldRef: - description: Select a field of the node. Supports `metadata.name`, - `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - required: - - nodeFieldRef - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - storagePools: - description: StoragePools is a list of storage pools to configure - on the node. - items: - properties: - filePool: - description: Configures a file system based storage pool, allocating - a regular file per volume. - properties: - directory: - description: Directory is the path to the host directory - used to store volume data. - type: string - type: object - fileThinPool: - description: Configures a file system based storage pool, allocating - a sparse file per volume. - properties: - directory: - description: Directory is the path to the host directory - used to store volume data. - type: string - type: object - lvmPool: - description: Configures a LVM Volume Group as storage pool. - properties: - volumeGroup: - type: string - type: object - lvmThinPool: - description: Configures a LVM Thin Pool as storage pool. - properties: - thinPool: - description: ThinPool is the name of the thinpool LV (without - VG prefix). - type: string - volumeGroup: - type: string - type: object - name: - description: Name of the storage pool in linstor. - minLength: 3 - type: string - properties: - description: Properties to set on the storage pool. - items: - properties: - expandFrom: - description: |- - ExpandFrom can reference multiple resource fields at once. - It either sets the property to an aggregate value based on matched resource fields, or expands to multiple - properties. - properties: - delimiter: - description: Delimiter used to join multiple key and - value pairs together. - type: string - nameTemplate: - description: |- - NameTemplate defines how the property key is expanded. - If set, the template is appended to the defined property name, creating multiple properties instead of one - aggregate. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - nodeFieldRef: - description: Select a field of the node. Supports - `metadata.name`, `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - valueTemplate: - description: |- - ValueTemplate defines how the property value is expanded. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - required: - - nodeFieldRef - type: object - name: - description: Name of the property to set. - minLength: 1 - type: string - optional: - description: Optional values are only set if they have - a non-empty value - type: boolean - value: - description: Value to set the property to. - type: string - valueFrom: - description: ValueFrom sets the value from an existing - resource. - properties: - nodeFieldRef: - description: Select a field of the node. Supports - `metadata.name`, `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - required: - - nodeFieldRef - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - source: - properties: - hostDevices: - description: HostDevices is a list of device paths used - to configure the given pool. - items: - type: string - minItems: 1 - type: array - type: object - zfsPool: - description: Configures a ZFS system based storage pool, allocating - zvols from the given zpool. - properties: - zPool: - description: ZPool is the name of the ZFS zpool. - type: string - type: object - zfsThinPool: - description: Configures a ZFS system based storage pool, allocating - sparse zvols from the given zpool. - properties: - zPool: - description: ZPool is the name of the ZFS zpool. - type: string - type: object - required: - - name - type: object - type: array - type: object - status: - description: LinstorSatelliteConfigurationStatus defines the observed - state of LinstorSatelliteConfiguration - properties: - appliedTo: - description: AppliedTo lists the LinstorSatellite resource this configuration - was applied to - items: - type: string - type: array - conditions: - description: Current LINSTOR Satellite Config state - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - matched: - description: Number of configured LinstorSatellite resource. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.18.0 - name: linstorsatellites.piraeus.io -spec: - group: piraeus.io - names: - kind: LinstorSatellite - listKind: LinstorSatelliteList - plural: linstorsatellites - singular: linstorsatellite - scope: Cluster - versions: - - additionalPrinterColumns: - - description: If the LINSTOR Satellite is connected - jsonPath: .status.conditions[?(@.type=='Available')].status - name: Connected - type: string - - description: If the LINSTOR Satellite is fully configured - jsonPath: .status.conditions[?(@.type=='Configured')].status - name: Configured - type: string - - description: The Satellite Configurations applied to this Satellite - jsonPath: .metadata.annotations.piraeus\.io/applied-configurations - name: Applied Configurations - priority: 10 - type: string - - description: The deletion policy of the Satellite - jsonPath: .spec.deletionPolicy - name: Deletion Policy - type: string - - description: The used capacity on the node - jsonPath: .status.capacity - name: Used Capacity - type: string - - description: The number of volumes on the node - jsonPath: .status.numberOfVolumes - name: Volumes - type: integer - - description: The number of snapshots on the node - jsonPath: .status.numberOfSnapshots - name: Snapshots - priority: 10 - type: integer - - description: The storage providers supported by the node - jsonPath: .status.storageProviders - name: Storage Providers - priority: 10 - type: string - - description: The device layers supported by the node - jsonPath: .status.deviceLayers - name: Device Layers - priority: 10 - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1 - schema: - openAPIV3Schema: - description: LinstorSatellite is the Schema for the linstorsatellites API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: LinstorSatelliteSpec defines the desired state of LinstorSatellite - properties: - clusterRef: - description: ClusterRef references the LinstorCluster used to create - this LinstorSatellite. - properties: - caReference: - description: |- - CAReference configures the CA certificate to use when validating TLS certificates. - If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate. - properties: - key: - default: ca.crt - description: |- - Key to select in the resource. - Defaults to ca.crt if not specified. - type: string - kind: - default: Secret - description: Kind of the resource containing the CA Certificate, - either a ConfigMap or Secret. - enum: - - ConfigMap - - Secret - type: string - name: - description: Name of the resource containing the CA Certificate. - type: string - optional: - description: Optional specifies whether the resource and its - key must exist. - type: boolean - required: - - name - type: object - clientSecretName: - description: ClientSecretName references the secret used by the - operator to validate the https endpoint. - type: string - externalController: - description: |- - ExternalController references an external controller. - When set, the Operator uses the external cluster to register satellites. - properties: - url: - description: URL of the external controller. - minLength: 3 - type: string - required: - - url - type: object - name: - description: Name of the LinstorCluster resource controlling this - satellite. - type: string - type: object - deletionPolicy: - default: Retain - description: |- - DeletionPolicy configures the way LinstorSatellite resources are deleted. - - A LinstorSatellite may be deleted because: - * It no longer matches the affinity and node selector of the LinstorCluster resource. - * The node it references has been removed from Kubernetes. - * It was manually deleted outside the Operator. - - A LinstorSatellite may store the last copy of a volume, in which case it is not desirable to unconditionally remove - the satellite from the cluster. For this reason, the following deletion policies exist: - - * DeletionPolicyEvacuate will start evacuation of the LINSTOR Satellite and wait until it completes before removing the LinstorSatellite object, comparable to the "linstor node evacuate" command. - * DeletionPolicyRetain will retain the LINSTOR Satellite, keeping it registered in LINSTOR, but removing associated Kubernetes resources. - * DeletionPolicyDelete will remove the LINSTOR Satellite from the LINSTOR Cluster without prior eviction, comparable to the "linstor node lost" command. - enum: - - Evacuate - - Retain - - Delete - type: string - evacuationStrategy: - description: EvacuationStrategy configures the evacuation of volumes - from a Satellite when DeletionPolicy "Evacuate" is used. - properties: - attachedVolumeReattachTimeout: - default: 5m - description: |- - AttachedVolumeReattachTimeout configures how long evacuation waits for attached volumes to reattach on - different nodes. Setting this to 0 disable this evacuation step. - type: string - unattachedVolumeAttachTimeout: - default: 5m - description: |- - UnattachedVolumeAttachTimeout configures how long evacuation waits for unattached volumes to attach on - different nodes. Setting this to 0 disable this evacuation step. - type: string - type: object - internalTLS: - description: |- - InternalTLS configures secure communication for the LINSTOR Satellite. - - If set, the control traffic between LINSTOR Controller and Satellite will be encrypted using mTLS. - The Controller will use the client key from `LinstorCluster.spec.internalTLS` when connecting. - nullable: true - properties: - caReference: - description: |- - CAReference configures the CA certificate to use when validating TLS certificates. - If not set, the TLS secret is expected to contain a "ca.crt" containing the CA certificate. - properties: - key: - default: ca.crt - description: |- - Key to select in the resource. - Defaults to ca.crt if not specified. - type: string - kind: - default: Secret - description: Kind of the resource containing the CA Certificate, - either a ConfigMap or Secret. - enum: - - ConfigMap - - Secret - type: string - name: - description: Name of the resource containing the CA Certificate. - type: string - optional: - description: Optional specifies whether the resource and its - key must exist. - type: boolean - required: - - name - type: object - certManager: - description: |- - CertManager references a cert-manager Issuer or ClusterIssuer. - If set, a Certificate resource will be created, provisioning the secret references in SecretName using the - issuer configured here. - properties: - group: - description: Group of the resource being referred to. - type: string - kind: - description: Kind of the resource being referred to. - type: string - name: - description: Name of the resource being referred to. - type: string - required: - - name - type: object - secretName: - description: SecretName references a secret holding the TLS key - and certificates. - type: string - tlsHandshakeDaemon: - description: |- - TLSHandshakeDaemon enables tlshd for establishing TLS sessions for use by DRBD. - - If enabled, adds a new sidecar to the LINSTOR Satellite that runs the tlshd handshake daemon. - The daemon uses the TLS certificate and key to establish secure connections on behalf of DRBD. - type: boolean - type: object - ipFamilies: - description: |- - IPFamilies configures the IP Family (IPv4 or IPv6) to use to connect to the LINSTOR Satellite. - - If set, the control traffic between LINSTOR Controller and Satellite will use only the given IP Family. - If not set, the Operator will configure all families found in the Satellites Pods' Status. - items: - description: IPFamily represents the IP Family (IPv4 or IPv6). - enum: - - IPv4 - - IPv6 - type: string - type: array - patches: - description: |- - Patches is a list of kustomize patches to apply. - - See https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/patches/ for how to create patches. - items: - description: Patch represent either a Strategic Merge Patch or a - JSON patch and its targets. - properties: - options: - description: Options is a list of options for the patch - properties: - allowKindChange: - description: AllowKindChange allows kind changes to the - resource. - type: boolean - allowNameChange: - description: AllowNameChange allows name changes to the - resource. - type: boolean - type: object - patch: - description: Patch is the content of a patch. - minLength: 1 - type: string - target: - description: Target points to the resources that the patch is - applied to - properties: - annotationSelector: - description: |- - AnnotationSelector is a string that follows the label selection expression - https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api - It matches against the resource annotations. - type: string - group: - type: string - kind: - type: string - labelSelector: - description: |- - LabelSelector is a string that follows the label selection expression - https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#api - It matches against the resource labels. - type: string - name: - description: Name of the resource. - type: string - namespace: - description: Namespace the resource belongs to, if it can - belong to a namespace. - type: string - version: - type: string - type: object - required: - - patch - type: object - type: array - properties: - description: Properties is a list of properties to set on the node. - items: - properties: - expandFrom: - description: |- - ExpandFrom can reference multiple resource fields at once. - It either sets the property to an aggregate value based on matched resource fields, or expands to multiple - properties. - properties: - delimiter: - description: Delimiter used to join multiple key and value - pairs together. - type: string - nameTemplate: - description: |- - NameTemplate defines how the property key is expanded. - If set, the template is appended to the defined property name, creating multiple properties instead of one - aggregate. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - nodeFieldRef: - description: Select a field of the node. Supports `metadata.name`, - `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - valueTemplate: - description: |- - ValueTemplate defines how the property value is expanded. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - required: - - nodeFieldRef - type: object - name: - description: Name of the property to set. - minLength: 1 - type: string - optional: - description: Optional values are only set if they have a non-empty - value - type: boolean - value: - description: Value to set the property to. - type: string - valueFrom: - description: ValueFrom sets the value from an existing resource. - properties: - nodeFieldRef: - description: Select a field of the node. Supports `metadata.name`, - `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - required: - - nodeFieldRef - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - repository: - description: Repository used to pull workload images. - type: string - storagePools: - description: StoragePools is a list of storage pools to configure - on the node. - items: - properties: - filePool: - description: Configures a file system based storage pool, allocating - a regular file per volume. - properties: - directory: - description: Directory is the path to the host directory - used to store volume data. - type: string - type: object - fileThinPool: - description: Configures a file system based storage pool, allocating - a sparse file per volume. - properties: - directory: - description: Directory is the path to the host directory - used to store volume data. - type: string - type: object - lvmPool: - description: Configures a LVM Volume Group as storage pool. - properties: - volumeGroup: - type: string - type: object - lvmThinPool: - description: Configures a LVM Thin Pool as storage pool. - properties: - thinPool: - description: ThinPool is the name of the thinpool LV (without - VG prefix). - type: string - volumeGroup: - type: string - type: object - name: - description: Name of the storage pool in linstor. - minLength: 3 - type: string - properties: - description: Properties to set on the storage pool. - items: - properties: - expandFrom: - description: |- - ExpandFrom can reference multiple resource fields at once. - It either sets the property to an aggregate value based on matched resource fields, or expands to multiple - properties. - properties: - delimiter: - description: Delimiter used to join multiple key and - value pairs together. - type: string - nameTemplate: - description: |- - NameTemplate defines how the property key is expanded. - If set, the template is appended to the defined property name, creating multiple properties instead of one - aggregate. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - nodeFieldRef: - description: Select a field of the node. Supports - `metadata.name`, `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - valueTemplate: - description: |- - ValueTemplate defines how the property value is expanded. - * $1 is replaced with the matched key. - * $2 is replaced with the matched value. - type: string - required: - - nodeFieldRef - type: object - name: - description: Name of the property to set. - minLength: 1 - type: string - optional: - description: Optional values are only set if they have - a non-empty value - type: boolean - value: - description: Value to set the property to. - type: string - valueFrom: - description: ValueFrom sets the value from an existing - resource. - properties: - nodeFieldRef: - description: Select a field of the node. Supports - `metadata.name`, `metadata.labels['']`, `metadata.annotations['']`. - minLength: 1 - type: string - required: - - nodeFieldRef - type: object - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - source: - properties: - hostDevices: - description: HostDevices is a list of device paths used - to configure the given pool. - items: - type: string - minItems: 1 - type: array - type: object - zfsPool: - description: Configures a ZFS system based storage pool, allocating - zvols from the given zpool. - properties: - zPool: - description: ZPool is the name of the ZFS zpool. - type: string - type: object - zfsThinPool: - description: Configures a ZFS system based storage pool, allocating - sparse zvols from the given zpool. - properties: - zPool: - description: ZPool is the name of the ZFS zpool. - type: string - type: object - required: - - name - type: object - type: array - required: - - clusterRef - type: object - status: - description: LinstorSatelliteStatus defines the observed state of LinstorSatellite - properties: - availableCapacityBytes: - description: The number of bytes in total in all storage pools on - this Satellite. - format: int64 - type: integer - capacity: - description: Capacity mirrors the information from TotalCapacityBytes - and FreeCapacityBytes in a human-readable string. - type: string - conditions: - description: Current LINSTOR Satellite state - items: - description: Condition contains details for one aspect of the current - state of this API Resource. - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - deviceLayers: - description: DeviceLayers lists the device layers (LUKS, CACHE, etc...) - this Satellite supports. - items: - type: string - type: array - freeCapacityBytes: - description: The number of bytes free in all storage pools on this - Satellite. - format: int64 - type: integer - numberOfSnapshots: - description: The number of snapshots on this Satellite. - format: int32 - type: integer - numberOfVolumes: - description: The number of volumes on this Satellite. - format: int32 - type: integer - storageProviders: - description: StorageProviders lists the storage providers (LVM, ZFS, - etc...) this Satellite supports. - items: - type: string - type: array - type: object - type: object - served: true - storage: true - subresources: - status: {} -{{ end }} diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/deployment.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/deployment.yaml index c67501972..0c3209e19 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/deployment.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/deployment.yaml @@ -3,6 +3,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "piraeus-operator.fullname" . }}-controller-manager + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} spec: @@ -70,6 +71,9 @@ spec: - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert readOnly: true + {{- with .Values.operator.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- if .Values.kubeRbacProxy.enabled }} - args: - --secure-listen-address=0.0.0.0:8443 @@ -91,8 +95,11 @@ spec: securityContext: {{- toYaml .Values.kubeRbacProxy.securityContext | nindent 12}} volumeMounts: - - mountPath: /etc/tls - name: cert + - mountPath: /etc/tls + name: cert + {{- with .Values.kubeRbacProxy.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} securityContext: runAsNonRoot: true @@ -100,6 +107,10 @@ spec: {{ toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ include "piraeus-operator.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} terminationGracePeriodSeconds: 10 priorityClassName: {{ .Values.priorityClassName | default "system-cluster-critical" }} nodeSelector: @@ -113,3 +124,6 @@ spec: secret: defaultMode: 420 secretName: {{ include "piraeus-operator.certifcateName" . }} + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 6 }} + {{- end }} diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/extra-objects.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/extra-objects.yaml new file mode 100644 index 000000000..b0ec6fad2 --- /dev/null +++ b/packages/system/piraeus-operator/charts/piraeus/templates/extra-objects.yaml @@ -0,0 +1,15 @@ +{{- /* Normalize extraObjects to a list, easier to loop over */ -}} +{{- $extraObjects := .Values.extraManifests | default (list) -}} + +{{- if kindIs "map" $extraObjects -}} + {{- $extraObjects = values $extraObjects -}} +{{- end -}} + +{{- range $extraObjects }} +--- + {{- if kindIs "map" . }} + {{- tpl (toYaml .) $ | nindent 0 }} + {{- else if kindIs "string" . }} + {{- tpl . $ | nindent 0 }} + {{- end }} +{{- end }} diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/metrics-service.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/metrics-service.yaml index d1a24e3ca..49dee196a 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/metrics-service.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/metrics-service.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Service metadata: name: {{ include "piraeus-operator.fullname" . }}-controller-manager-metrics-service + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} spec: diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/rbac.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/rbac.yaml index c532e02d4..56b7d019b 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/rbac.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/rbac.yaml @@ -15,7 +15,6 @@ rules: - events - persistentvolumes - pods - - secrets - serviceaccounts - services verbs: @@ -59,6 +58,13 @@ rules: - pods/eviction verbs: - create +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create + - get - apiGroups: - apiextensions.k8s.io resources: @@ -71,6 +77,12 @@ rules: - patch - update - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions/status + verbs: + - patch - apiGroups: - apps resources: @@ -235,7 +247,6 @@ rules: - snapshot.storage.k8s.io resources: - volumesnapshotclasses - - volumesnapshots verbs: - get - list @@ -257,6 +268,15 @@ rules: verbs: - patch - update +- apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - update + - watch - apiGroups: - storage.k8s.io resources: @@ -283,6 +303,7 @@ rules: - storage.k8s.io resources: - storageclasses + - volumeattributesclasses verbs: - get - list @@ -306,8 +327,30 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role +metadata: + name: {{ include "piraeus-operator.fullname" . }}-controller-manager + namespace: {{ .Release.Namespace }} + labels: + {{- include "piraeus-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role metadata: name: {{ include "piraeus-operator.fullname" . }}-leader-election + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} rules: diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/rolebindings.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/rolebindings.yaml index 97e53b952..fc5212cc4 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/rolebindings.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/rolebindings.yaml @@ -17,6 +17,22 @@ subjects: namespace: {{ .Release.Namespace }} --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "piraeus-operator.fullname" . }}-controller-manager + namespace: {{ .Release.Namespace }} + labels: + {{- include "piraeus-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "piraeus-operator.fullname" . }}-controller-manager +subjects: +- kind: ServiceAccount + name: {{ include "piraeus-operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: {{ include "piraeus-operator.fullname" . }}-controller-manager diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/serviceaccount.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/serviceaccount.yaml index 89315de95..c4997dd24 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/serviceaccount.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/serviceaccount.yaml @@ -4,6 +4,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "piraeus-operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} {{ end }} diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/validating-webhook-configuration.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/validating-webhook-configuration.yaml index 5a6c5c9e6..04d05bfaf 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/validating-webhook-configuration.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/validating-webhook-configuration.yaml @@ -9,6 +9,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: {{ include "piraeus-operator.fullname" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} spec: @@ -36,6 +37,7 @@ apiVersion: v1 kind: Secret metadata: name: {{ include "piraeus-operator.certifcateName" . }} + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} type: kubernetes.io/tls diff --git a/packages/system/piraeus-operator/charts/piraeus/templates/webhook-service.yaml b/packages/system/piraeus-operator/charts/piraeus/templates/webhook-service.yaml index d360269bc..57f8966c9 100644 --- a/packages/system/piraeus-operator/charts/piraeus/templates/webhook-service.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/templates/webhook-service.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Service metadata: name: {{ include "piraeus-operator.fullname" . }}-webhook-service + namespace: {{ .Release.Namespace }} labels: {{- include "piraeus-operator.labels" . | nindent 4 }} spec: diff --git a/packages/system/piraeus-operator/charts/piraeus/values.yaml b/packages/system/piraeus-operator/charts/piraeus/values.yaml index bcc4dc9af..ff0d92b5a 100644 --- a/packages/system/piraeus-operator/charts/piraeus/values.yaml +++ b/packages/system/piraeus-operator/charts/piraeus/values.yaml @@ -29,6 +29,12 @@ operator: # cpu: 100m # memory: 128Mi + ## Additional volume mounts + extraVolumeMounts: [ ] + # - name: clusterapi-kubeconfig + # mountPath: /etc/operator + # readOnly: true + kubeRbacProxy: enabled: false image: @@ -54,6 +60,16 @@ kubeRbacProxy: # cpu: 100m # memory: 128Mi + ## Additional volume mounts + extraVolumeMounts: [ ] + + +## Additional volumes +extraVolumes: [] +# - name: clusterapi-kubeconfig +# secret: +# secretName: super-secret + webhook: timeoutSeconds: 2 failurePolicy: Fail @@ -111,3 +127,30 @@ imageConfigOverride: [] # tag: my-custom-tag # Results in the image example.com/piraeus/linstor-csi:my-custom-tag being used. # See templates/config.yaml for available components. + +## Extra manifests to deploy. Can be of type dict or list. +## If dict, keys are ignored and only values are used. +## Items contained within extraObjects can be defined as dict or string and are passed through tpl. +extraManifests: null + # - apiVersion: v1 + # kind: ConfigMap + # metadata: + # labels: + # name: piraeus-extra + # data: + # extra-data: "value" + # + # can also be defined as a string, useful for templating field names + # - | + # apiVersion: v1 + # kind: Secret + # type: Opaque + # metadata: + # name: super-secret + # labels: + # {{- range $key, $value := .Values.commonLabels }} + # {{ $key }}: {{ $value }} + # {{- end }} + # data: + # plaintext: Zm9vYmFy +# templated: '{{ print "foobar" | upper | b64enc }}' diff --git a/packages/system/postgres-rd/cozyrds/postgres.yaml b/packages/system/postgres-rd/cozyrds/postgres.yaml index 0cd63edd2..052cb575d 100644 --- a/packages/system/postgres-rd/cozyrds/postgres.yaml +++ b/packages/system/postgres-rd/cozyrds/postgres.yaml @@ -30,7 +30,7 @@ spec: singular: postgres plural: postgreses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas. Under active autoscaling this is the resting/seed count, not the live count — KEDA drives the live count via `/scale`. Disabling autoscaling does NOT automatically bring the live count back to `replicas`: the client-side merge writes `instances` only when the rendered value changes, so a cluster KEDA grew past `replicas` stays there with no autoscaler left to reduce it — lower `replicas` to the count you want and it rebases down (shedding the surplus replicas + PVCs). Lowering it below the live count, or disabling/dry-running while it sits below the live count, sheds in one step past the scaleDown pacing; stage it to the live count first (suspend the HelmRelease and `kubectl scale` if needed) to avoid a transient. Must stay greater than `quorum.maxSyncReplicas`.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"flavor":{"description":"PostgreSQL image flavor, fixed when the cluster is created. `postgresql` uses the CloudNativePG PostgreSQL image. `postgis` uses the CloudNativePG PostGIS image, which adds postgis, postgis_raster, postgis_topology, postgis_sfcgal, postgis_tiger_geocoder and address_standardizer; enable them per database in `databases..extensions`. Changing the flavor of an existing cluster is refused, and a backup restores only into a cluster of the flavor it was taken from: to change flavor, create a new cluster and move the data with a logical dump (pg_dump / pg_restore).","type":"string","default":"postgresql","enum":["postgresql","postgis"]},"tls":{"description":"TLS configuration for server connections.","type":"object","default":{},"properties":{"enabled":{"description":"Tri-state switch controlling whether the chart injects the external hostnames into the operator-managed CNPG cert via spec.certificates.serverAltDNSNames. When omitted, the chart injects them if `external: true` and skips them otherwise. Set explicitly to `true` to inject regardless of `external` (no-op when `external: false` since there is no external hostname to add). Set to `false` to skip injection. Note that CNPG keeps its built-in TLS on the wire regardless of this flag — this toggle only controls the chart-side SAN injection; to disable PostgreSQL TLS entirely set `postgresql.parameters.ssl = \"off\"` at the CNPG layer.","type":"boolean"},"required":{"description":"Tri-state switch controlling whether the server refuses sessions that are not encrypted. When omitted, it is on if `external: true` and off otherwise. When on, the chart renders `spec.postgresql.pg_hba` with `hostssl all all all scram-sha-256` (`md5` on PostgreSQL 13, matching CloudNativePG's default rule) followed by `host all all all reject`, so a client that connects with `sslmode=disable` is refused instead of getting a plaintext session; in-cluster clients are held to the same rule, since the pod network cannot be told apart from an external client. Set to `false` to keep accepting plaintext sessions on an external instance, or to `true` to require TLS on an in-cluster one. Rendering fails if `postgresql.parameters.ssl` is off while this is in effect.","type":"boolean"}}},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters. Values may be strings or integers; integers are coerced to strings by the template (e.g. both `max_connections: 100` and `max_connections: \"100\"` are accepted). BLOCKED (enable arbitrary code execution): archive_command, restore_command, ssl_passphrase_command, archive_cleanup_command, recovery_end_command, dynamic_library_path, local_preload_libraries, session_preload_libraries, shared_preload_libraries. Do NOT override CloudNativePG-managed parameters: archive_mode, primary_conninfo, wal_level, max_replication_slots.","type":"object","default":{"max_connections":"100"},"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}},"sharedPreloadLibraries":{"description":"Libraries loaded at server start, restricted to an allowlist of modules already shipped in the PostgreSQL image. CloudNativePG also preloads `pg_stat_statements`, `auto_explain` and `pgaudit` by itself whenever `parameters` sets a key under their prefix (e.g. `pg_stat_statements.track`), so removing one of them from this list does not unload it while such a key is set; `pg_prewarm` is preloaded only through this list. Changing the list restarts PostgreSQL on every instance, the primary included, so writes pause briefly.","type":"array","items":{"type":"string","enum":["pg_stat_statements","auto_explain","pgaudit","pg_prewarm"]}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas). Under autoscaling it also sets the scale-down floor (the effective minimum is `max(minReplicas, maxSyncReplicas+1, 2)`); the chart fails to render if that floor exceeds `autoscaling.maxReplicas`.","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"autoscaling":{"description":"Read-replica autoscaling configuration.","type":"object","default":{},"required":["dryRun","enabled","maxReplicas","maxReplicationLagSeconds","metric","minReplicas","serverAddress","target","transition"],"properties":{"dryRun":{"description":"Recommendation mode: keep the static count and render the ScaledObject with both scaling directions paused, so KEDA keeps the HPA and keeps serving its trigger metric but actuates neither way. Note both scaling policies are disabled, so the HPA's desiredReplicas is frozen to the current count - read the recommendation from the matching read-load dashboard panel for your metric (connections or CPU): desired = 1 + ceil(read load / target), clamped to [effectiveMin..maxReplicas] where effectiveMin = max(minReplicas, maxSyncReplicas+1, 2), not from the desired-replicas line. Like disabling, flipping a LIVE autoscaled cluster into dryRun re-renders `instances: replicas` (the raw static value, which can be below the autoscaling floor); if `replicas` is below the live count, stage it to the live count first (see the enablement note) or the cluster sheds down to `replicas`. No shed occurs when the rendered value is unchanged (replicas already equals the active seed) — client-side merge is then a no-op and the live count is preserved. Permanent, unlike `transition`.","type":"boolean","default":false},"enabled":{"description":"Enable horizontal autoscaling of read replicas. Requires the platform `keda` package to be enabled by an administrator first; the chart fails to render (rather than break the release) if KEDA is not installed. Prerequisite: the queried vmselect must hold BOTH the CNPG metrics (cnpg_backends_total / container CPU) AND kube_pod_labels (kube-state-metrics). This holds for a tenant on the shared root monitoring stack (`monitoring: false`, both land in tenant-root). A tenant running its OWN Monitoring app (`monitoring: true`) has the CNPG metrics but not kube_pod_labels (KSM is root-only), so it must set `serverAddress` to a vmselect carrying both series or autoscaling never actuates.","type":"boolean","default":false},"maxReplicas":{"description":"Maximum total instances. Must be >= minReplicas and >= the synchronous-quorum floor (maxSyncReplicas+1); the chart fails to render (rather than silently run an unbounded instance count) if the effective floor exceeds it - raise maxReplicas. HAZARD: lowering maxReplicas below the live count makes the HPA cut straight to the new maximum in one step (the current>max branch bypasses the scale-down pacing), shedding the surplus replicas + PVCs at once; to lower it deliberately, suspend the HelmRelease and pin the live count with `kubectl scale` first.","type":"integer","default":6},"maxReplicationLagSeconds":{"description":"Freeze scaling while replication lag exceeds this (seconds) and the primary is writing. 0 disables the brake. Default 0: the freeze branch has only been validated on a live cluster in its non-braking (pass-through) form, so the lag brake is opt-in until the braking path is exercised under load; set a positive value (e.g. 30) to enable it.","type":"integer","default":0},"metric":{"description":"Read-load signal to scale on.","type":"string","default":"ReadConnections","enum":["ReadConnections","ReadCPUUtilization"]},"minReplicas":{"description":"Minimum total instances; raised to the synchronous-quorum floor (`maxSyncReplicas + 1`) and to 2 when either is higher.","type":"integer","default":2},"serverAddress":{"description":"Override the Prometheus/vmselect URL the ScaledObject queries. Empty (default) derives `http://vmselect-shortterm..svc:8481/select/0/prometheus`. Set it when the derived vmselect does not hold both the CNPG metrics and kube_pod_labels (e.g. `monitoring: true` tenants, or a non-default `metricsStorages` name). SSRF constraint: the host must be an in-cluster Service DNS name (`..svc[.]`) whose namespace is within the tenant's own subtree (its namespace, its parents, or `tenant-root`) — the cluster-privileged KEDA operator fetches this URL every poll, so it is refused a metadata endpoint, an external host, or a sibling/system namespace. Residual surface (accepted trade-off): `tenant-root` is always allowed because the derived default points there, so a tenant can still aim the operator at any Service in `tenant-root` on any port/path (a blind in-cluster reachability probe — the operator reads no response body back to the tenant); the egress NetworkPolicy does not narrow that.","type":"string","default":""},"target":{"description":"Target read load per read-serving replica (unit depends on `metric`: active connections, or CPU millicores). Dimensionless integer - a resource.Quantity here would accept `150m`, which PromQL reads as 150 minutes and KEDA cannot use as a threshold.","type":"integer","default":150},"transition":{"description":"Migration phase 1: render `.spec.instances` from `replicas` (which the operator stages to the current live count) AND pause the ScaledObject in both directions, so KEDA stands up with its HPA present but not actuating. Flip to false (phase 2) to hand the runtime count to KEDA. Because the both-direction pause keeps the HPA (not the HPA-deleting full pause), KEDA is genuinely warm at phase 2 — no HPA recreate — and because the active seed is max(replicas, effectiveMin), phase 2 re-renders the same instance count phase 1 did (when `replicas` is staged at/above the floor), so the flip is a merge no-op with no dip. The HPA's scaleUp stabilization window still smooths the first actuation.","type":"boolean","default":false}}},"users":{"description":"Users configuration map. Passwords are always auto-generated and stored in the `-credentials` Secret; they cannot be set from values — read a user's password from that Secret. A `password` left over in values from before the field was removed is ignored by the render and draws an admission warning, but it is not inert on an upgraded release: the chart preserves whatever password is already in the Secret, which on the first upgrade is the value that was set before removal, so that value stays the live credential until it is rotated (dedicated rotation is tracked in cozystack/community#72). The live password lives only in the Secret.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"bypassRls":{"description":"Whether the user can bypass row-level security policies (BYPASSRLS). A role can only create other roles carrying BYPASSRLS if it holds the attribute itself, so this is required alongside `createRole` to provision a privileged service role.","type":"boolean"},"createRole":{"description":"Whether the user can create and manage roles (CREATEROLE). Requires PostgreSQL 16 or newer. Setting it back to `false` withdraws the attribute only: the roles the user created, and its ADMIN and SET memberships in them, stay. Once any user has held it, the chart no longer adopts an existing role it did not create, so a user or database declared later under the name of such a role fails the init job until a platform administrator drops that role.","type":"boolean"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map. The PostgreSQL system databases `postgres`, `template0` and `template1` are refused.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility.","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointCA":{"description":"DEPRECATED. Pre-existing Secret with the CA bundle the barman-cloud plugin should trust when reaching a self-signed S3 endpoint. Used for both backup and bootstrap recovery in the legacy chart-managed flow.","type":"object","default":{},"properties":{"key":{"description":"Key within the Secret containing the CA bundle. Defaults to `ca.crt`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means no endpointCA is emitted (the plugin uses the system trust store).","type":"string","default":""}}},"endpointURL":{"description":"DEPRECATED. See `destinationPath`.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"DEPRECATED. Tenants no longer supply S3 keys; the system Bucket Secret is projected into the tenant namespace by the backup controller. Ignored when `s3CredentialsSecret.name` is set or `useSystemBucket` is true. The chart skips materialising `-s3-creds` whenever this field is empty so a default install does not leak placeholder credentials into the tenant namespace.","type":"string","default":""},"s3CredentialsSecret":{"description":"DEPRECATED. Pre-existing Secret with S3 credentials. Use the platform-managed `cozy-default` BackupClass instead. When set, the chart references this Secret directly (legacy chart-managed flow). The CNPG backup driver writes this field on restore so credentials never land in the CR `.spec`.","type":"object","default":{},"properties":{"accessKeyIDKey":{"description":"Key in the Secret holding the access key ID. Defaults to `AWS_ACCESS_KEY_ID`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means the chart materialises `-s3-creds` from `s3AccessKey`/`s3SecretKey`.","type":"string","default":""},"secretAccessKeyKey":{"description":"Key in the Secret holding the secret access key. Defaults to `AWS_SECRET_ACCESS_KEY`.","type":"string","default":""}}},"s3SecretKey":{"description":"DEPRECATED. See `s3AccessKey`.","type":"string","default":""},"schedule":{"description":"Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.","type":"string","default":""},"useSystemBucket":{"description":"Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`.","type":"boolean","default":false}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"backupID":{"description":"Barman ID of the base backup to recover from, passed as `recoveryTarget.backupID`. Empty lets the barman-cloud plugin pick one: the newest backup when `recoveryTime` is empty, otherwise the newest one ending at or before it, on any timeline. The CNPG backup driver sets this on restore.","type":"string","default":""},"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"newServerName":{"description":"WAL-archive server name (S3 path prefix) the RESTORED cluster writes to. Must differ from `serverName` (the recovery source) so a restored cluster archives to a fresh, empty prefix and barman-cloud-check-wal-archive passes while it recovers from the source's prefix. The CNPG backup driver sets this on restore; empty means the cluster archives under its own name.","type":"string","default":""},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Server name (S3 path prefix) used by the original cluster when writing backups; passed to the barman-cloud plugin via `externalClusters[].plugin.parameters.serverName`. Defaults to `bootstrap.oldName`. Set this only when the original cluster wrote backups under an explicit server name that differed from its Kubernetes resource name.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Postgres replicas. Under active autoscaling this is the resting/seed count, not the live count — KEDA drives the live count via `/scale`. Disabling autoscaling does NOT automatically bring the live count back to `replicas`: the client-side merge writes `instances` only when the rendered value changes, so a cluster KEDA grew past `replicas` stays there with no autoscaler left to reduce it — lower `replicas` to the count you want and it rebases down (shedding the surplus replicas + PVCs). Lowering it below the live count, or disabling/dry-running while it sits below the live count, sheds in one step past the scaleDown pacing; stage it to the live count first (suspend the HelmRelease and `kubectl scale` if needed) to avoid a transient. Must stay greater than `quorum.maxSyncReplicas`.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each PostgreSQL replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"10Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"version":{"description":"PostgreSQL major version to deploy","type":"string","default":"v18","enum":["v18","v17","v16","v15","v14","v13"]},"flavor":{"description":"PostgreSQL image flavor, fixed when the cluster is created. `postgresql` uses the CloudNativePG PostgreSQL image. `postgis` uses the CloudNativePG PostGIS image, which adds postgis, postgis_raster, postgis_topology, postgis_sfcgal, postgis_tiger_geocoder and address_standardizer; enable them per database in `databases..extensions`. Changing the flavor of an existing cluster is refused, and a backup restores only into a cluster of the flavor it was taken from: to change flavor, create a new cluster and move the data with a logical dump (pg_dump / pg_restore).","type":"string","default":"postgresql","enum":["postgresql","postgis"]},"tls":{"description":"TLS configuration for server connections.","type":"object","default":{},"properties":{"enabled":{"description":"Tri-state switch controlling whether the chart injects the external hostnames into the operator-managed CNPG cert via spec.certificates.serverAltDNSNames. When omitted, the chart injects them if `external: true` and skips them otherwise. Set explicitly to `true` to inject regardless of `external` (no-op when `external: false` since there is no external hostname to add). Set to `false` to skip injection. Note that CNPG keeps its built-in TLS on the wire regardless of this flag — this toggle only controls the chart-side SAN injection; to disable PostgreSQL TLS entirely set `postgresql.parameters.ssl = \"off\"` at the CNPG layer.","type":"boolean"},"required":{"description":"Tri-state switch controlling whether the server refuses sessions that are not encrypted. When omitted, it is on if `external: true` and off otherwise. When on, the chart renders `spec.postgresql.pg_hba` with `hostssl all all all scram-sha-256` (`md5` on PostgreSQL 13, matching CloudNativePG's default rule) followed by `host all all all reject`, so a client that connects with `sslmode=disable` is refused instead of getting a plaintext session; in-cluster clients are held to the same rule, since the pod network cannot be told apart from an external client. Set to `false` to keep accepting plaintext sessions on an external instance, or to `true` to require TLS on an in-cluster one. Rendering fails if `postgresql.parameters.ssl` is off while this is in effect.","type":"boolean"}}},"postgresql":{"description":"PostgreSQL server configuration.","type":"object","default":{},"properties":{"parameters":{"description":"PostgreSQL server parameters. Values may be strings or integers; integers are coerced to strings by the template (e.g. both `max_connections: 100` and `max_connections: \"100\"` are accepted). BLOCKED (enable arbitrary code execution): archive_command, restore_command, ssl_passphrase_command, archive_cleanup_command, recovery_end_command, dynamic_library_path, local_preload_libraries, session_preload_libraries, shared_preload_libraries. Do NOT override CloudNativePG-managed parameters: archive_mode, primary_conninfo, wal_level, max_replication_slots.","type":"object","default":{"max_connections":"100"},"additionalProperties":{"anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}},"sharedPreloadLibraries":{"description":"Libraries loaded at server start, restricted to an allowlist of modules already shipped in the PostgreSQL image. CloudNativePG also preloads `pg_stat_statements`, `auto_explain` and `pgaudit` by itself whenever `parameters` sets a key under their prefix (e.g. `pg_stat_statements.track`), so removing one of them from this list does not unload it while such a key is set; `pg_prewarm` is preloaded only through this list. Changing the list restarts PostgreSQL on every instance, the primary included, so writes pause briefly.","type":"array","items":{"type":"string","enum":["pg_stat_statements","auto_explain","pgaudit","pg_prewarm"]}}}},"quorum":{"description":"Quorum configuration for synchronous replication.","type":"object","default":{},"required":["maxSyncReplicas","minSyncReplicas"],"properties":{"maxSyncReplicas":{"description":"Maximum number of synchronous replicas allowed (must be less than total replicas). Under autoscaling it also sets the scale-down floor (the effective minimum is `max(minReplicas, maxSyncReplicas+1, 2)`); the chart fails to render if that floor exceeds `autoscaling.maxReplicas`.","type":"integer","default":0},"minSyncReplicas":{"description":"Minimum number of synchronous replicas required for commit.","type":"integer","default":0}}},"autoscaling":{"description":"Read-replica autoscaling configuration.","type":"object","default":{},"required":["dryRun","enabled","maxReplicas","maxReplicationLagSeconds","metric","minReplicas","serverAddress","target","transition"],"properties":{"dryRun":{"description":"Recommendation mode: keep the static count and render the ScaledObject with both scaling directions paused, so KEDA keeps the HPA and keeps serving its trigger metric but actuates neither way. Note both scaling policies are disabled, so the HPA's desiredReplicas is frozen to the current count - read the recommendation from the matching read-load dashboard panel for your metric (connections or CPU): desired = 1 + ceil(read load / target), clamped to [effectiveMin..maxReplicas] where effectiveMin = max(minReplicas, maxSyncReplicas+1, 2), not from the desired-replicas line. Like disabling, flipping a LIVE autoscaled cluster into dryRun re-renders `instances: replicas` (the raw static value, which can be below the autoscaling floor); if `replicas` is below the live count, stage it to the live count first (see the enablement note) or the cluster sheds down to `replicas`. No shed occurs when the rendered value is unchanged (replicas already equals the active seed) — client-side merge is then a no-op and the live count is preserved. Permanent, unlike `transition`.","type":"boolean","default":false},"enabled":{"description":"Enable horizontal autoscaling of read replicas. Requires the platform `keda` package to be enabled by an administrator first; the chart fails to render (rather than break the release) if KEDA is not installed. Prerequisite: the queried vmselect must hold BOTH the CNPG metrics (cnpg_backends_total / container CPU) AND kube_pod_labels (kube-state-metrics). This holds for a tenant on the shared root monitoring stack (`monitoring: false`, both land in tenant-root). A tenant running its OWN Monitoring app (`monitoring: true`) has the CNPG metrics but not kube_pod_labels (KSM is root-only), so it must set `serverAddress` to a vmselect carrying both series or autoscaling never actuates.","type":"boolean","default":false},"maxReplicas":{"description":"Maximum total instances. Must be >= minReplicas and >= the synchronous-quorum floor (maxSyncReplicas+1); the chart fails to render (rather than silently run an unbounded instance count) if the effective floor exceeds it - raise maxReplicas. HAZARD: lowering maxReplicas below the live count makes the HPA cut straight to the new maximum in one step (the current>max branch bypasses the scale-down pacing), shedding the surplus replicas + PVCs at once; to lower it deliberately, suspend the HelmRelease and pin the live count with `kubectl scale` first.","type":"integer","default":6},"maxReplicationLagSeconds":{"description":"Freeze scaling while replication lag exceeds this (seconds) and the primary is writing. 0 disables the brake. Default 0: the freeze branch has only been validated on a live cluster in its non-braking (pass-through) form, so the lag brake is opt-in until the braking path is exercised under load; set a positive value (e.g. 30) to enable it.","type":"integer","default":0},"metric":{"description":"Read-load signal to scale on.","type":"string","default":"ReadConnections","enum":["ReadConnections","ReadCPUUtilization"]},"minReplicas":{"description":"Minimum total instances; raised to the synchronous-quorum floor (`maxSyncReplicas + 1`) and to 2 when either is higher.","type":"integer","default":2},"serverAddress":{"description":"Override the Prometheus/vmselect URL the ScaledObject queries. Empty (default) derives `http://vmselect-shortterm..svc:8481/select/0/prometheus`. Set it when the derived vmselect does not hold both the CNPG metrics and kube_pod_labels (e.g. `monitoring: true` tenants, or a non-default `metricsStorages` name). SSRF constraint: the host must be an in-cluster Service DNS name (`..svc[.]`) whose namespace is within the tenant's own subtree (its namespace, its parents, or `tenant-root`) — the cluster-privileged KEDA operator fetches this URL every poll, so it is refused a metadata endpoint, an external host, or a sibling/system namespace. Residual surface (accepted trade-off): `tenant-root` is always allowed because the derived default points there, so a tenant can still aim the operator at any Service in `tenant-root` on any port/path (a blind in-cluster reachability probe — the operator reads no response body back to the tenant); the egress NetworkPolicy does not narrow that.","type":"string","default":""},"target":{"description":"Target read load per read-serving replica (unit depends on `metric`: active connections, or CPU millicores). Dimensionless integer - a resource.Quantity here would accept `150m`, which PromQL reads as 150 minutes and KEDA cannot use as a threshold.","type":"integer","default":150},"transition":{"description":"Migration phase 1: render `.spec.instances` from `replicas` (which the operator stages to the current live count) AND pause the ScaledObject in both directions, so KEDA stands up with its HPA present but not actuating. Flip to false (phase 2) to hand the runtime count to KEDA. Because the both-direction pause keeps the HPA (not the HPA-deleting full pause), KEDA is genuinely warm at phase 2 — no HPA recreate — and because the active seed is max(replicas, effectiveMin), phase 2 re-renders the same instance count phase 1 did (when `replicas` is staged at/above the floor), so the flip is a merge no-op with no dip. The HPA's scaleUp stabilization window still smooths the first actuation.","type":"boolean","default":false}}},"users":{"description":"Users configuration map. Passwords are always auto-generated and stored in the `-credentials` Secret; they cannot be set from values — read a user's password from that Secret. A `password` left over in values from before the field was removed is ignored by the render and draws an admission warning, but it is not inert on an upgraded release: the chart preserves whatever password is already in the Secret, which on the first upgrade is the value that was set before removal, so that value stays the live credential until it is rotated (dedicated rotation is tracked in cozystack/community#72). The live password lives only in the Secret.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"bypassRls":{"description":"Whether the user can bypass row-level security policies (BYPASSRLS). A role can only create other roles carrying BYPASSRLS if it holds the attribute itself, so this is required alongside `createRole` to provision a privileged service role.","type":"boolean"},"createRole":{"description":"Whether the user can create and manage roles (CREATEROLE). Requires PostgreSQL 16 or newer. Setting it back to `false` withdraws the attribute only: the roles the user created, and its ADMIN and SET memberships in them, stay. Once any user has held it, the chart no longer adopts an existing role it did not create, so a user or database declared later under the name of such a role fails the init job until a platform administrator drops that role.","type":"boolean"},"replication":{"description":"Whether the user has replication privileges.","type":"boolean"}}}},"databases":{"description":"Databases configuration map. The PostgreSQL system databases `postgres`, `template0` and `template1` are refused.","type":"object","default":{},"additionalProperties":{"type":"object","properties":{"extensions":{"description":"List of enabled PostgreSQL extensions.","type":"array","items":{"type":"string"}},"roles":{"description":"Roles assigned to users.","type":"object","properties":{"admin":{"description":"List of users with admin privileges.","type":"array","items":{"type":"string"}},"readonly":{"description":"List of users with read-only privileges.","type":"array","items":{"type":"string"}}}}}}},"backup":{"description":"Backup configuration.","type":"object","default":{},"required":["enabled"],"properties":{"destinationPath":{"description":"DEPRECATED. Per-tenant S3 configuration is superseded by the platform-managed `cozy-default` BackupClass and the `cozy-backups` system bucket. Leave empty for new installations; the BackupClass driver picks up the system-managed coordinates. Kept for in-place upgrade compatibility.","type":"string","default":"s3://bucket/path/to/folder/"},"enabled":{"description":"Enable regular backups.","type":"boolean","default":false},"endpointCA":{"description":"DEPRECATED. Pre-existing Secret with the CA bundle the barman-cloud plugin should trust when reaching a self-signed S3 endpoint. Used for both backup and bootstrap recovery in the legacy chart-managed flow.","type":"object","default":{},"properties":{"key":{"description":"Key within the Secret containing the CA bundle. Defaults to `ca.crt`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means no endpointCA is emitted (the plugin uses the system trust store).","type":"string","default":""}}},"endpointURL":{"description":"DEPRECATED. See `destinationPath`.","type":"string","default":"http://minio-gateway-service:9000"},"retentionPolicy":{"description":"Retention policy (e.g. \"30d\").","type":"string","default":"30d"},"s3AccessKey":{"description":"DEPRECATED. Tenants no longer supply S3 keys; the system Bucket Secret is projected into the tenant namespace by the backup controller. Ignored when `s3CredentialsSecret.name` is set or `useSystemBucket` is true. The chart skips materialising `-s3-creds` whenever this field is empty so a default install does not leak placeholder credentials into the tenant namespace.","type":"string","default":""},"s3CredentialsSecret":{"description":"DEPRECATED. Pre-existing Secret with S3 credentials. Use the platform-managed `cozy-default` BackupClass instead. When set, the chart references this Secret directly (legacy chart-managed flow). The CNPG backup driver writes this field on restore so credentials never land in the CR `.spec`.","type":"object","default":{},"properties":{"accessKeyIDKey":{"description":"Key in the Secret holding the access key ID. Defaults to `AWS_ACCESS_KEY_ID`.","type":"string","default":""},"name":{"description":"Name of the Secret in the application namespace. Empty means the chart materialises `-s3-creds` from `s3AccessKey`/`s3SecretKey`.","type":"string","default":""},"secretAccessKeyKey":{"description":"Key in the Secret holding the secret access key. Defaults to `AWS_SECRET_ACCESS_KEY`.","type":"string","default":""}}},"s3SecretKey":{"description":"DEPRECATED. See `s3AccessKey`.","type":"string","default":""},"schedule":{"description":"Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.","type":"string","default":""},"useSystemBucket":{"description":"Opt-in: when true, the chart-emitted `-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups///`.","type":"boolean","default":false}}},"bootstrap":{"description":"Bootstrap configuration.","type":"object","default":{},"required":["enabled","oldName"],"properties":{"backupID":{"description":"Barman ID of the base backup to recover from, passed as `recoveryTarget.backupID`. Empty lets the barman-cloud plugin pick one: the newest backup when `recoveryTime` is empty, otherwise the newest one ending at or before it, on any timeline. The CNPG backup driver sets this on restore.","type":"string","default":""},"enabled":{"description":"Whether to restore from a backup.","type":"boolean","default":false},"newServerName":{"description":"WAL-archive server name (S3 path prefix) the RESTORED cluster writes to. Must differ from `serverName` (the recovery source) so a restored cluster archives to a fresh, empty prefix and barman-cloud-check-wal-archive passes while it recovers from the source's prefix. The CNPG backup driver sets this on restore; empty means the cluster archives under its own name.","type":"string","default":""},"oldName":{"description":"Previous cluster name before deletion.","type":"string","default":""},"recoveryTime":{"description":"Timestamp (RFC3339) for point-in-time recovery; empty means latest.","type":"string","default":""},"serverName":{"description":"Server name (S3 path prefix) used by the original cluster when writing backups; passed to the barman-cloud plugin via `externalClusters[].plugin.parameters.serverName`. Defaults to `bootstrap.oldName`. Set this only when the original cluster wrote backups under an explicit server name that differed from its Kubernetes resource name.","type":"string","default":""}}}}} release: prefix: postgres- labels: diff --git a/packages/system/redis-rd/cozyrds/redis.yaml b/packages/system/redis-rd/cozyrds/redis.yaml index cfcaeb01d..8c3045b0f 100644 --- a/packages/system/redis-rd/cozyrds/redis.yaml +++ b/packages/system/redis-rd/cozyrds/redis.yaml @@ -8,7 +8,7 @@ spec: plural: redises singular: redis openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}} + {"title":"Chart Values","type":"object","properties":{"replicas":{"description":"Number of Redis replicas. Three Sentinels watch them, or a single one when `replicas` is 1, since a lone pod has no replica to fail over to.","type":"integer","default":2},"resources":{"description":"Explicit CPU and memory configuration for each Redis replica. When omitted, the preset defined in `resourcesPreset` is applied.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Default sizing preset used when `resources` is omitted.","type":"string","default":"t1.nano","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]},"size":{"description":"Persistent Volume Claim size available for application data.","default":"1Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"external":{"description":"Enable external access from outside the cluster. Can be enabled only when the instance is created, together with `tls.enabled: true`; turning it on for an existing instance, or changing `tls.enabled` while it is on, is refused at admission.","type":"boolean","default":false},"version":{"description":"Redis major version to deploy. The Redis and Sentinel pods run the same image.","type":"string","default":"v8","enum":["v8","v7"]},"tls":{"description":"TLS configuration. TLS is opt-in and is not inferred from `external`.","type":"object","default":{},"properties":{"authClients":{"description":"Maps to the Redis `tls-auth-clients` directive. Defaults to `no` — the server certificate is presented but client certificates are not validated. `yes` requires a client certificate signed by this release's CA, which the platform does not issue. Changing it restarts the Redis and Sentinel pods, because the directive is read from the config file once at startup.","type":"string","enum":["no","optional","yes"]},"enabled":{"description":"Enable TLS for Redis and Sentinel connections. Disabled unless set. Enabling it moves Redis and Sentinel to a TLS-only listener, so existing plaintext clients must be migrated at the same time. Encryption is provided by the redis-operator fork that mounts the certificate Secret into both Redis and Sentinel pods at `/tls`.","type":"boolean"}}},"authEnabled":{"description":"Enable password generation.","type":"boolean","default":true},"minReplicasToWrite":{"description":"Redis `min-replicas-to-write`: the master refuses writes unless at least this many replicas are connected and within `minReplicasMaxLag`. Unset by default, which leaves the Redis default (disabled). With `replicas: 2` and a value of 1 the master rejects writes while its only replica is down, trading availability for a bound on the writes lost if an isolated master is failed over. Replication stays asynchronous, so this bounds the loss, it does not remove it. Must be 0 or more and below `replicas` (a master with `replicas` pods has `replicas - 1` replicas), otherwise every write is refused; the chart refuses such a value.","type":"integer","minimum":0},"minReplicasMaxLag":{"description":"Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.","type":"integer","minimum":0},"maxmemory":{"description":"Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"maxmemoryPolicy":{"description":"Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.","type":"string","default":"noeviction","enum":["noeviction","allkeys-lru","allkeys-lfu","allkeys-random","volatile-lru","volatile-lfu","volatile-random","volatile-ttl"]},"snapshots":{"description":"Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save \"\"` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty.","type":"boolean","default":true},"sentinel":{"description":"Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.","type":"object","default":{},"properties":{"resources":{"description":"Explicit CPU and memory configuration for each Sentinel pod. Keys left out are filled from the preset in `sentinel.resourcesPreset`.","type":"object","properties":{"cpu":{"description":"CPU available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available to each replica.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Sizing preset for the Sentinel pods. Defaults to `resourcesPreset`.","type":"string","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}} release: prefix: redis- labels: @@ -25,7 +25,7 @@ spec: tags: - cache icon: PHN2ZyB3aWR0aD0iMTQ0IiBoZWlnaHQ9IjE0NCIgdmlld0JveD0iMCAwIDE0NCAxNDQiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIxNDQiIGhlaWdodD0iMTQ0IiByeD0iMjQiIGZpbGw9InVybCgjcGFpbnQwX2xpbmVhcl82ODNfMzIxMykiLz4KPHBhdGggZD0iTTEyMC4xNDkgOTUuNTQ5MUMxMTQuNTg2IDk4LjQ0ODUgODUuNzcwOSAxMTAuMjk2IDc5LjYzNjMgMTEzLjQ5NUM3My41MDE2IDExNi42OTMgNzAuMDkzNyAxMTYuNjYyIDY1LjI0NzIgMTE0LjM0NkM2MC40MDEyIDExMi4wMjkgMjkuNzM2NCA5OS42NDIzIDI0LjIxMjUgOTcuMDAxOUMyMS40NTE5IDk1LjY4MjcgMjAgOTQuNTY4NyAyMCA5My41MTY2VjgyLjk4MDlDMjAgODIuOTgwOSA1OS45MjIgNzQuMjkwMSA2Ni4zNjY5IDcxLjk3NzhDNzIuODExNSA2OS42NjU2IDc1LjA0NzYgNjkuNTgyMSA4MC41MzIgNzEuNTkxQzg2LjAxNzMgNzMuNjAwOCAxMTguODEyIDc5LjUxNzYgMTI0LjIzMyA4MS41MDI5TDEyNC4yMyA5MS44ODk2QzEyNC4yMzEgOTIuOTMxMSAxMjIuOTggOTQuMDczNiAxMjAuMTQ5IDk1LjU0OTFaIiBmaWxsPSIjOTEyNjI2Ii8+CjxwYXRoIGQ9Ik0xMjAuMTQ3IDg1LjA3NTJDMTE0LjU4NSA4Ny45NzM0IDg1Ljc3IDk5LjgyMTggNzkuNjM1NCAxMDMuMDJDNzMuNTAxMSAxMDYuMjE5IDcwLjA5MzIgMTA2LjE4NyA2NS4yNDcyIDEwMy44NzFDNjAuNDAwNyAxMDEuNTU1IDI5LjczNzEgODkuMTY2OCAyNC4yMTM2IDg2LjUyOEMxOC42OTAxIDgzLjg4NzYgMTguNTc0NCA4Mi4wNzA0IDI0LjAwMDMgNzkuOTQ1OEMyOS40MjYxIDc3LjgyMDUgNTkuOTIxNSA2NS44NTYxIDY2LjM2NzIgNjMuNTQzOEM3Mi44MTE4IDYxLjIzMjQgNzUuMDQ3NSA2MS4xNDgxIDgwLjUzMTkgNjMuMTU3OEM4Ni4wMTY4IDY1LjE2NjggMTE0LjY2IDc2LjU2NzYgMTIwLjA3OSA3OC41NTI1QzEyNS41MDEgODAuNTM5OSAxMjUuNzA5IDgyLjE3NjMgMTIwLjE0NyA4NS4wNzUyWiIgZmlsbD0iI0M2MzAyQiIvPgo8cGF0aCBkPSJNMTIwLjE0OSA3OC41MDJDMTE0LjU4NiA4MS40MDE4IDg1Ljc3MDkgOTMuMjQ5MyA3OS42MzYzIDk2LjQ0ODhDNzMuNTAxNiA5OS42NDYyIDcwLjA5MzcgOTkuNjE1MiA2NS4yNDcyIDk3LjI5ODVDNjAuNDAwOCA5NC45ODMgMjkuNzM2NCA4Mi41OTUyIDI0LjIxMjUgNzkuOTU0N0MyMS40NTE5IDc4LjYzNTUgMjAgNzcuNTIzMiAyMCA3Ni40NzA3VjY1LjkzMzhDMjAgNjUuOTMzOCA1OS45MjIgNTcuMjQzNCA2Ni4zNjY5IDU0LjkzMTFDNzIuODExNSA1Mi42MTg5IDc1LjA0NzYgNTIuNTM1IDgwLjUzMiA1NC41NDQzQzg2LjAxNzcgNTYuNTUzNiAxMTguODEzIDYyLjQ2OTMgMTI0LjIzMyA2NC40NTVMMTI0LjIzIDc0Ljg0MjhDMTI0LjIzMSA3NS44ODQgMTIyLjk4IDc3LjAyNjQgMTIwLjE0OSA3OC41MDJaIiBmaWxsPSIjOTEyNjI2Ii8+CjxwYXRoIGQ9Ik0xMjAuMTQ3IDY4LjAyODJDMTE0LjU4NSA3MC45MjcxIDg1Ljc3IDgyLjc3NDcgNzkuNjM1NCA4NS45NzM3QzczLjUwMTEgODkuMTcxNiA3MC4wOTMyIDg5LjE0MDIgNjUuMjQ3MiA4Ni44MjM1QzYwLjQwMDcgODQuNTA4NCAyOS43MzcxIDcyLjEyMDEgMjQuMjEzNiA2OS40ODA5QzE4LjY5MDEgNjYuODQxMyAxOC41NzQ0IDY1LjAyMzcgMjQuMDAwMyA2Mi44OTg0QzI5LjQyNjEgNjAuNzc0MiA1OS45MjE5IDQ4LjgwOSA2Ni4zNjcyIDQ2LjQ5NzJDNzIuODExOCA0NC4xODUzIDc1LjA0NzUgNDQuMTAxNCA4MC41MzE5IDQ2LjExMDhDODYuMDE2OCA0OC4xMTk3IDExNC42NiA1OS41MTk3IDEyMC4wNzkgNjEuNTA1NUMxMjUuNTAxIDYzLjQ5MjQgMTI1LjcwOSA2NS4xMjkyIDEyMC4xNDcgNjguMDI4MloiIGZpbGw9IiNDNjMwMkIiLz4KPHBhdGggZD0iTTEyMC4xNDkgNjAuODIyNEMxMTQuNTg2IDYzLjcyMTQgODUuNzcwOSA3NS41Njk4IDc5LjYzNjMgNzguNzY5MkM3My41MDE2IDgxLjk2NzEgNzAuMDkzNyA4MS45MzU3IDY1LjI0NzIgNzkuNjE5QzYwLjQwMDggNzcuMzAzNSAyOS43MzY0IDY0LjkxNTIgMjQuMjEyNSA2Mi4yNzZDMjEuNDUxOSA2MC45NTU2IDIwIDU5Ljg0MjggMjAgNTguNzkxNVY0OC4yNTQyQzIwIDQ4LjI1NDIgNTkuOTIyIDM5LjU2NDIgNjYuMzY2OSAzNy4yNTI0QzcyLjgxMTUgMzQuOTM5NyA3NS4wNDc2IDM0Ljg1NjcgODAuNTMyIDM2Ljg2NTZDODYuMDE3NyAzOC44NzQ5IDExOC44MTMgNDQuNzkwNSAxMjQuMjMzIDQ2Ljc3NjNMMTI0LjIzIDU3LjE2MzdDMTI0LjIzMSA1OC4yMDQgMTIyLjk4IDU5LjM0NjUgMTIwLjE0OSA2MC44MjI0WiIgZmlsbD0iIzkxMjYyNiIvPgo8cGF0aCBkPSJNMTIwLjE0NyA1MC4zNDlDMTE0LjU4NSA1My4yNDc5IDg1Ljc2OTggNjUuMDk2MyA3OS42MzUyIDY4LjI5NDFDNzMuNTAwOSA3MS40OTIgNzAuMDkzIDcxLjQ2MDYgNjUuMjQ2OSA2OS4xNDUxQzYwLjQwMDkgNjYuODI4MyAyOS43MzY5IDU0LjQ0MDkgMjQuMjEzOCA1MS44MDEzQzE4LjY4OTkgNDkuMTYyMSAxOC41NzQ2IDQ3LjM0NDEgMjQgNDUuMjE5MkMyOS40MjU5IDQzLjA5NDYgNTkuOTIxNyAzMS4xMzEgNjYuMzY3IDI4LjgxODRDNzIuODExNiAyNi41MDYxIDc1LjA0NzMgMjYuNDIzIDgwLjUzMTcgMjguNDMyNEM4Ni4wMTY2IDMwLjQ0MTcgMTE0LjY1OSA0MS44NDE4IDEyMC4wNzkgNDMuODI3NUMxMjUuNTAxIDQ1LjgxMjggMTI1LjcwOSA0Ny40NSAxMjAuMTQ3IDUwLjM0OVoiIGZpbGw9IiNDNjMwMkIiLz4KPHBhdGggZD0iTTg0Ljg1NDEgNDAuMDk5NEw3NS44OTI2IDQxLjAyOThMNzMuODg2NSA0NS44NTdMNzAuNjQ2MyA0MC40NzAzTDYwLjI5ODMgMzkuNTQwNEw2OC4wMTk3IDM2Ljc1NThMNjUuNzAzIDMyLjQ4MTRMNzIuOTMyMSAzNS4zMDg4TDc5Ljc0NzEgMzMuMDc3NUw3Ny45MDUyIDM3LjQ5NzJMODQuODU0MSA0MC4wOTk0Wk03My4zNTE1IDYzLjUxODRMNTYuNjI2NiA1Ni41ODE2TDgwLjU5MiA1Mi45MDI5TDczLjM1MTUgNjMuNTE4NFpNNTAuMTYzNyA0Mi43ODI2QzU3LjIzODEgNDIuNzgyNiA2Mi45NzMgNDUuMDA1NyA2Mi45NzMgNDcuNzQ3NUM2Mi45NzMgNTAuNDkwMSA1Ny4yMzgxIDUyLjcxMjggNTAuMTYzNyA1Mi43MTI4QzQzLjA4OTMgNTIuNzEyOCAzNy4zNTQ1IDUwLjQ4OTcgMzcuMzU0NSA0Ny43NDc1QzM3LjM1NDUgNDUuMDA1NyA0My4wODkzIDQyLjc4MjYgNTAuMTYzNyA0Mi43ODI2WiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTk1LjQ0MzQgNDEuNDE3NEwxMDkuNjI3IDQ3LjAyMjRMOTUuNDU1NiA1Mi42MjJMOTUuNDQzNCA0MS40MTc0WiIgZmlsbD0iIzYyMUIxQyIvPgo8cGF0aCBkPSJNNzkuNzUyOSA0Ny42MjYxTDk1LjQ0NDkgNDEuNDE4OUw5NS40NTcxIDUyLjYyMzZMOTMuOTE4NCA1My4yMjU0TDc5Ljc1MjkgNDcuNjI2MVoiIGZpbGw9IiM5QTI5MjgiLz4KPGRlZnM+CjxsaW5lYXJHcmFkaWVudCBpZD0icGFpbnQwX2xpbmVhcl82ODNfMzIxMyIgeDE9IjE4OSIgeTE9IjIxMC41IiB4Mj0iMCIgeTI9IjAiIGdyYWRpZW50VW5pdHM9InVzZXJTcGFjZU9uVXNlIj4KPHN0b3Agc3RvcC1jb2xvcj0iI0E4MDAwMCIvPgo8c3RvcCBvZmZzZXQ9IjEiIHN0b3AtY29sb3I9IiNGRkNGQ0YiLz4KPC9saW5lYXJHcmFkaWVudD4KPC9kZWZzPgo8L3N2Zz4K - keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "tls"], ["spec", "authEnabled"], ["spec", "sentinel"]] + keysOrder: [["apiVersion"], ["appVersion"], ["kind"], ["metadata"], ["metadata", "name"], ["spec", "replicas"], ["spec", "resources"], ["spec", "resourcesPreset"], ["spec", "size"], ["spec", "storageClass"], ["spec", "external"], ["spec", "version"], ["spec", "tls"], ["spec", "authEnabled"], ["spec", "maxmemoryPolicy"], ["spec", "snapshots"], ["spec", "sentinel"]] secrets: # Backstop for the label-only include below, which matches on the label # alone with no name constraint. These two Secrets are the ones whose diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml index 1ff0e7768..11d7fb779 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml @@ -39,4 +39,13 @@ driverName: {{ .Values.cosi.driverName }} authenticationType: KEY parameters: accessPolicy: readonly +--- +kind: BucketAccessClass +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: {{ .Values.cosi.bucketClassName }}-public +driverName: {{ .Values.cosi.driverName }} +authenticationType: KEY +parameters: + accessPolicy: public-read {{- end }} diff --git a/packages/system/seaweedfs/images/seaweedfs/Dockerfile b/packages/system/seaweedfs/images/seaweedfs/Dockerfile index 8f42f3ea2..d52ee7329 100644 --- a/packages/system/seaweedfs/images/seaweedfs/Dockerfile +++ b/packages/system/seaweedfs/images/seaweedfs/Dockerfile @@ -16,6 +16,15 @@ # seaweedfs/seaweedfs#11674 after 4.48; drop the patch on the first vendored # release that carries it. # +# patches/volume-layout-crowded-lock.diff makes SetVolumeCrowded take the write +# lock of the volume layout. Upstream 4.45 mutates the crowded map under the +# read lock, while Assign reads it under the same read lock through +# GetWritableVolumeCount, so under a steady stream of writes the master dies on +# a fatal "concurrent map read and map write", which recover cannot catch, and +# S3 answers 500 until it is back. Backport of seaweedfs/seaweedfs#11216, +# released in 4.46; drop the patch on the first vendored release that carries +# it. +# # `make update` re-pins VERSION from the vendored chart, SOURCE_SHA256 from the # source archive of that tag, UPSTREAM_DIGEST from the upstream image of that # tag and the builder from upstream's docker/Dockerfile.go_build, and checks diff --git a/packages/system/seaweedfs/images/seaweedfs/patches/volume-layout-crowded-lock.diff b/packages/system/seaweedfs/images/seaweedfs/patches/volume-layout-crowded-lock.diff new file mode 100644 index 000000000..12e09ca67 --- /dev/null +++ b/packages/system/seaweedfs/images/seaweedfs/patches/volume-layout-crowded-lock.diff @@ -0,0 +1,87 @@ +diff --git a/weed/topology/crowded_writable_test.go b/weed/topology/crowded_writable_test.go +--- a/weed/topology/crowded_writable_test.go ++++ b/weed/topology/crowded_writable_test.go +@@ -1,6 +1,7 @@ + package topology + + import ( ++ "sync" + "testing" + "time" + +@@ -73,3 +74,52 @@ func TestCrowdedCountStillMatchesWritables(t *testing.T) { + t.Errorf("writable=%d crowded=%d, want 1 and 1", writable, crowded) + } + } ++ ++// SetVolumeCrowded mutates the crowded map while GetWritableVolumeCount reads ++// it on the Assign hot path. Under RLock both could run at once and the Go ++// runtime aborts with "concurrent map read and map write". Run with -race to ++// reproduce the original bug; the write lock makes this pass. ++func TestSetVolumeCrowdedNoRaceWithGetWritableVolumeCount(t *testing.T) { ++ rp, _ := super_block.NewReplicaPlacementFromString("000") ++ vl := NewVolumeLayout(rp, needle.EMPTY_TTL, types.HardDriveType, 10000, false) ++ // GetWritableVolumeCount iterates vl.writables, so give it something to ++ // read while the crowded map is being mutated. ++ vl.writables = []needle.VolumeId{1, 2, 3} ++ ++ var wg sync.WaitGroup ++ stop := make(chan struct{}) ++ for range 8 { ++ wg.Add(1) ++ go func() { ++ defer wg.Done() ++ for { ++ select { ++ case <-stop: ++ return ++ default: ++ vl.SetVolumeCrowded(needle.VolumeId(2)) ++ } ++ } ++ }() ++ } ++ for range 8 { ++ wg.Add(1) ++ go func() { ++ defer wg.Done() ++ for { ++ select { ++ case <-stop: ++ return ++ default: ++ vl.GetWritableVolumeCount() ++ } ++ } ++ }() ++ } ++ ++ // The race detector needs only a brief window to flag a concurrent ++ // map read/write; let the goroutines hammer the lock briefly. ++ time.Sleep(50 * time.Millisecond) ++ close(stop) ++ wg.Wait() ++} +diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go +--- a/weed/topology/volume_layout.go ++++ b/weed/topology/volume_layout.go +@@ -1019,11 +1019,14 @@ func (vl *VolumeLayout) setVolumeCrowded(vid needle.VolumeId) { + } + + func (vl *VolumeLayout) SetVolumeCrowded(vid needle.VolumeId) { +- // since delete is guarded by accessLock.Lock(), +- // and is always called in sequential order, +- // RLock() should be safe enough +- vl.accessLock.RLock() +- defer vl.accessLock.RUnlock() ++ // setVolumeCrowded mutates the crowded map, and GetWritableVolumeCount ++ // reads it under RLock on the Assign hot path. Two concurrent RLock ++ // holders with one writing the map triggers a fatal ++ // "concurrent map read and map write". Take the write lock: this path ++ // is a low-frequency single consumer driven by the crowded-volume ++ // event loop, and every other mutation of crowded already holds Lock(). ++ vl.accessLock.Lock() ++ defer vl.accessLock.Unlock() + + vl.setVolumeCrowded(vid) + } diff --git a/packages/system/seaweedfs/patches/cosi-bucket-class-lock-readonly.patch b/packages/system/seaweedfs/patches/cosi-bucket-class-lock-readonly.patch index ddca7873e..84ca4fcf2 100644 --- a/packages/system/seaweedfs/patches/cosi-bucket-class-lock-readonly.patch +++ b/packages/system/seaweedfs/patches/cosi-bucket-class-lock-readonly.patch @@ -1,8 +1,8 @@ diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml -index 4d19cc128..1ff0e7768 100644 +index 4d19cc128..11d7fb779 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml -@@ -11,10 +11,32 @@ parameters: +@@ -11,10 +11,41 @@ {{- toYaml . | nindent 2 }} {{- end }} --- @@ -34,4 +34,13 @@ index 4d19cc128..1ff0e7768 100644 +authenticationType: KEY +parameters: + accessPolicy: readonly ++--- ++kind: BucketAccessClass ++apiVersion: objectstorage.k8s.io/v1alpha1 ++metadata: ++ name: {{ .Values.cosi.bucketClassName }}-public ++driverName: {{ .Values.cosi.driverName }} ++authenticationType: KEY ++parameters: ++ accessPolicy: public-read {{- end }} diff --git a/packages/system/seaweedfs/tests/cosi_image_test.yaml b/packages/system/seaweedfs/tests/cosi_image_test.yaml index d43aa176e..4057c989e 100644 --- a/packages/system/seaweedfs/tests/cosi_image_test.yaml +++ b/packages/system/seaweedfs/tests/cosi_image_test.yaml @@ -7,8 +7,8 @@ suite: seaweedfs COSI driver image pin # BucketAccess and ignores the accessPolicy parameter. The chart ships a # -readonly BucketAccessClass with parameters.accessPolicy: readonly, so on # v0.1.2 a credential meant to be read-only is silently granted read-write. -# The package values.yaml overrides cosi.image to v0.4.0, which honours -# accessPolicy: readonly. +# The package values.yaml overrides cosi.image to v0.5.0, which honours +# accessPolicy: readonly and public-read. # # This test does not set the image itself; it relies on the package values.yaml # override, so it goes red if that override is removed and the image falls back @@ -25,7 +25,7 @@ release: namespace: cozy-seaweedfs tests: - - it: pins the cosi-driver container to v0.4.0 (read-only access enforcement) + - it: pins the cosi-driver container to v0.5.0 (read-only and public-read access policies) template: charts/seaweedfs/templates/cosi/cosi-deployment.yaml set: seaweedfs.cosi.enabled: true @@ -35,4 +35,4 @@ tests: value: seaweedfs-cosi-driver - equal: path: spec.template.spec.containers[0].image - value: ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.4.0 + value: ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.5.0 diff --git a/packages/system/seaweedfs/tests/cosi_public_class_test.yaml b/packages/system/seaweedfs/tests/cosi_public_class_test.yaml new file mode 100644 index 000000000..72c815e0d --- /dev/null +++ b/packages/system/seaweedfs/tests/cosi_public_class_test.yaml @@ -0,0 +1,25 @@ +suite: vendored COSI classes include public-read +templates: + - charts/seaweedfs/templates/cosi/cosi-bucket-class.yaml + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + - it: renders a -public BucketAccessClass with the public-read policy + set: + seaweedfs: + cosi: + enabled: true + bucketClassName: tenant-root + driverName: tenant-root.seaweedfs.objectstorage.k8s.io + documentSelector: + path: metadata.name + value: tenant-root-public + asserts: + - isKind: + of: BucketAccessClass + - equal: + path: parameters.accessPolicy + value: public-read diff --git a/packages/system/seaweedfs/tests/lifecycle_test.yaml b/packages/system/seaweedfs/tests/lifecycle_test.yaml index 7f3cec8ff..0fd02b8a8 100644 --- a/packages/system/seaweedfs/tests/lifecycle_test.yaml +++ b/packages/system/seaweedfs/tests/lifecycle_test.yaml @@ -57,6 +57,36 @@ tests: path: spec.template.spec.containers[0].command[2] pattern: '-admin=seaweedfs-admin\.tenant-root:23646\.33646 ' + # Both are the weed binary under a fixed memory limit the parent chart does + # not set, so their GOMEMLIMIT is pinned here rather than derived: without + # one the heap has no ceiling and a large lifecycle pass grows it past the + # limit. + - it: caps the admin heap below its memory limit + template: charts/seaweedfs/templates/admin/admin-statefulset.yaml + set: *enabled + asserts: + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 256Mi + - contains: + path: spec.template.spec.containers[0].env + content: + name: GOMEMLIMIT + value: 192MiB + + - it: caps the worker heap below its memory limit + template: charts/seaweedfs/templates/worker/worker-deployment.yaml + set: *enabled + asserts: + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 512Mi + - contains: + path: spec.template.spec.containers[0].env + content: + name: GOMEMLIMIT + value: 384MiB + # The worker executes every job type it registers for. Lifecycle is the only # one this release asks for: vacuum, balance and erasure coding move data and # are not something enabling lifecycle should start doing as a side effect. diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 64e2562ba..c6e4c26b2 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -313,8 +313,9 @@ seaweedfs: # actually issues read-only (Read,List) credentials instead of read-write. # v0.4.0 also writes each grant to the per-identity IAM file: v0.3.1 # rewrites only the legacy identity.json, which the S3 gateways migrate - # concurrently, so a grant could be lost for good. - image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.4.0" + # concurrently, so a grant could be lost for good. v0.5.0 adds the + # public-read accessPolicy the -public BucketAccessClass relies on. + image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.5.0" podLabels: policy.cozystack.io/allow-to-apiserver: "true" driverName: "seaweedfs.objectstorage.k8s.io" @@ -349,6 +350,11 @@ seaweedfs: limits: cpu: "500m" memory: "256Mi" + # 75% of the limit above, as cozy-lib.resources.goMemLimit derives it for + # the components whose resources the parent chart sets. Without it the + # heap has no absolute ceiling and grows past the limit. + extraEnvironmentVars: + GOMEMLIMIT: "192MiB" worker: enabled: false # Vacuum, balance and erasure coding move data; enabling lifecycle must @@ -378,6 +384,9 @@ seaweedfs: limits: cpu: "1" memory: "512Mi" + # 75% of the limit above; see admin.extraEnvironmentVars. + extraEnvironmentVars: + GOMEMLIMIT: "384MiB" certificates: commonName: "SeaweedFS CA" ipAddresses: [] diff --git a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml index 6b8c82675..0ff338b48 100644 --- a/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml +++ b/packages/system/vertical-pod-autoscaler/tests/vpa_test.yaml @@ -242,3 +242,38 @@ tests: count: 0 + + - it: runs two admission controllers spread over nodes + template: charts/vertical-pod-autoscaler/templates/admission-controller/deployment.yaml + asserts: + - equal: + path: spec.replicas + value: 2 + - equal: + path: spec.template.spec.topologySpreadConstraints[0].topologyKey + value: kubernetes.io/hostname + - equal: + path: spec.template.spec.topologySpreadConstraints[0].maxSkew + value: 1 + - equal: + path: spec.template.spec.topologySpreadConstraints[0].whenUnsatisfiable + value: ScheduleAnyway + - equal: + path: spec.template.spec.topologySpreadConstraints[0].labelSelector.matchLabels + value: + app.kubernetes.io/name: vertical-pod-autoscaler + app.kubernetes.io/instance: vpa + app.kubernetes.io/component: admission-controller + + - it: keeps one admission controller up through a voluntary disruption + template: charts/vertical-pod-autoscaler/templates/admission-controller/pdb.yaml + asserts: + - equal: + path: spec.minAvailable + value: 1 + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: vertical-pod-autoscaler + app.kubernetes.io/instance: vpa + app.kubernetes.io/component: admission-controller diff --git a/packages/system/vertical-pod-autoscaler/values.yaml b/packages/system/vertical-pod-autoscaler/values.yaml index 46678903e..456b99912 100644 --- a/packages/system/vertical-pod-autoscaler/values.yaml +++ b/packages/system/vertical-pod-autoscaler/values.yaml @@ -37,6 +37,18 @@ vertical-pod-autoscaler: prometheus-cadvisor-job-name: cadvisor admissionController: + replicaCount: 2 + pdb: + create: true + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: vertical-pod-autoscaler + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/component: admission-controller resources: limits: memory: 256Mi diff --git a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml index e91a334a1..dc533ee59 100644 --- a/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml +++ b/packages/system/vm-instance-rd/cozyrds/vm-instance.yaml @@ -12,7 +12,7 @@ spec: singular: vminstance plural: vminstances openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"externalAllowICMP":{"description":"Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.","type":"boolean","default":true},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type. Ignored when `resources` sizes the VM, which needs whole-number cpu and sockets plus memory; the whole type is dropped then, so any dedicated CPU placement, hugepages, NUMA or realtime settings it carries go with it. While this field is set, a `resources` block that sizes only part of the VM is rejected outright instead of ignored, so complete it or clear it.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","x-cozystack-options":{"source":"instanceprofile"}},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string","x-cozystack-options":{"source":"vmdisk"}}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string","x-cozystack-options":{"source":"network"}}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string","x-cozystack-options":{"source":"network"}}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"firmware":{"description":"Firmware and boot configuration (UEFI/BIOS selection, Secure Boot, persistent EFI NVRAM).","type":"object","default":{},"properties":{"bootloader":{"description":"Bootloader to boot the VM with: \"uefi\" (OVMF) or \"bios\" (SeaBIOS). Empty inherits the instanceProfile default.","type":"string"},"efiPersistent":{"description":"Persist EFI NVRAM (e.g. enrolled Secure Boot keys, such as an updated Microsoft UEFI CA) across reboots. Only applies when bootloader is \"uefi\".","type":"boolean"},"secureBoot":{"description":"Enable UEFI Secure Boot. Only applies when bootloader is \"uefi\".","type":"boolean"}}},"resources":{"description":"Resource configuration for the virtual machine. Set whole-number cpu and sockets together with memory to size the VM directly, which overrides `instanceType`. Alongside an `instanceType`, a block that sizes only part of the VM is rejected; a whole-number cpu or sockets on its own sizes nothing and is ignored.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated, as a positive whole number such as `2`. Millicore or other unit-suffixed, fractional, zero, negative, above 4294967295 and quoted leading-zero values are rejected at render time, because the value becomes the VM's unsigned integer `domain.cpu.cores`.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated, as a quantity of at least `1M` such as `8Gi`. Less than 1M is rejected at render time, because KubeVirt refuses a non-zero amount below 1M and a zero or negative amount sizes nothing.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology), as a positive whole number such as `1`. Millicore or other unit-suffixed, fractional, zero, negative, above 4294967295 and quoted leading-zero values are rejected at render time, because the value becomes the VM's unsigned integer `domain.cpu.sockets`.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"netplanDhcp":{"description":"Supply cloud-init network-data that requests DHCP on every `en*` and `eth*` interface and keeps the DNS search domains DHCP offers. Needed for a netplan guest (Ubuntu, Debian) to address a secondary network such as a VPC subnet. Leave it off for a guest rendered by anything other than netplan, which would lose DHCP on every interface, and for a guest that configures an interface statically through netplan, which this configuration would shadow. Turning it on for a VM that has neither `cloudInit` nor `sshKeys` adds a cloud-init disk, and the guest's next boot is then treated as a first boot.","type":"boolean","default":false},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} + {"title":"Chart Values","type":"object","properties":{"external":{"description":"Enable external access from outside the cluster.","type":"boolean","default":false},"externalMethod":{"description":"Method to pass through traffic to the VM.","type":"string","default":"PortList","enum":["PortList","WholeIP"]},"externalPorts":{"description":"Ports to forward from outside the cluster.","type":"array","default":[22],"items":{"type":"integer"}},"externalAllowICMP":{"description":"Whether to accept ICMP traffic to the VM in PortList mode (preserves ping and PMTU discovery). No effect in WholeIP mode. Default true so ping behaves as users expect even when port filtering is in effect.","type":"boolean","default":true},"runStrategy":{"description":"Requested running state of the VirtualMachineInstance","type":"string","default":"Always","enum":["Always","Halted","Manual","RerunOnFailure","Once"]},"instanceType":{"description":"Virtual Machine instance type. Ignored when `resources` sizes the VM, which needs whole-number cpu and sockets plus memory; the whole type is dropped then, so any dedicated CPU placement, hugepages, NUMA or realtime settings it carries go with it. While this field is set, a `resources` block that sizes only part of the VM is rejected outright instead of ignored, so complete it or clear it.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"instanceProfile":{"description":"Virtual Machine preferences profile.","type":"string","default":"ubuntu","x-cozystack-options":{"source":"instanceprofile"}},"disks":{"description":"List of disks to attach.","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"bootOrder":{"description":"Boot priority of the disk, 1 being tried first. Unset falls back to the disk's position in `disks` (the first disk gets 1), so set it to boot from a disk other than the first, such as an attached installer or rescue ISO, without reordering the list. Effective boot orders must be unique across disks: when one disk is given a value that another takes from its position, set both.","type":"integer","minimum":1},"bus":{"description":"Disk bus type (e.g. \"sata\").","type":"string"},"name":{"description":"Disk name.","type":"string","x-cozystack-options":{"source":"vmdisk"}}}}},"networks":{"description":"Networks to attach the VM to.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string","x-cozystack-options":{"source":"network"}}}}},"subnets":{"description":"Deprecated: use networks instead.","type":"array","default":[],"items":{"type":"object","properties":{"name":{"description":"Network attachment name.","type":"string","x-cozystack-options":{"source":"network"}}}}},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","default":[],"items":{"type":"object","required":["name"],"properties":{"name":{"description":"The name of the GPU resource to attach.","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"cpuModel":{"description":"Model specifies the CPU model inside the VMI. List of available models https://github.com/libvirt/libvirt/tree/master/src/cpu_map","type":"string","default":""},"firmware":{"description":"Firmware and boot configuration (UEFI/BIOS selection, Secure Boot, persistent EFI NVRAM).","type":"object","default":{},"properties":{"bootloader":{"description":"Bootloader to boot the VM with: \"uefi\" (OVMF) or \"bios\" (SeaBIOS). Empty inherits the instanceProfile default.","type":"string"},"efiPersistent":{"description":"Persist EFI NVRAM (e.g. enrolled Secure Boot keys, such as an updated Microsoft UEFI CA) across reboots. Only applies when bootloader is \"uefi\".","type":"boolean"},"secureBoot":{"description":"Enable UEFI Secure Boot. Only applies when bootloader is \"uefi\".","type":"boolean"}}},"resources":{"description":"Resource configuration for the virtual machine. Set whole-number cpu and sockets together with memory to size the VM directly, which overrides `instanceType`. Alongside an `instanceType`, a block that sizes only part of the VM is rejected; a whole-number cpu or sockets on its own sizes nothing and is ignored.","type":"object","default":{},"properties":{"cpu":{"description":"Number of CPU cores allocated, as a positive whole number such as `2`. Millicore or other unit-suffixed, fractional, zero, negative, above 4294967295 and quoted leading-zero values are rejected at render time, because the value becomes the VM's unsigned integer `domain.cpu.cores`.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Amount of memory allocated, as a quantity of at least `1M` such as `8Gi`. Less than 1M is rejected at render time, because KubeVirt refuses a non-zero amount below 1M and a zero or negative amount sizes nothing.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"sockets":{"description":"Number of CPU sockets (vCPU topology), as a positive whole number such as `1`. Millicore or other unit-suffixed, fractional, zero, negative, above 4294967295 and quoted leading-zero values are rejected at render time, because the value becomes the VM's unsigned integer `domain.cpu.sockets`.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"sshKeys":{"description":"List of SSH public keys for authentication.","type":"array","default":[],"items":{"type":"string"}},"cloudInit":{"description":"Cloud-init user data.","type":"string","default":""},"netplanDhcp":{"description":"Supply cloud-init network-data that requests DHCP on every `en*` and `eth*` interface and keeps the DNS search domains DHCP offers. Needed for a netplan guest (Ubuntu, Debian) to address a secondary network such as a VPC subnet. Leave it off for a guest rendered by anything other than netplan, which would lose DHCP on every interface, and for a guest that configures an interface statically through netplan, which this configuration would shadow. Turning it on for a VM that has neither `cloudInit` nor `sshKeys` adds a cloud-init disk, and the guest's next boot is then treated as a first boot.","type":"boolean","default":false},"cloudInitSeed":{"description":"Seed string to generate SMBIOS UUID for the VM.","type":"string","default":""}}} release: prefix: vm-instance- labels: diff --git a/packages/tests/cozy-lib-tests/templates/tests/resources-defaulting.yaml b/packages/tests/cozy-lib-tests/templates/tests/resources-defaulting.yaml new file mode 100644 index 000000000..3e6d9a9eb --- /dev/null +++ b/packages/tests/cozy-lib-tests/templates/tests/resources-defaulting.yaml @@ -0,0 +1,11 @@ +{{- with .Values.defaulting }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ $.Release.Name }}-defaulting +spec: + containers: + - name: main + image: busybox + resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .preset .resources $) | nindent 6 }} +{{- end }} diff --git a/packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml b/packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml new file mode 100644 index 000000000..f9d3739b0 --- /dev/null +++ b/packages/tests/cozy-lib-tests/templates/tests/resources-gomemlimit.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-gomemlimit +data: +{{- range $name, $resources := .Values.sanitizedResources }} + {{ $name }}: {{ include "cozy-lib.resources.goMemLimit" $resources | quote }} +{{- end }} diff --git a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml new file mode 100644 index 000000000..1753f925f --- /dev/null +++ b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_test.yaml @@ -0,0 +1,62 @@ +# ./tests/resources_gomemlimit_test.yaml +suite: resources goMemLimit helper + +# The helper turns a memory limit into a GOMEMLIMIT the Go runtime accepts, +# at 75% of the limit. Two properties matter and neither is visible from a +# single value: the result carries the MiB suffix Go parses, and it stays +# strictly under the limit it protects, whatever unit the quantity came in. +# A value that came out at or above the limit would leave the runtime with +# no headroom and the container would still be killed. + +templates: + - templates/tests/resources-gomemlimit.yaml + +tests: + - it: takes 75% of the limit and keeps the unit Go parses + values: + - resources_gomemlimit_values.yaml + + release: + name: myrelease + namespace: default + + asserts: + - equal: + path: data.binaryGi + value: "6144MiB" + + - equal: + path: data.binaryMi + value: "384MiB" + + # A decimal quantity is smaller than its binary lookalike: 1G is + # 1000000000 bytes, so 75% of it lands below 768MiB. + - equal: + path: data.decimalG + value: "715MiB" + + - equal: + path: data.bareBytes + value: "768MiB" + + # The smallest limit that still yields a usable value: 75% of + # 1398102 bytes (4/3 Mi, rounded up) floors to 1MiB. + - equal: + path: data.smallestUsable + value: "1MiB" + + # Integer division floors, so a limit one byte under 4/3 Mi comes out + # at 0MiB. Go accepts that and then collects almost without pause, + # slowing the component instead of protecting it, so the render stops + # instead. + - it: refuses a limit too small to leave a Go heap + values: + - resources_gomemlimit_values.yaml + set: + sanitizedResources: + belowOneMiB: + limits: + memory: "1398101" + asserts: + - failedTemplate: + errorPattern: "leaves no room for a Go heap" diff --git a/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml new file mode 100644 index 000000000..f6be917e0 --- /dev/null +++ b/packages/tests/cozy-lib-tests/tests/resources_gomemlimit_values.yaml @@ -0,0 +1,16 @@ +sanitizedResources: + binaryGi: + limits: + memory: "8Gi" + binaryMi: + limits: + memory: "512Mi" + decimalG: + limits: + memory: "1G" + bareBytes: + limits: + memory: "1073741824" + smallestUsable: + limits: + memory: "1398102" diff --git a/packages/tests/cozy-lib-tests/tests/resources_memory_floor_test.yaml b/packages/tests/cozy-lib-tests/tests/resources_memory_floor_test.yaml new file mode 100644 index 000000000..485829054 --- /dev/null +++ b/packages/tests/cozy-lib-tests/tests/resources_memory_floor_test.yaml @@ -0,0 +1,203 @@ +# ./tests/resources_memory_floor_test.yaml +suite: resources memory floor + +# A quantity without a unit counts bytes, so a memory of 8 typed for 8Gi would +# size the container at 8 bytes. sanitize refuses anything below 1Mi, which +# every preset clears by far. flatten renders tenant quotas without the floor, +# because a quota of 0 memory is a valid way to allow none. + +templates: + - templates/tests/resources.yaml + - templates/tests/resources-defaulting.yaml + - templates/tests/quota.yaml + +release: + name: myrelease + namespace: default + +tests: + - it: refuses a quoted number without a unit + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: "8" + asserts: + - failedTemplate: + errorMessage: memory (8) must be at least 1Mi. A value without a unit is read as bytes, so add the unit, such as 512Mi or 8Gi. + + - it: refuses an unquoted number without a unit + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 8 + asserts: + - failedTemplate: + errorPattern: '^memory \(8\) must be at least 1Mi\.' + + - it: refuses a zero memory + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 0 + asserts: + - failedTemplate: + errorPattern: '^memory \(0\) must be at least 1Mi\.' + + - it: refuses a byte count one below 1Mi + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: "1048575" + asserts: + - failedTemplate: + errorPattern: '^memory \(1048575\) must be at least 1Mi\.' + + - it: refuses a suffixed value below 1Mi + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 1023Ki + asserts: + - failedTemplate: + errorPattern: '^memory \(1023Ki\) must be at least 1Mi\.' + + - it: refuses a negative memory + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: -1Gi + asserts: + - failedTemplate: + errorPattern: '^memory \(-1Gi\) must be at least 1Mi\.' + + - it: renders exactly 1Mi + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 1Mi + asserts: + - equal: + path: spec.containers[0].resources.limits.memory + value: 1Mi + - equal: + path: spec.containers[0].resources.requests.memory + value: "1048576" + + - it: renders an unquoted byte count of exactly 1Mi + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 1048576 + asserts: + - notFailedTemplate: {} + - equal: + path: spec.containers[0].resources.requests.memory + value: "1048576" + + - it: renders 512Mi + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 512Mi + asserts: + - equal: + path: spec.containers[0].resources.limits.memory + value: 512Mi + - equal: + path: spec.containers[0].resources.requests.memory + value: "536870912" + + - it: renders 8Gi + template: templates/tests/resources.yaml + set: + resources: + cpu: "1" + memory: 8Gi + asserts: + - equal: + path: spec.containers[0].resources.limits.memory + value: 8Gi + - equal: + path: spec.containers[0].resources.requests.memory + value: "8589934592" + + - it: leaves a map without memory alone + template: templates/tests/resources.yaml + set: + resources: + ephemeral-storage: 2Gi + asserts: + - notFailedTemplate: {} + - notExists: + path: spec.containers[0].resources.limits.memory + + - it: refuses a number without a unit set over a preset + template: templates/tests/resources-defaulting.yaml + set: + defaulting: + preset: t1.nano + resources: + memory: "8" + asserts: + - failedTemplate: + errorPattern: '^memory \(8\) must be at least 1Mi\.' + + - it: renders the smallest preset unchanged + template: templates/tests/resources-defaulting.yaml + set: + defaulting: + preset: t1.nano + asserts: + - equal: + path: spec.containers[0].resources.limits.memory + value: 128Mi + - equal: + path: spec.containers[0].resources.requests.memory + value: "134217728" + + - it: renders the smallest legacy preset unchanged + template: templates/tests/resources-defaulting.yaml + set: + defaulting: + preset: nano + asserts: + - equal: + path: spec.containers[0].resources.limits.memory + value: 128Mi + + - it: renders an explicit memory with a unit over a preset + template: templates/tests/resources-defaulting.yaml + set: + defaulting: + preset: t1.nano + resources: + memory: 8Gi + asserts: + - equal: + path: spec.containers[0].resources.limits.memory + value: 8Gi + - equal: + path: spec.containers[0].resources.limits.cpu + value: 250m + + - it: still renders a tenant quota of 0 memory + template: templates/tests/quota.yaml + set: + quota: + memory: 0 + asserts: + - equal: + path: spec.hard["limits.memory"] + value: "0" + - equal: + path: spec.hard["requests.memory"] + value: "0" diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index c3555e46c..06f024bc6 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -78,7 +78,7 @@ func init() { // Register HelmRelease types. if err := helmv2.AddToScheme(mgrScheme); err != nil { - panic(fmt.Errorf("Failed to add HelmRelease types to scheme: %w", err)) + panic(fmt.Errorf("failed to add HelmRelease types to scheme: %w", err)) } // Register the in-tree CiliumNetworkPolicy mirror backing SecurityGroup. @@ -87,10 +87,10 @@ func init() { } if err := corev1.AddToScheme(mgrScheme); err != nil { - panic(fmt.Errorf("Failed to add core types to scheme: %w", err)) + panic(fmt.Errorf("failed to add core types to scheme: %w", err)) } if err := rbacv1.AddToScheme(mgrScheme); err != nil { - panic(fmt.Errorf("Failed to add RBAC types to scheme: %w", err)) + panic(fmt.Errorf("failed to add RBAC types to scheme: %w", err)) } // Register Cozystack types for WorkloadMonitor queries. diff --git a/pkg/cmd/server/openapi.go b/pkg/cmd/server/openapi.go index 90af89cd0..d0cbe3289 100644 --- a/pkg/cmd/server/openapi.go +++ b/pkg/cmd/server/openapi.go @@ -319,7 +319,7 @@ func BuildPostProcessV3(kindSchemas map[string]string) func(*spec3.OpenAPI) (*sp base, ok1 := doc.Components.Schemas[baseRef] list, ok2 := doc.Components.Schemas[baseListRef] stat, ok3 := doc.Components.Schemas[baseStatusRef] - if !(ok1 && ok2 && ok3) { + if !ok1 || !ok2 || !ok3 { return doc, nil // not the apps GV — nothing to patch } @@ -463,7 +463,7 @@ func BuildPostProcessV2(kindSchemas map[string]string) func(*spec.Swagger) (*spe base, ok1 := defs[baseRef] list, ok2 := defs[baseListRef] stat, ok3 := defs[baseStatusRef] - if !(ok1 && ok2 && ok3) { + if !ok1 || !ok2 || !ok3 { return sw, nil // not the apps GV — nothing to patch } diff --git a/pkg/cmd/server/watch_initial_events_test.go b/pkg/cmd/server/watch_initial_events_test.go index 331417545..75d43c3e1 100644 --- a/pkg/cmd/server/watch_initial_events_test.go +++ b/pkg/cmd/server/watch_initial_events_test.go @@ -104,7 +104,7 @@ func watchStream(t *testing.T, ts *httptest.Server, cli client.Client, query str if err != nil { t.Fatalf("watch: %v", err) } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { t.Fatalf("watch: status %d", resp.StatusCode) } diff --git a/pkg/lineage/helmrelease_test.go b/pkg/lineage/helmrelease_test.go new file mode 100644 index 000000000..c8ef2676e --- /dev/null +++ b/pkg/lineage/helmrelease_test.go @@ -0,0 +1,70 @@ +package lineage + +import ( + "context" + "fmt" + "strings" + "testing" + + helmv2 "github.com/fluxcd/helm-controller/api/v2" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" +) + +// namedAppMapper maps a HelmRelease to the Kubernetes application it names; +// a HelmRelease missing from the map fails to map. +type namedAppMapper map[string]string + +func (m namedAppMapper) Map(hr *helmv2.HelmRelease) (string, string, string, error) { + app, ok := m[hr.Name] + if !ok { + return "", "", "", fmt.Errorf("no application for HelmRelease %s", hr.Name) + } + return "apps.cozystack.io/v1alpha1", "Kubernetes", strings.TrimSuffix(hr.Name, app), nil +} + +// A HelmRelease that cannot be resolved to its application hands the walk to +// the HelmRelease its label names; one that resolves ends the walk there. +func TestWalkHelmReleaseFallsBackToItsLabel(t *testing.T) { + parent := map[string]string{HRLabel: "kubernetes-b"} + unconvertible := capkObject(HRAPIVersion, HRKind, "kubernetes-a", parent, nil) + unconvertible.Object["spec"] = "not an object" + for _, tc := range []struct { + name string + hr *unstructured.Unstructured + mapped namedAppMapper + want []string + }{ + {name: "mapped", hr: capkObject(HRAPIVersion, HRKind, "kubernetes-a", parent, nil), mapped: namedAppMapper{"kubernetes-a": "a", "kubernetes-b": "b"}, want: []string{"a"}}, + {name: "unmappable", hr: capkObject(HRAPIVersion, HRKind, "kubernetes-a", parent, nil), mapped: namedAppMapper{"kubernetes-b": "b"}, want: []string{"b"}}, + {name: "application missing", hr: capkObject(HRAPIVersion, HRKind, "kubernetes-a", parent, nil), mapped: namedAppMapper{"kubernetes-a": "absent", "kubernetes-b": "b"}, want: []string{"b"}}, + {name: "unconvertible", hr: unconvertible, mapped: namedAppMapper{"kubernetes-a": "a", "kubernetes-b": "b"}, want: []string{"b"}}, + {name: "unmappable without a label", hr: capkObject(HRAPIVersion, HRKind, "kubernetes-a", nil, nil), mapped: namedAppMapper{"kubernetes-b": "b"}, want: nil}, + } { + t.Run(tc.name, func(t *testing.T) { + objs := []runtime.Object{ + capkObject("apps.cozystack.io/v1alpha1", "Kubernetes", "a", nil, nil), + capkObject("apps.cozystack.io/v1alpha1", "Kubernetes", "b", nil, nil), + capkObject(HRAPIVersion, HRKind, "kubernetes-b", nil, nil), + } + mapper := meta.NewDefaultRESTMapper(nil) + for _, o := range objs { + mapper.Add(o.GetObjectKind().GroupVersionKind(), meta.RESTScopeNamespace) + } + dyn := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), objs...) + var got []string + for _, id := range WalkOwnershipGraph(context.Background(), dyn, mapper, tc.mapped, tc.hr) { + got = append(got, id.Kind+"/"+id.Name) + } + var want []string + for _, name := range tc.want { + want = append(want, "Kubernetes/"+name) + } + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("WalkOwnershipGraph(%s) = %v, want %v", tc.name, got, want) + } + }) + } +} diff --git a/pkg/lineage/lineage.go b/pkg/lineage/lineage.go index 1546df647..8d8614b68 100644 --- a/pkg/lineage/lineage.go +++ b/pkg/lineage/lineage.go @@ -113,36 +113,8 @@ func WalkOwnershipGraph( return } - // I want "if err1 != nil go to next block, if err2 != nil, go to next block, etc semantics", - // like an early return from a function, but if all checks succeed, I don't want to do the rest - // of the function, so it's a `for { if err { break } if othererr { break } if allgood { return } - for { - if obj.GetAPIVersion() != HRAPIVersion || obj.GetKind() != HRKind { - break - } - hr := helmReleaseFromUnstructured(obj) - if hr == nil { - break - } - a, k, p, err := appMapper.Map(hr) - if err != nil { - l.Error(err, "failed to map HelmRelease to app") - break - } - ownerObj, err := getUnstructuredObject(ctx, client, mapper, state.cache, a, k, obj.GetNamespace(), strings.TrimPrefix(obj.GetName(), p)) - if err != nil { - l.Error(err, "couldn't get unstructured object", "APIVersion", a, "Kind", k, "Name", strings.TrimPrefix(obj.GetName(), p)) - break - } - // successfully mapped a HelmRelease to a custom app, no need to continue - out = append(out, - ObjectID{ - APIVersion: ownerObj.GetAPIVersion(), - Kind: ownerObj.GetKind(), - Namespace: ownerObj.GetNamespace(), - Name: ownerObj.GetName(), - }, - ) + if app, ok := helmReleaseApp(ctx, client, mapper, appMapper, state.cache, obj); ok { + out = append(out, app) return } @@ -160,6 +132,33 @@ func WalkOwnershipGraph( return } +// helmReleaseApp resolves a HelmRelease to the application it deploys. When +// it cannot, the walk goes on through the HelmRelease's own label instead. +func helmReleaseApp(ctx context.Context, client dynamic.Interface, mapper meta.RESTMapper, appMapper AppMapper, cache *ObjectCache, obj *unstructured.Unstructured) (ObjectID, bool) { + hr := helmReleaseFromUnstructured(obj) + if hr == nil { + return ObjectID{}, false + } + l := log.FromContext(ctx) + a, k, p, err := appMapper.Map(hr) + if err != nil { + l.Error(err, "failed to map HelmRelease to app") + return ObjectID{}, false + } + name := strings.TrimPrefix(obj.GetName(), p) + app, err := getUnstructuredObject(ctx, client, mapper, cache, a, k, obj.GetNamespace(), name) + if err != nil { + l.Error(err, "couldn't get unstructured object", "APIVersion", a, "Kind", k, "Name", name) + return ObjectID{}, false + } + return ObjectID{ + APIVersion: app.GetAPIVersion(), + Kind: app.GetKind(), + Namespace: app.GetNamespace(), + Name: app.GetName(), + }, true +} + // kubevirtMachineName returns the KubevirtMachine a CAPK-created // VirtualMachine belongs to. It only follows a KubevirtMachine in the VM's own // namespace: CAPK can also run against an external infrastructure cluster, diff --git a/pkg/ovnstatus/ovnstatus.go b/pkg/ovnstatus/ovnstatus.go index e6d682e51..b50c3f7ed 100644 --- a/pkg/ovnstatus/ovnstatus.go +++ b/pkg/ovnstatus/ovnstatus.go @@ -277,33 +277,6 @@ func defaultRunner(ctx context.Context, bin string, args ...string) (string, err return out, nil } -/************** Helpers **************/ - -func parseClusterStatusJSON(out string) (ClusterStatus, bool) { - var cs ClusterStatus - if json.Unmarshal([]byte(out), &cs) == nil && len(cs.Servers) > 0 { - return cs, true - } - var wrap struct { - Data ClusterStatus `json:"data"` - } - if json.Unmarshal([]byte(out), &wrap) == nil && len(wrap.Data.Servers) > 0 { - return wrap.Data, true - } - return ClusterStatus{}, false -} - -func portOf(db DB) string { - switch db { - case DBNorthbound: - return "6641" - case DBSouthbound: - return "6642" - default: - return "0" - } -} - /************** Minimal text fallback for "Servers" **************/ // Accepts variants like: diff --git a/pkg/ovnstatus/ovnstatus_test.go b/pkg/ovnstatus/ovnstatus_test.go index 8dde66b3a..8c44f5413 100644 --- a/pkg/ovnstatus/ovnstatus_test.go +++ b/pkg/ovnstatus/ovnstatus_test.go @@ -28,9 +28,7 @@ var expectedServersBlock = `` + func TestExtractServersBlock(t *testing.T) { if actual := extractServersBlock(testStdout); actual != expectedServersBlock { - fmt.Println([]byte(actual)) - fmt.Println([]byte(expectedServersBlock)) - t.Errorf("error extracting servers block from following string:\n%s\nexpected:\n%s\ngot:\n%s\n", testStdout, expectedServersBlock, actual) + t.Errorf("error extracting servers block from following string:\n%s\nexpected:\n%q\ngot:\n%q\n", testStdout, expectedServersBlock, actual) } } diff --git a/pkg/registry/apps/application/external_tls_policy_test.go b/pkg/registry/apps/application/external_tls_policy_test.go index 1cbc808fc..ff5704a23 100644 --- a/pkg/registry/apps/application/external_tls_policy_test.go +++ b/pkg/registry/apps/application/external_tls_policy_test.go @@ -41,14 +41,14 @@ import ( // expressions a cluster actually receives rather than a copy of them. const externalTLSPolicyPath = "../../../../packages/system/cozystack-basics/templates/database-external-tls-policy.yaml" -// loadExternalTLSPolicy returns the policy and its binding from the chart -// template. The template's only directives are the capability guard lines -// around the manifests; any other directive would mean the test is no longer -// looking at what renders, so it fails instead of guessing. -func loadExternalTLSPolicy(t *testing.T) (*admissionregistrationv1.ValidatingAdmissionPolicy, *admissionregistrationv1.ValidatingAdmissionPolicyBinding) { +// loadPolicy returns the policy and its binding from a chart template. The +// template's only directives are the capability guard lines around the +// manifests; any other directive would mean the test is no longer looking at +// what renders, so it fails instead of guessing. +func loadPolicy(t *testing.T, path string) (*admissionregistrationv1.ValidatingAdmissionPolicy, *admissionregistrationv1.ValidatingAdmissionPolicyBinding) { t.Helper() - raw, err := os.ReadFile(externalTLSPolicyPath) + raw, err := os.ReadFile(path) if err != nil { t.Fatalf("read policy template: %v", err) } @@ -92,11 +92,11 @@ func loadExternalTLSPolicy(t *testing.T) (*admissionregistrationv1.ValidatingAdm return policy, binding } -// compileExternalTLSPolicy mirrors compilePolicy in +// compilePolicy mirrors compilePolicy in // k8s.io/apiserver/pkg/admission/plugin/policy/validating, which is // unexported: the same composition environment, the same variable, match // condition, validation and message compilers. -func compileExternalTLSPolicy(t *testing.T, policy *admissionregistrationv1.ValidatingAdmissionPolicy) validating.Validator { +func compilePolicy(t *testing.T, policy *admissionregistrationv1.ValidatingAdmissionPolicy) validating.Validator { t.Helper() compiler, err := plugincel.NewCompositedCompiler(environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion())) @@ -139,7 +139,7 @@ func compileExternalTLSPolicy(t *testing.T, policy *admissionregistrationv1.Vali } func TestDatabaseExternalTLSPolicy_Binding(t *testing.T) { - policy, binding := loadExternalTLSPolicy(t) + policy, binding := loadPolicy(t, externalTLSPolicyPath) if binding.Spec.PolicyName != policy.Name { t.Errorf("binding names policy %q, want %q", binding.Spec.PolicyName, policy.Name) @@ -171,8 +171,8 @@ func TestDatabaseExternalTLSPolicy_Binding(t *testing.T) { } func TestDatabaseExternalTLSPolicy_Decisions(t *testing.T) { - policy, _ := loadExternalTLSPolicy(t) - validator := compileExternalTLSPolicy(t, policy) + policy, _ := loadPolicy(t, externalTLSPolicyPath) + validator := compilePolicy(t, policy) tlsOn := map[string]any{"enabled": true} tlsOff := map[string]any{"enabled": false} diff --git a/pkg/registry/apps/application/rest.go b/pkg/registry/apps/application/rest.go index aa0e24a62..6cd544994 100644 --- a/pkg/registry/apps/application/rest.go +++ b/pkg/registry/apps/application/rest.go @@ -2155,7 +2155,7 @@ const kubernetesKind = "Kubernetes" // upgraded cluster keeps them in its values until re-edited, and the adoption // migration deliberately leaves them in place rather than rewriting parent // values mid-upgrade -- but they no longer render anything on the Kubernetes CR. -var removedKubernetesFields = []string{"nodeGroups", "nodeHealthCheck", "maxNodeProvisionTime"} +var removedKubernetesFields = []string{"nodeGroups", "nodeHealthCheck", "maxNodeProvisionTime", "talos"} // warnRemovedKubernetesFields emits a client-facing admission warning for each // Phase 2-removed field still present in a Kubernetes CR's values, so an diff --git a/pkg/registry/apps/application/rest_name_schema_test.go b/pkg/registry/apps/application/rest_name_schema_test.go index a4bc67ec2..27f0795fe 100644 --- a/pkg/registry/apps/application/rest_name_schema_test.go +++ b/pkg/registry/apps/application/rest_name_schema_test.go @@ -36,7 +36,7 @@ func TestNameCapComesFromTheShippedSchema(t *testing.T) { cap int explains string }{ - {"Kubernetes", "kubernetes-", "../../../../packages/system/kubernetes-rd/cozyrds/kubernetes.yaml", 32, "md0"}, + {"Kubernetes", "kubernetes-", "../../../../packages/system/kubernetes-rd/cozyrds/kubernetes.yaml", 32, "kubernetes-nodes--"}, {"Kafka", "kafka-", "../../../../packages/system/kafka-rd/cozyrds/kafka.yaml", 44, "KRaft"}, } { t.Run(tc.kind, func(t *testing.T) { diff --git a/pkg/registry/apps/application/rest_nullstrip_admission_test.go b/pkg/registry/apps/application/rest_nullstrip_admission_test.go index 46b150fa1..9fdd252cd 100644 --- a/pkg/registry/apps/application/rest_nullstrip_admission_test.go +++ b/pkg/registry/apps/application/rest_nullstrip_admission_test.go @@ -45,14 +45,14 @@ var nullChartCases = []nullChartCase{ {name: "nullable schema default", schema: `{"type":"object","required":["name"],"properties":{"name":{"type":"string","nullable":true,"default":"example"}}}`, values: `{"name":"example"}`, input: `{"name":null}`}, {name: "item schema defaults", schema: `{"type":"object","properties":{"entries":{"type":"object","additionalProperties":{"type":"object","required":["name"],"properties":{"name":{"type":"string","default":"example"}}}},"items":{"type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"type":"string","default":"example"}}}}}}`, values: `{"entries":{},"items":[{"name":"chart-default"}]}`, input: `{"entries":{"mine":{"name":null}},"items":[{"name":null}]}`}, - {name: "nested defaults", chart: "apps/kubernetes", rd: "kubernetes", input: `{"talos":{"registryMirrors":null,"version":null},"controlPlane":{"replicas":null,"apiServer":{"resourcesPreset":null}},"nodeGroups":{"legacy":{"instanceType":null}}}`, want: `{"talos":{},"controlPlane":{"apiServer":{}},"nodeGroups":{"legacy":{"instanceType":null}}}`, paths: []string{"controlPlane.apiServer.resourcesPreset", "controlPlane.replicas", "talos.registryMirrors", "talos.version"}}, + {name: "nested defaults", chart: "apps/kubernetes", rd: "kubernetes", input: `{"oidc":{"mode":null},"addons":{"cilium":{"valuesOverride":null}},"controlPlane":{"replicas":null,"apiServer":{"resourcesPreset":null}},"nodeGroups":{"legacy":{"instanceType":null}}}`, want: `{"oidc":{},"addons":{"cilium":{}},"controlPlane":{"apiServer":{}},"nodeGroups":{"legacy":{"instanceType":null}}}`, paths: []string{"addons.cilium.valuesOverride", "controlPlane.apiServer.resourcesPreset", "controlPlane.replicas", "oidc.mode"}}, {name: "mariadb users", chart: "apps/mariadb", rd: "mariadb", input: `{"users":{"alice":{"password":null,"maxUserConnections":null},"blank":null}}`}, {name: "rabbitmq vhosts", chart: "apps/rabbitmq", rd: "rabbitmq", input: `{"vhosts":{"vh1":{"roles":null},"blank":null}}`}, {name: "seaweedfs pools", chart: "extra/seaweedfs", rd: "seaweedfs", input: `{"volume":{"pools":{"ssd":{"diskType":null},"blank":null}}}`}, {name: "vm source http", chart: "apps/vm-disk", rd: "vm-disk", input: `{"source":{"http":{"url":null}}}`}, {name: "vm source disk", chart: "apps/vm-disk", rd: "vm-disk", input: `{"source":{"disk":{"name":null}}}`}, {name: "vm source image", chart: "apps/vm-disk", rd: "vm-disk", input: `{"source":{"image":{"name":null}}}`}, - {name: "optional section", chart: "apps/kubernetes", rd: "kubernetes", input: `{"talos":null,"storageClass":null}`}, + {name: "optional section", chart: "apps/kubernetes", rd: "kubernetes", input: `{"oidc":null,"storageClass":null}`}, } func chartSpecSchema(t *testing.T, rd string) *structuralschema.Structural { diff --git a/pkg/registry/apps/application/rest_nullstrip_test.go b/pkg/registry/apps/application/rest_nullstrip_test.go index 7d62844a4..ce56eb0bb 100644 --- a/pkg/registry/apps/application/rest_nullstrip_test.go +++ b/pkg/registry/apps/application/rest_nullstrip_test.go @@ -64,15 +64,16 @@ func normalize(t *testing.T, s *structuralschema.Structural, spec string) ([]str func TestNormalizeSpecNulls_RequiredFields(t *testing.T) { s := kubernetesSpecSchema(t) - paths, got := normalize(t, s, `{"host":"k8s","talos":{"registryMirrors":null,"version":null,"imageFactoryURL":"https://factory.example.invalid"}}`) + paths, got := normalize(t, s, `{"host":"k8s","addons":{"cilium":{"valuesOverride":null}},"oidc":{"mode":null}}`) - want := []string{"talos.registryMirrors", "talos.version"} + want := []string{"addons.cilium.valuesOverride", "oidc.mode"} if !reflect.DeepEqual(paths, want) { t.Errorf("stripped paths = %v, want %v", paths, want) } wantSpec := map[string]any{ - "host": "k8s", - "talos": map[string]any{"imageFactoryURL": "https://factory.example.invalid"}, + "host": "k8s", + "addons": map[string]any{"cilium": map[string]any{}}, + "oidc": map[string]any{}, } if !reflect.DeepEqual(got, wantSpec) { t.Errorf("normalized spec = %v, want %v", got, wantSpec) @@ -82,12 +83,12 @@ func TestNormalizeSpecNulls_RequiredFields(t *testing.T) { func TestNormalizeSpecNulls_OptionalFieldsUntouched(t *testing.T) { s := kubernetesSpecSchema(t) - paths, got := normalize(t, s, `{"talos":null,"storageClass":null,"nodeGroups":null,"version":null}`) + paths, got := normalize(t, s, `{"oidc":null,"storageClass":null,"nodeGroups":null,"version":null}`) if len(paths) != 0 { t.Errorf("stripped paths = %v, want none: the Kubernetes schema requires none of these at the root", paths) } - for _, key := range []string{"talos", "storageClass", "nodeGroups", "version"} { + for _, key := range []string{"oidc", "storageClass", "nodeGroups", "version"} { v, ok := got[key] if !ok || v != nil { t.Errorf("%s = %v (present=%v), want a preserved null", key, v, ok) @@ -96,20 +97,20 @@ func TestNormalizeSpecNulls_OptionalFieldsUntouched(t *testing.T) { } // TestNormalizeSpecNulls_FreeFormSubtreeUntouched pins the boundary: inside -// registryMirrors the keys are registry hostnames the user invents, and the +// addons.cilium.valuesOverride the keys are Helm values the user invents, and the // schema declares nothing there. A null in that position may be the payload, so // it is left exactly as written rather than guessed at. func TestNormalizeSpecNulls_FreeFormSubtreeUntouched(t *testing.T) { s := kubernetesSpecSchema(t) - paths, got := normalize(t, s, `{"talos":{"registryMirrors":{"ghcr.io":null}}}`) + paths, got := normalize(t, s, `{"addons":{"cilium":{"valuesOverride":{"ghcr.io":null}}}}`) if len(paths) != 0 { t.Errorf("stripped paths = %v, want none", paths) } - mirrors := got["talos"].(map[string]any)["registryMirrors"].(map[string]any) - if _, ok := mirrors["ghcr.io"]; !ok { - t.Errorf("free-form key ghcr.io was dropped: %v", mirrors) + overrides := got["addons"].(map[string]any)["cilium"].(map[string]any)["valuesOverride"].(map[string]any) + if _, ok := overrides["ghcr.io"]; !ok { + t.Errorf("free-form key ghcr.io was dropped: %v", overrides) } } @@ -143,12 +144,12 @@ func TestNormalizeSpecNulls_NoDefaultOrUserCollection(t *testing.T) { } func TestNormalizeSpecNulls_PreservesOtherValuesExactly(t *testing.T) { - app := &appsv1alpha1.Application{Spec: &apiextv1.JSON{Raw: []byte(`{"talos":{"registryMirrors":null},"large":9007199254740993,"negative":-9007199254740993,"text":"null","bool":false,"empty":[],"object":{}}`)}} + app := &appsv1alpha1.Application{Spec: &apiextv1.JSON{Raw: []byte(`{"addons":{"cilium":{"valuesOverride":null}},"large":9007199254740993,"negative":-9007199254740993,"text":"null","bool":false,"empty":[],"object":{}}`)}} r := &REST{specSchema: kubernetesSpecSchema(t)} if _, err := r.normalizeSpecNulls(app); err != nil { t.Fatal(err) } - if string(app.Spec.Raw) != `{"bool":false,"empty":[],"large":9007199254740993,"negative":-9007199254740993,"object":{},"talos":{},"text":"null"}` { + if string(app.Spec.Raw) != `{"addons":{"cilium":{}},"bool":false,"empty":[],"large":9007199254740993,"negative":-9007199254740993,"object":{},"text":"null"}` { t.Fatalf("unrelated values changed: %s", app.Spec.Raw) } } @@ -182,8 +183,8 @@ func TestNormalizeSpecNulls_NoSchemaOrNoNulls(t *testing.T) { } func TestEmptyFieldsWarning(t *testing.T) { - short := emptyFieldsWarning([]string{"talos.version", "talos.registryMirrors"}) - if !strings.Contains(short, "talos.version, talos.registryMirrors") || + short := emptyFieldsWarning([]string{"oidc.mode", "addons.cilium.valuesOverride"}) + if !strings.Contains(short, "oidc.mode, addons.cilium.valuesOverride") || !strings.Contains(short, "the chart default applies") { t.Errorf("unexpected warning: %q", short) } diff --git a/pkg/registry/apps/application/rest_validation_test.go b/pkg/registry/apps/application/rest_validation_test.go index f48f40d26..51a9f48ce 100644 --- a/pkg/registry/apps/application/rest_validation_test.go +++ b/pkg/registry/apps/application/rest_validation_test.go @@ -466,6 +466,12 @@ func TestWarnRemovedKubernetesFields(t *testing.T) { specJSON: `{"nodeGroups":{},"nodeHealthCheck":{"maxUnhealthy":"50%"},"maxNodeProvisionTime":"10m"}`, wantKeys: []string{"nodeGroups", "nodeHealthCheck", "maxNodeProvisionTime"}, }, + { + name: "kubernetes with a stored talos block warns", + kindName: "Kubernetes", + specJSON: `{"version":"v1.35","talos":{"version":"v1.13.6"}}`, + wantKeys: []string{"talos"}, + }, { name: "kubernetes without removed fields does not warn", kindName: "Kubernetes", diff --git a/pkg/registry/apps/application/tenant_namespace_delete_policy_test.go b/pkg/registry/apps/application/tenant_namespace_delete_policy_test.go new file mode 100644 index 000000000..6c8870ea2 --- /dev/null +++ b/pkg/registry/apps/application/tenant_namespace_delete_policy_test.go @@ -0,0 +1,163 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package application + +import ( + "context" + "slices" + "strings" + "testing" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/admission/plugin/policy/validating" + celconfig "k8s.io/apiserver/pkg/apis/cel" + "k8s.io/apiserver/pkg/authentication/user" +) + +const tenantNamespaceDeletePolicyPath = "../../../../packages/system/cozystack-basics/templates/tenant-namespace-delete-policy.yaml" + +func TestTenantNamespaceDeletePolicy_Binding(t *testing.T) { + policy, binding := loadPolicy(t, tenantNamespaceDeletePolicyPath) + + if binding.Spec.PolicyName != policy.Name { + t.Errorf("binding names policy %q, want %q", binding.Spec.PolicyName, policy.Name) + } + if !slices.Equal(binding.Spec.ValidationActions, []admissionregistrationv1.ValidationAction{admissionregistrationv1.Deny}) { + t.Errorf("binding validationActions = %v, want [Deny]", binding.Spec.ValidationActions) + } + if policy.Spec.FailurePolicy == nil || *policy.Spec.FailurePolicy != admissionregistrationv1.Fail { + t.Errorf("failurePolicy must be Fail") + } + + rules := policy.Spec.MatchConstraints.ResourceRules + if len(rules) != 1 { + t.Fatalf("want one resource rule, got %d", len(rules)) + } + rule := rules[0] + if !slices.Equal(rule.APIGroups, []string{""}) || !slices.Equal(rule.APIVersions, []string{"v1"}) { + t.Errorf("rule must match core v1 only, got apiGroups %v apiVersions %v", rule.APIGroups, rule.APIVersions) + } + if !slices.Equal(rule.Operations, []admissionregistrationv1.OperationType{admissionregistrationv1.Delete}) { + t.Errorf("operations = %v, want [DELETE]", rule.Operations) + } + if !slices.Equal(rule.Resources, []string{"namespaces"}) { + t.Errorf("resources = %v, want [namespaces]", rule.Resources) + } + + if binding.Spec.MatchResources == nil || binding.Spec.MatchResources.ObjectSelector == nil { + t.Fatal("binding must select namespaces by label") + } + selector, err := metav1.LabelSelectorAsSelector(binding.Spec.MatchResources.ObjectSelector) + if err != nil { + t.Fatalf("binding objectSelector: %v", err) + } + selected := []struct { + name string + labels labels.Set + want bool + }{ + {name: "tenant namespace under tenant-root", labels: labels.Set{"tenant.cozystack.io/tenant-root": "", "tenant.cozystack.io/tenant-foo": "", "namespace.cozystack.io/host": "foo.example.org"}, want: true}, + {name: "nested tenant namespace", labels: labels.Set{"tenant.cozystack.io/tenant-root": "", "tenant.cozystack.io/tenant-foo": "", "tenant.cozystack.io/tenant-foo-bar": ""}, want: true}, + {name: "tenant-root namespace", labels: labels.Set{"platform.cozystack.io/no-delete": "true", "namespace.cozystack.io/host": "example.org", "namespace.cozystack.io/gateway": "tenant-root"}}, + {name: "namespace named tenant-* without tenant labels", labels: labels.Set{"namespace.cozystack.io/host": "case-probe.example.org"}}, + {name: "unlabelled namespace", labels: labels.Set{}}, + } + for _, tc := range selected { + if got := selector.Matches(tc.labels); got != tc.want { + t.Errorf("%s: selected = %v, want %v", tc.name, got, tc.want) + } + } +} + +func TestTenantNamespaceDeletePolicy_Decisions(t *testing.T) { + policy, _ := loadPolicy(t, tenantNamespaceDeletePolicyPath) + validator := compilePolicy(t, policy) + + fluxGroups := []string{"system:serviceaccounts", "system:serviceaccounts:cozy-fluxcd", "system:authenticated"} + kubeSystemGroups := []string{"system:serviceaccounts", "system:serviceaccounts:kube-system", "system:authenticated"} + adminGroups := []string{"system:masters", "system:authenticated"} + + cases := []struct { + name string + namespace string + user string + groups []string + denied bool + tenant string + parent string + }{ + {name: "helm-controller uninstalling a Tenant", namespace: "tenant-foo", user: "system:serviceaccount:cozy-fluxcd:flux", groups: fluxGroups}, + {name: "helm-controller uninstalling a nested Tenant", namespace: "tenant-foo-bar", user: "system:serviceaccount:cozy-fluxcd:flux", groups: fluxGroups}, + {name: "garbage collector following a deleted parent namespace", namespace: "tenant-foo-bar", user: "system:serviceaccount:kube-system:generic-garbage-collector", groups: kubeSystemGroups}, + {name: "cluster admin deleting a tenant namespace", namespace: "tenant-foo", user: "admin", groups: adminGroups, denied: true, tenant: "foo", parent: "tenant-root"}, + {name: "cluster admin deleting a nested tenant namespace", namespace: "tenant-foo-bar", user: "admin", groups: adminGroups, denied: true, tenant: "bar", parent: "tenant-foo"}, + {name: "cluster admin deleting a third-level tenant namespace", namespace: "tenant-foo-bar-baz", user: "admin", groups: adminGroups, denied: true, tenant: "baz", parent: "tenant-foo-bar"}, + {name: "cluster admin deleting a labelled namespace without a dash", namespace: "standalone", user: "admin", groups: adminGroups, denied: true, tenant: "standalone", parent: "tenant-root"}, + {name: "OIDC user bound to cluster-admin", namespace: "tenant-foo", user: "cluster-admin-user", groups: []string{"cozystack-cluster-admin", "system:authenticated"}, denied: true, tenant: "foo", parent: "tenant-root"}, + {name: "cozy-system service account", namespace: "tenant-foo", user: "system:serviceaccount:cozy-system:cozystack-api", groups: []string{"system:serviceaccounts", "system:serviceaccounts:cozy-system", "system:authenticated"}, denied: true, tenant: "foo", parent: "tenant-root"}, + {name: "tenant service account", namespace: "tenant-foo-bar", user: "system:serviceaccount:tenant-foo:default", groups: []string{"system:serviceaccounts", "system:serviceaccounts:tenant-foo", "system:authenticated"}, denied: true, tenant: "bar", parent: "tenant-foo"}, + {name: "user without groups", namespace: "tenant-foo", user: "no-groups", denied: true, tenant: "foo", parent: "tenant-root"}, + } + + gvk := schema.GroupVersionKind{Version: "v1", Kind: "Namespace"} + gvr := schema.GroupVersionResource{Version: "v1", Resource: "namespaces"} + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + oldObj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", + "kind": "Namespace", + "metadata": map[string]any{ + "name": tc.namespace, + "labels": map[string]any{"tenant.cozystack.io/tenant-root": ""}, + }, + }} + var newObj runtime.Object + attrs := admission.NewAttributesRecord(newObj, oldObj, gvk, tc.namespace, tc.namespace, gvr, "", admission.Delete, &metav1.DeleteOptions{}, false, &user.DefaultInfo{Name: tc.user, Groups: tc.groups}) + versioned := &admission.VersionedAttributes{Attributes: attrs, VersionedObject: admission.NewLazyObject(newObj), VersionedOldObject: admission.NewLazyObject(oldObj), VersionedKind: gvk} + + result := validator.Validate(context.Background(), gvr, versioned, nil, nil, celconfig.RuntimeCELCostBudget, nil) + var denials []validating.PolicyDecision + for _, d := range result.Decisions { + if d.Evaluation == validating.EvalError { + t.Fatalf("evaluation error: %s", d.Message) + } + if d.Action == validating.ActionDeny { + denials = append(denials, d) + } + } + if got := len(denials) > 0; got != tc.denied { + t.Fatalf("denied = %v, want %v (decisions %+v)", got, tc.denied, result.Decisions) + } + for _, d := range denials { + for _, want := range []string{ + "kubectl delete tenants.apps.cozystack.io " + tc.tenant + " --namespace " + tc.parent + ".", + "kubectl label namespace " + tc.namespace + " tenant.cozystack.io/tenant-root-", + } { + if !strings.Contains(d.Message, want) { + t.Errorf("denial message must carry %q, got %q", want, d.Message) + } + } + } + }) + } +} diff --git a/pkg/registry/write_options_test.go b/pkg/registry/write_options_test.go index 488d7a51a..8d6b375f9 100644 --- a/pkg/registry/write_options_test.go +++ b/pkg/registry/write_options_test.go @@ -17,7 +17,9 @@ limitations under the License. package registry import ( + "encoding/json" "reflect" + "strings" "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,7 +28,6 @@ import ( func TestClientWriteOptionsPreserveAPISemantics(t *testing.T) { grace := int64(7) - orphan := true propagation := metav1.DeletePropagationForeground uid := types.UID("expected-uid") rv := "42" @@ -74,11 +75,20 @@ func TestClientWriteOptionsPreserveAPISemantics(t *testing.T) { t.Fatalf("conversion mutated API options: create=%#v update=%#v delete=%#v", create, update, deleteOptions) } - orphanDelete := &metav1.DeleteOptions{OrphanDependents: &orphan, DryRun: []string{metav1.DryRunAll}} + // orphanDependents is deprecated in Go but still accepted on the wire, + // which is how a raw-only option reaches the registry. + orphanDelete := &metav1.DeleteOptions{} + if err := json.Unmarshal([]byte(`{"orphanDependents":true,"dryRun":["All"]}`), orphanDelete); err != nil { + t.Fatal(err) + } orphanExpected := orphanDelete.DeepCopy() - if got := ClientDeleteOptions(orphanDelete).AsDeleteOptions(); !reflect.DeepEqual(got, orphanExpected) { + got := ClientDeleteOptions(orphanDelete).AsDeleteOptions() + if !reflect.DeepEqual(got, orphanExpected) { t.Fatalf("raw-only delete option lost: got %#v, want %#v", got, orphanExpected) } + if wire, err := json.Marshal(got); err != nil || !strings.Contains(string(wire), `"orphanDependents":true`) { + t.Fatalf("converted delete options = %s (%v), want orphanDependents kept", wire, err) + } if !reflect.DeepEqual(orphanDelete, orphanExpected) { t.Fatalf("raw-only delete conversion mutated input: got %#v, want %#v", orphanDelete, orphanExpected) } diff --git a/pkg/reservation/evaluator.go b/pkg/reservation/evaluator.go index 6927b0bf2..40d5c2ced 100644 --- a/pkg/reservation/evaluator.go +++ b/pkg/reservation/evaluator.go @@ -284,9 +284,9 @@ func mulQuantity(q, n ref.Val) ref.Val { if !ok { return types.MaybeNoSuchOverloadErr(n) } - out := qq.Quantity.DeepCopy() + out := qq.DeepCopy() if !out.Mul(int64(nn)) { - return types.NewErr("quantity %s * %d overflows", qq.Quantity.String(), int64(nn)) + return types.NewErr("quantity %s * %d overflows", qq.String(), int64(nn)) } return apiservercel.Quantity{Quantity: &out} } @@ -294,7 +294,7 @@ func mulQuantity(q, n ref.Val) ref.Val { func toQuantity(v ref.Val) (resource.Quantity, error) { switch t := v.(type) { case apiservercel.Quantity: - return t.Quantity.DeepCopy(), nil + return t.DeepCopy(), nil case types.Int: return *resource.NewQuantity(int64(t), resource.DecimalSI), nil case types.Uint: diff --git a/pkg/reservation/instancetype_test.go b/pkg/reservation/instancetype_test.go index 0d072b548..f837af8e3 100644 --- a/pkg/reservation/instancetype_test.go +++ b/pkg/reservation/instancetype_test.go @@ -56,7 +56,9 @@ func shippedInstancetypes(t *testing.T) []map[string]any { out = append(out, obj) } } - fh.Close() + if err := fh.Close(); err != nil { + t.Fatalf("%s: %v", f, err) + } } return out } diff --git a/test/check-readiness/main_test.go b/test/check-readiness/main_test.go index 622340c7f..01e6c1505 100644 --- a/test/check-readiness/main_test.go +++ b/test/check-readiness/main_test.go @@ -13,6 +13,7 @@ package checkreadiness_test import ( "bytes" + "context" "errors" "flag" "fmt" @@ -23,6 +24,7 @@ import ( "runtime" "strings" "testing" + "time" ) var update = flag.Bool("update", false, "regenerate golden files") @@ -335,3 +337,80 @@ func repoRoot(t *testing.T) string { } return root } + +// A result that never reaches stdout fails the run, whatever the cluster's +// state, and the watch loop stops instead of refreshing into the void. stdout +// is a read-only descriptor, so every write fails without a SIGPIPE. +func TestUnwritableStdoutFails(t *testing.T) { + root := repoRoot(t) + fakeBin := filepath.Join(root, "test", "check-readiness", "fakekubectl") + base := filepath.Join(root, "test", "check-readiness", "testdata", "fixtures", "_base") + for _, tc := range []struct { + name string + args []string + }{ + {name: "once", args: []string{"--no-color"}}, + {name: "wait", args: []string{"--wait", "--no-color"}}, + {name: "watch", args: []string{"-w", "1", "--no-color"}}, + } { + t.Run(tc.name, func(t *testing.T) { + stdout, err := os.Open(os.DevNull) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdout.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + t.Cleanup(cancel) + argv := subjectArgv(tc.args) + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Env = testEnv(fakeBin, base, base, filepath.Join(t.TempDir(), "kubectl.log")) + var stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = stdout, &stderr + runErr := cmd.Run() + if ctx.Err() != nil { + t.Fatalf("still running after its output failed\nstderr:\n%s", stderr.String()) + } + if code := exitCode(t, runErr); code != 1 { + t.Errorf("exit code = %d, want 1\nstderr:\n%s", code, stderr.String()) + } + if !strings.Contains(stderr.String(), "writing output") { + t.Errorf("stderr does not say the output was lost:\n%s", stderr.String()) + } + }) + } +} + +// A wait that times out ends with its verdict: the table comes first and the +// timeout line last, also when the table could not be written. +func TestWaitTimeoutVerdictComesLast(t *testing.T) { + root := repoRoot(t) + fakeBin := filepath.Join(root, "test", "check-readiness", "fakekubectl") + fixtures := filepath.Join(root, "test", "check-readiness", "testdata", "fixtures") + args := []string{"-w", "1", "--wait", "--timeout", "1s", "--no-color"} + for _, unwritable := range []bool{false, true} { + t.Run(fmt.Sprintf("unwritable=%v", unwritable), func(t *testing.T) { + cmd := scriptCommand(args) + cmd.Env = testEnv(fakeBin, filepath.Join(fixtures, "helmrelease-not-ready"), filepath.Join(fixtures, "_base"), filepath.Join(t.TempDir(), "kubectl.log")) + var combined bytes.Buffer + cmd.Stdout, cmd.Stderr = &combined, &combined + if unwritable { + stdout, err := os.Open(os.DevNull) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = stdout.Close() }) + cmd.Stdout = stdout + } + if code := exitCode(t, cmd.Run()); code != 1 { + t.Errorf("exit code = %d, want 1\n%s", code, combined.String()) + } + lines := strings.Split(strings.TrimRight(combined.String(), "\n"), "\n") + if last := lines[len(lines)-1]; !strings.HasPrefix(last, "Timeout after") { + t.Errorf("last line = %q, want the timeout verdict\n%s", last, combined.String()) + } + if unwritable && !strings.Contains(combined.String(), "writing output") { + t.Errorf("stderr does not say the table was lost:\n%s", combined.String()) + } + }) + } +} diff --git a/test/check-readiness/testdata/golden/help-long.stdout b/test/check-readiness/testdata/golden/help-long.stdout index f1f289e08..428f135aa 100644 --- a/test/check-readiness/testdata/golden/help-long.stdout +++ b/test/check-readiness/testdata/golden/help-long.stdout @@ -21,3 +21,7 @@ Options: --no-color Disable color output (auto-disabled on non-TTY) -h, --help Show this help +Exit status: 0 when everything is ready, 1 when something is not ready or a +write to stdout fails, 2 on invalid arguments. A closed pipe ends the process +with SIGPIPE, like other Unix tools. + diff --git a/test/check-readiness/testdata/golden/help-short.stdout b/test/check-readiness/testdata/golden/help-short.stdout index f1f289e08..428f135aa 100644 --- a/test/check-readiness/testdata/golden/help-short.stdout +++ b/test/check-readiness/testdata/golden/help-short.stdout @@ -21,3 +21,7 @@ Options: --no-color Disable color output (auto-disabled on non-TTY) -h, --help Show this help +Exit status: 0 when everything is ready, 1 when something is not ready or a +write to stdout fails, 2 on invalid arguments. A closed pipe ends the process +with SIGPIPE, like other Unix tools. + diff --git a/test/check-readiness/testdata/golden/unknown-arg.stderr b/test/check-readiness/testdata/golden/unknown-arg.stderr index d6ec7c0db..ca652d9ea 100644 --- a/test/check-readiness/testdata/golden/unknown-arg.stderr +++ b/test/check-readiness/testdata/golden/unknown-arg.stderr @@ -22,3 +22,7 @@ Options: --no-color Disable color output (auto-disabled on non-TTY) -h, --help Show this help +Exit status: 0 when everything is ready, 1 when something is not ready or a +write to stdout fails, 2 on invalid arguments. A closed pipe ends the process +with SIGPIPE, like other Unix tools. + diff --git a/test/check-readiness/testdata/golden/watch-nonnumeric-interval.stderr b/test/check-readiness/testdata/golden/watch-nonnumeric-interval.stderr index 3cb2c43bb..cc23fc457 100644 --- a/test/check-readiness/testdata/golden/watch-nonnumeric-interval.stderr +++ b/test/check-readiness/testdata/golden/watch-nonnumeric-interval.stderr @@ -22,3 +22,7 @@ Options: --no-color Disable color output (auto-disabled on non-TTY) -h, --help Show this help +Exit status: 0 when everything is ready, 1 when something is not ready or a +write to stdout fails, 2 on invalid arguments. A closed pipe ends the process +with SIGPIPE, like other Unix tools. +