mirror of
https://github.com/cozystack/cozystack.git
synced 2026-11-21 12:13:55 +09:00
chore(tests): merge upstream main into resource selector branch
Preserve the Helm selector guard alongside main's independently gated backport tests when refreshing the branch's upstream base. Validated with actionlint and 31 focused Bats tests. Signed-off-by: Ahmed Kamal <email.ahmedkamal@googlemail.com> Assisted-by: LLM
This commit is contained in:
commit
b1e067f6e1
375 files changed
+18128
-5730
No files matched your search
@@ -13,6 +13,7 @@
|
||||
|
||||
## What this PR does
|
||||
|
||||
<!-- Link the issues this PR addresses. For a complete fix targeting the default branch, use Closes #N (Closes owner/repository#N for another repository). Otherwise use a non-closing reference; release-branch PRs need manual issue linking and closure when appropriate. -->
|
||||
|
||||
### Screenshots
|
||||
|
||||
|
||||
@@ -61,7 +61,8 @@ jobs:
|
||||
# base-repo PR file list — NOT from the presence/absence of an artifact the
|
||||
# fork's own build controls. Fail-closed if the fork build failed or the
|
||||
# pr-patch artifact is missing; green-and-stop only for a PR whose file list is
|
||||
# entirely under docs/ (verified against the base-repo API).
|
||||
# entirely under docs/ or the CI-only tools cmd/backport-batch/ and
|
||||
# cmd/api-gate/ (verified against the base-repo API).
|
||||
resolve:
|
||||
name: Resolve fork PR
|
||||
# Only fork PRs: head_repository differs from the base repository. Same-repo
|
||||
@@ -266,15 +267,21 @@ jobs:
|
||||
await decide('failure', 'PR file list hit the 3000-file API cap — cannot verify docs-only');
|
||||
return;
|
||||
}
|
||||
// Paths with nothing to build and nothing for e2e to install: docs,
|
||||
// and the CI-only Go tools no image builds. pull-requests.yaml's
|
||||
// plan job answers the same question with the same prefixes, and
|
||||
// hack/gate-concurrency-contract.bats holds the two lists equal.
|
||||
//
|
||||
// A rename reports the NEW path in `filename` and the old one in
|
||||
// `previous_filename`. Checking only `filename` would read a
|
||||
// packages/… -> docs/… rename as docs-only, when it in fact removes
|
||||
// a code path. BOTH sides must be under docs/ to count as docs-only.
|
||||
const isDocs = p => typeof p === 'string' && p.startsWith('docs/');
|
||||
const hasNonDocs = files.some(f =>
|
||||
!isDocs(f.filename) || (f.previous_filename && !isDocs(f.previous_filename)));
|
||||
if (!hasNonDocs) {
|
||||
await decide('success', 'docs-only PR (verified via PR file list) — e2e not required');
|
||||
// a code path. BOTH sides must be under these prefixes to count.
|
||||
const noRuntimePrefixes = ['docs/', 'cmd/backport-batch/', 'cmd/api-gate/'];
|
||||
const isNoRuntime = p => typeof p === 'string' && noRuntimePrefixes.some(x => p.startsWith(x));
|
||||
const hasRuntime = files.some(f =>
|
||||
!isNoRuntime(f.filename) || (f.previous_filename && !isNoRuntime(f.previous_filename)));
|
||||
if (!hasRuntime) {
|
||||
await decide('success', 'docs or CI-only tools (verified via PR file list) — e2e not required');
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -75,6 +75,8 @@ jobs:
|
||||
statuses: write
|
||||
outputs:
|
||||
code: ${{ steps.p.outputs.code }}
|
||||
docs: ${{ steps.p.outputs.docs }}
|
||||
backport_batch: ${{ steps.p.outputs.backport_batch }}
|
||||
matrix: ${{ steps.p.outputs.matrix }}
|
||||
any: ${{ steps.p.outputs.any }}
|
||||
touched: ${{ steps.p.outputs.touched }}
|
||||
@@ -167,11 +169,27 @@ jobs:
|
||||
# alone here rather than widened by a side effect.
|
||||
git diff --name-only --no-renames "origin/${BASE_REF}...HEAD" > /tmp/changed_norenames.txt
|
||||
echo "Changed files:"; sed 's/^/ /' /tmp/changed_norenames.txt
|
||||
if grep -qvE '^docs/' /tmp/changed_norenames.txt; then
|
||||
# `code` is false when every path is under docs/ or in a CI-only Go
|
||||
# tool that no image builds (cmd/backport-batch, cmd/api-gate): there
|
||||
# is nothing to build and nothing for e2e to install. e2e-fork.yaml
|
||||
# answers the same question from the PR file list with the same
|
||||
# prefixes, and hack/gate-concurrency-contract.bats holds the two lists
|
||||
# equal.
|
||||
if grep -qvE '^(docs/|cmd/backport-batch/|cmd/api-gate/)' /tmp/changed_norenames.txt; then
|
||||
echo "code=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "code=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
# The Bats contracts read docs, so a docs change runs them even when
|
||||
# `code` is false; a change confined to the CI-only tools does not.
|
||||
if grep -qE '^docs/' /tmp/changed_norenames.txt; then
|
||||
echo "docs=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "docs=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
backport_batch="$(hack/backport-batch-tests-needed.sh /tmp/changed_norenames.txt)"
|
||||
echo "backport-batch tests needed: $backport_batch"
|
||||
echo "backport_batch=$backport_batch" >> "$GITHUB_OUTPUT"
|
||||
matrix="$(hack/build-matrix.sh /tmp/changed.txt)"
|
||||
echo "Build matrix: $matrix"
|
||||
echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
|
||||
@@ -293,9 +311,11 @@ jobs:
|
||||
contents: read
|
||||
needs: ["plan"]
|
||||
# Unit contracts read both code and documentation. Run this job for every
|
||||
# non-release PR: code changes take the complete unit path, while docs-only
|
||||
# changes run the Bats lane that owns those cross-tree contracts. Builds and
|
||||
# E2E remain gated on `code` below.
|
||||
# non-release PR: code changes take the complete unit path, docs changes run
|
||||
# the Bats lane that owns those cross-tree contracts, and the backport-batch
|
||||
# tests run whenever one of their inputs changed and at no other time. A
|
||||
# change confined to the CI-only tools runs no other lane. Builds and E2E
|
||||
# remain gated on `code` below.
|
||||
if: |
|
||||
!contains(github.event.pull_request.labels.*.name, 'release')
|
||||
steps:
|
||||
@@ -306,20 +326,21 @@ jobs:
|
||||
# The module cache is restored from the key build-main.yaml writes on
|
||||
# main, which every PR run can read, so a PR's first run starts warm. The
|
||||
# download fetches whatever the cache lacks, the whole set on a miss. The
|
||||
# Go steps are for code changes only: a docs-only run is the Bats lane alone.
|
||||
# Go steps are for code changes and the backport-batch tests only: a
|
||||
# docs-only run is the Bats lane alone.
|
||||
- name: Locate Go module cache
|
||||
if: needs.plan.outputs.code == 'true'
|
||||
if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true'
|
||||
id: gomodcache
|
||||
run: echo "dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
|
||||
- name: Cache Go modules
|
||||
if: needs.plan.outputs.code == 'true'
|
||||
if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true'
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: ${{ steps.gomodcache.outputs.dir }}
|
||||
key: go-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
|
||||
restore-keys: go-mod-${{ runner.os }}-
|
||||
- name: Download Go modules
|
||||
if: needs.plan.outputs.code == 'true'
|
||||
if: needs.plan.outputs.code == 'true' || needs.plan.outputs.backport_batch == 'true'
|
||||
run: |
|
||||
for mod in $(git ls-files go.mod '*/go.mod' | xargs -n1 dirname); do
|
||||
(cd "$mod" && "$GITHUB_WORKSPACE/hack/go-mod-download.sh")
|
||||
@@ -327,8 +348,9 @@ jobs:
|
||||
# Ephemeral runners don't carry the helm-unittest plugin the persistent
|
||||
# self-hosted runner accreted. The plugin embeds its own Helm, so its
|
||||
# version is pinned: a plugin release must not change test results. Bats is
|
||||
# installed on every run, docs-only ones included.
|
||||
# installed whenever a Bats lane runs, docs-only ones included.
|
||||
- name: Set up test toolchain
|
||||
if: needs.plan.outputs.code == 'true' || needs.plan.outputs.docs == 'true'
|
||||
env:
|
||||
CODE_CHANGED: ${{ needs.plan.outputs.code }}
|
||||
HELM_UNITTEST_VERSION: "1.1.2"
|
||||
@@ -374,8 +396,15 @@ jobs:
|
||||
if: needs.plan.outputs.code == 'true'
|
||||
run: make unit-tests test-controllers test-controllers-envtest -j4 -k --output-sync=target
|
||||
- name: Run Bats unit tests for docs-only changes
|
||||
if: needs.plan.outputs.code == 'false'
|
||||
if: needs.plan.outputs.code == 'false' && needs.plan.outputs.docs == 'true'
|
||||
run: make bats-unit-tests
|
||||
# Not part of `make unit-tests`: hack/backport-batch-tests-needed.sh
|
||||
# decides from cmd/backport-batch/test-inputs.txt, whatever `code` is.
|
||||
# `!cancelled()` for the reason the unit lane runs with -k: a red step
|
||||
# above must not hide this suite's own result.
|
||||
- name: Run backport-batch tests
|
||||
if: ${{ !cancelled() && needs.plan.outputs.backport_batch == 'true' }}
|
||||
run: make test-backport-batch
|
||||
- name: Check new Helm document indices
|
||||
if: ${{ !cancelled() }}
|
||||
env:
|
||||
@@ -1432,7 +1461,7 @@ jobs:
|
||||
description = `release packages candidate did not verify (result: ${CANDIDATE_RESULT})`;
|
||||
} else if (CODE !== 'true') {
|
||||
state = 'success';
|
||||
description = 'docs-only PR — e2e not required';
|
||||
description = 'docs or CI-only tools — e2e not required';
|
||||
} else if (IS_RELEASE === 'true' && IS_FULL_E2E !== 'true') {
|
||||
// Release promotion PRs deliberately do not repeat e2e: the rc was
|
||||
// already verified against the immutable candidate whose digests
|
||||
|
||||
@@ -36,7 +36,7 @@ This file provides structured guidance for AI coding assistants and agents worki
|
||||
|
||||
- **E2E tests and E2E CI** (e.g., "write/fix an e2e test", "stabilize a flaky test", "add a bats test", "change the e2e workflow", "why did e2e fail")
|
||||
- Read: [`e2e-testing.md`](./docs/agents/e2e-testing.md)
|
||||
- Action: Read the entire file and follow the conventions exactly — no new retries on deterministic steps, event-driven backstops before every `kubectl wait`, no test-level `EXIT`/`RETURN` traps (a self-contained one inside a Chainsaw `script` step or an explicit subshell is fine — see the doc), no assertion beginning with `!` (errexit exempts an inverted status, so it cannot fail — write `if cmd; then echo "FAIL: …"; false; fi`), fail-fast on HelmRelease readiness
|
||||
- Action: Read the entire file and follow the conventions exactly — no new retries on deterministic steps, event-driven backstops before every `kubectl wait`, no test-level `EXIT`/`RETURN` traps (a self-contained one inside a Chainsaw `script` step or an explicit subshell is fine — see the doc), no assertion beginning with `!` (errexit exempts an inverted status, so it can be silently ignored — write `if cmd; then echo "FAIL: …"; false; fi`), fail-fast on HelmRelease readiness
|
||||
|
||||
- **General questions about contributing**
|
||||
- Read: [`contributing.md`](./docs/agents/contributing.md)
|
||||
@@ -51,10 +51,22 @@ This file provides structured guidance for AI coding assistants and agents worki
|
||||
- PR titles: a Conventional Commits header (`type(scope): description`, types from [`contributing.md`](./docs/agents/contributing.md)) auto-applies `kind/*` and `area/*` via `.github/workflows/pr-labeler.yaml`. Append `!` (or add a `BREAKING CHANGE:` footer) to apply `kind/breaking-change`
|
||||
|
||||
**Important rules:**
|
||||
|
||||
- **Match scope, then read first.** Load a process file only when the task genuinely falls in its scope, and load it before starting work — not midway through. Tasks outside a file's scope should not pull it into context.
|
||||
- **Follow it exactly.** Do not assume you already know the process, and do not skip steps (`changelog.md` especially). Where the documentation and your instinct disagree, the documentation wins.
|
||||
- **[`overview.md`](./docs/agents/overview.md) is the standing reference** for structure and conventions, useful even when no route above points at it.
|
||||
|
||||
## Working Principles
|
||||
|
||||
These hold for every task, whichever document the routes above point at.
|
||||
|
||||
- **A causal claim needs a mechanism.** A failure that appeared after a commit, a bump or a rerun was not thereby caused by it, and matching symptoms prove nothing either: unrelated faults often look identical from outside. Back a root cause with a log line, a reproducer, a bisect, or a ruled-out alternative; without one, name at least two hypotheses and what would tell them apart. Before tying a problem to a change, check whether it predates the change (an older run, a condition's `lastTransitionTime`, `git blame`). Close a hypothesis when evidence rules it out. Missing evidence means inconclusive, not coincidence: record that limit, stop treating the hypothesis as a cause, and revisit it only with new evidence or a concrete discriminating check.
|
||||
- **Verify, do not recall.** Versions, CLI flags, API fields and defaults drift faster than a model's training data. Check the upstream source, the documentation at the pinned version, or `--help` before writing code or advice that depends on them. When a problem is not solved after two attempts, stop and research it instead of trying a third variation.
|
||||
- **Long flag names.** New and changed commands in docs, scripts, CI and PR text spell options out (`--namespace`, `--output`, `--filename`, `--message`), so that a reader does not need to know what `-f` means to this particular tool. An option with no long form is the only exception.
|
||||
- **Name the cluster.** Management and tenant clusters are different targets. Every `kubectl` call, including reads, carries `--context`; cluster-facing Helm commands carry `--kube-context`. Use the human-approved target and the task's kubeconfig where one was supplied, not the ambient current context. Before running wrappers or cluster-facing tests, verify that they propagate the chosen target. Tests that need no cluster must not inherit live cluster credentials.
|
||||
- **Respect the reader's time.** Lead with the answer or decision; prefer direct wording over formalities, filler and repeated conclusions. Put necessary detail after a short summary or in a linked document.
|
||||
- **Public text.** Everything that reaches GitHub (commits, PR and issue bodies, review comments, code comments) is in English. It names no private infrastructure (cluster, customer or internal namespace names) and none of an agent's private tooling (custom skills, slash commands). Nothing is published on someone's behalf without their approval, unless they have delegated it.
|
||||
|
||||
## Project Overview
|
||||
|
||||
**Cozystack** is a Kubernetes-based platform for building cloud infrastructure with managed services (databases, VMs, K8s clusters), multi-tenancy, and GitOps delivery.
|
||||
@@ -62,6 +74,7 @@ This file provides structured guidance for AI coding assistants and agents worki
|
||||
## Quick Reference
|
||||
|
||||
### Code Structure
|
||||
|
||||
- `packages/core/` - Core platform charts (installer, platform)
|
||||
- `packages/system/` - System components (CSI, CNI, operators)
|
||||
- `packages/apps/` - User-facing applications in catalog
|
||||
@@ -70,6 +83,7 @@ This file provides structured guidance for AI coding assistants and agents worki
|
||||
- `api/` - Kubernetes CRDs
|
||||
|
||||
### Conventions
|
||||
|
||||
- **Helm Charts**: Umbrella pattern, vendored upstream charts in `charts/`
|
||||
- **Go Code**: Controller-runtime patterns, kubebuilder style
|
||||
- **Git Commits**: Conventional Commits (`type(scope): description`) with `--signoff`; an agent-assisted commit adds one `Assisted-by: LLM` trailer
|
||||
@@ -77,6 +91,7 @@ This file provides structured guidance for AI coding assistants and agents worki
|
||||
- **Prose**: One continuous line per paragraph — see [Prose Formatting](#prose-formatting)
|
||||
|
||||
### What NOT to Do
|
||||
|
||||
- ❌ Edit `/vendor/`, `zz_generated.*.go`, upstream charts directly
|
||||
- ❌ Modify `go.mod`/`go.sum` manually (use `go get`)
|
||||
- ❌ Force push to main/master
|
||||
|
||||
@@ -120,7 +120,7 @@ test:
|
||||
make -C packages/core/testing apply
|
||||
make -C packages/core/testing e2e
|
||||
|
||||
unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness test-backport-batch migrations-target-check
|
||||
unit-tests: helm-unit-tests bats-unit-tests bats-posix-compat-tests go-unit-tests go-module-tests rd-presets-check test-check-readiness migrations-target-check
|
||||
|
||||
helm-unit-tests:
|
||||
hack/helm-unit-tests.sh
|
||||
@@ -139,13 +139,14 @@ rd-presets-check:
|
||||
migrations-target-check:
|
||||
hack/check-migrations-target.sh
|
||||
|
||||
# Scoped go test over the cozystack-api surface that this repo owns. Kept
|
||||
# narrow intentionally - running `go test ./...` pulls in generated code
|
||||
# round-trip suites whose behavior depends on tool versions outside this
|
||||
# repo's control (kubebuilder, openapi-gen, etc.) and is better exercised
|
||||
# from their generator workflows.
|
||||
# Go tests of the cozystack-api surface and of the standalone packages and
|
||||
# commands listed here, whose tests need no cluster. Kept narrow
|
||||
# intentionally - running `go test ./...` pulls in generated code round-trip
|
||||
# suites whose behavior depends on tool versions outside this repo's control
|
||||
# (kubebuilder, openapi-gen, etc.) and is better exercised from their
|
||||
# generator workflows.
|
||||
go-unit-tests:
|
||||
go test ./pkg/registry/... ./pkg/config/... ./pkg/cmd/server/...
|
||||
go test ./pkg/registry/... ./pkg/config/... ./pkg/cmd/server/... ./pkg/lineage/... ./pkg/ovnstatus/... ./pkg/reservation/... ./cmd/cozypkg/... ./cmd/check-readiness/...
|
||||
|
||||
# The nested Go modules (image sources and the published API types) are
|
||||
# outside ./... of the root module, so no other target reaches their tests.
|
||||
@@ -158,9 +159,7 @@ go-module-tests:
|
||||
done
|
||||
|
||||
# Go tests for the controllers and supporting packages under ./internal.
|
||||
# Excludes ./pkg/... and ./cmd/... — those are run separately by
|
||||
# go-unit-tests above (pkg subset) and skipped (cmd) until their tests
|
||||
# stabilise. CI schedules this target in the same four-slot make invocation as
|
||||
# CI schedules this target in the same four-slot make invocation as
|
||||
# unit-tests; locally invoke it directly or chain the two targets.
|
||||
test-controllers:
|
||||
go test ./internal/... -count=1
|
||||
@@ -195,7 +194,10 @@ test-controllers-envtest: $(SETUP_ENVTEST)
|
||||
test-check-readiness:
|
||||
go test ./test/check-readiness/ -count=1
|
||||
|
||||
# ./cmd/... is excluded from go-unit-tests, so the batch tool needs its own target.
|
||||
# The batch tool's suite drives real git repositories for minutes, so it is
|
||||
# kept out of go-unit-tests in a target of its own. It is not a prerequisite
|
||||
# of unit-tests: the checks job in pull-requests.yaml runs it only when the
|
||||
# diff touches a path listed in cmd/backport-batch/test-inputs.txt.
|
||||
test-backport-batch:
|
||||
go test ./cmd/backport-batch/... -count=1
|
||||
|
||||
|
||||
Generated
+3
@@ -25,6 +25,9 @@ type ConfigSpec struct {
|
||||
// Deploys the web UI (s3manager) with its Service and its Ingress or HTTPRoute at `<bucket>.<tenant host>`. Set to `false` to run the bucket without them.
|
||||
// +kubebuilder:default:=true
|
||||
Ui bool `json:"ui"`
|
||||
// Serves the bucket's objects to anyone without credentials at `<endpoint>/<bucket>/<key>`. Anonymous callers can read objects but cannot list the bucket or write to it.
|
||||
// +kubebuilder:default:=false
|
||||
PublicRead bool `json:"publicRead"`
|
||||
// Users configuration map.
|
||||
// +kubebuilder:default:={}
|
||||
Users map[string]User `json:"users,omitempty"`
|
||||
|
||||
Generated
+3
-24
@@ -25,10 +25,10 @@ type ConfigSpec struct {
|
||||
// Kubernetes major.minor version to deploy
|
||||
// +kubebuilder:default:="v1.35"
|
||||
Version Version `json:"version"`
|
||||
// External hostname for Kubernetes cluster. Defaults to `<cluster-name>.<tenant-host>` if empty.
|
||||
// External hostname for Kubernetes cluster. Defaults to `kubernetes-<cluster-name>.<tenant-host>` if empty.
|
||||
// +kubebuilder:default:=""
|
||||
Host string `json:"host"`
|
||||
// Which infrastructure provider backs this cluster's worker VMs. `kubevirt` runs them inside this cluster; `proxmox` runs them on an external Proxmox VE cluster through capmox. The control plane is Kamaji either way — this selects the infrastructure half only. Must match the `substrate` of every KubernetesNodes pool attached to this cluster: the pools reference this cluster's infrastructure object by kind, and a mismatch leaves their Machines unreconciled. Switching an existing cluster is not supported; create a new one.
|
||||
// Which infrastructure provider backs this cluster's worker VMs. `kubevirt` runs them inside this cluster; `proxmox` runs them on an external Proxmox VE cluster through capmox. The control plane is Kamaji either way — this selects the infrastructure half only. Every KubernetesNodes pool of this cluster must set the same `substrate`: a pool set to the other one refuses to render once this cluster exists. Switching an existing cluster is not supported; create a new one.
|
||||
// +kubebuilder:default:="kubevirt"
|
||||
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="substrate is immutable"
|
||||
Substrate string `json:"substrate"`
|
||||
@@ -41,9 +41,6 @@ type ConfigSpec struct {
|
||||
// Kubernetes control-plane configuration.
|
||||
// +kubebuilder:default:={}
|
||||
ControlPlane ControlPlane `json:"controlPlane"`
|
||||
// Talos worker image configuration.
|
||||
// +kubebuilder:default:={}
|
||||
Talos Talos `json:"talos"`
|
||||
// OIDC authentication and per-user RBAC for the tenant kube-apiserver. See https://cozystack.io/docs/v1.6/kubernetes/oidc-authentication/ for the operator guide.
|
||||
// +kubebuilder:default:={}
|
||||
Oidc OIDC `json:"oidc"`
|
||||
@@ -53,7 +50,7 @@ type APIServer struct {
|
||||
// Admission plugins the tenant kube-apiserver enables, passed through to KamajiControlPlane `spec.admissionControllers`, which the control plane renders as `--enable-admission-plugins`. A non-empty list replaces the control plane's default set instead of adding to it. The kube-apiserver enables these plugins in addition to its own default ones, so leaving a plugin out does not turn it off; to turn one off, name it in `--disable-admission-plugins` in `controlPlane.apiServer.extraArgs` and keep it out of this list, since the apiserver refuses to start with a plugin in both, and the control plane's default set names several. Empty by default, which keeps that default set, taken from the TenantControlPlane CRD: CertificateApproval, CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, DefaultStorageClass, DefaultTolerationSeconds, LimitRanger, MutatingAdmissionWebhook, NamespaceLifecycle, PersistentVolumeClaimResize, Priority, ResourceQuota, RuntimeClass, ServiceAccount, StorageObjectInUseProtection, TaintNodesByCondition, ValidatingAdmissionWebhook. `NodeRestriction` is not in it.
|
||||
// +kubebuilder:default:={}
|
||||
AdmissionControllers []AdmissionController `json:"admissionControllers,omitempty"`
|
||||
// Extra command-line flags appended to the tenant kube-apiserver, passed through to KamajiControlPlane `spec.apiServer.extraArgs`. For OIDC use `spec.oidc.mode` — this passthrough is the escape hatch for unrelated apiserver flags. The tenant control plane sets each of the following flags itself and drops an entry naming one before the apiserver sees it. The chart moves an `--enable-admission-plugins=` entry into `controlPlane.apiServer.admissionControllers` and a `--kubelet-preferred-address-types=` entry into `controlPlane.kubelet.preferredAddressTypes` while that field is empty, and refuses any other entry naming one of them: `--advertise-address`, `--client-ca-file`, `--enable-admission-plugins`, `--service-cluster-ip-range`, `--kubelet-client-certificate`, `--kubelet-client-key`, `--kubelet-preferred-address-types`, `--proxy-client-cert-file`, `--proxy-client-key-file`, `--requestheader-allowed-names`, `--requestheader-client-ca-file`, `--secure-port`, `--service-account-key-file`, `--service-account-signing-key-file`, `--tls-cert-file`, `--tls-private-key-file`, `--etcd-servers`, `--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, `--etcd-prefix`, `--etcd-compaction-interval`, `--egress-selector-config-file`. When `spec.oidc.mode` is not `None` the chart also renders OIDC flags here, reads what you have written on them, and refuses some combinations outright; the Aggregated API servers section of the Kubernetes OIDC guide on cozystack.io carries which flags, which combinations, and why. Do NOT add legacy `--oidc-*` flags here when `spec.oidc.mode` is not `None`; the chart injects `--authentication-config` and the apiserver refuses to boot with both. Empty by default (no change to current behavior).
|
||||
// Extra command-line flags appended to the tenant kube-apiserver, passed through to KamajiControlPlane `spec.apiServer.extraArgs`. For OIDC use `spec.oidc.mode` — this passthrough is the escape hatch for unrelated apiserver flags. The tenant control plane sets each of the following flags itself and drops an entry naming one before the apiserver sees it. The chart moves an `--enable-admission-plugins=` entry into `controlPlane.apiServer.admissionControllers` and a `--kubelet-preferred-address-types=` entry into `controlPlane.kubelet.preferredAddressTypes` while that field is empty, and refuses any other entry naming one of them: `--advertise-address`, `--client-ca-file`, `--enable-admission-plugins`, `--service-cluster-ip-range`, `--kubelet-client-certificate`, `--kubelet-client-key`, `--kubelet-preferred-address-types`, `--proxy-client-cert-file`, `--proxy-client-key-file`, `--requestheader-allowed-names`, `--requestheader-client-ca-file`, `--secure-port`, `--service-account-key-file`, `--service-account-signing-key-file`, `--tls-cert-file`, `--tls-private-key-file`, `--etcd-servers`, `--etcd-cafile`, `--etcd-certfile`, `--etcd-keyfile`, `--etcd-prefix`, `--etcd-compaction-interval`, `--egress-selector-config-file`. When `spec.oidc.mode` is not `None` the chart also renders OIDC flags here, reads what you have written on them, and refuses some combinations outright; the Aggregated API servers section of the Kubernetes OIDC guide on cozystack.io carries which flags, which combinations, and why. Do NOT add legacy `--oidc-*` flags here when `spec.oidc.mode` is not `None`; the chart injects `--authentication-config` and the apiserver refuses to boot with both. Empty by default.
|
||||
// +kubebuilder:default:={}
|
||||
ExtraArgs []string `json:"extraArgs,omitempty"`
|
||||
// Extra volume mounts added to the tenant kube-apiserver container, passed through to KamajiControlPlane `spec.apiServer.extraVolumeMounts`. Each `name` must reference a volume declared in `extraVolumes`; the chart-managed talos secret volumes cannot be mounted. Each item is a core/v1 VolumeMount. Empty by default.
|
||||
@@ -349,24 +346,6 @@ type Scheduler struct {
|
||||
ResourcesPreset ResourcesPreset `json:"resourcesPreset"`
|
||||
}
|
||||
|
||||
type Talos struct {
|
||||
// Base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash.
|
||||
// +kubebuilder:default:="https://factory.talos.dev"
|
||||
ImageFactoryURL string `json:"imageFactoryURL"`
|
||||
// OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `<installerRepository>/<schematicID>:<version>`. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.
|
||||
// +kubebuilder:default:="factory.talos.dev/installer"
|
||||
InstallerRepository string `json:"installerRepository"`
|
||||
// Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap.
|
||||
// +kubebuilder:default:={}
|
||||
RegistryMirrors k8sRuntime.RawExtension `json:"registryMirrors"`
|
||||
// Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here.
|
||||
// +kubebuilder:default:="ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"
|
||||
SchematicID string `json:"schematicID"`
|
||||
// Talos release used for worker OS image and installer. Must satisfy the chart's Talos<->Kubernetes support matrix against the chosen `version`.
|
||||
// +kubebuilder:default:="v1.13.6"
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
type VeleroAddon struct {
|
||||
// Enable Velero.
|
||||
// +kubebuilder:default:=false
|
||||
|
||||
@@ -154,7 +154,6 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) {
|
||||
in.Proxmox.DeepCopyInto(&out.Proxmox)
|
||||
in.Addons.DeepCopyInto(&out.Addons)
|
||||
in.ControlPlane.DeepCopyInto(&out.ControlPlane)
|
||||
in.Talos.DeepCopyInto(&out.Talos)
|
||||
in.Oidc.DeepCopyInto(&out.Oidc)
|
||||
}
|
||||
|
||||
@@ -570,22 +569,6 @@ func (in *Scheduler) DeepCopy() *Scheduler {
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *Talos) DeepCopyInto(out *Talos) {
|
||||
*out = *in
|
||||
in.RegistryMirrors.DeepCopyInto(&out.RegistryMirrors)
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Talos.
|
||||
func (in *Talos) DeepCopy() *Talos {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(Talos)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *VeleroAddon) DeepCopyInto(out *VeleroAddon) {
|
||||
*out = *in
|
||||
|
||||
+23
-23
@@ -18,11 +18,11 @@ type Config struct {
|
||||
}
|
||||
|
||||
type ConfigSpec struct {
|
||||
// Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-<cluster>`; the KubernetesNodes CR must be named `<cluster>-<pool>`. The pool name (the part after `<cluster>-`) must not collide with a nodeGroup still managed by the parent kubernetes chart (e.g. the default `md0`), or the render fails on ownership conflict.
|
||||
// Name of the parent Kubernetes cluster (the `Kubernetes` CR) in the same namespace this node pool attaches to. Required. The pool's CAPI objects are wired to the parent cluster `kubernetes-<cluster>`; the KubernetesNodes CR must be named `<cluster>-<pool>`. The pool's objects are named `kubernetes-<cluster>-<pool>`; when a MachineDeployment of that name already exists and belongs to another Helm release, the render fails instead of taking it over.
|
||||
// +kubebuilder:default:=""
|
||||
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="cluster is immutable"
|
||||
Cluster string `json:"cluster"`
|
||||
// StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster.
|
||||
// StorageClass for the worker node system disk. When empty, the cluster default applies. Worker VMs live-migrate, so their disks need ReadWriteMany — the RWX access mode is supplied by the chosen StorageClass's CDI StorageProfile — and linstor-csi rejects RWX volumes that are not on a DRBD-backed StorageClass, so prefer a replicated/DRBD class. On an `osImage.builtin` pool this field also decides whether the pool renders at all: CDI takes the storage-layer clone path only when the worker disk and the golden share a StorageClass, and falls back to a host-assisted copy over the pod network across classes, so the chart refuses a pool whose class differs from its golden's. Leaving it empty resolves to the cluster's default StorageClass, and the same comparison is made against that. On a `builtin` pool the resolved class is then written onto the worker disk rather than left for the cluster to fill in at bind time, so a pool that has already cloned keeps binding on the class its disks are on even after an admin moves the default annotation; on the HTTP path an empty class is still left to the cluster. Has no effect on `substrate: proxmox`, where the worker disk is cloned from the VM template onto the template's storage or `proxmox.storage`.
|
||||
// +kubebuilder:default:="replicated"
|
||||
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="storageClass is immutable"
|
||||
StorageClass string `json:"storageClass"`
|
||||
@@ -32,10 +32,10 @@ type ConfigSpec struct {
|
||||
// Maximum number of replicas in this pool. Used as the cluster-autoscaler ceiling.
|
||||
// +kubebuilder:default:=10
|
||||
MaxReplicas int `json:"maxReplicas"`
|
||||
// Virtual machine instance type.
|
||||
// KubeVirt instance type that sizes each worker VM when `resources` is not set. Ignored when `resources.cpu` and `resources.memory` are both set, and therefore on `substrate: proxmox`, which requires them.
|
||||
// +kubebuilder:default:="u1.medium"
|
||||
InstanceType string `json:"instanceType"`
|
||||
// System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events.
|
||||
// System disk size for the worker VM. Carries the Talos OS image, kubelet state, containerd image cache, and any local-path PVCs. How the OS image reaches the disk follows this pool's `osImage`: the default and `osImage.factory` stream the Image Factory raw artifact in over HTTP, while `osImage.builtin` makes the disk a storage-layer CDI clone of a shared golden image, with no HTTP fetch. A `builtin` pool must keep diskSize at or above the golden's storage size (6Gi in the shipped worker image catalog), because a CDI clone target cannot be smaller than its source -- the chart rejects a smaller value at render time rather than letting the clone stall. Sizing a `builtin` pool also has to allow for transient headroom the finished disk does not show: CDI populates a larger target by cloning the golden at the source size and then growing it, so a worker being provisioned occupies a temporary clone PVC at the golden's size on top of this diskSize, and both are multiplied by the replica count of a replicated StorageClass. The golden itself is a separate one-time cost for the whole cluster, not a per-worker one. So the peak is the golden, plus `diskSize + golden storage` for each worker cloning at that moment. A pool whose storage backend cannot spare that peak does not fail: the clone waits for space, the VM never boots, and the reason is visible only in the storage layer's own events. Has no effect on `substrate: proxmox`, where the worker disk is a clone of the VM template's disk and keeps its size.
|
||||
// +kubebuilder:default:="20Gi"
|
||||
DiskSize resource.Quantity `json:"diskSize"`
|
||||
// List of node roles. Each role `r` labels nodes with `node-role.kubernetes.io/<r>`. Use `[ingress-nginx]` for a pool that hosts the tenant ingress-nginx controller.
|
||||
@@ -44,16 +44,16 @@ type ConfigSpec struct {
|
||||
// Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. On the kubevirt substrate `cpu` is written into the VM as spelled and decoded as YAML, so a leading zero is read as octal, as in any Kubernetes manifest: `"010"` is eight cores, and the autoscaler capacity counts eight. The kubelet reservations still read the value as a decimal, so `"010"` reserves what `"10"` does.
|
||||
// +kubebuilder:default:={}
|
||||
Resources Resources `json:"resources,omitempty"`
|
||||
// CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.
|
||||
// CPU limit for the worker VM's virt-launcher `compute` container, which is the ceiling the guest vCPU threads, the QEMU emulator thread and the IO threads share. Distinct from `resources.cpu`, which sizes the guest itself. Cannot be combined with `instanceType` sizing: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.limits.cpu`, so the chart rejects that combination at render time. Sizing the pool by `resources` (both `cpu` and `memory`) is what drops the instancetype from the VM, which is where the limit is usable. When unset, KubeVirt derives the limit from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so a worker sized at N vCPUs runs under a ceiling of N with the emulation overhead taken out of the same N. Setting a value declares the limit on the VM directly, which is both what lands it on the container and what stops KubeVirt deriving one. It moves the CPU request with it: KubeVirt copies a declared CPU limit into the CPU request when the VM declares no request of its own, so a pool that sets only this field asks the scheduler for its whole ceiling instead of the fraction of the vCPU count it asked for before, and a ceiling the nodes cannot spare leaves the workers Pending on `Insufficient cpu`. Pair it with `podCpuRequest` to move the ceiling without moving the request. Zero means no ceiling at all rather than the tightest one, since the kubelet turns a zero CPU limit into no CFS quota while the declared key still stops KubeVirt deriving one — but it is only accepted together with `podCpuRequest: 0`, because KubeVirt keeps deriving a request from the vCPU count when the VM declares none and a request above a declared limit of zero is rejected when the worker's Pod is created. A negative value is refused at render time. Changing it rolls the pool, because the worker VM template is named by a hash of its content.
|
||||
PodCpuLimit resource.Quantity `json:"podCpuLimit,omitempty"`
|
||||
// CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content. Keep in sync with the parent kubernetes chart.
|
||||
// CPU request for the worker VM's virt-launcher `compute` container, which is what the scheduler reserves for a worker on its host and what weights it against neighbours on a contended one. Distinct from `resources.cpu`, which sizes the guest itself. It exists to be paired with `podCpuLimit`: KubeVirt copies a declared CPU limit into the request when the VM declares no request, so raising the ceiling alone raises what every worker in the pool asks the scheduler for, and this field is what holds the request still while the ceiling moves. Cannot be combined with `instanceType` sizing, for the same reason as `podCpuLimit`: KubeVirt refuses a VM that references an instancetype and also sets `domain.resources.requests.cpu`, so the chart rejects that combination at render time, as it does a value above `podCpuLimit`. Usable without a limit, where it only changes the scheduler's weighting: KubeVirt still derives the ceiling from the guest vCPU count in any namespace carrying a ResourceQuota that limits CPU, so keep the request at or below that count. A request above that derived ceiling is refused only when the worker's Pod is created, and deliberately not at render time: the ceiling exists only where such a ResourceQuota does, so refusing here would take away a configuration that works in a namespace without one. Zero is accepted on its own and reserves nothing, but zero beside a `podCpuLimit` is refused, because KubeVirt cannot tell an explicit zero request from an absent one and replaces both with the limit — which is the outcome this field exists to avoid, arrived at silently. A negative value is refused as well. When unset, KubeVirt computes the request from the guest vCPU count and the cluster CPU allocation ratio. Changing it rolls the pool, because the worker VM template is named by a hash of its content.
|
||||
PodCpuRequest resource.Quantity `json:"podCpuRequest,omitempty"`
|
||||
// List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).
|
||||
// +kubebuilder:default:={}
|
||||
Gpus []GPU `json:"gpus,omitempty"`
|
||||
// Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later. The automatic set is itself such a change, on the upgrade that introduces this field, for every existing pool that holds an `nvidia.com/*` GPU and leaves the field unset: that pool rolls.
|
||||
// Kernel modules loaded on every worker in this pool, emitted as Talos `machine.kernel.modules`. A Talos system extension installs a module but does not load it, so an extension-provided driver needs its modules declared here. Leave unset to let the chart decide: a pool holding at least one `nvidia.com/*` GPU gets `nvidia` (with `NVreg_NvLinkDisable=1`, which the gpu-operator driver container applies on non-Talos workers; on Talos that container has to be turned off or it clashes with the system extension, and nothing then mounts the ConfigMap carrying the parameter), `nvidia_uvm`, `nvidia_drm`, `nvidia_modeset` (that order — Talos loads the list in sequence and the last three depend on the first), and any other pool gets nothing. Set a non-empty list to replace the chart's choice entirely, or `[]` to opt out and emit no modules even on a GPU pool. The field carries no default, and unset is not the same value as `[]`: a generated parameter table that shows `[]` in its Default column is printing what it prints for any array with no default, not the effective value — leaving the field out selects the automatic set, writing `[]` disables it. The module still has to be in the image, and which extensions the image carries is the pool's schematic: a GPU pool points `osImage.factory.schematicID` (or `osImage.builtin.schematicID`) at a schematic with the NVIDIA extensions, and on Blackwell (GB202) that must be the open-kernel-modules extension. The automatic set keys on the `nvidia.com/` resource prefix alone and nothing cross-checks it against that schematic; on Talos v1.13 a declared module the image does not ship leaves its controller retrying rather than failing the boot, so the symptom is a missing driver, not a dead node. This value is part of the worker machine config, whose content hash names the `TalosConfigTemplate`, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.
|
||||
KernelModules []KernelModule `json:"kernelModules,omitempty"`
|
||||
// Kubelet resource reservations for this pool.
|
||||
// Kubelet resource reservations for this pool. An empty reservation is computed from the worker's size: `resources` when set, otherwise the `instanceType`.
|
||||
// +kubebuilder:default:={}
|
||||
Kubelet Kubelet `json:"kubelet,omitempty"`
|
||||
// Stream each worker VM's guest serial console into a `guest-console-log` container beside virt-launcher, readable with `kubectl logs`. The only view of a worker that stalls during boot before Talos apid answers, when neither a Node nor a certificate request exists to diagnose from. Two costs the field name does not show. Changing the value rolls the pool, because it is part of the worker VM template whose name is a hash of its content -- including the stuck VM whose console was wanted, so it buys visibility into the next boot rather than the current one. And it overrides the platform's cluster-wide `disableSerialConsoleLog`, set because that container has been seen holding virt-launcher in `PodInitializing` (kubevirt/kubevirt#15989), which would take the pool down rather than explain it -- so check that virt-launcher Pods reach Running after enabling it.
|
||||
@@ -62,25 +62,25 @@ type ConfigSpec struct {
|
||||
// Maximum number of unhealthy nodes tolerated in this pool before remediation is paused. The MachineHealthCheck admission webhook accepts either a bare integer ("0", "1", ...) or a percentage ("0%", "50%"); bare numeric strings are rejected, so the safer default is a percentage. Drop to "0%" once the pool is stable. Watch how a percentage behaves in a small group: the MachineHealthCheck controller scales it against the number of machines the check covers and rounds down, then remediates only while the unhealthy count is at or below that result. At "50%" a group of one therefore never remediates, because half of one rounds down to zero, and a group of two remediates one unhealthy machine but not two.
|
||||
// +kubebuilder:default:="50%"
|
||||
MaxUnhealthy string `json:"maxUnhealthy"`
|
||||
// Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (Talos image fetch from factory.talos.dev or a busy storage class on the kubevirt-csi PVC populator). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.
|
||||
// Maximum time a Machine is allowed to spend reaching the Ready condition before it is remediated. Raise for slow first boots (the Talos image import from the Image Factory, or a storage class slow to provision or clone the worker disk). The MachineHealthCheck measures it from the later of the Machine's creation and its InfrastructureReady transition, so the clock restarts when the infrastructure reports ready: a worker gets this long for the disk image import and the VM to come up, and then this long again for the guest to boot and its kubelet to register with the tenant apiserver. Either leg stretches on a host under load, and a Machine remediated here is deleted before it ever joins, so its replacement starts the same sequence from the beginning.
|
||||
// +kubebuilder:default:="20m"
|
||||
NodeStartupTimeout string `json:"nodeStartupTimeout"`
|
||||
// How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.
|
||||
// How long a worker Node may report `Ready=Unknown` before the MachineHealthCheck remediates the Machine, which for a KubeVirt worker means deleting the VM. `Ready=Unknown` is what the node lifecycle controller writes once it stops hearing from a kubelet, on a grace period of its own, so the wall clock from a guest going quiet to a deleted worker is this value plus that grace period. A guest starved of CPU stops answering long before it is dead, so a value of a few tens of seconds deletes workers that would have come back on their own. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.
|
||||
// +kubebuilder:default:="5m"
|
||||
ReadyUnknownTimeout string `json:"readyUnknownTimeout"`
|
||||
// How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips. Keep in sync with the parent kubernetes chart.
|
||||
// How long a worker Node may report `Ready=False` before the MachineHealthCheck remediates the Machine. A Node that has registered but has no working CNI yet reports `Ready=False`, so this value is the whole budget a first boot gets for the CNI rollout to reach that node, and a worker deleted mid-rollout takes the rollout with it while its replacement starts over. Raising it delays replacing a worker that is genuinely dead, but not rescheduling its workload: Pods leave an unreachable Node on the node lifecycle controller's taint-based eviction and the toleration that carries, not on this timer. Validated at render time like maxNodeProvisionTime: the CAPI webhook does not check these timeouts, and a zero or negative value would remediate the moment the condition flips.
|
||||
// +kubebuilder:default:="10m"
|
||||
ReadyFalseTimeout string `json:"readyFalseTimeout"`
|
||||
// How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours ("30m", "1h30m"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to ("20m0s", "1h0m0s") is accepted. Keep in sync with the parent kubernetes chart.
|
||||
// How long the cluster-autoscaler waits for a newly created worker to register as a Node before it treats that machine as long-unregistered. Rendered onto this pool's MachineDeployment as the `cluster.x-k8s.io/autoscaling-options-maxnodeprovisiontime` annotation, which overrides the autoscaler's built-in default for this pool only. Two things change when the timer fires, and both make a slow join worse rather than better: the machine stops counting towards the pool's upcoming capacity, so the autoscaler asks for a replacement for capacity that is already on its way, and the machine becomes eligible for removal. Size it above the slowest join a healthy worker can take, counting the Talos image import, the guest boot and the CNI rollout. Accepted as a whole number of seconds, minutes or hours ("30m", "1h30m"), which is narrower than Go's duration syntax on purpose: the autoscaler ignores a value it cannot parse and falls back to its own default without failing, and the narrower shape cannot express a value that overflows the parser or a fraction it rounds to zero. The canonical form a duration serializes back to ("20m0s", "1h0m0s") is accepted.
|
||||
// +kubebuilder:default:="30m"
|
||||
MaxNodeProvisionTime string `json:"maxNodeProvisionTime"`
|
||||
// Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool -- and must satisfy the Talos<->Kubernetes support matrix against `talos.version`.
|
||||
// Kubernetes major.minor version the pool joins. Must not be ahead of the parent cluster's minor (workers may not lead the apiserver); it may lag behind during a rolling upgrade -- bump the parent Kubernetes CR first, then each pool. The render fails when it is ahead. Its pairing with the Talos release the pool resolves (`talos.version`, or the `osImage` override) is checked against the chart's Talos<->Kubernetes support matrix: for a Talos minor the matrix lists, a Kubernetes minor outside that minor's list is refused; a Talos minor the matrix does not list is not checked.
|
||||
// +kubebuilder:default:="v1.35"
|
||||
Version Version `json:"version"`
|
||||
// Worker OS image source for this pool. Default (omitted): import over HTTP from the `talos.*` Image Factory. Set `osImage.builtin` to clone a golden from the opt-in worker image catalog instead. Editing this on an existing pool re-images that pool.
|
||||
// +kubebuilder:default:={}
|
||||
OsImage WorkerImage `json:"osImage,omitempty"`
|
||||
// Talos worker image configuration. Keep in sync with the parent cluster's `talos`.
|
||||
// Talos worker image configuration for this pool.
|
||||
// +kubebuilder:default:={}
|
||||
Talos Talos `json:"talos"`
|
||||
// Which infrastructure provider backs this pool's worker VMs. `kubevirt` runs them inside this cluster on KubeVirt, sized by `instanceType` and booted from a Talos disk image CDI streams from the image factory. `proxmox` runs them on an external Proxmox VE cluster through capmox, sized by `resources`, and cloned from a Talos VM template that already exists on the hypervisor. The two substrates take different Talos platform images and different sizing inputs, so switching an existing pool is refused: the chart reads the live MachineDeployment's template kind and fails the render rather than rolling every worker to the other hypervisor and deleting the template the draining MachineSet still references. Create a new pool on the other substrate and scale this one down.
|
||||
@@ -111,13 +111,13 @@ type Kubelet struct {
|
||||
// Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).
|
||||
// +kubebuilder:default:="10%"
|
||||
EvictionSoftMemory string `json:"evictionSoftMemory,omitempty"`
|
||||
// CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.
|
||||
// CPU reserved for kubelet and container runtime. When empty, 5% of the worker's CPU, clamped to 50m..500m.
|
||||
KubeReservedCpu string `json:"kubeReservedCpu,omitempty"`
|
||||
// Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.
|
||||
// Memory reserved for kubelet and container runtime. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.
|
||||
KubeReservedMemory string `json:"kubeReservedMemory,omitempty"`
|
||||
// CPU reserved for host OS. Auto-computed from instanceType if empty.
|
||||
// CPU reserved for host OS. When empty, 5% of the worker's CPU, clamped to 50m..500m.
|
||||
SystemReservedCpu string `json:"systemReservedCpu,omitempty"`
|
||||
// Memory reserved for host OS. Auto-computed from instanceType if empty.
|
||||
// Memory reserved for host OS. When empty, 5% of the worker's memory, clamped to 256Mi..1Gi.
|
||||
SystemReservedMemory string `json:"systemReservedMemory,omitempty"`
|
||||
}
|
||||
|
||||
@@ -160,13 +160,13 @@ type Resources struct {
|
||||
}
|
||||
|
||||
type Talos struct {
|
||||
// Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there.
|
||||
// Complete PEM certificate block(s) CDI must trust when importing this pool's worker OS disk image over HTTPS — the no-osImage default and `osImage.factory` path. Empty by default, so the importer uses only its system trust store, which is what the public factory needs. Set this when the factory in use (`imageFactoryURL` or `osImage.factory.imageFactoryURL`) is a self-hosted factory or mirror served by an internal CA, which the importer otherwise rejects with `x509: certificate signed by unknown authority`. The chart renders it into a ConfigMap in the release namespace — CDI requires certConfigMap to sit alongside the DataVolume — and references that from the worker DataVolume's `source.http.certConfigMap`; CDI appends it to the system pool rather than replacing it, so a publicly-trusted endpoint verifies either way. A pool on `osImage.builtin` clones a golden image and builds no HTTP source, so this field has no effect there. On `substrate: proxmox` it has no effect on the workers either: the ConfigMap is still rendered, but nothing references it.
|
||||
// +kubebuilder:default:=""
|
||||
ImageFactoryCA string `json:"imageFactoryCA,omitempty"`
|
||||
// Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal.
|
||||
// Default base URL of the Talos Image Factory that serves the worker OS disk image (the `openstack-amd64.raw.xz` raw artifact streamed in by CDI over HTTP). Applies only when this pool imports over HTTP — the default and `osImage.factory`, which overrides it via `osImage.factory.imageFactoryURL`. A pool on `osImage.builtin` clones a golden image from the worker image catalog and does not use this field at all. Defaults to the public factory. Point at a self-hosted Image Factory, a caching mirror, or an internal HTTP file server for air-gapped, rate-limited, or flaky-egress environments. No trailing slash. It is interpolated into the worker DataVolume source URL, so on those HTTP paths it must be a plain `http(s)` URL and the render refuses anything else; a pool on `osImage.builtin` builds no such URL and is not held to it, so blanking this field there is legal. On `substrate: proxmox` the workers clone a VM template that already carries the OS, so this field has no effect there beyond that format check.
|
||||
// +kubebuilder:default:="https://factory.talos.dev"
|
||||
ImageFactoryURL string `json:"imageFactoryURL"`
|
||||
// OCI repository prefix for the Talos installer image used by the in-guest `talos-reconcile` upgrade Job. Resolved as `<installerRepository>/<schematicID>:<version>` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.
|
||||
// OCI repository prefix for the Talos installer image, written into the worker machine config as `machine.install.image`. Resolved as `<installerRepository>/<schematicID>:<version>` against whatever this pool's `osImage` pins, falling back to `talos.schematicID` and `talos.version` — so the installer matches the OS the pool actually booted. The prefix itself has no `osImage.*` override, so an air-gapped pool needs it pointed at a reachable registry as well. Defaults to the public factory's installer path. Override for air-gapped or mirrored registries. No trailing slash.
|
||||
// +kubebuilder:default:="factory.talos.dev/installer"
|
||||
InstallerRepository string `json:"installerRepository"`
|
||||
// Talos `machine.registries.mirrors` passthrough for worker nodes: a map of upstream registry host to `{ endpoints: [ ... ] }`. Empty by default, so workers pull container images (the Talos `kubelet` image included) directly from the upstream registry. Point a host such as `ghcr.io` at an in-cluster pull-through mirror for air-gapped, rate-limited, or flaky-egress environments so a worker's boot does not depend on live public egress. Talos still falls back to the upstream registry unless a host also sets `skipFallback: true`, so a mirror alone does not enforce air-gap. This value is part of the worker machine config, whose content hash names the TalosConfigTemplate, so changing it rolls every existing worker in the pool rather than only affecting nodes created later.
|
||||
@@ -175,7 +175,7 @@ type Talos struct {
|
||||
// Talos image-factory schematic ID. Defaults to the cozystack-tested vanilla schematic. Operators using custom schematics (system extensions, kernel args) override here. The pool can pin its own via `osImage.builtin.schematicID` or `osImage.factory.schematicID`; when `osImage` omits it, this value applies. Whichever of the two applies is interpolated into an image URL and a DataVolume name, so it must be a plain lowercase alphanumeric identifier -- the render refuses anything else. A custom schematic on `osImage.builtin` also needs its own catalog entry, since the golden is named from the schematic ID.
|
||||
// +kubebuilder:default:="ce4c980550dd2ab1b17bbf2b08801c7eb59418eafe8f279833297925d67c7515"
|
||||
SchematicID string `json:"schematicID"`
|
||||
// Default Talos release for this pool's worker OS image and in-guest installer. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix rejects a pairing it lists as unsupported; it lists the Talos minors this chart ships and passes any other minor unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.
|
||||
// Default Talos release for this pool's worker OS image and for the installer image written into the worker machine config. The pool can pin its own release via `osImage.builtin.version` or `osImage.factory.version`; when `osImage` omits it, this value applies. Both ways in are checked against the chart's Talos<->Kubernetes support matrix, against whichever release the pool actually resolves to, so an override cannot slip past a check `talos.version` would have failed. The matrix lists, for each Talos minor the chart knows, the Kubernetes minors it supports and refuses a pool `version` outside that list; a Talos minor it does not list passes unchecked, so pointing either field at a Talos release the chart does not know about is the operator's call, not a checked one. Whichever release applies is interpolated into an image URL and a DataVolume name, so it must be a full `vMAJOR.MINOR.PATCH` release, optionally with a pre-release suffix -- the render refuses anything else, a minor-only `v1.13` included. The values schema types this as a bare string, so a malformed one is accepted by the CR and fails the pool's HelmRelease instead.
|
||||
// +kubebuilder:default:="v1.13.6"
|
||||
Version string `json:"version"`
|
||||
}
|
||||
@@ -195,7 +195,7 @@ type WorkerImageBuiltin struct {
|
||||
}
|
||||
|
||||
type WorkerImageFactory struct {
|
||||
// Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. NOTE: this redirects only the worker boot-disk import; the in-guest upgrade installer image is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.
|
||||
// Base URL of the Image Factory serving the boot raw image. Defaults to `talos.imageFactoryURL`. No trailing slash. This redirects only the worker boot-disk import; the installer image in the worker machine config is always sourced from `talos.installerRepository`, so an air-gapped pool must also have `talos.installerRepository` pointed at a reachable registry.
|
||||
ImageFactoryURL string `json:"imageFactoryURL,omitempty"`
|
||||
// Talos schematic ID. Defaults to `talos.schematicID`.
|
||||
SchematicID string `json:"schematicID,omitempty"`
|
||||
|
||||
Generated
+1
-1
@@ -124,7 +124,7 @@ type Backup struct {
|
||||
// DEPRECATED. See `s3AccessKey`.
|
||||
// +kubebuilder:default:=""
|
||||
S3SecretKey string `json:"s3SecretKey,omitempty"`
|
||||
// Legacy. Cron schedule (CNPG 6-field format) for the chart-emitted ScheduledBackup. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.
|
||||
// Legacy. Cron schedule for the chart-emitted ScheduledBackup, either a standard five-field cron such as `0 2 * * *` for every day at 02:00, or six fields with leading seconds such as `0 0 2 * * *` for the same time. Empty means no chart-managed schedule, which is the recommended setup when a `BackupClass` from `backups.cozystack.io` already drives backup orchestration. In the legacy chart-managed flow `spec.plugins` plus the barman-cloud ObjectStore is rendered when `backup.enabled=true` AND `useSystemBucket=false` AND `destinationPath` is non-empty AND inline-or-external creds are supplied; in the platform `useSystemBucket=true` flow the chart skips emitting `spec.plugins` and the CNPG driver SSA-applies the ObjectStore and patches `spec.plugins` onto the live Cluster at first BackupJob time.
|
||||
// +kubebuilder:default:=""
|
||||
Schedule string `json:"schedule,omitempty"`
|
||||
// Opt-in: when true, the chart-emitted `<release>-s3-creds` Secret is skipped AND `spec.plugins` (plus the barman-cloud ObjectStore) is left UNSET in the chart-rendered Cluster — the cozy-default BackupClass driver SSA-applies an ObjectStore (carrying destinationPath/endpointURL/credentials) and patches `spec.plugins` on the live Cluster when the first BackupJob runs. Consequence: plugin WAL archiving is NOT active until that first BackupJob fires; WAL accumulates on the PVC in the meantime, so fire an ad-hoc BackupJob immediately after enabling the flag on existing releases. Use together with the platform `cozy-default` BackupClass — tenants do not need to fill `s3AccessKey`/`s3SecretKey` or `destinationPath`/`endpointURL`. The destination path automatically scopes to `s3://cozy-backups/<namespace>/<release>/`.
|
||||
|
||||
Generated
+11
@@ -51,6 +51,14 @@ type ConfigSpec struct {
|
||||
// Redis `min-replicas-max-lag`, in seconds: a replica whose last acknowledgement is older than this does not count towards `minReplicasToWrite`. Unset by default, which leaves the Redis default (10). Has no effect unless `minReplicasToWrite` is set. Must be at least 1 when `minReplicasToWrite` is set: Redis treats 0 as disabling the check, so the chart refuses it.
|
||||
// +kubebuilder:validation:Minimum=0
|
||||
MinReplicasMaxLag *int `json:"minReplicasMaxLag,omitempty"`
|
||||
// Redis `maxmemory`: the dataset size at which Redis applies `maxmemoryPolicy`. Unset by default, which renders `maxmemory 0` (no limit, the Redis default), so a value set by hand with `CONFIG SET` is reset on the next reconcile. Must be at most 90% of the memory request of each Redis pod (from `resources` or `resourcesPreset`): the operator silently drops a larger value, so the chart refuses it. Leave headroom below that for replication and snapshot buffers.
|
||||
Maxmemory resource.Quantity `json:"maxmemory,omitempty"`
|
||||
// Redis `maxmemory-policy`: what Redis does once `maxmemory` is reached. `noeviction` refuses writes; `allkeys-lru` evicts the least recently used keys, which suits an instance used purely as a cache. Has no effect while `maxmemory` is unset.
|
||||
// +kubebuilder:default:="noeviction"
|
||||
MaxmemoryPolicy MaxmemoryPolicy `json:"maxmemoryPolicy"`
|
||||
// Write RDB snapshots to the persistent volume (the Redis `save` directive, with the Redis default schedule). Set to `false` for an instance used purely as a cache: `save ""` is rendered and the volume is kept, so no further scheduled snapshot is written while replicas still have room for the full-sync RDB they write to disk. An RDB file already on the volume is not removed, and Redis loads it on restart, so turning snapshots off does not by itself empty the dataset. Always off when `size` is empty.
|
||||
// +kubebuilder:default:=true
|
||||
Snapshots bool `json:"snapshots"`
|
||||
// Sentinel configuration. When `sentinel.resources` is empty and `sentinel.resourcesPreset` is unset, the Sentinel pods are sized like the Redis pods (`resources` and `resourcesPreset`). An empty `sentinel.resources` map counts as unset, as an empty top-level `resources` does.
|
||||
// +kubebuilder:default:={}
|
||||
Sentinel Sentinel `json:"sentinel,omitempty"`
|
||||
@@ -80,6 +88,9 @@ type TLS struct {
|
||||
// +kubebuilder:validation:Enum="no";"optional";"yes"
|
||||
type AuthClients string
|
||||
|
||||
// +kubebuilder:validation:Enum="noeviction";"allkeys-lru";"allkeys-lfu";"allkeys-random";"volatile-lru";"volatile-lfu";"volatile-random";"volatile-ttl"
|
||||
type MaxmemoryPolicy string
|
||||
|
||||
// +kubebuilder:validation:Enum="t1.nano";"t1.micro";"t1.small";"t1.medium";"t1.large";"t1.xlarge";"t1.2xlarge";"t1.4xlarge";"c1.nano";"c1.micro";"c1.small";"c1.medium";"c1.large";"c1.xlarge";"c1.2xlarge";"c1.4xlarge";"s1.nano";"s1.micro";"s1.small";"s1.medium";"s1.large";"s1.xlarge";"s1.2xlarge";"s1.4xlarge";"u1.nano";"u1.micro";"u1.small";"u1.medium";"u1.large";"u1.xlarge";"u1.2xlarge";"u1.4xlarge";"m1.nano";"m1.micro";"m1.small";"m1.medium";"m1.large";"m1.xlarge";"m1.2xlarge";"m1.4xlarge";"nano";"micro";"small";"medium";"large";"xlarge";"2xlarge"
|
||||
type ResourcesPreset string
|
||||
|
||||
|
||||
@@ -66,6 +66,7 @@ func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) {
|
||||
*out = new(int)
|
||||
**out = **in
|
||||
}
|
||||
out.Maxmemory = in.Maxmemory.DeepCopy()
|
||||
in.Sentinel.DeepCopyInto(&out.Sentinel)
|
||||
}
|
||||
|
||||
|
||||
@@ -39,6 +39,10 @@ type WorkloadMonitorSpec struct {
|
||||
// that is not operational because a DataVolume it selects is not ready.
|
||||
const WorkloadMonitorReasonDataVolumeNotReady = "DataVolumeNotReady"
|
||||
|
||||
// WorkloadMonitorReasonBucketClaimNotReady is the status Reason of a monitor
|
||||
// that is not operational because a BucketClaim it selects is not provisioned.
|
||||
const WorkloadMonitorReasonBucketClaimNotReady = "BucketClaimNotReady"
|
||||
|
||||
// WorkloadMonitorStatus defines the observed state of WorkloadMonitor
|
||||
type WorkloadMonitorStatus struct {
|
||||
// Operational indicates if the workload meets all operational requirements
|
||||
|
||||
@@ -27,6 +27,8 @@ Requires `git` and an authenticated `gh` on PATH, and a remote called `origin`.
|
||||
|
||||
Build the binary wherever an exit code matters: `go run` turns any non-zero exit into its own `1`, so under it a check that could not be completed looks the same as one that found something missing.
|
||||
|
||||
Output is written before a push or a publish. Once a write to stdout or stderr fails, a closed pipe included, the run changes nothing more: no push, no GitHub write, no local branch move, and no new pick; a pick already in flight still finishes its rollback. It exits 2 where it would have exited 0, saying on stderr, while it still can, that the output was lost. A confirmation lost after a push or a publish fails the run the same way.
|
||||
|
||||
### run
|
||||
|
||||
Switches the current working tree to `backport-batch/<line>-<yyyymmdd>`, created at `origin/<line>`, and picks onto it one original at a time, in the order they merged into `main`, as one unit each on the branch's first-parent line. A pull request merged into `main` with a merge commit `M` goes on as its own commits, `git rev-list --reverse --topo-order --no-merges M^1..M^2`, each picked with `git cherry-pick -x --signoff`, so each keeps its author, message and sign-offs and gains the `(cherry picked from commit …)` line; the merges that brought `main` into its branch are left out, since `main` already has what they brought. A merge commit then joins the picks to the batch: its first parent is the batch before the original, its second the last pick, and its message is `M`'s with a paragraph of its own, `Backported-from: M`. A pull request squash-merged into `main` goes on as one pick. When any commit of an original conflicts, the pick is aborted, the branch goes back to where it was before that original, and the original is recorded with the commit that failed and its files; the run goes on with the next one. A commit the branch already carries is skipped as empty, and an original none of whose commits changes anything is reported as carried. The tree has to be clean. A local branch of that name is reset to where the run starts; when that drops commits only it has, the run first prints its tip and the `git branch` command that restores them, and so does `--defer`. Signing follows your git configuration; hooks are off for the picks and the merge, so each commit carries its message as git writes it.
|
||||
|
||||
@@ -99,7 +99,7 @@ func apiSnapshot(commit string) (apigate.Snapshot, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
defer func() { _ = os.RemoveAll(dir) }()
|
||||
for name, blob := range paths {
|
||||
file := filepath.Join(dir, filepath.FromSlash(name))
|
||||
if err := os.MkdirAll(filepath.Dir(file), 0o700); err != nil {
|
||||
|
||||
@@ -153,7 +153,7 @@ func (a *app) run(args []string) int {
|
||||
}
|
||||
// The scheduled job has half an hour for every line; this says where it
|
||||
// goes.
|
||||
fmt.Fprintf(a.stderr, "prepared %s in %s, %s of it judging partial units and what originals build on\n", l.branch(), time.Since(started).Round(time.Millisecond), b.detecting.Round(time.Millisecond))
|
||||
_, _ = fmt.Fprintf(a.stderr, "prepared %s in %s, %s of it judging partial units and what originals build on\n", l.branch(), time.Since(started).Round(time.Millisecond), b.detecting.Round(time.Millisecond))
|
||||
if drops != nil {
|
||||
return a.redrop(b, drops, reason, *undropArg != "", *push)
|
||||
}
|
||||
@@ -273,7 +273,7 @@ func (a *app) run(args []string) int {
|
||||
continue
|
||||
}
|
||||
}
|
||||
p, err := pickOriginal(v, func(base, head string) ([]string, error) {
|
||||
p, err := a.pickUnit(v, func(base, head string) ([]string, error) {
|
||||
omitted, diag, err := b.omittedEffects(v, base, head)
|
||||
v.addDiag(diag...)
|
||||
return omitted, err
|
||||
@@ -449,20 +449,20 @@ func (a *app) finish(b *batch, r *report, branch string, date time.Time, push, r
|
||||
}
|
||||
body, left := r.body(template)
|
||||
p.body = body
|
||||
fmt.Fprint(a.stdout, body)
|
||||
_, _ = fmt.Fprint(a.stdout, body)
|
||||
skipped := 0
|
||||
for _, items := range r.skipped {
|
||||
skipped += len(items)
|
||||
}
|
||||
for _, rule := range left {
|
||||
for _, v := range r.skipped[rule] {
|
||||
fmt.Fprintf(a.stderr, "skipped by %s: %s %s (%s)\n", ruleName(rule), v.o.url(), terminalSafe(v.o.title), terminalSafe(v.reason))
|
||||
_, _ = fmt.Fprintf(a.stderr, "skipped by %s: %s %s (%s)\n", ruleName(rule), v.o.url(), terminalSafe(v.o.title), terminalSafe(v.reason))
|
||||
}
|
||||
}
|
||||
for _, p := range r.waiting {
|
||||
fmt.Fprintf(a.stderr, "waiting for a maintainer: %s %s changes %s\n", p.v.o.url(), terminalSafe(p.v.o.title), terminalSafe(strings.Join(p.files, ", ")))
|
||||
_, _ = fmt.Fprintf(a.stderr, "waiting for a maintainer: %s %s changes %s\n", p.v.o.url(), terminalSafe(p.v.o.title), terminalSafe(strings.Join(p.files, ", ")))
|
||||
}
|
||||
fmt.Fprintf(a.stderr, "%s at %.12s: %d on the branch, %d deferred, %d waiting for a maintainer, %d skipped\ntitle: %s\nreport: %s\n",
|
||||
_, _ = fmt.Fprintf(a.stderr, "%s at %.12s: %d on the branch, %d deferred, %d waiting for a maintainer, %d skipped\ntitle: %s\nreport: %s\n",
|
||||
branch, head, len(r.onBranch), len(r.deferred)+len(r.held), len(r.waiting), skipped, p.title, doc.ID)
|
||||
if !push {
|
||||
return 0
|
||||
@@ -471,8 +471,8 @@ func (a *app) finish(b *batch, r *report, branch string, date time.Time, push, r
|
||||
// Picking nothing is not a failure of the run; check still reports
|
||||
// whatever the line lacks. Merged and closed batches are never
|
||||
// written, so there is nowhere to keep what the run found.
|
||||
fmt.Fprintf(a.stderr, "%s carries nothing beyond %s; nothing to push\n", branch, b.line.branch())
|
||||
fmt.Fprintf(a.stderr, "no batch pull request is open on %s; this run's classification is not retained\n", b.line.branch())
|
||||
_, _ = fmt.Fprintf(a.stderr, "%s carries nothing beyond %s; nothing to push\n", branch, b.line.branch())
|
||||
_, _ = fmt.Fprintf(a.stderr, "no batch pull request is open on %s; this run's classification is not retained\n", b.line.branch())
|
||||
return 0
|
||||
}
|
||||
if err := a.publish(b, p, rebuilt); err != nil {
|
||||
@@ -529,6 +529,9 @@ func cleanTree() error {
|
||||
// the move drops commits of and how to get them back: a clean tree says
|
||||
// nothing about commits only a local branch has.
|
||||
func (a *app) warnReset(branch, start string) error {
|
||||
if err := a.mayAct(); err != nil {
|
||||
return err
|
||||
}
|
||||
old, err := lookup("refs/heads/" + branch)
|
||||
if err != nil || old == "" || old == start {
|
||||
return err
|
||||
@@ -537,7 +540,11 @@ func (a *app) warnReset(branch, start string) error {
|
||||
if err != nil || kept {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(a.stderr, "resetting the local %s from %s, dropping commits %.12s does not carry; git branch %s-%.12s %s restores them\n", branch, old, start, branch, old, old)
|
||||
// The hint is the only way back to the dropped commits, so the reset does
|
||||
// not run when it could not be written.
|
||||
if _, err := fmt.Fprintf(a.stderr, "resetting the local %s from %s, dropping commits %.12s does not carry; git branch %s-%.12s %s restores them\n", branch, old, start, branch, old, old); err != nil {
|
||||
return fmt.Errorf("not resetting the local %s: %w", branch, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -719,6 +726,9 @@ func (a *app) publish(b *batch, p publication, rebuilt bool) error {
|
||||
}
|
||||
}
|
||||
if pushed != p.head {
|
||||
if err := a.mayAct(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := git(pushArgs(p.branch, p.head, lease)...); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -727,13 +737,13 @@ func (a *app) publish(b *batch, p publication, rebuilt bool) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(name)
|
||||
defer func() { _ = os.Remove(name) }()
|
||||
if b.pr == nil {
|
||||
n, url, err := a.gh.createPull(b.line.branch(), p.branch, p.title, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(a.stderr, "opened draft %s\n", url)
|
||||
_, _ = fmt.Fprintf(a.stderr, "opened draft %s\n", url)
|
||||
if err := a.postReport(n, p.report); err != nil {
|
||||
return fmt.Errorf("posting report %.12s on #%d: %w; the body keeps it, and the next run --push posts it", p.report.doc.ID, n, err)
|
||||
}
|
||||
@@ -745,7 +755,7 @@ func (a *app) publish(b *batch, p publication, rebuilt bool) error {
|
||||
if err := a.gh.editPull(b.pr.Number, p.title, name); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(a.stderr, "updated %s\n", pullURL(b.pr))
|
||||
_, _ = fmt.Fprintf(a.stderr, "updated %s\n", pullURL(b.pr))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -828,7 +838,7 @@ func (a *app) closeBatch(b *batch, p publication) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(name)
|
||||
defer func() { _ = os.Remove(name) }()
|
||||
if err := a.stillOpen(c.PR); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -861,7 +871,7 @@ func (a *app) finishClose(l line, url string, c closing) error {
|
||||
if err := a.deleteBranch(l, c); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(a.stderr, "closed %s: without what is deferred it carries nothing beyond %s; the next batch tries them again\n", url, l.branch())
|
||||
_, _ = fmt.Fprintf(a.stderr, "closed %s: without what is deferred it carries nothing beyond %s; the next batch tries them again\n", url, l.branch())
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -878,6 +888,9 @@ func (a *app) deleteBranch(l line, c closing) error {
|
||||
case h != c.Head:
|
||||
return movedBranch(c, h)
|
||||
}
|
||||
if err := a.mayAct(); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := git(deleteArgs(c.Branch, c.Head)...); err != nil {
|
||||
return fmt.Errorf("deleting %s, left by the close of #%d: %w; backport-batch run --line %s --push deletes it", c.Branch, c.PR, err, l.branch())
|
||||
}
|
||||
@@ -933,7 +946,7 @@ func (a *app) resumeClose(b *batch, push, changing bool) (bool, int) {
|
||||
if err := a.deleteBranch(b.line, c); err != nil {
|
||||
return false, a.fail(err)
|
||||
}
|
||||
fmt.Fprintf(a.stderr, "deleted %s, which the close of #%d left\n", c.Branch, c.PR)
|
||||
_, _ = fmt.Fprintf(a.stderr, "deleted %s, which the close of #%d left\n", c.Branch, c.PR)
|
||||
return false, a.finished(c, changing)
|
||||
}
|
||||
|
||||
|
||||
@@ -73,14 +73,14 @@ func (a *app) check(args []string) int {
|
||||
case d.kind == outcomeWarning:
|
||||
a.warn("%s", d.text)
|
||||
case d.kind == outcomeNote:
|
||||
fmt.Fprintf(a.stderr, "note: %s\n", d.text)
|
||||
_, _ = fmt.Fprintf(a.stderr, "note: %s\n", d.text)
|
||||
}
|
||||
}
|
||||
if len(missing)+len(failing) == 0 {
|
||||
if *cut {
|
||||
fmt.Fprintf(a.stdout, "%s: no change the policy takes is missing before the cut\n", l.branch())
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: no change the policy takes is missing before the cut\n", l.branch())
|
||||
} else {
|
||||
fmt.Fprintf(a.stdout, "%s: every change the policy takes is on the line or on the batch\n", l.branch())
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: every change the policy takes is on the line or on the batch\n", l.branch())
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -89,13 +89,13 @@ func (a *app) check(args []string) int {
|
||||
where = "nor on batch pull request #" + strconv.Itoa(b.pr.Number)
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
fmt.Fprintf(a.stdout, "%s: %d changes the policy takes are not on the line, %s:\n", l.branch(), len(missing), where)
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: %d changes the policy takes are not on the line, %s:\n", l.branch(), len(missing), where)
|
||||
for _, text := range missing {
|
||||
fmt.Fprintf(a.stdout, " %s\n", text)
|
||||
_, _ = fmt.Fprintf(a.stdout, " %s\n", text)
|
||||
}
|
||||
}
|
||||
for _, text := range failing {
|
||||
fmt.Fprintf(a.stdout, "%s: %s\n", l.branch(), text)
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: %s\n", l.branch(), text)
|
||||
}
|
||||
return 1
|
||||
}
|
||||
@@ -326,10 +326,10 @@ func (a *app) fold(args []string) int {
|
||||
return a.fail(err)
|
||||
}
|
||||
if why != "" {
|
||||
fmt.Fprintf(a.stdout, "refused: %s\n", why)
|
||||
_, _ = fmt.Fprintf(a.stdout, "refused: %s\n", why)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintln(a.stdout, "ok")
|
||||
_, _ = fmt.Fprintln(a.stdout, "ok")
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
@@ -102,7 +102,7 @@ func (a *app) reassemble(b *batch, out, hold map[int]string, drops []droppedEntr
|
||||
return a.fail(err)
|
||||
}
|
||||
for _, u := range replay {
|
||||
failed, files, err := replayUnit(u)
|
||||
failed, files, err := a.replayNext(u)
|
||||
switch {
|
||||
case err != nil:
|
||||
return a.fail(err)
|
||||
|
||||
@@ -367,7 +367,7 @@ func (b *batch) alreadyApplied(pre, cur, post string) (applied, unsupported bool
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
defer func() { _ = os.RemoveAll(dir) }()
|
||||
names := map[string]string{}
|
||||
for _, f := range []struct{ name, oid string }{{"cur", cur}, {"pre", pre}, {"post", post}} {
|
||||
names[f.name] = filepath.Join(dir, f.name)
|
||||
|
||||
@@ -53,7 +53,7 @@ func emptyBatch(f *fixture, gh *fakeGH, body string) *foldWorld {
|
||||
// and every check of the batch's shape takes the old name as the same file:
|
||||
// verify, a hand fold, and a rebuild by --defer.
|
||||
func TestRenamePickThroughFoldAndVerify(t *testing.T) {
|
||||
f, mc5 := renameWorld(t)
|
||||
f, _ := renameWorld(t)
|
||||
f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"})
|
||||
gh := &fakeGH{}
|
||||
head := openBatch(t, f, gh)
|
||||
@@ -74,7 +74,7 @@ func TestRenamePickThroughFoldAndVerify(t *testing.T) {
|
||||
t.Errorf("verify after the rebuild = %d\n%s\n%s", code, out, stderr)
|
||||
}
|
||||
|
||||
f, mc5 = renameWorld(t)
|
||||
f, mc5 := renameWorld(t)
|
||||
w := emptyBatch(f, &fakeGH{}, "")
|
||||
pick := w.pick(w.head, f.git("rev-parse", mc5+"^2"), map[string]string{"app/old.txt": strings.Replace(appFile, "three", "THREE", 1)})
|
||||
fold := w.merge(w.head, pick, "fix(app): fix the app file (#5)\n\n"+batchTrailer+": "+mc5)
|
||||
|
||||
@@ -522,15 +522,6 @@ func (f *fixture) batchPR(n int, base, branch, body string) pullRequest {
|
||||
return pr
|
||||
}
|
||||
|
||||
// botBatchPR is batchPR opened by the bot the scheduled batch runs as.
|
||||
func (f *fixture) botBatchPR(gh *fakeGH, n int, base, branch, body string) pullRequest {
|
||||
f.t.Helper()
|
||||
gh.bot = defaultBatchBot
|
||||
pr := f.batchPR(n, base, branch, body)
|
||||
pr.User.Login, pr.User.Type, pr.AuthorAssociation = defaultBatchBot, "Bot", "CONTRIBUTOR"
|
||||
return pr
|
||||
}
|
||||
|
||||
// mergeBatch merges the open batch into release-1.1 with a merge commit, as
|
||||
// GitHub does, and deletes its branch, as the repository's
|
||||
// delete_branch_on_merge does. The pull request stays, merged, with its body
|
||||
|
||||
@@ -137,6 +137,54 @@ type github interface {
|
||||
batchBot() string
|
||||
}
|
||||
|
||||
// guardedGitHub refuses every GitHub write once the run's output was lost.
|
||||
type guardedGitHub struct {
|
||||
github
|
||||
a *app
|
||||
}
|
||||
|
||||
func (g guardedGitHub) createPull(base, head, title, bodyFile string) (int, string, error) {
|
||||
if err := g.a.mayAct(); err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
return g.github.createPull(base, head, title, bodyFile)
|
||||
}
|
||||
|
||||
func (g guardedGitHub) editPull(number int, title, bodyFile string) error {
|
||||
if err := g.a.mayAct(); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.github.editPull(number, title, bodyFile)
|
||||
}
|
||||
|
||||
func (g guardedGitHub) editBody(number int, bodyFile string) error {
|
||||
if err := g.a.mayAct(); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.github.editBody(number, bodyFile)
|
||||
}
|
||||
|
||||
func (g guardedGitHub) closePull(number int) error {
|
||||
if err := g.a.mayAct(); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.github.closePull(number)
|
||||
}
|
||||
|
||||
func (g guardedGitHub) createComment(number int, bodyFile string) error {
|
||||
if err := g.a.mayAct(); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.github.createComment(number, bodyFile)
|
||||
}
|
||||
|
||||
func (g guardedGitHub) editComment(id int64, bodyFile string) error {
|
||||
if err := g.a.mayAct(); err != nil {
|
||||
return err
|
||||
}
|
||||
return g.github.editComment(id, bodyFile)
|
||||
}
|
||||
|
||||
type ghCLI struct{ bot string }
|
||||
|
||||
func (g ghCLI) batchBot() string { return g.bot }
|
||||
|
||||
@@ -142,11 +142,11 @@ func TestOptionInjection(t *testing.T) {
|
||||
// no bidi override reorders it and no zero-width character hides in it.
|
||||
func TestDeferredBlockCleansFileNames(t *testing.T) {
|
||||
l, _ := parseLine("release-1.6")
|
||||
files := []string{"app/txt.exe", "app/zerowidth.yaml"}
|
||||
files := []string{"app/\u202etxt.exe", "app/zero\u200bwidth.yaml"}
|
||||
conflict := pick{v: &verdict{o: &original{number: 5}}, files: files, failed: strings.Repeat("a", 40)}
|
||||
waiting := pick{v: &verdict{o: &original{number: 6}}, files: files}
|
||||
block := deferredBlock(l, []pick{conflict}, nil, []pick{waiting})
|
||||
if strings.ContainsAny(block, "") {
|
||||
if strings.ContainsAny(block, "\u202e\u200b") {
|
||||
t.Errorf("the block carries format characters: %q", block)
|
||||
}
|
||||
want := []string{"app/txt.exe", "app/zerowidth.yaml"}
|
||||
@@ -154,7 +154,7 @@ func TestDeferredBlockCleansFileNames(t *testing.T) {
|
||||
if err != nil || !slices.Equal(got[5].Files, want) || !slices.Equal(got[6].Files, want) {
|
||||
t.Errorf("parseDeferred = %+v, %v; want both originals with %q", got, err, want)
|
||||
}
|
||||
if files[0] != "app/txt.exe" {
|
||||
if files[0] != "app/\u202etxt.exe" {
|
||||
t.Errorf("deferredBlock rewrote the pick's own file names: %q", files)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const modulePath = "github.com/cozystack/cozystack/"
|
||||
|
||||
// readTestInputs returns the entries of test-inputs.txt, the list CI reads to
|
||||
// decide whether a pull request runs these tests.
|
||||
func readTestInputs(t *testing.T) []string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile("test-inputs.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var entries []string
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, line)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
t.Fatal("test-inputs.txt lists no path")
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
// covered reports whether a change to path matches an entry the way the plan
|
||||
// job matches it: a trailing slash is a directory prefix, anything else is
|
||||
// one file.
|
||||
func covered(entries []string, path string) bool {
|
||||
for _, e := range entries {
|
||||
if strings.HasSuffix(e, "/") && strings.HasPrefix(path, e) || path == e {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// A pull request that changes an input of these tests has to run them, and CI
|
||||
// knows the inputs only from test-inputs.txt. An import added here, or added to
|
||||
// a package this tool imports, lands in a pull request that touches a listed
|
||||
// path, so it runs this test, which fails until the new package is listed.
|
||||
func TestInputsListed(t *testing.T) {
|
||||
entries := readTestInputs(t)
|
||||
|
||||
out, err := exec.Command("go", "list", "-deps", "-test", "-f", "{{.ImportPath}}", ".").Output()
|
||||
if err != nil {
|
||||
t.Fatalf("go list: %v", err)
|
||||
}
|
||||
var deps int
|
||||
for _, pkg := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
// A test variant prints as "pkg [pkg.test]"; the synthesized main
|
||||
// package as "pkg.test". Both are this directory.
|
||||
pkg, _, _ = strings.Cut(pkg, " ")
|
||||
pkg = strings.TrimSuffix(pkg, ".test")
|
||||
rel, ok := strings.CutPrefix(pkg, modulePath)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
deps++
|
||||
if !covered(entries, rel+"/x.go") {
|
||||
t.Errorf("these tests import %s, which test-inputs.txt does not list: add %s/", pkg, rel)
|
||||
}
|
||||
}
|
||||
if deps == 0 {
|
||||
t.Fatalf("go list named no package of %s, so nothing was checked:\n%s", modulePath, out)
|
||||
}
|
||||
|
||||
for _, page := range []string{policyPage, "docs/release/backport-tasks/x.md"} {
|
||||
if !covered(entries, page) {
|
||||
t.Errorf("the tests read %s, which test-inputs.txt does not cover", page)
|
||||
}
|
||||
}
|
||||
for _, f := range []string{"go.mod", "go.sum"} {
|
||||
if !covered(entries, f) {
|
||||
t.Errorf("test-inputs.txt does not list %s", f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A listed path that no longer exists makes the list read as broader than it
|
||||
// is, and usually means a dependency moved and its new home is missing.
|
||||
func TestInputsExist(t *testing.T) {
|
||||
for _, e := range readTestInputs(t) {
|
||||
if _, err := os.Stat("../../" + e); err != nil {
|
||||
t.Errorf("test-inputs.txt lists %s: %v", e, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,9 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -111,18 +113,42 @@ type app struct {
|
||||
policyRef string
|
||||
// tool is the commit backport-batch was built from, for the record.
|
||||
tool string
|
||||
// unwritten is the first error writing to stdout or stderr. Output goes
|
||||
// out before a push or a publish, so a run that lost it acts on nothing,
|
||||
// and one that lost a confirmation afterwards fails.
|
||||
unwritten error
|
||||
}
|
||||
|
||||
func main() {
|
||||
catchSIGPIPE()
|
||||
a := &app{gh: ghCLI{bot: batchBotLogin(os.LookupEnv)}, stdout: os.Stdout, stderr: os.Stderr, now: time.Now, tool: toolVersion()}
|
||||
os.Exit(a.main(os.Args[1:]))
|
||||
}
|
||||
|
||||
// catchSIGPIPE makes a write to a closed stdout or stderr fail with EPIPE like
|
||||
// any other write, so the run stops before it pushes or publishes and says
|
||||
// why, instead of dying silently. signal.Ignore would do the same here but
|
||||
// would also leave SIGPIPE ignored in the git and gh children, across exec.
|
||||
func catchSIGPIPE() {
|
||||
signal.Notify(make(chan os.Signal, 1), syscall.SIGPIPE)
|
||||
}
|
||||
|
||||
func (a *app) main(args []string) int {
|
||||
a.stdout, a.stderr = textWriter{a.stdout}, textWriter{a.stderr}
|
||||
a.gh = &permCache{github: a.gh, perms: map[string]string{}}
|
||||
a.stdout, a.stderr = textWriter{w: a.stdout, failed: &a.unwritten}, textWriter{w: a.stderr, failed: &a.unwritten}
|
||||
code := a.command(args)
|
||||
if a.unwritten != nil {
|
||||
_, _ = fmt.Fprintf(a.stderr, "backport-batch: output was lost: %v\n", a.unwritten)
|
||||
if code == 0 {
|
||||
code = 2
|
||||
}
|
||||
}
|
||||
return code
|
||||
}
|
||||
|
||||
func (a *app) command(args []string) int {
|
||||
a.gh = guardedGitHub{github: &permCache{github: a.gh, perms: map[string]string{}}, a: a}
|
||||
if len(args) == 0 {
|
||||
fmt.Fprint(a.stderr, usageText)
|
||||
_, _ = fmt.Fprint(a.stderr, usageText)
|
||||
return 2
|
||||
}
|
||||
if bot := a.gh.batchBot(); bot != "" && !botLoginRE.MatchString(bot) {
|
||||
@@ -144,20 +170,26 @@ func (a *app) main(args []string) int {
|
||||
case "lines":
|
||||
return a.lines(args[1:])
|
||||
case "-h", "--help", "help":
|
||||
fmt.Fprint(a.stdout, usageText)
|
||||
_, _ = fmt.Fprint(a.stdout, usageText)
|
||||
return 0
|
||||
}
|
||||
fmt.Fprintf(a.stderr, "unknown command %q\n\n%s", args[0], usageText)
|
||||
_, _ = fmt.Fprintf(a.stderr, "unknown command %q\n\n%s", args[0], usageText)
|
||||
return 2
|
||||
}
|
||||
|
||||
// textWriter keeps output fit for a terminal. Titles, file names and the
|
||||
// messages git relays come from other people, and a control sequence among
|
||||
// them would move the cursor or rewrite what was printed before.
|
||||
type textWriter struct{ w io.Writer }
|
||||
type textWriter struct {
|
||||
w io.Writer
|
||||
failed *error
|
||||
}
|
||||
|
||||
func (t textWriter) Write(p []byte) (int, error) {
|
||||
if _, err := io.WriteString(t.w, clean(string(p), true)); err != nil {
|
||||
if t.failed != nil && *t.failed == nil {
|
||||
*t.failed = err
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
return len(p), nil
|
||||
@@ -166,7 +198,7 @@ func (t textWriter) Write(p []byte) (int, error) {
|
||||
func (a *app) flags(name string) *flag.FlagSet {
|
||||
fs := flag.NewFlagSet(name, flag.ContinueOnError)
|
||||
fs.SetOutput(a.stderr)
|
||||
fs.Usage = func() { fmt.Fprint(a.stderr, usageText) }
|
||||
fs.Usage = func() { _, _ = fmt.Fprint(a.stderr, usageText) }
|
||||
return fs
|
||||
}
|
||||
|
||||
@@ -188,13 +220,24 @@ func parse(fs *flag.FlagSet, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// mayAct refuses any further change once output was lost: the run would act
|
||||
// on what it printed while nobody read it. Every push and every move of a
|
||||
// local batch branch go through it, and every GitHub write through
|
||||
// guardedGitHub.
|
||||
func (a *app) mayAct() error {
|
||||
if a.unwritten != nil {
|
||||
return fmt.Errorf("output was lost: %w; nothing more is pushed or written to GitHub", a.unwritten)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *app) fail(err error) int {
|
||||
fmt.Fprintf(a.stderr, "backport-batch: %v\n", err)
|
||||
_, _ = fmt.Fprintf(a.stderr, "backport-batch: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
|
||||
func (a *app) warn(format string, args ...any) {
|
||||
fmt.Fprintf(a.stderr, "warning: "+format+"\n", args...)
|
||||
_, _ = fmt.Fprintf(a.stderr, "warning: "+format+"\n", args...)
|
||||
}
|
||||
|
||||
// verdict is what the policy says about one original for the line.
|
||||
|
||||
@@ -102,10 +102,10 @@ func (a *app) notify(args []string) int {
|
||||
var conflicts, waiting []int
|
||||
after := map[int][]int{}
|
||||
for n, e := range deferred {
|
||||
switch place := placement(deferred, n, map[int]bool{}); {
|
||||
case place == "workflow":
|
||||
switch placement(deferred, n, map[int]bool{}) {
|
||||
case "workflow":
|
||||
waiting = append(waiting, n)
|
||||
case place == "conflict":
|
||||
case "conflict":
|
||||
conflicts = append(conflicts, n)
|
||||
}
|
||||
if len(e.Needs) > 0 {
|
||||
@@ -135,11 +135,11 @@ func (a *app) notify(args []string) int {
|
||||
stalled := days >= stalledAfter
|
||||
var text []string
|
||||
if stalled {
|
||||
fmt.Fprintf(a.stdout, "%s: batch pull request #%d has been open since %s, %d days\n", l.branch(), pr.Number, opened.Format(time.DateOnly), days)
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: batch pull request #%d has been open since %s, %d days\n", l.branch(), pr.Number, opened.Format(time.DateOnly), days)
|
||||
text = append(text, fmt.Sprintf("This batch has been open since %s, %d days, through at least one scheduled run. Merge it, or close it and the next run starts a new one; until then each run adds what the rules take to it.", opened.Format(time.DateOnly), days))
|
||||
}
|
||||
if len(partial) > 0 {
|
||||
fmt.Fprintf(a.stdout, "%s: %d units on #%d carry only part of their originals\n", l.branch(), len(partial), pr.Number)
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: %d units on #%d carry only part of their originals\n", l.branch(), len(partial), pr.Number)
|
||||
var b strings.Builder
|
||||
b.WriteString("These are on the batch but carry only part of their originals on this line; take each out with `backport-batch run --defer` before the cut, and port what applies as a pull request of its own:\n")
|
||||
for _, n := range partial {
|
||||
@@ -148,7 +148,7 @@ func (a *app) notify(args []string) int {
|
||||
text = append(text, b.String())
|
||||
}
|
||||
if len(conflicts) > 0 {
|
||||
fmt.Fprintf(a.stdout, "%s: %d deferred conflicts on #%d wait for a maintainer\n", l.branch(), count(conflicts), pr.Number)
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: %d deferred conflicts on #%d wait for a maintainer\n", l.branch(), count(conflicts), pr.Number)
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "These conflicts wait for a maintainer, as [resolving a conflict](%sresolve-conflict.md) describes; unresolved, each goes to the next batch, and so does what waits on one:\n", tasksURL)
|
||||
for _, n := range conflicts {
|
||||
@@ -158,9 +158,9 @@ func (a *app) notify(args []string) int {
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
if count(waiting) == len(waiting) {
|
||||
fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s and wait for a maintainer\n", l.branch(), len(waiting), pr.Number, workflowsDir)
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s and wait for a maintainer\n", l.branch(), len(waiting), pr.Number, workflowsDir)
|
||||
} else {
|
||||
fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s or wait on one that does\n", l.branch(), len(waiting), pr.Number, workflowsDir)
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: %d originals on #%d change %s or wait on one that does\n", l.branch(), len(waiting), pr.Number, workflowsDir)
|
||||
}
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "These change `%s`, which the batch's token may not push, and wait for a maintainer to bring them over, as [changes to workflows](%s#changes-to-workflows) describes, or wait on one that does:\n", workflowsDir, policyURL)
|
||||
@@ -170,7 +170,7 @@ func (a *app) notify(args []string) int {
|
||||
text = append(text, b.String())
|
||||
}
|
||||
if len(parks) > 0 {
|
||||
fmt.Fprintf(a.stdout, "%s: %d parked originals on #%d, named once\n", l.branch(), len(parks), pr.Number)
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: %d parked originals on #%d, named once\n", l.branch(), len(parks), pr.Number)
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "These cannot land on this line whole, so the batch parked them; each is named here once. A maintainer ports what applies as a pull request of its own, as [fixes that exist only on a release line](%s#fixes-that-exist-only-on-a-release-line) describes, or drops the original with `backport-batch run --drop`:\n", policyURL)
|
||||
for _, p := range parks {
|
||||
@@ -229,7 +229,7 @@ func (a *app) announced(pr *pullRequest, rec *lineRecord, parks []parkedEntry) e
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(name)
|
||||
defer func() { _ = os.Remove(name) }()
|
||||
return a.gh.editBody(pr.Number, name)
|
||||
}
|
||||
|
||||
@@ -259,10 +259,10 @@ func placement(deferred map[int]deferredEntry, n int, seen map[int]bool) string
|
||||
seen[n] = true
|
||||
place := ""
|
||||
for _, f := range e.Needs {
|
||||
switch p := placement(deferred, f, seen); {
|
||||
case p == "workflow":
|
||||
switch p := placement(deferred, f, seen); p {
|
||||
case "workflow":
|
||||
return p
|
||||
case p == "conflict":
|
||||
case "conflict":
|
||||
place = p
|
||||
}
|
||||
}
|
||||
@@ -309,9 +309,9 @@ func (a *app) comment(pr *pullRequest, mention string, today time.Time, text []s
|
||||
if err != nil {
|
||||
return a.fail(err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
defer func() { _ = os.Remove(f.Name()) }()
|
||||
if _, err := f.WriteString(b.String()); err != nil {
|
||||
f.Close()
|
||||
_ = f.Close()
|
||||
return a.fail(err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
@@ -354,7 +354,7 @@ func (a *app) lines(args []string) int {
|
||||
return a.fail(err)
|
||||
}
|
||||
for _, l := range slices.Backward(lines) {
|
||||
fmt.Fprintln(a.stdout, l.branch())
|
||||
_, _ = fmt.Fprintln(a.stdout, l.branch())
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -41,12 +41,12 @@ func TestOntoContinues(t *testing.T) {
|
||||
f.lines()
|
||||
head := pushed(t, f)
|
||||
f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"})
|
||||
code, stdout, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head, "--date", "2026-10-29")
|
||||
code, _, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head, "--date", "2026-10-29")
|
||||
if code != 2 || !strings.Contains(stderr, "not the head of backport-batch/release-1.1-20261029") {
|
||||
t.Errorf("--onto with another date = %d (%s), want a refusal", code, stderr)
|
||||
}
|
||||
f.now = f.now.AddDate(0, 0, 3)
|
||||
code, stdout, stderr = f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head)
|
||||
code, stdout, stderr := f.exec(&fakeGH{}, "run", "--line", "release-1.1", "--onto", head)
|
||||
if code != 0 {
|
||||
t.Fatalf("run --onto = %d\n%s", code, stderr)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,385 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type failingWriter struct{}
|
||||
|
||||
func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("broken pipe") }
|
||||
|
||||
// Output that never reached its reader fails the command, and stderr says why.
|
||||
func TestUnwritableOutputFailsTheCommand(t *testing.T) {
|
||||
var stderr strings.Builder
|
||||
a := &app{gh: &fakeGH{}, stdout: failingWriter{}, stderr: &stderr}
|
||||
if code := a.main([]string{"help"}); code != 2 {
|
||||
t.Fatalf("help with an unwritable stdout = %d, want 2", code)
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "broken pipe") {
|
||||
t.Errorf("stderr does not name the write failure:\n%s", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A command that already failed keeps its own exit code when its output is
|
||||
// lost as well: check still says something is missing.
|
||||
func TestUnwritableOutputKeepsTheCommandsFailure(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.lines()
|
||||
f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"})
|
||||
var stderr strings.Builder
|
||||
a := &app{gh: &fakeGH{remoteHead: f.remoteHead}, stdout: failingWriter{}, stderr: &stderr, now: func() time.Time { return f.now }, tool: f.tool}
|
||||
if code := a.main([]string{"check", "--line", "release-1.1"}); code != 1 {
|
||||
t.Fatalf("check with #6 missing and an unwritable stdout = %d, want 1\n%s", code, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "broken pipe") {
|
||||
t.Errorf("stderr does not name the write failure:\n%s", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A closed pipe on stdout does not kill the run before it can say what
|
||||
// already happened: the write fails like any other, and the run exits 2.
|
||||
func TestClosedStdoutPipeIsReported(t *testing.T) {
|
||||
if os.Getenv("BACKPORT_BATCH_CLOSED_STDOUT_HELPER") == "1" {
|
||||
os.Args = []string{"backport-batch", "help"}
|
||||
main()
|
||||
return
|
||||
}
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = r.Close()
|
||||
t.Cleanup(func() { _ = w.Close() })
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestClosedStdoutPipeIsReported$")
|
||||
cmd.Env = append(os.Environ(), "BACKPORT_BATCH_CLOSED_STDOUT_HELPER=1")
|
||||
var stderr strings.Builder
|
||||
cmd.Stdout, cmd.Stderr = w, &stderr
|
||||
err = cmd.Run()
|
||||
var exit *exec.ExitError
|
||||
if !errors.As(err, &exit) || exit.ExitCode() != 2 {
|
||||
t.Fatalf("help into a closed pipe = %v, want exit 2\nstderr:\n%s", err, stderr.String())
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "output was lost") {
|
||||
t.Errorf("stderr does not say the output was lost:\n%s", stderr.String())
|
||||
}
|
||||
}
|
||||
|
||||
// failOn fails every write that contains want, and takes every other one.
|
||||
type failOn struct {
|
||||
want string
|
||||
buf strings.Builder
|
||||
}
|
||||
|
||||
func (w *failOn) Write(p []byte) (int, error) {
|
||||
if strings.Contains(string(p), w.want) {
|
||||
return 0, errors.New("broken pipe")
|
||||
}
|
||||
return w.buf.Write(p)
|
||||
}
|
||||
|
||||
func runWith(f *fixture, gh *fakeGH, stdout, stderr io.Writer, args ...string) int {
|
||||
f.t.Helper()
|
||||
if gh.remoteHead == nil {
|
||||
gh.remoteHead = f.remoteHead
|
||||
}
|
||||
a := &app{gh: gh, stdout: stdout, stderr: stderr, now: func() time.Time { return f.now }, tool: f.tool}
|
||||
return a.main(args)
|
||||
}
|
||||
|
||||
// Output is written before the push; when it is lost, on either stream,
|
||||
// nothing is pushed and no pull request is opened.
|
||||
func TestLostOutputPushesNothing(t *testing.T) {
|
||||
for _, stream := range []string{"stdout", "stderr"} {
|
||||
t.Run(stream, func(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.lines()
|
||||
f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"})
|
||||
gh := &fakeGH{}
|
||||
var stdout, stderr io.Writer = &strings.Builder{}, &strings.Builder{}
|
||||
if stream == "stdout" {
|
||||
stdout = failingWriter{}
|
||||
} else {
|
||||
stderr = failingWriter{}
|
||||
}
|
||||
if code := runWith(f, gh, stdout, stderr, "run", "--line", "release-1.1", "--push"); code != 2 {
|
||||
t.Fatalf("run --push with a failing %s = %d, want 2", stream, code)
|
||||
}
|
||||
if head := f.remoteHead(batchBranch1); head != "" {
|
||||
t.Errorf("the batch was pushed at %.12s", head)
|
||||
}
|
||||
if len(gh.prs) != 0 {
|
||||
t.Errorf("a pull request was opened: %+v", gh.prs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A confirmation lost after the push fails the run: the pull request was
|
||||
// opened, and the run does not exit 0 without having said so.
|
||||
func TestLostConfirmationFailsTheRun(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.lines()
|
||||
f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"})
|
||||
gh := &fakeGH{}
|
||||
if code := runWith(f, gh, &strings.Builder{}, &failOn{want: "opened draft"}, "run", "--line", "release-1.1", "--push"); code != 2 {
|
||||
t.Fatalf("run --push losing its confirmation = %d, want 2", code)
|
||||
}
|
||||
if len(gh.prs) != 1 {
|
||||
t.Errorf("%d pull requests, want the one the run opened", len(gh.prs))
|
||||
}
|
||||
if len(gh.issueComments) != 0 {
|
||||
t.Errorf("the report was posted after the confirmation was lost: %+v", gh.issueComments)
|
||||
}
|
||||
}
|
||||
|
||||
// notify writes what it found before it comments; when that is lost, it
|
||||
// does not comment.
|
||||
func TestLostOutputCommentsNothing(t *testing.T) {
|
||||
gh := &fakeGH{prs: []pullRequest{notifyPR("20261019", "")}}
|
||||
var stderr strings.Builder
|
||||
a := &app{gh: gh, stdout: failingWriter{}, stderr: &stderr, now: func() time.Time { return time.Date(2026, 10, 26, 6, 23, 0, 0, time.UTC) }}
|
||||
if code := a.main([]string{"notify", "--line", "release-1.1", "--push"}); code != 2 {
|
||||
t.Fatalf("notify --push with a failing stdout = %d, want 2\n%s", code, stderr.String())
|
||||
}
|
||||
if len(gh.issueComments) != 0 {
|
||||
t.Errorf("notify commented: %+v", gh.issueComments)
|
||||
}
|
||||
}
|
||||
|
||||
// writesGitHub records the GitHub writes that reach it.
|
||||
type writesGitHub struct {
|
||||
github
|
||||
writes []string
|
||||
}
|
||||
|
||||
func (g *writesGitHub) createPull(string, string, string, string) (int, string, error) {
|
||||
g.writes = append(g.writes, "createPull")
|
||||
return 1, "", nil
|
||||
}
|
||||
func (g *writesGitHub) editPull(int, string, string) error {
|
||||
g.writes = append(g.writes, "editPull")
|
||||
return nil
|
||||
}
|
||||
func (g *writesGitHub) editBody(int, string) error {
|
||||
g.writes = append(g.writes, "editBody")
|
||||
return nil
|
||||
}
|
||||
func (g *writesGitHub) closePull(int) error {
|
||||
g.writes = append(g.writes, "closePull")
|
||||
return nil
|
||||
}
|
||||
func (g *writesGitHub) createComment(int, string) error {
|
||||
g.writes = append(g.writes, "createComment")
|
||||
return nil
|
||||
}
|
||||
func (g *writesGitHub) editComment(int64, string) error {
|
||||
g.writes = append(g.writes, "editComment")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Every GitHub write is refused once output was lost. Every method of
|
||||
// github is either a read or in this table, so a write added later without
|
||||
// a guard fails here.
|
||||
func TestEveryGitHubWriteIsGuarded(t *testing.T) {
|
||||
reads := map[string]bool{"openPulls": true, "recentPulls": true, "permission": true, "labelled": true, "pull": true, "comments": true, "batchBot": true}
|
||||
inner := &writesGitHub{}
|
||||
g := guardedGitHub{github: inner, a: &app{unwritten: errors.New("broken pipe")}}
|
||||
writes := map[string]func() error{
|
||||
"createPull": func() error { _, _, err := g.createPull("b", "h", "t", "f"); return err },
|
||||
"editPull": func() error { return g.editPull(1, "t", "f") },
|
||||
"editBody": func() error { return g.editBody(1, "f") },
|
||||
"closePull": func() error { return g.closePull(1) },
|
||||
"createComment": func() error { return g.createComment(1, "f") },
|
||||
"editComment": func() error { return g.editComment(1, "f") },
|
||||
}
|
||||
iface := reflect.TypeFor[github]()
|
||||
for i := range iface.NumMethod() {
|
||||
name := iface.Method(i).Name
|
||||
if _, ok := writes[name]; !ok && !reads[name] {
|
||||
t.Errorf("github.%s is neither a known read nor a guarded write", name)
|
||||
}
|
||||
}
|
||||
for name, write := range writes {
|
||||
if err := write(); err == nil || !strings.Contains(err.Error(), "output was lost") {
|
||||
t.Errorf("%s with output lost = %v, want a refusal", name, err)
|
||||
}
|
||||
}
|
||||
if len(inner.writes) != 0 {
|
||||
t.Errorf("writes reached GitHub after output was lost: %q", inner.writes)
|
||||
}
|
||||
}
|
||||
|
||||
// A warning lost while the open batch is being updated stops the update:
|
||||
// nothing is pushed, posted or edited.
|
||||
func TestLostWarningLeavesTheOpenBatchAlone(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.lines()
|
||||
f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"})
|
||||
gh := &fakeGH{}
|
||||
mustRun(t, f, gh, "run", "--line", "release-1.1", "--push")
|
||||
f.pr(6, "fix(app): six", "", map[string]string{"app/six.txt": "6\n"})
|
||||
bad := issueComment{ID: 8, Body: reportMarker + "names none\n"}
|
||||
bad.User.Login, bad.User.Type = "maintainer", "User"
|
||||
gh.issueComments = append(gh.issueComments, bad)
|
||||
head, comments, body := f.remoteHead(batchBranch1), len(gh.issueComments), gh.find(90).Body
|
||||
f.now = f.now.AddDate(0, 0, 1)
|
||||
if code := runWith(f, gh, &strings.Builder{}, &failOn{want: "does not count as a report"}, "run", "--line", "release-1.1", "--continue", "--push"); code != 2 {
|
||||
t.Fatalf("run --continue --push losing a warning = %d, want 2", code)
|
||||
}
|
||||
if got := f.remoteHead(batchBranch1); got != head {
|
||||
t.Errorf("the batch moved to %.12s", got)
|
||||
}
|
||||
if len(gh.issueComments) != comments {
|
||||
t.Errorf("a comment was posted: %d comments, want %d", len(gh.issueComments), comments)
|
||||
}
|
||||
if gh.find(90).Body != body {
|
||||
t.Error("the batch pull request was edited")
|
||||
}
|
||||
}
|
||||
|
||||
// A warning lost while a batch is being closed leaves it open, with no
|
||||
// closing record half written.
|
||||
func TestLostWarningLeavesTheBatchUnclosed(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.lines()
|
||||
f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"})
|
||||
gh := &fakeGH{}
|
||||
mustRun(t, f, gh, "run", "--line", "release-1.1", "--push")
|
||||
bad := issueComment{ID: 8, Body: reportMarker + "names none\n"}
|
||||
bad.User.Login, bad.User.Type = "maintainer", "User"
|
||||
gh.issueComments = append(gh.issueComments, bad)
|
||||
body := gh.find(90).Body
|
||||
if code := runWith(f, gh, &strings.Builder{}, &failOn{want: "does not count as a report"}, "run", "--line", "release-1.1", "--defer", "5", "--reason", "x", "--push"); code != 2 {
|
||||
t.Fatalf("run --defer losing a warning = %d, want 2", code)
|
||||
}
|
||||
if gh.find(90).Body != body {
|
||||
t.Error("the batch pull request was edited")
|
||||
}
|
||||
if !gh.isOpen(90) {
|
||||
t.Error("the batch pull request was closed")
|
||||
}
|
||||
}
|
||||
|
||||
// The git and gh children keep the default SIGPIPE: catching it here must not
|
||||
// leave it ignored in them, as signal.Ignore would through exec.
|
||||
func TestChildrenKeepTheDefaultSIGPIPE(t *testing.T) {
|
||||
catchSIGPIPE()
|
||||
out, err := exec.Command("sh", "-c", "kill -PIPE $$; echo survived").Output()
|
||||
if err == nil || strings.Contains(string(out), "survived") {
|
||||
t.Fatalf("a child survived SIGPIPE: %q, %v", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The branch a close left is not deleted once output was lost: a deletion
|
||||
// is a push like any other.
|
||||
func TestLostOutputDeletesNoBranch(t *testing.T) {
|
||||
f, gh, h1 := closeWorld(t)
|
||||
f.git("switch", "-q", "main")
|
||||
l, err := parseLine("release-1.1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := &app{gh: gh, unwritten: errors.New("broken pipe")}
|
||||
if err := a.deleteBranch(l, closing{PR: 90, Branch: batchBranch1, Head: h1}); err == nil || !strings.Contains(err.Error(), "output was lost") {
|
||||
t.Fatalf("deleteBranch with output lost = %v, want a refusal", err)
|
||||
}
|
||||
if got := f.remoteHead(batchBranch1); got != h1 {
|
||||
t.Errorf("the branch is at %q, want it left at %.12s", got, h1)
|
||||
}
|
||||
}
|
||||
|
||||
// A park line lost on stdout stops notify before it writes: no comment, and
|
||||
// the record is not marked as announced.
|
||||
func TestLostParkLineWritesNothing(t *testing.T) {
|
||||
park := parkedEntry{Original: 5, Merge: strings.Repeat("c", 40), Kind: parkDependency, Needs: []int{3}}
|
||||
gh := &fakeGH{prs: []pullRequest{notifyPR("20261026", parkedBody(park))}}
|
||||
body := gh.prs[0].Body
|
||||
var stderr strings.Builder
|
||||
a := &app{gh: gh, stdout: &failOn{want: "parked originals"}, stderr: &stderr, now: func() time.Time { return time.Date(2026, 10, 26, 6, 23, 0, 0, time.UTC) }}
|
||||
if code := a.main([]string{"notify", "--line", "release-1.1", "--push"}); code != 2 {
|
||||
t.Fatalf("notify --push losing the park line = %d, want 2\n%s", code, stderr.String())
|
||||
}
|
||||
if len(gh.issueComments) != 0 {
|
||||
t.Errorf("notify commented: %+v", gh.issueComments)
|
||||
}
|
||||
if gh.prs[0].Body != body {
|
||||
t.Error("notify marked the park as announced")
|
||||
}
|
||||
}
|
||||
|
||||
// The hint naming the commits a reset drops is the only way back to them, so
|
||||
// the local branch is not reset when the hint is lost.
|
||||
func TestLostResetHintKeepsTheBranch(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.lines()
|
||||
f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"})
|
||||
f.git("switch", "-q", "-C", batchBranch1, "origin/release-1.1")
|
||||
f.write("app/mine.txt", "unpushed\n")
|
||||
f.git("add", "-A")
|
||||
f.git("commit", "-q", "-m", "an unpushed resolution")
|
||||
tip := f.git("rev-parse", "HEAD")
|
||||
f.git("switch", "-q", "main")
|
||||
if code := runWith(f, &fakeGH{}, &strings.Builder{}, &failOn{want: "resetting the local"}, "run", "--line", "release-1.1"); code != 2 {
|
||||
t.Fatalf("run losing the reset hint = %d, want 2", code)
|
||||
}
|
||||
if got := f.git("rev-parse", batchBranch1); got != tip {
|
||||
t.Errorf("the local %s moved to %.12s, want it kept at %.12s", batchBranch1, got, tip)
|
||||
}
|
||||
}
|
||||
|
||||
// Once output is lost the run moves no local branch, even one a reset would
|
||||
// only fast-forward.
|
||||
func TestLostOutputMovesNoLocalBranch(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.lines()
|
||||
f.pr(5, "fix(app): five", "", map[string]string{"app/five.txt": "5\n"})
|
||||
f.git("branch", "-f", batchBranch1, "origin/release-1.1^")
|
||||
before := f.git("rev-parse", batchBranch1)
|
||||
if code := runWith(f, &fakeGH{}, &strings.Builder{}, &failOn{want: "prepared"}, "run", "--line", "release-1.1"); code != 2 {
|
||||
t.Fatalf("run losing an early line = %d, want 2", code)
|
||||
}
|
||||
if got := f.git("rev-parse", batchBranch1); got != before {
|
||||
t.Errorf("the local %s moved to %.12s, want it kept at %.12s", batchBranch1, got, before)
|
||||
}
|
||||
}
|
||||
|
||||
// No new unit starts once output is lost: neither a pick nor a replay touches
|
||||
// the branch.
|
||||
func TestLostOutputStartsNoUnit(t *testing.T) {
|
||||
f := newFixture(t)
|
||||
f.lines()
|
||||
before := f.git("rev-parse", "HEAD")
|
||||
a := &app{unwritten: errors.New("broken pipe")}
|
||||
if _, err := a.pickUnit(&verdict{}, nil); err == nil || !strings.Contains(err.Error(), "output was lost") {
|
||||
t.Errorf("pickUnit with output lost = %v, want a refusal", err)
|
||||
}
|
||||
if _, _, err := a.replayNext(unit{}); err == nil || !strings.Contains(err.Error(), "output was lost") {
|
||||
t.Errorf("replayNext with output lost = %v, want a refusal", err)
|
||||
}
|
||||
if got := f.git("rev-parse", "HEAD"); got != before {
|
||||
t.Errorf("HEAD moved to %.12s", got)
|
||||
}
|
||||
}
|
||||
@@ -176,7 +176,7 @@ func TestPolicyPageMatchesCode(t *testing.T) {
|
||||
t.Errorf("the page's block gives %d released lines, want 2 written out", pol.released)
|
||||
}
|
||||
|
||||
rule2 := regexp.MustCompile("(?m)^2\\. Its title type is (.*)$").FindStringSubmatch(page)
|
||||
rule2 := regexp.MustCompile(`(?m)^2\. Its title type is (.*)$`).FindStringSubmatch(page)
|
||||
if rule2 == nil {
|
||||
t.Fatal("the page has no rule 2")
|
||||
}
|
||||
@@ -187,7 +187,7 @@ func TestPolicyPageMatchesCode(t *testing.T) {
|
||||
if !slices.Equal(types, takenTypes) {
|
||||
t.Errorf("rule 2 on the page takes %v, the code %v", types, takenTypes)
|
||||
}
|
||||
never := regexp.MustCompile("(?m)^### What waits for the next minor\\n\\n(.*)$").FindStringSubmatch(page)
|
||||
never := regexp.MustCompile(`(?m)^### What waits for the next minor\n\n(.*)$`).FindStringSubmatch(page)
|
||||
if never == nil {
|
||||
t.Fatal("the page has no list of what never goes into a patch release")
|
||||
}
|
||||
|
||||
@@ -708,7 +708,7 @@ func (a *app) report(args []string) int {
|
||||
return a.fail(fmt.Errorf("batch pull request #%d keeps no report %.12s", pr.Number, *id))
|
||||
}
|
||||
if *asJSON {
|
||||
fmt.Fprintln(a.stdout, s.raw)
|
||||
_, _ = fmt.Fprintln(a.stdout, s.raw)
|
||||
return 0
|
||||
}
|
||||
a.printReport(&pr, s, inBody, inComment)
|
||||
@@ -721,29 +721,29 @@ func (a *app) report(args []string) int {
|
||||
func (a *app) printReport(pr *pullRequest, s storedReport, inBody *storedReport, inComment bool) {
|
||||
d, b := s.doc, s.doc.Built
|
||||
safe := terminalSafe
|
||||
fmt.Fprintf(a.stdout, "report %s for %s, made %s by a %s run", safe(d.ID), safe(d.Line), safe(b.At), safe(b.Options.Mode))
|
||||
_, _ = fmt.Fprintf(a.stdout, "report %s for %s, made %s by a %s run", safe(d.ID), safe(d.Line), safe(b.At), safe(b.Options.Mode))
|
||||
if b.Options.NoWorkflowFiles {
|
||||
fmt.Fprint(a.stdout, " with --no-workflow-files")
|
||||
_, _ = fmt.Fprint(a.stdout, " with --no-workflow-files")
|
||||
}
|
||||
fmt.Fprintln(a.stdout)
|
||||
fmt.Fprintf(a.stdout, " main %s, line %s, newer line %s, policy %s\n batch %s, tool %s, run %s\n", safe(b.Main), safe(b.Line), safe(or(b.Newer, "none")), safe(b.Policy), safe(b.Batch), safe(or(b.Tool, "unknown")), safe(or(b.Run, "none")))
|
||||
_, _ = fmt.Fprintln(a.stdout)
|
||||
_, _ = fmt.Fprintf(a.stdout, " main %s, line %s, newer line %s, policy %s\n batch %s, tool %s, run %s\n", safe(b.Main), safe(b.Line), safe(or(b.Newer, "none")), safe(b.Policy), safe(b.Batch), safe(or(b.Tool, "unknown")), safe(or(b.Run, "none")))
|
||||
current := "none"
|
||||
if inBody != nil {
|
||||
current = safe(inBody.doc.ID)
|
||||
}
|
||||
switch {
|
||||
case inBody != nil && inBody.doc.ID == d.ID:
|
||||
fmt.Fprintf(a.stdout, " #%d's body carries this report\n", pr.Number)
|
||||
_, _ = fmt.Fprintf(a.stdout, " #%d's body carries this report\n", pr.Number)
|
||||
case pr.Head.SHA == b.Batch:
|
||||
fmt.Fprintf(a.stdout, " #%d's head is %.12s, the batch this report describes, but its body carries report %.12s\n", pr.Number, safe(b.Batch), current)
|
||||
_, _ = fmt.Fprintf(a.stdout, " #%d's head is %.12s, the batch this report describes, but its body carries report %.12s\n", pr.Number, safe(b.Batch), current)
|
||||
default:
|
||||
fmt.Fprintf(a.stdout, " publication unconfirmed: #%d's head is %.12s, not %.12s, and its body carries report %.12s\n", pr.Number, safe(pr.Head.SHA), safe(b.Batch), current)
|
||||
_, _ = fmt.Fprintf(a.stdout, " publication unconfirmed: #%d's head is %.12s, not %.12s, and its body carries report %.12s\n", pr.Number, safe(pr.Head.SHA), safe(b.Batch), current)
|
||||
}
|
||||
if !inComment {
|
||||
if prState(pr) == "open" {
|
||||
fmt.Fprintln(a.stdout, " comment not posted; the next backport-batch run --push posts it")
|
||||
_, _ = fmt.Fprintln(a.stdout, " comment not posted; the next backport-batch run --push posts it")
|
||||
} else {
|
||||
fmt.Fprintf(a.stdout, " comment not posted; kept in the body only, since nothing is written to a %s pull request\n", prState(pr))
|
||||
_, _ = fmt.Fprintf(a.stdout, " comment not posted; kept in the body only, since nothing is written to a %s pull request\n", prState(pr))
|
||||
}
|
||||
}
|
||||
type line struct {
|
||||
@@ -780,8 +780,8 @@ func (a *app) printReport(pr *pullRequest, s storedReport, inBody *storedReport,
|
||||
}
|
||||
for _, e := range d.Primary.Parked {
|
||||
text := "parked: "
|
||||
switch {
|
||||
case e.Kind == parkPartial:
|
||||
switch e.Kind {
|
||||
case parkPartial:
|
||||
text += "only part of it applies, it would leave " + strings.Join(e.Files, ", ") + " without effect"
|
||||
default:
|
||||
var needs []string
|
||||
@@ -802,22 +802,22 @@ func (a *app) printReport(pr *pullRequest, s storedReport, inBody *storedReport,
|
||||
}
|
||||
slices.SortFunc(lines, func(x, y line) int { return cmp.Compare(x.n, y.n) })
|
||||
for _, l := range lines {
|
||||
fmt.Fprintf(a.stdout, "#%d %s\n", l.n, terminalSafe(l.text))
|
||||
_, _ = fmt.Fprintf(a.stdout, "#%d %s\n", l.n, terminalSafe(l.text))
|
||||
}
|
||||
overlay := func(name string, list []int) {
|
||||
if len(list) > 0 {
|
||||
fmt.Fprintf(a.stdout, "%s: %s\n", name, strings.Trim(fmt.Sprint(list), "[]"))
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: %s\n", name, strings.Trim(fmt.Sprint(list), "[]"))
|
||||
}
|
||||
}
|
||||
overlay("stale, still on the batch", d.Overlays.Stale)
|
||||
overlay("only part of it applies, still on the batch", d.Overlays.Partial)
|
||||
for _, e := range d.Overlays.Dropped {
|
||||
fmt.Fprintf(a.stdout, "dropped: #%d, merged as %.12s, on #%d: %s\n", e.Original, terminalSafe(e.Merge), e.On, terminalSafe(e.Reason))
|
||||
_, _ = fmt.Fprintf(a.stdout, "dropped: #%d, merged as %.12s, on #%d: %s\n", e.Original, terminalSafe(e.Merge), e.On, terminalSafe(e.Reason))
|
||||
}
|
||||
overlay("changed in the pick", d.Overlays.Changed)
|
||||
for _, n := range slices.Sorted(maps.Keys(d.Diag)) {
|
||||
for _, ln := range d.Diag[n] {
|
||||
fmt.Fprintf(a.stdout, "#%s: %s\n", terminalSafe(n), terminalSafe(ln))
|
||||
_, _ = fmt.Fprintf(a.stdout, "#%s: %s\n", terminalSafe(n), terminalSafe(ln))
|
||||
}
|
||||
}
|
||||
if len(d.Other) > 0 {
|
||||
@@ -825,7 +825,7 @@ func (a *app) printReport(pr *pullRequest, s storedReport, inBody *storedReport,
|
||||
for i, sha := range d.Other {
|
||||
other[i] = safe(sha)
|
||||
}
|
||||
fmt.Fprintf(a.stdout, "commits on main that merged no pull request: %s\n", strings.Join(other, ", "))
|
||||
_, _ = fmt.Fprintf(a.stdout, "commits on main that merged no pull request: %s\n", strings.Join(other, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -850,7 +850,7 @@ func (a *app) postReport(n int, s storedReport) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(name)
|
||||
defer func() { _ = os.Remove(name) }()
|
||||
return a.gh.createComment(n, name)
|
||||
}
|
||||
|
||||
@@ -873,12 +873,12 @@ func writeTemp(pattern, content string) (string, error) {
|
||||
return "", err
|
||||
}
|
||||
if _, err := f.WriteString(content); err != nil {
|
||||
f.Close()
|
||||
os.Remove(f.Name())
|
||||
_ = f.Close()
|
||||
_ = os.Remove(f.Name())
|
||||
return "", err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
os.Remove(f.Name())
|
||||
_ = os.Remove(f.Name())
|
||||
return "", err
|
||||
}
|
||||
return f.Name(), nil
|
||||
|
||||
@@ -224,7 +224,9 @@ func TestReportPartition(t *testing.T) {
|
||||
}
|
||||
t.Setenv("GIT_COMMITTER_DATE", "2026-01-01T00:00:00Z")
|
||||
f.pr(4, "fix(app): four, before the start", "", map[string]string{"app/four.txt": "4\n"})
|
||||
os.Unsetenv("GIT_COMMITTER_DATE")
|
||||
if err := os.Unsetenv("GIT_COMMITTER_DATE"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.setPolicy("release-1.0: 2026-06-01\n")
|
||||
f.onLine("release-1.0", "fix(app): a line-only eleven", map[string]string{"app/eleven.txt": "line\n"})
|
||||
pr(5, "fix(app): five", map[string]string{"app/five.txt": "5\n"}, true)
|
||||
|
||||
@@ -114,6 +114,23 @@ func mergeMessage(message, merge string) string {
|
||||
return strings.TrimRight(message, " \t\r\n") + "\n\n" + batchTrailer + ": " + merge + "\n"
|
||||
}
|
||||
|
||||
// pickUnit and replayNext start the next unit only while the run's output is
|
||||
// intact: once it was lost no new unit starts, and a unit already in flight
|
||||
// still finishes its rollback.
|
||||
func (a *app) pickUnit(v *verdict, validate func(base, head string) ([]string, error)) (pick, error) {
|
||||
if err := a.mayAct(); err != nil {
|
||||
return pick{}, err
|
||||
}
|
||||
return pickOriginal(v, validate)
|
||||
}
|
||||
|
||||
func (a *app) replayNext(u unit) (string, []string, error) {
|
||||
if err := a.mayAct(); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return replayUnit(u)
|
||||
}
|
||||
|
||||
// commitMerge makes the merge of tip onto base, whose tree is tip's since tip
|
||||
// descends from base, and moves the current branch from tip to it.
|
||||
func commitMerge(base, tip, message string) (string, error) {
|
||||
@@ -121,9 +138,9 @@ func commitMerge(base, tip, message string) (string, error) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
defer func() { _ = os.Remove(f.Name()) }()
|
||||
if _, err := f.WriteString(message); err != nil {
|
||||
f.Close()
|
||||
_ = f.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# Every path whose change can change the result of
|
||||
# `go test ./cmd/backport-batch/...`. The plan job in
|
||||
# .github/workflows/pull-requests.yaml runs those tests when, and only when, the
|
||||
# pull request touches one of these paths.
|
||||
#
|
||||
# One repository-relative path per line. A trailing slash names a directory and
|
||||
# matches everything below it; anything else matches that file alone. Blank lines
|
||||
# and lines starting with # are ignored.
|
||||
#
|
||||
# TestInputsListed in inputs_test.go keeps this list in step with the code: it
|
||||
# fails when the tests' in-repo package dependencies (go list -deps -test) or the
|
||||
# docs pages they read are missing from it.
|
||||
|
||||
cmd/backport-batch/
|
||||
go.mod
|
||||
go.sum
|
||||
|
||||
# In-repo packages the tool and its tests import.
|
||||
internal/apigate/
|
||||
pkg/apis/apps/v1alpha1/
|
||||
pkg/apis/apps/validation/
|
||||
pkg/config/
|
||||
|
||||
# Pages the tests read from the tree.
|
||||
docs/release/backports.md
|
||||
docs/release/backport-tasks/
|
||||
@@ -66,7 +66,7 @@ func (a *app) verify(args []string) int {
|
||||
return a.fail(err)
|
||||
}
|
||||
if pr == nil {
|
||||
fmt.Fprintf(a.stdout, "%s: no batch pull request is open; nothing to verify\n", l.branch())
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: no batch pull request is open; nothing to verify\n", l.branch())
|
||||
return 0
|
||||
}
|
||||
if head, err = batchHead(pr); err != nil {
|
||||
@@ -102,7 +102,7 @@ func (a *app) verify(args []string) int {
|
||||
}
|
||||
if why != "" {
|
||||
refused = append(refused, why)
|
||||
fmt.Fprintf(a.stdout, "refused %.12s: %s\n", u.commit, why)
|
||||
_, _ = fmt.Fprintf(a.stdout, "refused %.12s: %s\n", u.commit, why)
|
||||
continue
|
||||
}
|
||||
seen[v] = u.commit
|
||||
@@ -110,13 +110,13 @@ func (a *app) verify(args []string) int {
|
||||
if !u.merge() {
|
||||
n = 1
|
||||
}
|
||||
fmt.Fprintf(a.stdout, "ok %.12s #%d, %d %s\n", u.commit, v.o.number, n, plural(n, "commit"))
|
||||
_, _ = fmt.Fprintf(a.stdout, "ok %.12s #%d, %d %s\n", u.commit, v.o.number, n, plural(n, "commit"))
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
fmt.Fprintf(a.stdout, "%s: %d of %d units at %.12s are not originals in the batch's shape\n", l.branch(), len(refused), len(units), head)
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: %d of %d units at %.12s are not originals in the batch's shape\n", l.branch(), len(refused), len(units), head)
|
||||
return 1
|
||||
}
|
||||
fmt.Fprintf(a.stdout, "%s: %d %s at %.12s, each one original in the batch's shape\n", l.branch(), len(units), plural(len(units), "unit"), head)
|
||||
_, _ = fmt.Fprintf(a.stdout, "%s: %d %s at %.12s, each one original in the batch's shape\n", l.branch(), len(units), plural(len(units), "unit"), head)
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
+40
-15
@@ -22,6 +22,7 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
@@ -58,6 +59,10 @@ Options:
|
||||
--no-color Disable color output (auto-disabled on non-TTY)
|
||||
-h, --help Show this help
|
||||
|
||||
Exit status: 0 when everything is ready, 1 when something is not ready or a
|
||||
write to stdout fails, 2 on invalid arguments. A closed pipe ends the process
|
||||
with SIGPIPE, like other Unix tools.
|
||||
|
||||
`
|
||||
|
||||
// entry is a resource catalog item.
|
||||
@@ -181,10 +186,16 @@ func run(args []string) int {
|
||||
cfg.coreOnly = true
|
||||
i++
|
||||
case "-h", "--help":
|
||||
fmt.Fprint(os.Stdout, usageText)
|
||||
if _, err := fmt.Fprint(os.Stdout, usageText); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "writing usage: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
case "--version":
|
||||
fmt.Fprintln(os.Stdout, Version)
|
||||
if _, err := fmt.Fprintln(os.Stdout, Version); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "writing version: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "Unknown argument: %s\n", a)
|
||||
@@ -213,14 +224,14 @@ func run(args []string) int {
|
||||
cfg.timeoutS = sec
|
||||
|
||||
if cfg.waitMode {
|
||||
return cfg.runWait()
|
||||
return cfg.runWait(os.Stdout)
|
||||
}
|
||||
if cfg.watch {
|
||||
cfg.runWatch()
|
||||
return 0
|
||||
return cfg.runWatch(os.Stdout)
|
||||
}
|
||||
if writeOutput(os.Stdout, cfg.runOnce()) != nil {
|
||||
return 1
|
||||
}
|
||||
out := cfg.runOnce()
|
||||
fmt.Print(out)
|
||||
if cfg.foundIssues {
|
||||
return 1
|
||||
}
|
||||
@@ -281,18 +292,31 @@ func parseTimeout(raw string) (int, bool) {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func (cfg *config) runWait() int {
|
||||
// writeOutput prints a result. A result its reader never got fails the run,
|
||||
// whatever the resources' state, since the exit code alone would vouch for it.
|
||||
func writeOutput(w io.Writer, parts ...string) error {
|
||||
for _, part := range parts {
|
||||
if _, err := io.WriteString(w, part); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "writing output: %v\n", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cfg *config) runWait(w io.Writer) int {
|
||||
start := time.Now()
|
||||
for {
|
||||
output := cfg.runOnce()
|
||||
elapsed := int(time.Since(start).Seconds())
|
||||
if strings.Contains(output, "All resources are ready.") {
|
||||
fmt.Print(output)
|
||||
fmt.Printf("%sReady after %ds.%s\n", cfg.green, elapsed, cfg.reset)
|
||||
if writeOutput(w, output, fmt.Sprintf("%sReady after %ds.%s\n", cfg.green, elapsed, cfg.reset)) != nil {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if elapsed >= cfg.timeoutS {
|
||||
fmt.Print(output)
|
||||
_ = writeOutput(w, output)
|
||||
fmt.Fprintf(os.Stderr, "%s%sTimeout after %ds — resources still not ready.%s\n", cfg.red, cfg.bold, cfg.timeoutS, cfg.reset)
|
||||
return 1
|
||||
}
|
||||
@@ -300,13 +324,14 @@ func (cfg *config) runWait() int {
|
||||
}
|
||||
}
|
||||
|
||||
func (cfg *config) runWatch() {
|
||||
func (cfg *config) runWatch(w io.Writer) int {
|
||||
for {
|
||||
output := cfg.runOnce()
|
||||
// Smooth clear is cosmetic and untested; just reprint.
|
||||
fmt.Printf("%sLast updated: %s (refreshing every %ds, Ctrl+C to stop)%s\n", cfg.bold, time.Now().Format(time.UnixDate), cfg.interval, cfg.reset)
|
||||
fmt.Println()
|
||||
fmt.Print(output)
|
||||
header := fmt.Sprintf("%sLast updated: %s (refreshing every %ds, Ctrl+C to stop)%s\n\n", cfg.bold, time.Now().Format(time.UnixDate), cfg.interval, cfg.reset)
|
||||
if writeOutput(w, header, output) != nil {
|
||||
return 1
|
||||
}
|
||||
time.Sleep(time.Duration(cfg.interval) * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHelpAndVersionSucceedOnWritableStdout(t *testing.T) {
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = r.Close() })
|
||||
stdout := os.Stdout
|
||||
os.Stdout = w
|
||||
t.Cleanup(func() { os.Stdout = stdout })
|
||||
read := make(chan string, 1)
|
||||
go func() {
|
||||
b, _ := io.ReadAll(r)
|
||||
read <- string(b)
|
||||
}()
|
||||
|
||||
for _, arg := range []string{"--help", "--version"} {
|
||||
if code := run([]string{arg}); code != 0 {
|
||||
t.Errorf("run(%s) = %d, want 0", arg, code)
|
||||
}
|
||||
}
|
||||
_ = w.Close()
|
||||
if out := <-read; !strings.Contains(out, "Usage:") || !strings.Contains(out, Version) {
|
||||
t.Errorf("stdout = %q, want the usage text and the version", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelpAndVersionFailOnUnwritableStdout(t *testing.T) {
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = r.Close()
|
||||
_ = w.Close()
|
||||
stdout := os.Stdout
|
||||
os.Stdout = w
|
||||
t.Cleanup(func() { os.Stdout = stdout })
|
||||
|
||||
for _, arg := range []string{"--help", "--version"} {
|
||||
if code := run([]string{arg}); code == 0 {
|
||||
t.Errorf("run(%s) with an unwritable stdout = 0, want a failure", arg)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -390,7 +390,7 @@ func createPackageFromFile(ctx context.Context, k8sClient client.Client, filePat
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("Package %s not found in file", packageName)
|
||||
return fmt.Errorf("no Package %s in file", packageName)
|
||||
}
|
||||
|
||||
func installPackage(ctx context.Context, k8sClient client.Client, packageSourceName string) error {
|
||||
|
||||
+27
-21
@@ -19,7 +19,7 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"io"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
|
||||
@@ -78,24 +78,25 @@ Use --components flag to show components on separate lines.`,
|
||||
}
|
||||
|
||||
if listCmdFlags.installed {
|
||||
return listPackages(ctx, k8sClient, listCmdFlags.components)
|
||||
return listPackages(ctx, k8sClient, cmd.OutOrStdout(), listCmdFlags.components)
|
||||
}
|
||||
return listPackageSources(ctx, k8sClient, listCmdFlags.components)
|
||||
return listPackageSources(ctx, k8sClient, cmd.OutOrStdout(), listCmdFlags.components)
|
||||
},
|
||||
}
|
||||
|
||||
func listPackageSources(ctx context.Context, k8sClient client.Client, showComponents bool) error {
|
||||
func listPackageSources(ctx context.Context, k8sClient client.Client, out io.Writer, showComponents bool) error {
|
||||
var psList cozyv1alpha1.PackageSourceList
|
||||
if err := k8sClient.List(ctx, &psList); err != nil {
|
||||
return fmt.Errorf("failed to list PackageSources: %w", err)
|
||||
}
|
||||
|
||||
// Use tabwriter for better column alignment
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
|
||||
defer w.Flush()
|
||||
w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0)
|
||||
|
||||
// Print header
|
||||
fmt.Fprintln(w, "NAME\tVARIANTS\tREADY\tSTATUS")
|
||||
if _, err := fmt.Fprintln(w, "NAME\tVARIANTS\tREADY\tSTATUS"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Print rows
|
||||
for _, ps := range psList.Items {
|
||||
@@ -123,24 +124,26 @@ func listPackageSources(ctx context.Context, k8sClient client.Client, showCompon
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", ps.Name, variantsStr, ready, status)
|
||||
if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", ps.Name, variantsStr, ready, status); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Show components if requested
|
||||
if showComponents {
|
||||
for _, variant := range ps.Spec.Variants {
|
||||
for _, component := range variant.Components {
|
||||
fmt.Fprintf(w, " %s\t%s\t\t\n",
|
||||
fmt.Sprintf("%s.%s", ps.Name, component.Name),
|
||||
variant.Name)
|
||||
if _, err := fmt.Fprintf(w, " %s.%s\t%s\t\t\n", ps.Name, component.Name, variant.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return w.Flush()
|
||||
}
|
||||
|
||||
func listPackages(ctx context.Context, k8sClient client.Client, showComponents bool) error {
|
||||
func listPackages(ctx context.Context, k8sClient client.Client, out io.Writer, showComponents bool) error {
|
||||
var pkgList cozyv1alpha1.PackageList
|
||||
if err := k8sClient.List(ctx, &pkgList); err != nil {
|
||||
return fmt.Errorf("failed to list Packages: %w", err)
|
||||
@@ -160,11 +163,12 @@ func listPackages(ctx context.Context, k8sClient client.Client, showComponents b
|
||||
}
|
||||
|
||||
// Use tabwriter for better column alignment
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
|
||||
defer w.Flush()
|
||||
w := tabwriter.NewWriter(out, 0, 0, 3, ' ', 0)
|
||||
|
||||
// Print header
|
||||
fmt.Fprintln(w, "NAME\tVARIANT\tREADY\tSTATUS")
|
||||
if _, err := fmt.Fprintln(w, "NAME\tVARIANT\tREADY\tSTATUS"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Print rows
|
||||
for _, pkg := range pkgList.Items {
|
||||
@@ -187,7 +191,9 @@ func listPackages(ctx context.Context, k8sClient client.Client, showComponents b
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", pkg.Name, variant, ready, status)
|
||||
if _, err := fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", pkg.Name, variant, ready, status); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Show components if requested
|
||||
if showComponents {
|
||||
@@ -197,9 +203,9 @@ func listPackages(ctx context.Context, k8sClient client.Client, showComponents b
|
||||
for _, v := range ps.Spec.Variants {
|
||||
if v.Name == variant {
|
||||
for _, component := range v.Components {
|
||||
fmt.Fprintf(w, " %s\t%s\t\t\n",
|
||||
fmt.Sprintf("%s.%s", pkg.Name, component.Name),
|
||||
variant)
|
||||
if _, err := fmt.Fprintf(w, " %s.%s\t%s\t\t\n", pkg.Name, component.Name, variant); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
@@ -208,7 +214,7 @@ func listPackages(ctx context.Context, k8sClient client.Client, showComponents b
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
return w.Flush()
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
/*
|
||||
Copyright 2026 The Cozystack Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
|
||||
"github.com/spf13/cobra"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
)
|
||||
|
||||
type failingWriter struct{}
|
||||
|
||||
func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("broken pipe") }
|
||||
|
||||
func TestPrintReportReturnsWriteError(t *testing.T) {
|
||||
r := &Report{Findings: []Finding{{Severity: SeverityError, Code: "X", Message: "m"}}}
|
||||
if err := printReport(r, failingWriter{}); err == nil {
|
||||
t.Fatal("printReport to an unwritable writer returned nil")
|
||||
}
|
||||
}
|
||||
|
||||
// The table is buffered until the tabwriter flushes, so a failing reader shows
|
||||
// up only as a Flush error, which list has to return.
|
||||
func TestListReturnsWriteError(t *testing.T) {
|
||||
s := runtime.NewScheme()
|
||||
utilruntime.Must(cozyv1alpha1.AddToScheme(s))
|
||||
ps := &cozyv1alpha1.PackageSource{ObjectMeta: metav1.ObjectMeta{Name: "acme.app"}}
|
||||
pkg := &cozyv1alpha1.Package{ObjectMeta: metav1.ObjectMeta{Name: "acme.app"}}
|
||||
cl := fake.NewClientBuilder().WithScheme(s).WithObjects(ps, pkg).Build()
|
||||
|
||||
for name, list := range map[string]func(io.Writer) error{
|
||||
"sources": func(w io.Writer) error { return listPackageSources(context.Background(), cl, w, true) },
|
||||
"packages": func(w io.Writer) error { return listPackages(context.Background(), cl, w, true) },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if err := list(failingWriter{}); err == nil {
|
||||
t.Fatal("list to an unwritable writer returned nil")
|
||||
}
|
||||
var out strings.Builder
|
||||
if err := list(&out); err != nil || !strings.Contains(out.String(), "acme.app") {
|
||||
t.Fatalf("list = %v, output %q; want the row", err, out.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A command that cannot print its validation report still fails, and a
|
||||
// refusal keeps validation as its reason with the write error joined to it.
|
||||
func TestValidationCommandsReportUnwritableReport(t *testing.T) {
|
||||
fakeFlux := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(fakeFlux, "flux"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", fakeFlux+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
|
||||
emptyRepo := func(t *testing.T) string {
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "packages"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return root
|
||||
}
|
||||
run := func(c *cobra.Command, args ...string) error {
|
||||
c.SetOut(failingWriter{})
|
||||
t.Cleanup(func() { c.SetOut(nil) })
|
||||
return c.RunE(c, args)
|
||||
}
|
||||
push := func(root string) error {
|
||||
pushCmdFlags.path = root
|
||||
t.Cleanup(func() { pushCmdFlags.path = "." })
|
||||
return run(pushCmd, "oci://registry.example/acme/apps:v1")
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
run func(t *testing.T) error
|
||||
refusal string
|
||||
}{
|
||||
{name: "validate, valid", run: func(t *testing.T) error { return run(validateCmd, validRepo(t)) }},
|
||||
{name: "validate, invalid", run: func(t *testing.T) error { return run(validateCmd, emptyRepo(t)) }, refusal: "validation failed"},
|
||||
{name: "push, valid", run: func(t *testing.T) error { return push(validRepo(t)) }, refusal: "nothing was pushed"},
|
||||
{name: "push, invalid", run: func(t *testing.T) error { return push(emptyRepo(t)) }, refusal: "refusing to push"},
|
||||
{name: "tap, invalid", run: func(t *testing.T) error { return run(tapCmd, "oci://registry.example/acme/apps:v1") }, refusal: "refusing to tap"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := tc.run(t)
|
||||
if err == nil || !strings.Contains(err.Error(), "broken pipe") {
|
||||
t.Fatalf("err = %v, want the write error", err)
|
||||
}
|
||||
if tc.refusal != "" && !strings.Contains(err.Error(), tc.refusal) {
|
||||
t.Errorf("err = %v, want the refusal %q kept", err, tc.refusal)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A tap that applied is reported as applied even when its summary is lost,
|
||||
// and the lost summary still fails the command.
|
||||
func TestTapSummaryReportsUnwritableOutput(t *testing.T) {
|
||||
err := writeTapSummary(failingWriter{}, "oci://registry.example/acme/apps:v1", "tap-acme", []string{"acme.app"})
|
||||
if err == nil || !strings.Contains(err.Error(), "tap applied") || !strings.Contains(err.Error(), "broken pipe") {
|
||||
t.Fatalf("err = %v, want the write error with the tap reported as applied", err)
|
||||
}
|
||||
var out strings.Builder
|
||||
if err := writeTapSummary(&out, "oci://registry.example/acme/apps:v1", "tap-acme", []string{"acme.app"}); err != nil || !strings.Contains(out.String(), "cozypkg add acme.app") {
|
||||
t.Fatalf("writeTapSummary = %v, output %q", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A clean report has no finding lines, so its summary is its only write and
|
||||
// alone decides the result.
|
||||
func TestPrintReportReturnsSummaryWriteError(t *testing.T) {
|
||||
if err := printReport(&Report{}, failingWriter{}); err == nil {
|
||||
t.Fatal("printReport of a clean report to an unwritable writer returned nil")
|
||||
}
|
||||
}
|
||||
|
||||
// push runs flux only after its status line was written, with validation
|
||||
// skipped and after a report that was written.
|
||||
func TestPushStopsBeforeFluxWhenItsStatusIsLost(t *testing.T) {
|
||||
fakeFlux := t.TempDir()
|
||||
ran := filepath.Join(t.TempDir(), "flux-ran")
|
||||
if err := os.WriteFile(filepath.Join(fakeFlux, "flux"), []byte("#!/bin/sh\ntouch "+ran+"\n"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("PATH", fakeFlux+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
for _, skip := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("skip-validate=%v", skip), func(t *testing.T) {
|
||||
pushCmdFlags.path, pushCmdFlags.skipValidate = validRepo(t), skip
|
||||
pushCmdFlags.source, pushCmdFlags.revision = "https://example.com/acme.git", "v1"
|
||||
t.Cleanup(func() {
|
||||
pushCmdFlags.path, pushCmdFlags.skipValidate = ".", false
|
||||
pushCmdFlags.source, pushCmdFlags.revision = "", ""
|
||||
})
|
||||
pushCmd.SetOut(&failOn{want: "Pushing"})
|
||||
t.Cleanup(func() { pushCmd.SetOut(nil) })
|
||||
err := pushCmd.RunE(pushCmd, []string{"oci://registry.example/acme/apps:v1"})
|
||||
if err == nil || !strings.Contains(err.Error(), "nothing was pushed") {
|
||||
t.Fatalf("push losing its status = %v, want nothing pushed", err)
|
||||
}
|
||||
if _, statErr := os.Stat(ran); statErr == nil {
|
||||
t.Fatal("flux ran after the status line was lost")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// failOn fails every write that contains want, and takes every other one.
|
||||
type failOn struct {
|
||||
want string
|
||||
buf strings.Builder
|
||||
}
|
||||
|
||||
func (w *failOn) Write(p []byte) (int, error) {
|
||||
if strings.Contains(string(p), w.want) {
|
||||
return 0, errors.New("broken pipe")
|
||||
}
|
||||
return w.buf.Write(p)
|
||||
}
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -106,10 +107,13 @@ published. Source URL and revision are derived from git when not given.`,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printReport(rep, cmd.OutOrStdout())
|
||||
printErr := printReport(rep, cmd.OutOrStdout())
|
||||
if rep.HasErrors() {
|
||||
errs, _, _ := rep.Counts()
|
||||
return fmt.Errorf("refusing to push: validation failed with %d error(s) (use --skip-validate to override)", errs)
|
||||
return errors.Join(fmt.Errorf("refusing to push: validation failed with %d error(s) (use --skip-validate to override)", errs), printErr)
|
||||
}
|
||||
if printErr != nil {
|
||||
return fmt.Errorf("nothing was pushed: writing the validation report failed: %w", printErr)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,7 +133,9 @@ published. Source URL and revision are derived from git when not given.`,
|
||||
}
|
||||
|
||||
fluxArgs := buildFluxPushArgs(ref, packagesDir, source, revision, pushCmdFlags.reproducible)
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Pushing %s (source=%s revision=%s)\n", ref, source, revision)
|
||||
if _, err := fmt.Fprintf(cmd.OutOrStdout(), "Pushing %s (source=%s revision=%s)\n", ref, source, revision); err != nil {
|
||||
return fmt.Errorf("nothing was pushed: writing the push status failed: %w", err)
|
||||
}
|
||||
flux := exec.Command("flux", fluxArgs...)
|
||||
flux.Stdout = cmd.OutOrStdout()
|
||||
flux.Stderr = cmd.ErrOrStderr()
|
||||
|
||||
+22
-12
@@ -18,6 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
@@ -402,9 +403,8 @@ runs its charts in your management cluster: tap only sources you trust.`,
|
||||
return err
|
||||
}
|
||||
if rep.HasErrors() {
|
||||
printReport(rep, cmd.OutOrStdout())
|
||||
errs, _, _ := rep.Counts()
|
||||
return fmt.Errorf("refusing to tap: artifact failed validation with %d error(s) (use --skip-validate to override)", errs)
|
||||
return errors.Join(fmt.Errorf("refusing to tap: artifact failed validation with %d error(s) (use --skip-validate to override)", errs), printReport(rep, cmd.OutOrStdout()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -487,19 +487,29 @@ runs its charts in your management cluster: tap only sources you trust.`,
|
||||
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "warning: could not reset materialization state on OCIRepository/%s: %v\n", srcName, err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "Tapped %s\n OCIRepository/%s\n", fullRef, srcName)
|
||||
for _, n := range names {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), " PackageSource/%s\n", n)
|
||||
}
|
||||
_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Tapping installs nothing. Install an app with:\n")
|
||||
for _, n := range names {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), " cozypkg add %s\n", n)
|
||||
}
|
||||
_, _ = fmt.Fprintf(cmd.OutOrStdout(), "A privileged install component is not installed until you confirm it with 'cozypkg add --allow-privileged'.\n")
|
||||
return nil
|
||||
return writeTapSummary(cmd.OutOrStdout(), fullRef, srcName, names)
|
||||
},
|
||||
}
|
||||
|
||||
// writeTapSummary reports what a tap created. The tap is applied by then, so a
|
||||
// summary that cannot be written fails the command while saying so.
|
||||
func writeTapSummary(w io.Writer, fullRef, srcName string, names []string) error {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "Tapped %s\n OCIRepository/%s\n", fullRef, srcName)
|
||||
for _, n := range names {
|
||||
fmt.Fprintf(&b, " PackageSource/%s\n", n)
|
||||
}
|
||||
b.WriteString("Tapping installs nothing. Install an app with:\n")
|
||||
for _, n := range names {
|
||||
fmt.Fprintf(&b, " cozypkg add %s\n", n)
|
||||
}
|
||||
b.WriteString("A privileged install component is not installed until you confirm it with 'cozypkg add --allow-privileged'.\n")
|
||||
if _, err := io.WriteString(w, b.String()); err != nil {
|
||||
return fmt.Errorf("tap applied, but writing its summary failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var untapCmd = &cobra.Command{
|
||||
Use: "untap <packagesource-name>",
|
||||
Short: "Remove a tapped repository",
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
@@ -535,16 +536,19 @@ func pullOCIArtifact(ref string) (string, func(), error) {
|
||||
return dir, cleanup, nil
|
||||
}
|
||||
|
||||
func printReport(r *Report, w io.Writer) {
|
||||
func printReport(r *Report, w io.Writer) error {
|
||||
for _, f := range r.Findings {
|
||||
loc := f.Location
|
||||
if loc == "" {
|
||||
loc = "-"
|
||||
}
|
||||
fmt.Fprintf(w, "[%s] %s (%s): %s\n", f.Severity, loc, f.Code, f.Message)
|
||||
if _, err := fmt.Fprintf(w, "[%s] %s (%s): %s\n", f.Severity, loc, f.Code, f.Message); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
errs, warns, infos := r.Counts()
|
||||
fmt.Fprintf(w, "\nSummary: %d error(s), %d warning(s), %d info\n", errs, warns, infos)
|
||||
_, err := fmt.Fprintf(w, "\nSummary: %d error(s), %d warning(s), %d info\n", errs, warns, infos)
|
||||
return err
|
||||
}
|
||||
|
||||
var validateCmdFlags struct {
|
||||
@@ -663,10 +667,13 @@ runs "helm lint" on every component chart.`,
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printReport(rep, cmd.OutOrStdout())
|
||||
printErr := printReport(rep, cmd.OutOrStdout())
|
||||
if rep.HasErrors() {
|
||||
errs, _, _ := rep.Counts()
|
||||
return fmt.Errorf("validation failed: %d error(s)", errs)
|
||||
return errors.Join(fmt.Errorf("validation failed: %d error(s)", errs), printErr)
|
||||
}
|
||||
if printErr != nil {
|
||||
return fmt.Errorf("writing the validation report failed: %w", printErr)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
|
||||
@@ -663,7 +663,7 @@ func validateGitRef(refMap map[string]string) error {
|
||||
return fmt.Errorf("commit SHA should be at least 7 characters, got: %s", commit)
|
||||
}
|
||||
for _, c := range commit {
|
||||
if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) {
|
||||
if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') {
|
||||
return fmt.Errorf("commit SHA should be a hexadecimal string, got: %s", commit)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,9 +14,10 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Command observability-proxy serves tenant logs from the VictoriaLogs
|
||||
// stores behind a Kubernetes TokenReview and SubjectAccessReview, with the
|
||||
// tenant's namespace scope forced into every upstream request.
|
||||
// Command observability-proxy serves tenant logs and metrics from the
|
||||
// VictoriaLogs and VictoriaMetrics stores behind a Kubernetes TokenReview
|
||||
// and SubjectAccessReview, with the tenant's namespace scope forced into
|
||||
// every upstream request.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -37,6 +38,9 @@ import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"k8s.io/apimachinery/pkg/util/validation"
|
||||
"k8s.io/client-go/dynamic"
|
||||
"k8s.io/client-go/dynamic/dynamicinformer"
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
@@ -56,6 +60,7 @@ type options struct {
|
||||
tailIdleTimeout time.Duration
|
||||
shutdownTimeout time.Duration
|
||||
logs observabilityproxy.LogsLimits
|
||||
metrics observabilityproxy.MetricsConfig
|
||||
limiter observabilityproxy.LimiterConfig
|
||||
}
|
||||
|
||||
@@ -81,8 +86,23 @@ func parseFlags(args []string) (options, error) {
|
||||
fs.IntVar(&o.logs.DefaultLimit, "logs-default-limit", 1000, "entries returned when a request sets no limit")
|
||||
fs.IntVar(&o.logs.MaxLimit, "logs-max-limit", 10000, "largest limit a request may set")
|
||||
fs.DurationVar(&o.logs.MaxTailResume, "logs-max-tail-resume", 15*time.Minute, "how far back a tail may resume with since or start_offset")
|
||||
// Store names and retention follow metricsStorages in
|
||||
// packages/system/monitoring/values.yaml.
|
||||
fs.StringVar(&o.metrics.ShortTermStore, "metrics-shortterm-store", "shortterm", "VMCluster name of the short-term metrics store of the root tenant")
|
||||
fs.StringVar(&o.metrics.LongTermStore, "metrics-longterm-store", "longterm", "VMCluster name of the long-term metrics store of the root tenant")
|
||||
fs.DurationVar(&o.metrics.ShortTermRetention, "metrics-shortterm-retention", 72*time.Hour, "retention of the short-term store; older windows are read from the long-term one")
|
||||
fs.DurationVar(&o.metrics.Limits.MaxRange, "metrics-max-range", 30*24*time.Hour, "longest [start, end] and step a metrics request may use")
|
||||
fs.IntVar(&o.metrics.Limits.MaxPoints, "metrics-max-points", 11000, "most points per series a range query may return")
|
||||
fs.DurationVar(&o.metrics.Limits.MaxTimeout, "metrics-max-timeout", 30*time.Second, "timeout forwarded to vmselect; a longer client timeout is clamped; must be below --request-timeout when that is set")
|
||||
// Only the refusal threshold: the lookup itself spans the window the
|
||||
// query reads, and that estimate is a loose upper bound (every "("
|
||||
// counts as an implicit subquery), so the default is long enough to
|
||||
// serve month-long panels at a dashboard step.
|
||||
fs.DurationVar(&o.metrics.App.Lookback, "metrics-app-lookback", 30*24*time.Hour, "how far before the window an application metrics query may read; one reading further back, or by an unknown amount, answers 422")
|
||||
fs.IntVar(&o.metrics.App.MaxPods, "metrics-app-max-pods", 500, "most pods of one application in the lookup window; more answer 422")
|
||||
fs.IntVar(&o.metrics.App.MaxPVCs, "metrics-app-max-pvcs", 500, "most PVCs of one application in the lookup window; more answer 422")
|
||||
fs.IntVar(&o.limiter.MaxConcurrentPerUser, "max-concurrent-per-user", 10, "concurrent non-tail requests per user; 0 disables")
|
||||
fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained requests per second per user; 0 disables")
|
||||
fs.Float64Var(&o.limiter.RatePerUser, "rate-limit-per-user", 10, "sustained client requests per second per user; an application metrics request makes three store calls (pod lookup, PVC lookup, query); 0 disables")
|
||||
fs.IntVar(&o.limiter.RateBurst, "rate-limit-burst", 20, "request burst per user")
|
||||
fs.IntVar(&o.limiter.MaxTailsPerUser, "max-tails-per-user", 5, "open tails per user; 0 disables")
|
||||
fs.IntVar(&o.limiter.MaxTails, "max-tails", 200, "open tails in total; 0 disables")
|
||||
@@ -94,6 +114,13 @@ func parseFlags(args []string) (options, error) {
|
||||
if fs.NArg() > 0 {
|
||||
return o, fmt.Errorf("unexpected argument %q", fs.Arg(0))
|
||||
}
|
||||
o.metrics.RootTenant = o.rootTenant
|
||||
for _, store := range []string{o.metrics.ShortTermStore, o.metrics.LongTermStore} {
|
||||
// The name becomes part of a Service host name.
|
||||
if errs := validation.IsDNS1123Label("vmselect-" + store); store == "" || len(errs) > 0 {
|
||||
return o, fmt.Errorf("invalid metrics store name %q", store)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case o.rootTenant == "":
|
||||
return o, errors.New("--root-tenant must not be empty")
|
||||
@@ -110,8 +137,19 @@ func parseFlags(args []string) (options, error) {
|
||||
return o, errors.New("--logs-max-range must be at least 3s")
|
||||
case o.logs.DefaultLimit <= 0 || o.logs.MaxLimit < o.logs.DefaultLimit:
|
||||
return o, errors.New("--logs-default-limit must be positive and not above --logs-max-limit")
|
||||
case o.metrics.ShortTermRetention <= 0 || o.metrics.Limits.MaxRange <= 0 || o.metrics.Limits.MaxTimeout <= 0:
|
||||
return o, errors.New("--metrics-shortterm-retention, --metrics-max-range and --metrics-max-timeout must be positive")
|
||||
case o.metrics.Limits.MaxPoints < 2:
|
||||
return o, errors.New("--metrics-max-points must be at least 2")
|
||||
case o.metrics.App.Lookback <= 0 || o.metrics.App.MaxPods < 1 || o.metrics.App.MaxPVCs < 1:
|
||||
return o, errors.New("--metrics-app-lookback, --metrics-app-max-pods and --metrics-app-max-pvcs must be positive")
|
||||
case o.logs.MaxTailResume < 0:
|
||||
return o, errors.New("--logs-max-tail-resume must not be negative")
|
||||
// The proxy's deadline starts before the request reaches vmselect, so
|
||||
// at an equal or longer timeout it cuts a query vmselect is still
|
||||
// allowed to run and answers 504 for a store that is within its time.
|
||||
case o.requestTimeout > 0 && o.metrics.Limits.MaxTimeout >= o.requestTimeout:
|
||||
return o, errors.New("--metrics-max-timeout must be below --request-timeout")
|
||||
case o.requestTimeout < 0 || o.tailIdleTimeout < 0 || o.authCacheTTL < 0 || o.shutdownTimeout < 0:
|
||||
return o, errors.New("timeouts and the auth cache TTL must not be negative")
|
||||
// NaN passes every comparison and +Inf never empties a bucket; either
|
||||
@@ -151,13 +189,24 @@ func run(o options) error {
|
||||
return fmt.Errorf("building client: %w", err)
|
||||
}
|
||||
|
||||
dynClient, err := dynamic.NewForConfig(restConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("building dynamic client: %w", err)
|
||||
}
|
||||
|
||||
factory := informers.NewSharedInformerFactory(client, 0)
|
||||
namespaces := factory.Core().V1().Namespaces()
|
||||
lister := namespaces.Lister()
|
||||
appDefs := dynamicinformer.NewDynamicSharedInformerFactory(dynClient, 0).
|
||||
ForResource(observabilityproxy.ApplicationDefinitionsResource).Informer()
|
||||
if err := appDefs.AddIndexers(observabilityproxy.ApplicationPluralIndexers()); err != nil {
|
||||
return fmt.Errorf("indexing application definitions: %w", err)
|
||||
}
|
||||
factory.Start(ctx.Done())
|
||||
go appDefs.Run(ctx.Done())
|
||||
var ready atomic.Bool
|
||||
go func() {
|
||||
if cache.WaitForCacheSync(ctx.Done(), namespaces.Informer().HasSynced) {
|
||||
if cache.WaitForCacheSync(ctx.Done(), namespaces.Informer().HasSynced, appDefs.HasSynced) {
|
||||
ready.Store(true)
|
||||
}
|
||||
}()
|
||||
@@ -167,16 +216,18 @@ func run(o options) error {
|
||||
proxy := observabilityproxy.NewServer(observabilityproxy.Config{
|
||||
ClusterDomain: o.clusterDomain,
|
||||
Logs: o.logs,
|
||||
Metrics: o.metrics,
|
||||
Limiter: o.limiter,
|
||||
RequestTimeout: o.requestTimeout,
|
||||
TailIdleTimeout: o.tailIdleTimeout,
|
||||
}, observabilityproxy.NewAuthorizer(client, o.authCacheTTL), observabilityproxy.NewStoreResolver(lister, namespaces.Informer().HasSynced, o.rootTenant), reg)
|
||||
}, observabilityproxy.NewAuthorizer(client, o.authCacheTTL), observabilityproxy.NewStoreResolver(lister, namespaces.Informer().HasSynced, o.rootTenant),
|
||||
observabilityproxy.NewApplicationKinds(appDefs.GetIndexer(), appDefs.HasSynced), reg)
|
||||
|
||||
probes := http.NewServeMux()
|
||||
probes.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "ok") })
|
||||
probes.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
if !ready.Load() {
|
||||
http.Error(w, "namespace cache not synced", http.StatusServiceUnavailable)
|
||||
http.Error(w, "namespace or application definition cache not synced", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
_, _ = io.WriteString(w, "ok")
|
||||
|
||||
@@ -32,6 +32,20 @@ func TestParseFlagsDefaults(t *testing.T) {
|
||||
if o.authCacheTTL != 10*time.Second || o.logs.MaxTailResume != 15*time.Minute || o.logs.MaxRange != 7*24*time.Hour {
|
||||
t.Errorf("limit defaults changed: %+v", o)
|
||||
}
|
||||
// Retention and store names of packages/system/monitoring/values.yaml.
|
||||
m := o.metrics
|
||||
if m.ShortTermStore != "shortterm" || m.LongTermStore != "longterm" || m.ShortTermRetention != 72*time.Hour {
|
||||
t.Errorf("metrics store defaults changed: %+v", m)
|
||||
}
|
||||
if m.Limits.MaxRange != 30*24*time.Hour || m.Limits.MaxPoints != 11000 || m.Limits.MaxTimeout != 30*time.Second {
|
||||
t.Errorf("metrics limit defaults changed: %+v", m.Limits)
|
||||
}
|
||||
if m.App.Lookback != 30*24*time.Hour || m.App.MaxPods != 500 || m.App.MaxPVCs != 500 {
|
||||
t.Errorf("application scope defaults changed: %+v", m.App)
|
||||
}
|
||||
if m.RootTenant != o.rootTenant {
|
||||
t.Errorf("metrics root tenant = %q, want --root-tenant", m.RootTenant)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFlagsOverrides(t *testing.T) {
|
||||
@@ -48,6 +62,23 @@ func TestParseFlagsOverrides(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFlagsMetricsOverrides(t *testing.T) {
|
||||
o, err := parseFlags([]string{
|
||||
"--root-tenant=tenant-x", "--metrics-shortterm-store=fast", "--metrics-longterm-store=slow",
|
||||
"--metrics-shortterm-retention=24h", "--metrics-max-range=168h", "--metrics-max-points=500", "--metrics-max-timeout=10s",
|
||||
"--metrics-app-lookback=2h", "--metrics-app-max-pods=7", "--metrics-app-max-pvcs=3",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := o.metrics
|
||||
if m.RootTenant != "tenant-x" || m.ShortTermStore != "fast" || m.LongTermStore != "slow" || m.ShortTermRetention != 24*time.Hour ||
|
||||
m.Limits.MaxRange != 168*time.Hour || m.Limits.MaxPoints != 500 || m.Limits.MaxTimeout != 10*time.Second ||
|
||||
m.App.Lookback != 2*time.Hour || m.App.MaxPods != 7 || m.App.MaxPVCs != 3 {
|
||||
t.Errorf("overrides not applied: %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFlagsInvalid(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"--root-tenant="},
|
||||
@@ -69,7 +100,23 @@ func TestParseFlagsInvalid(t *testing.T) {
|
||||
{"--max-concurrent-per-user=-1"},
|
||||
{"--max-tails-per-user=-1"},
|
||||
{"--max-tails=-1"},
|
||||
{"--metrics-shortterm-store="},
|
||||
{"--metrics-longterm-store=Long.Term"},
|
||||
{"--metrics-shortterm-retention=0s"},
|
||||
{"--metrics-max-range=0s"},
|
||||
{"--metrics-max-points=1"},
|
||||
{"--metrics-max-timeout=0s"},
|
||||
{"--metrics-app-lookback=-1h"},
|
||||
// Every query reads some time before its window, so a zero lookback
|
||||
// would refuse every application metrics query.
|
||||
{"--metrics-app-lookback=0s"},
|
||||
{"--metrics-app-max-pods=0"},
|
||||
{"--metrics-app-max-pvcs=0"},
|
||||
{"--unknown"},
|
||||
// The proxy would cut the request while vmselect is still within
|
||||
// the timeout it was given.
|
||||
{"--request-timeout=10s", "--metrics-max-timeout=30s"},
|
||||
{"--request-timeout=30s", "--metrics-max-timeout=30s"},
|
||||
{"unexpected", "--root-tenant=tenant-x"},
|
||||
} {
|
||||
if _, err := parseFlags(args); err == nil {
|
||||
@@ -77,3 +124,16 @@ func TestParseFlagsInvalid(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The vmselect timeout must stay strictly below a request timeout, and is
|
||||
// free when the request has none.
|
||||
func TestParseFlagsTimeoutPairs(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{"--request-timeout=31s", "--metrics-max-timeout=30s"},
|
||||
{"--request-timeout=0", "--metrics-max-timeout=10m"},
|
||||
} {
|
||||
if _, err := parseFlags(args); err != nil {
|
||||
t.Errorf("parseFlags(%v) refused: %v", args, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
+46
-66
@@ -42,14 +42,14 @@ After every change, verify the relevant artifacts before opening the PR:
|
||||
- **Helm templates changed:** `make show` in the package directory renders cleanly.
|
||||
- **Go code changed:** `make generate && make manifests` at the repo root, then commit the regenerated files.
|
||||
- **`values.yaml` annotations changed:** `make generate` in the package; review the `values.schema.json` diff.
|
||||
- **Migration scripts:** confirm what data actually exists first with `kubectl get <resource> -A -o jsonpath='{...}'` on a dev cluster before writing the logic.
|
||||
- **Migration scripts:** when the task supplies a kubeconfig, export its path as `KUBECONFIG`. Set `KUBE_CONTEXT` to the human-approved dev-cluster context and confirm the existing data with `kubectl --context "${KUBE_CONTEXT:?}" get <resource> --all-namespaces --output=jsonpath='{...}'` before writing the logic.
|
||||
|
||||
## Commit Format
|
||||
|
||||
Follow [Conventional Commits](https://www.conventionalcommits.org/) with `--signoff`:
|
||||
|
||||
```bash
|
||||
git commit --signoff -m "type(scope): brief description"
|
||||
git commit --signoff --message "type(scope): brief description"
|
||||
```
|
||||
|
||||
**Types:** `feat`, `fix`, `security`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, `chore`
|
||||
@@ -67,12 +67,23 @@ Breaking changes: append `!` after type/scope (`feat(api)!: ...`) or add a `BREA
|
||||
The subject is imperative and specific; `fix`, `wip` and `update` are not subjects. The body, when there is one, explains why the change is made and only that: the diff already shows what changed and how. Write for a reader who has only `git log`, so no review-iteration or planning vocabulary (`address review comments`, `pass 2`, `batch 1`, `as discussed`) and no bare issue or ticket number in place of the explanation.
|
||||
|
||||
**Examples:**
|
||||
|
||||
```bash
|
||||
git commit --signoff -m "feat(dashboard): add config hash annotations to restart pods on config changes"
|
||||
git commit --signoff -m "fix(postgres): update operator to version 1.2.3"
|
||||
git commit --signoff -m "docs(contributing): add installation guide"
|
||||
git commit --signoff --message "feat(dashboard): add config hash annotations to restart pods on config changes"
|
||||
git commit --signoff --message "fix(postgres): update operator to version 1.2.3"
|
||||
git commit --signoff --message "docs(contributing): add installation guide"
|
||||
```
|
||||
|
||||
### One Commit, One Logical Change
|
||||
|
||||
A commit is the unit a backport cherry-picks and a revert undoes, so it carries exactly one logical change: one fix, one feature, one refactor, together with the regenerated files and tests that belong to it. When the content of a commit changes during review, rewrite that commit (`git commit --amend`, or a `fixup` in `git rebase --interactive`) rather than stacking a correction on top; the branch should read as if each change had been right the first time. If another commit in the series names the rewritten commit in a `Fixes:` trailer, update that reference to its new hash and subject.
|
||||
|
||||
### References to Commits and Issues
|
||||
|
||||
A commit that fixes a defect introduced by an earlier commit carries a kernel-style `Fixes:` trailer with at least 12 characters of the hash and the full subject, on one unbroken line, which `git log --max-count=1 --abbrev=12 --format='Fixes: %h ("%s")' <sha>` prints. `Fixes: #N` is not used, so that the `Fixes:` trailer always points at a commit.
|
||||
|
||||
Use `Closes: #N` only when the change fully resolves the issue. For partial work, use a non-closing reference such as `Refs: #N`; a closing commit trailer can close the issue when it reaches the default branch even if the PR body uses a non-closing link. For an issue in another repository, qualify the reference as `owner/repository#N`, including in a closing trailer (`Closes: owner/repository#N`). Reference trailers go after the body and before `Assisted-by:` and `Signed-off-by:`.
|
||||
|
||||
### AI Agent Attribution
|
||||
|
||||
When an AI agent authors or materially assists with a commit, add exactly one trailer line next to the `Signed-off-by:` trailer that `--signoff` produces:
|
||||
@@ -104,13 +115,13 @@ Editing the title does not re-run it. The `edited` trigger was dropped in #3211
|
||||
|
||||
**Type → `kind/*`:**
|
||||
|
||||
| type | label |
|
||||
| --------- | ------------------ |
|
||||
| feat | kind/feature |
|
||||
| fix | kind/bug |
|
||||
| docs | kind/documentation |
|
||||
| chore | kind/cleanup |
|
||||
| refactor | kind/cleanup |
|
||||
| type | label |
|
||||
| --- | --- |
|
||||
| feat | kind/feature |
|
||||
| fix | kind/bug |
|
||||
| docs | kind/documentation |
|
||||
| chore | kind/cleanup |
|
||||
| refactor | kind/cleanup |
|
||||
| security, style, perf, test, build, ci, revert | (no kind label) |
|
||||
|
||||
**Scope → `area/*`** (full mapping in `.github/workflows/pr-labeler.yaml`):
|
||||
@@ -150,13 +161,17 @@ If the branch has extra commits, clean it up:
|
||||
git fetch upstream
|
||||
git checkout -b my-feature upstream/main
|
||||
git cherry-pick <your-commit-hash>
|
||||
git push -f origin my-feature
|
||||
git push --force origin my-feature
|
||||
```
|
||||
|
||||
## Pull Request Body
|
||||
|
||||
Fill in the template at [`.github/PULL_REQUEST_TEMPLATE.md`](../../.github/PULL_REQUEST_TEMPLATE.md). It includes the required `release-note` block and the [Downstream Repositories](#downstream-repositories) checklist.
|
||||
|
||||
Link the issues the PR addresses in its body. Use `Closes #N` when it fully resolves an issue, or `Closes owner/repository#N` for another repository; link partial work without a closing keyword. A closing keyword in a commit can close the issue on merge into the default branch, but does not create the PR's linked-issue entry. Put the reference in the PR body even when the commit already carries it.
|
||||
|
||||
GitHub interprets closing keywords in a PR description only when it targets the default branch. For a release-branch PR, use a non-closing reference and have a maintainer with write access link the issue manually and close it separately when appropriate. See [GitHub's issue-linking rules](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/linking-a-pull-request-to-an-issue).
|
||||
|
||||
Create the PR with `gh pr create --title "type(scope): brief description" --body-file <file>`.
|
||||
|
||||
`--body` and `--body-file` replace the body wholesale, so the template is not applied and its checklists are silently dropped. Start the body file from the template instead:
|
||||
@@ -167,6 +182,14 @@ cp .github/PULL_REQUEST_TEMPLATE.md /tmp/pr-body.md
|
||||
gh pr create --draft --title "type(scope): brief description" --body-file /tmp/pr-body.md
|
||||
```
|
||||
|
||||
### What Goes in the Commit, What Goes in the PR Body
|
||||
|
||||
The commit message is the permanent record: a reader of `git log` or `git blame` years later has it and nothing else. It carries what broke or was missing and for whom, the exact error string verbatim when there is one, so that `git log --grep` finds it, why this design and not the obvious alternative, why it is safe where the diff does not show that, and one line on how it was tested, including the failure path when the change adds one.
|
||||
|
||||
The PR body is read once, next to those commits, so it does not repeat them. Its first screen says what is broken or missing, where, and what that costs. After that come one sentence per commit and only what no single commit can carry: results that need the whole series, the environment they came from, decisions the reviewer has to make across commits. Questions to reviewers and open doubts belong in the PR body and never in a commit, where they go stale the day it merges. A fact that fits both goes in the commit.
|
||||
|
||||
Size a commit body against accepted commits touching the same files, not against a fixed line cap, which cuts the answers before the narrative. After shortening any of these texts, check that every must-keep fact survived, that every number still sits next to the version or setup it was measured on, that nothing new was added, and that a claim about a set ("every caller", "never happens") is backed by a grep or rewritten as a mechanism.
|
||||
|
||||
## Downstream Repositories
|
||||
|
||||
Cozystack is upstream for repositories that are **not** kept in sync with it automatically. A change here can break them silently, because nothing in CI compares the two sides.
|
||||
@@ -292,60 +315,17 @@ The release process keeps a **separate** list of repositories, in [`changelog.md
|
||||
|
||||
- A bug-fix PR should include a behavioural regression test, not just a single field assertion — prove the bug can no longer recur.
|
||||
- Keep the PR description aligned with the template: `## What this PR does` plus the `release-note` block.
|
||||
- Before posting a claim about a set ("all callers", "the only user", "never happens"), grep for the counterexample, or state the mechanism instead.
|
||||
- Disagreement carries a mechanism: a code path, a number, a spec clause, a failing test. Time spent and urgency are not arguments.
|
||||
- Answer the point directly and explain why it is accepted or declined. Read brief feedback literally rather than inferring tone, and avoid unnecessary notifications. Follow the [prose guidance](../../AGENTS.md#working-principles).
|
||||
|
||||
## Fetching Unresolved Review Comments
|
||||
## Collecting Review Feedback
|
||||
|
||||
Cozystack uses GitHub review threads with resolution status. Only unresolved threads are actionable — resolved threads are already handled.
|
||||
Read the feedback from these sources, including existing replies:
|
||||
|
||||
The REST endpoint `/pulls/{pr}/reviews` returns review summaries, not individual review comments. Use the GraphQL API to access `reviewThreads` with `isResolved` status:
|
||||
- Unresolved inline review threads.
|
||||
- Review submission bodies, even when the review also has inline comments.
|
||||
- General PR conversation comments.
|
||||
- Comments posted directly on the PR's commits.
|
||||
|
||||
```bash
|
||||
gh api graphql -F owner=cozystack -F repo=cozystack -F pr=<PR_NUMBER> -f query='
|
||||
query($owner: String!, $repo: String!, $pr: Int!) {
|
||||
repository(owner: $owner, name: $repo) {
|
||||
pullRequest(number: $pr) {
|
||||
reviewThreads(first: 100) {
|
||||
nodes {
|
||||
isResolved
|
||||
comments(first: 100) {
|
||||
nodes {
|
||||
id
|
||||
path
|
||||
line
|
||||
author { login }
|
||||
bodyText
|
||||
url
|
||||
createdAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}' --jq '.data.repository.pullRequest.reviewThreads.nodes[] | select(.isResolved == false) | .comments.nodes[]'
|
||||
```
|
||||
|
||||
Compact one-line variant:
|
||||
|
||||
```bash
|
||||
gh api graphql -F owner=cozystack -F repo=cozystack -F pr=<PR_NUMBER> -f query='
|
||||
query($owner: String!, $repo: String!, $pr: Int!) {
|
||||
repository(owner: $owner, name: $repo) {
|
||||
pullRequest(number: $pr) {
|
||||
reviewThreads(first: 100) {
|
||||
nodes {
|
||||
isResolved
|
||||
comments(first: 100) {
|
||||
nodes {
|
||||
path
|
||||
line
|
||||
author { login }
|
||||
bodyText
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}' --jq '.data.repository.pullRequest.reviewThreads.nodes[] | select(.isResolved == false) | .comments.nodes[] | "\(.path):\(.line // "N/A") - \(.author.login): \(.bodyText[:150])"'
|
||||
```
|
||||
Verify each human or automated finding against the current code before deciding whether it needs a change. Do not post a duplicate answer to a point already addressed.
|
||||
@@ -81,8 +81,7 @@ Trap-based cleanup was the single biggest source of false failures in the BATS s
|
||||
|
||||
The install test fails if **any** HelmRelease is not Ready. A toothless gate (backgrounded `kubectl wait` discarding child exit codes + a bare `echo` instead of `exit 1`) once shipped a permanently-failing platform HR through green CI for weeks.
|
||||
|
||||
- Use a single `kubectl wait hr --all -A`, then an outcome-based re-list (covers HRs created after the snapshot), and `exit 1` on any non-Ready HR.
|
||||
- Dump the full Ready-condition message per non-Ready HR so the real error is in the test output. See `hack/e2e-install-cozystack.bats`.
|
||||
Use `cozy_wait_all_helmreleases_ready` from `hack/e2e-wait-helmreleases.sh`, as `hack/e2e-install-cozystack.bats` does, and preserve its failure status and diagnostics.
|
||||
|
||||
### 6. Assert no HelmRelease failed an action it retries in place — via `status.history`
|
||||
|
||||
@@ -121,7 +120,7 @@ Be precise about TIA's scope — it is narrower than "skip E2E for unrelated PRs
|
||||
- **It trims only which suites `chainsaw test` runs, not the expensive stages.** `Prepare environment` and `Install Cozystack` (the full platform install) run regardless of the selection. TIA only decides the `CHAINSAW_SUITES` passed to the final E2E step. A perfect narrow saves the suite tail, not the bulk of wall-time. (Package image builds are scoped separately by `hack/build-matrix.sh`; the Talos leg runs when its direct or shared inputs changed and remains unconditional for forks, while installer/finalize runs for every non-docs PR. That is a distinct mechanism from TIA's suite trimming.)
|
||||
- **`Select E2E tests` runs *after* `Install Cozystack` and has no `always()`.** GitHub implicitly ANDs a step's `if:` with `success()`, so if install fails the selector (and `Run E2E tests`) are skipped entirely. On a red install you will not see TIA narrow anything — that is the ordering, not a selector bug.
|
||||
- **Most PRs legitimately escalate to the full suite.** Platform manifests live under `packages/core/`, Go under `api/`/`internal/`, helpers under `hack/`, plus the `Makefile` — touching any one triggers the full suite. So "all suites selected" is usually correct, not a failure to narrow.
|
||||
- **Two skip layers, with a gap.** The `plan` job computes a coarse docs-only gate (`code`, true for any change outside `docs/**`) that skips the whole build+E2E pipeline for docs-only PRs, plus a per-package build matrix (`hack/build-matrix.sh`) that scopes which images build. TIA's finer skip (`select-e2e.sh` ignoring `docs/`, `dashboards/`, `*.md` and the rest of `inert_config_pattern`) only trims the suite selection. A PR touching only `dashboards/*.json` therefore still runs install, then skips just the Chainsaw suites — near-zero savings. If you need such PRs to skip the pipeline, widen the `plan` docs-gate, not `select-e2e.sh`.
|
||||
- **Two skip layers, with a gap.** The `plan` job computes a coarse gate (`code`, false only when every changed path is under `docs/` or one of the CI-only Go tools no image builds, `cmd/backport-batch/` and `cmd/api-gate/`) that skips the whole build+E2E pipeline for such PRs, and `e2e-fork.yaml` skips on the same prefixes (`hack/gate-concurrency-contract.bats` holds the two lists equal), plus a per-package build matrix (`hack/build-matrix.sh`) that scopes which images build. TIA's finer skip (`select-e2e.sh` ignoring `docs/`, `dashboards/`, `*.md` and the rest of `inert_config_pattern`) only trims the suite selection. A PR touching only `dashboards/*.json` therefore still runs install, then skips just the Chainsaw suites — near-zero savings. If you need such PRs to skip the pipeline, widen the `plan` docs-gate, not `select-e2e.sh`.
|
||||
|
||||
### 8. Failure must be self-explanatory — diagnostics are first-class
|
||||
|
||||
@@ -160,6 +159,7 @@ A fork's `pull_request` run carries no registry credentials, so it cannot push t
|
||||
|
||||
Consequences to keep in mind when touching either workflow:
|
||||
|
||||
- **A `workflow_run` workflow always runs the default branch's copy of itself.** An edit to `e2e-fork.yaml` therefore has no effect on any fork PR, the one carrying the edit included, until it is merged to `main`. To see it work, merge first and then push to a fork PR: re-running an existing `e2e-fork` run replays the commit and workflow file it was first triggered with.
|
||||
- **Fork builds export, they do not push (`OCI_EXPORT_DIR`).** The fork's unprivileged build sets `OCI_EXPORT_DIR` (see `hack/common-envs.mk`) so every image is written to a per-image OCI archive artifact instead of being pushed; the privileged `e2e-fork.yaml` later pushes those archives to the registry **by digest**. Any build recipe that shells out to push directly — e.g. a `skopeo copy … docker://…` that bypasses the `image-tags`/`oci-output` macros — must honour `OCI_EXPORT_DIR` too, or a fork build dies with an anonymous-push `denied` (#3257). This is covered by `hack/common-envs_test.bats`.
|
||||
- **"E2E Tests" is a commit status — not a job, and not a check-run.** The required merge-gate context named `E2E Tests` is a GitHub **commit status**, posted by the `plan`/`e2e-report` pair for same-repo PRs and by `e2e-fork.yaml`'s `resolve`/`report` pair for forks — **not** the `e2e` job itself. Each lane opens the context with `pending` and concludes it when the suite is done, and the opener is not optional: a commit status never expires, so a head SHA that already went green would otherwise keep reading green from the moment a re-run, a reopen or a `full-e2e` label starts a new run until that run reports, leaving the pull request mergeable across a suite that may be about to fail. While the gate was a job, a fresh non-success check run appeared automatically and closed that window. The two lanes close it at different speeds, and the fork lane does not close it entirely. Same-repo, the opener is `plan`'s first step, so the window lasts seconds. On a fork the opener is `resolve`, which cannot run until the unprivileged build has finished, so re-running an already-green fork head leaves the old green readable for the length of that build. The gap is structural rather than an oversight: a fork's `pull_request` token is read-only and cannot write a status at all, which is the same property the rest of this design relies on. It is bounded by the build and it is the one case where "the opener closes the window" is not literally true. Two reasons it is a status:
|
||||
- *Not a job:* a required *job* that is skipped (as the `e2e` job is for forks) counts as passed by branch protection — exactly how fork PRs used to merge with e2e never run (#3257). A status is `success` only when the report step posts it; a never-posted required status stays "Expected" and blocks the merge. If you rename the `e2e` job, do **not** name it `E2E Tests` and do not wire branch protection to a job name — the status must stay the gate.
|
||||
@@ -181,9 +181,9 @@ Consequences to keep in mind when touching either workflow:
|
||||
|
||||
### 11. Never negate a BATS assertion with `!`
|
||||
|
||||
An assertion spelled `! cmd` inside an `@test` body cannot fail, whichever runner executes it. POSIX exempts a command whose return value is inverted with `!` from `set -e`, so the failing status is dropped where it is produced. `hack/cozytest.sh` then appends `return 0` to every test function, so it never resurfaces as the test's own status. The `bats` binary looks stricter and mostly is not: its ERR trap carries the identical exemption, and `bash -c 'set -eE; trap "echo TRAP" ERR; ! true; echo REACHED'` prints `REACHED`, runs no trap, and exits zero. The one shape `bats` does catch is a negated assertion standing **last** in the body, where the inverted status is what the function returns — put any statement below it and `bats` passes the test too. This is the same family as the ban on test-level `EXIT`/`RETURN` traps, and unlike that one it is not settled by preferring the stricter runner, because there is no stricter runner to prefer.
|
||||
An assertion spelled `! cmd` inside an `@test` body can silently pass. POSIX exempts a command whose return value is inverted with `!` from `set -e`, so a failing inverted status does not stop the test body. `hack/cozytest.sh` then appends `return 0` to every test function, so it never resurfaces as the test's own status. The `bats` binary looks stricter and mostly is not: its ERR trap carries the identical exemption, and `bash -c 'set -eE; trap "echo TRAP" ERR; ! true; echo REACHED'` prints `REACHED`, runs no trap, and exits zero. The one shape `bats` does catch is a negated assertion standing **last** in the body, where the inverted status is what the function returns — put any statement below it and `bats` passes the test too. This is the same family as the ban on test-level `EXIT`/`RETURN` traps, and unlike that one it is not settled by preferring the stricter runner, because there is no stricter runner to prefer.
|
||||
|
||||
The exemption is also wider than the "negated pipeline" phrasing found in older comments: a bare `! true`, a negated `[ … ]`, one inside a `for` body, and one following `;`, `&&` or `||` all pass. Write the absence so that it fails — `if cmd; then echo "FAIL: <what must not have happened>"; false; fi` — and let the `false` carry the verdict. `hack/bats-no-negated-assert.bats` enforces this across every `hack/**/*.bats`, subdirectories included. Inverted **conditions** (`if !`, `elif !`, `while !`, `until !`) are untouched, because `set -e` is not what decides a condition. `[ ! -f x ]` is untouched for a different reason and stays a normal, failing assertion: the `!` there is an operator of `test`, not negation of a command, so nothing exempts its status. The scan is lexical, with the limits that implies: a statement assembled at runtime is invisible to it, and the form is counted inside a heredoc or a fixture string where no command exists — which reports a file holding nothing rather than passing one that does, and is why that guard's own fixtures build the token from a separate `printf` argument.
|
||||
The exemption applies to any inverted command, not just pipelines: `! true`, negated `[ … ]`, negation inside a `for` body, and negation following `;`, `&&` or `||` do not trigger errexit. A later successful statement can hide their failure. Write the absence so that it fails — `if cmd; then echo "FAIL: <what must not have happened>"; false; fi` — and let the `false` carry the verdict. `hack/bats-no-negated-assert.bats` enforces this across every `hack/**/*.bats`, subdirectories included. Inverted **conditions** (`if !`, `elif !`, `while !`, `until !`) are untouched, because `set -e` is not what decides a condition. `[ ! -f x ]` is untouched for a different reason and stays a normal, failing assertion: the `!` there is an operator of `test`, not negation of a command, so nothing exempts its status. The scan is lexical, with the limits that implies: a statement assembled at runtime is invisible to it, and the form is counted inside a heredoc or a fixture string where no command exists — which reports a file holding nothing rather than passing one that does, and is why that guard's own fixtures build the token from a separate `printf` argument.
|
||||
|
||||
## Chainsaw v0.2.15 gotchas
|
||||
|
||||
@@ -209,7 +209,7 @@ The suite is pinned to Chainsaw **v0.2.15** (the latest release as of May 2026);
|
||||
7. Failure path attaches scoped diagnostics via a `catch:` block, never a silent pass. The node-join deadline is no exception, and no longer has one: with the soft-red marker gone it fails its suite like any other deadline and its diagnostics run from the same `catch:`.
|
||||
8. If it touches HelmRelease install/upgrade behavior, add the `status.history` guard: fail the run on a `failed` or `uninstalled` Snapshot of a release whose install strategy is `RetryOnFailure`, fail it on a parent whose strategy is anything else, and report the footprint of every other release without failing. What is reusable today is the predicate and the jsonpath in `hack/e2e-chainsaw/_lib/remediation-guard.sh`; the policy that weighs the footprint against `.spec.install.strategy.name`, judges an empty history against the release's own readiness, and keeps each read's exit status apart from its output lives in `cozy_guard_helmrelease` in `hack/e2e-chainsaw/_lib/run-kubernetes.sh` — read it before rebuilding any of that for another suite. If the suite has a failure path that exits before the guard, give that path `cozy_report_helmrelease_remediation` as well, or the runs the guard exists for are the runs it does not see.
|
||||
9. A test that creates a tenant `Kubernetes` cluster waits for its release to reach `Ready` **before the assertions it ends on**, with a `--timeout` the guard can read as positive and no longer than a day. Reads earlier in the run are expected and are not what this bounds. The mechanism, how to size the budget and what the guard does with it are under "Do not mask cleanup or teardown failures". One thing to carry from there rather than look up: the naming is part of the rule, not a detail of the check, so an end-assertion added under any name outside the `cozy_assert_*` / `cozy_switch_and_assert_*` convention is invisible to it and passes before the wait without reddening anything — as long as one recognised call site is left behind it. Rename every recognised call and the guard reds instead, on finding none at all, which is the one shape of this mistake it does catch.
|
||||
10. No BATS assertion begins with `!` — an inverted status is exempt from `set -e` and the assertion cannot fail; use `if cmd; then echo "FAIL: …"; false; fi`.
|
||||
10. No BATS assertion begins with `!` — an inverted status is exempt from `set -e` and can be silently ignored; use `if cmd; then echo "FAIL: …"; false; fi`.
|
||||
|
||||
## The container lane: what the srv nodes are, and what that costs
|
||||
|
||||
|
||||
+42
-3
@@ -16,7 +16,7 @@ The platform exposes infrastructure services via the Kubernetes API with ready-m
|
||||
|
||||
## Code Layout
|
||||
|
||||
```
|
||||
```text
|
||||
.
|
||||
├── packages/ # Main directory for cozystack packages
|
||||
│ ├── core/ # Core platform logic charts (installer, platform)
|
||||
@@ -50,7 +50,7 @@ The platform exposes infrastructure services via the Kubernetes API with ready-m
|
||||
|
||||
Every package is a Helm chart following the umbrella chart pattern:
|
||||
|
||||
```
|
||||
```text
|
||||
packages/<category>/<package-name>/
|
||||
├── Chart.yaml # Chart definition and parameter docs
|
||||
├── Makefile # Development workflow targets
|
||||
@@ -106,17 +106,39 @@ A chart also declares the limits on its own resource name there, with `## @name
|
||||
|
||||
## Testing
|
||||
|
||||
- **Helm and BATS unit tests:** `make helm-unit-tests` runs `hack/helm-unit-tests.sh` over every package that defines a `test` target. A `test` target is expected to run helm-unittest: the sweep fails a package whose run reports no suite, because exiting 0 having asserted nothing is indistinguishable from passing. A target that drives something else belongs under another name, as `packages/core/testing` already does with its sandbox flow. `make bats-unit-tests` requires bats-core 1.5 or newer; GNU parallel is optional and the target runs serially without it. Run the complete local BATS and POSIX-compatibility pass with `pre-commit run bats-unit-tests --hook-stage manual --all-files`; pull-request CI runs both lanes for code changes and the BATS contracts for docs-only changes. On a host whose `/bin/sh` differs from CI, `make BATS_POSIX_SHELL=dash bats-posix-compat-tests` selects the same interpreter explicitly. `make unit-tests` runs the full unit suite — Helm, BATS, the targeted POSIX-shell compatibility pass, Go, and the preset/readiness checks. It does not include the controller tests under `./internal/...`; run `make test-controllers` as well. CI invokes both targets together.
|
||||
- **Helm and BATS unit tests:** `make helm-unit-tests` runs `hack/helm-unit-tests.sh` over every package that defines a `test` target. A `test` target is expected to run helm-unittest: the sweep fails a package whose run reports no suite, because exiting 0 having asserted nothing is indistinguishable from passing. A target that drives something else belongs under another name, as `packages/core/testing` already does with its sandbox flow. `make bats-unit-tests` requires bats-core 1.5 or newer; GNU parallel is optional and the target runs serially without it. Run the complete local BATS and POSIX-compatibility pass with `pre-commit run bats-unit-tests --hook-stage manual --all-files`; pull-request CI runs both lanes for code changes and the BATS contracts for docs-only changes. On a host whose `/bin/sh` differs from CI, `make BATS_POSIX_SHELL=dash bats-posix-compat-tests` selects the same interpreter explicitly. `make unit-tests` runs the full unit suite — Helm, BATS, the targeted POSIX-shell compatibility pass, Go, and the preset/readiness checks. It does not include the controller tests under `./internal/...`; run `make test-controllers` as well. CI invokes both targets together. The backport-batch tests are not part of it: run `make test-backport-batch` after changing anything listed in `cmd/backport-batch/test-inputs.txt`, which is also what CI keys them on.
|
||||
- **E2E tests:** Kyverno Chainsaw suites in `hack/e2e-chainsaw/` (one directory per app), run with `chainsaw test`. Cluster bootstrap (`hack/e2e-install-cozystack.bats`) and the OpenAPI checks (`hack/e2e-test-openapi.bats`) remain BATS. Conventions for writing and stabilising them — and the CI that runs them — live in [`e2e-testing.md`](./e2e-testing.md).
|
||||
- **Go tests:** standard `go test`, with Ginkgo/Gomega for controllers.
|
||||
|
||||
### Test first, failure path first
|
||||
|
||||
These rules apply to changes with testable behavior. Documentation-only edits and unmodified upstream updates need the relevant checks, not an artificial failing test. First-party adaptations made during a dependency bump remain covered by these rules.
|
||||
|
||||
1. Write the test before changing the implementation and watch it fail for the intended reason, not on a typo or a missing fixture. Then make it green, then refactor under it.
|
||||
2. Cover relevant failure paths before the happy path: empty or null values, invalid input, a number where a string is expected, a missing resource, a timeout or an API error. Assert the resulting message, exit status and post-failure state required by the operation's contract.
|
||||
3. Fix regressions in the implementation. Do not skip, weaken or delete a test, or silence a linter, merely to get a green run. When the intended contract changes, update or replace the obsolete assertions and retain coverage of applicable failure cases.
|
||||
4. A stub needs a failing test for the intended behavior during development. Finish that behavior and make the test pass before merge; do not make the test accept an unintended not-implemented result just to get green CI.
|
||||
5. A bug found along the way that is out of the PR's scope gets an issue and a test pinning the current broken behaviour, marked `FIXME(#N)`. The PR that fixes the bug flips the test to pin the fixed behaviour.
|
||||
|
||||
Do not negate a BATS assertion with `!`: it can silently pass. Bats catches a failing negation only when it is the last command in the test body; `hack/cozytest.sh` appends a successful return and misses it there too. See [§11 of `e2e-testing.md`](./e2e-testing.md#11-never-negate-a-bats-assertion-with-).
|
||||
|
||||
### Linters
|
||||
|
||||
Fix every linter finding before pushing, the cosmetic ones included. When a rule genuinely conflicts with the project, change the linter configuration and state the reason there, instead of suppressing the finding inline.
|
||||
|
||||
Markdown follows these markdownlint rules: every code fence names a language (`text` when nothing else fits), table separators have spaces around the pipes (`| --- |`), and there is a blank line before and after every list and after every heading. Line length (MD013) is the exception: prose stays one line per paragraph, as [Prose Formatting](../../AGENTS.md#prose-formatting) explains.
|
||||
|
||||
## Conventions
|
||||
|
||||
### Helm Charts
|
||||
|
||||
- Follow **umbrella chart** pattern for system components
|
||||
- Include upstream charts in `charts/` directory (vendored, not referenced)
|
||||
- Override configuration in root `values.yaml`
|
||||
- Use `values.schema.json` for input validation and dashboard UI rendering
|
||||
- An unquoted numeric value (`name: 123`) reaches a template as a number: `len` on it aborts the render with `len of type float64` (`int64` for a value given with `--set`) before any `fail` message is reached, and `if not $v` treats `0` as unset. For fields validated as text that permit numeric inputs, reject missing or null values with `kindIs "invalid"`, then coerce with `printf "%v"` and check emptiness with `eq $v ""`. Coercing null first produces the non-empty string `<nil>`.
|
||||
- For integer fields, keep string inputs on their lexical validation path. Validate numeric inputs for type, integrality and bounds before conversion, then format the validated integer in decimal. Generic `printf "%v"` can emit exponent notation for a float (`4200000000` becomes `4.2e+09`), so it is not integer normalization.
|
||||
- For nested values, validate or normalize each parent map before reading its children; checking only the leaf cannot prevent a nil-pointer error from a missing or null parent.
|
||||
|
||||
**No `lookup` that copies another object into the render.** A `lookup` reads cluster state at render time, and nothing re-renders the HelmRelease when that state changes, so whatever it copied into the output (a bucket name, an endpoint, a CA, a list of StorageClasses) stays stale until something unrelated upgrades the release. Lookups of this kind already in the charts have caused failures, and several have since been moved to values, so they are not precedent: do not add one of this kind unless a maintainer has authorized that specific one.
|
||||
|
||||
@@ -124,7 +146,18 @@ A `lookup` is fine when a repeated render gives the same output and carries noth
|
||||
|
||||
To use a fact about the cluster instead, read it from values: `_cluster` and `_namespace` in the `cozystack-values` Secret, or `valuesFrom` on a specific Secret, whose content change is a values change and so upgrades the release. Ask whether an API is served with `.Capabilities.APIVersions.Has`, since a `lookup` on an unserved group aborts the whole render. Otherwise check at run time in a hook or Job, or let a controller own the object.
|
||||
|
||||
### Container images
|
||||
|
||||
For a new container image built from source maintained in this repository, name the build definition `Containerfile` and pass it to `docker buildx build` with `--file` in the package `Makefile`. Keep existing `Dockerfile`s and vendored build definitions unchanged.
|
||||
|
||||
### Examples and fixtures
|
||||
|
||||
Illustrative addresses and names in docs, examples and test fixtures come from the reserved documentation ranges, never from a routable value that merely looks realistic: IPv4 from RFC 5737 (`192.0.2.0/24`, `198.51.100.0/24`, `203.0.113.0/24`), IPv6 from RFC 3849 (`2001:db8::/32`), domains from RFC 2606 and RFC 6761 (`example.com`, `.example`, `.test`, `.invalid`), ASNs from RFC 5398 (`64496`–`64511`, `65536`–`65551`), MAC addresses from RFC 7042 (`00:00:5E:00:53:00`–`00:00:5E:00:53:FF`). Where no reserved range exists, use an obviously synthetic placeholder such as `<NODE_IP>`.
|
||||
|
||||
When a test depends on a protocol-defined address's behavior, use that value, such as `127.0.0.1` or `localhost` for a local test server. Keep network tests isolated; this exception does not permit arbitrary live endpoints.
|
||||
|
||||
### Go Code
|
||||
|
||||
- Follow standard **Go conventions** and idioms
|
||||
- Use **controller-runtime** patterns for Kubernetes controllers
|
||||
- Prefer **kubebuilder** for API definitions and controllers
|
||||
@@ -158,6 +191,8 @@ When a document could go in either `docs/` or the website, the test is whether a
|
||||
|
||||
Documenting a feature's current limitations is useful and belongs with the feature. Documenting the *development process* around it does not: no "What is NOT in Phase N", no "out of scope for this PR", no "this PR" phrasing, no action plans or review negotiation. A merged document describes what the code does now, and states a limitation as a limitation rather than as a roadmap position. Scope, phasing and follow-ups belong in the pull request description or an issue.
|
||||
|
||||
An app's README and the field descriptions in its `values.yaml` describe what the app does now, for the person using it: no history ("moves from", "previously", "used to", rollout phase names) and no account of how another chart behaved before. A change users must act on is announced by a `!` in the pull request title and described on the website's per-version upgrade page (`content/en/docs/next/operations/cluster/upgrade-to-vX.Y.md`), so an app README carries no history or breaking-changes section. A description says what the field does, what it accepts and what it refuses, not what it replaced, and does not tell the reader to keep it in sync with another chart; when two charts have to agree, a test says so. A change that touches a README or a description fixes what is wrong in that place while it is there.
|
||||
|
||||
A document that names an unmerged pull request or an unreleased dependency needs a tracking issue, because nothing re-checks that claim later and it silently becomes false once the thing ships.
|
||||
|
||||
### Domain references for specific tasks
|
||||
@@ -170,6 +205,7 @@ Before working on these areas, read the relevant doc first:
|
||||
## Things Agents Should Not Do
|
||||
|
||||
### Never Edit These
|
||||
|
||||
- Do not modify files in `/vendor/` (Go dependencies)
|
||||
- Do not edit generated files: `zz_generated.*.go`
|
||||
- Do not change `go.mod`/`go.sum` manually (use `go get`)
|
||||
@@ -177,13 +213,16 @@ Before working on these areas, read the relevant doc first:
|
||||
- Do not modify image digests in `values.yaml` (generated by build)
|
||||
|
||||
### Version Control
|
||||
|
||||
- Do not commit built artifacts from `_out`
|
||||
- Do not commit test artifacts or temporary files
|
||||
|
||||
### Git Operations
|
||||
|
||||
- Do not force push to main/master
|
||||
- Do not update git config
|
||||
- Do not perform destructive operations without explicit request
|
||||
|
||||
### Core Components
|
||||
|
||||
- Do not modify `packages/core/platform/` without understanding migration impact
|
||||
@@ -0,0 +1,82 @@
|
||||
<!--
|
||||
https://github.com/cozystack/cozystack/releases/tag/v1.7.1
|
||||
-->
|
||||
|
||||
# Cozystack v1.7.1
|
||||
|
||||
Cozystack v1.7.1 is a patch release on the v1.7 line. It fixes the COSI driver bug that could leave freshly granted S3 keys rejected for good, makes marketplace taps digest-pinned, makes MariaDB, PostgreSQL and ClickHouse backups report their real outcome, and repairs several tenant-facing issues: tenant cluster metrics, SecurityGroups on Kubernetes workers, Redis with a single replica, the Proxmox cloud-controller-manager and the tcp-balancer. Read the Migration and Upgrades section before upgrading.
|
||||
|
||||
## Fixes
|
||||
|
||||
* **SeaweedFS COSI driver v0.4.0**: bucket users granted together could get S3 keys that never work (`403 InvalidAccessKeyId`) because the v0.3.1 driver rewrote a legacy IAM file the S3 gateways were migrating. The driver is bumped to v0.4.0, which writes each grant to a per-identity IAM file. Keys already lost on running clusters stay broken: remove the user from the Bucket, wait until its BucketAccess is gone, and add it back ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5284, backport #5290).
|
||||
* **Marketplace taps are pinned to a digest**: a tapped repository no longer follows a mutable tag, so moving the tag upstream cannot change what runs in the cluster. `tap connect` accepts `oci://...@sha256:...` references and an optional keyless cosign verify policy, anchored to an exact identity and preserved across re-connects; with the verify flags the CLI verifies the signature and pins the verified digest at tap time. Community index entries can record a digest too ([**@IvanHunters**](https://github.com/IvanHunters) in #5082, backport #5246).
|
||||
* **Tenant cluster metrics reach the tenant's Monitoring**: the vmagent of tenant Kubernetes clusters now writes to the tenant's vminsert and labels series with the owning tenant instead of `cozy-monitoring`; before, no tenant cluster metric reached the tenant's Monitoring ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5024, backport #5246).
|
||||
* **SecurityGroups apply to Kubernetes worker nodes**: the virt-launcher pods of Kubernetes worker nodes are now resolved to their Kubernetes application and marked as managed by Cozystack, so a SecurityGroup attached to a Kubernetes cluster applies to its workers, not only to its control plane ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5124, backport #5246).
|
||||
* **Membership-only SecurityGroups**: a SecurityGroup with attachments and no rules no longer leaves an invalid CiliumNetworkPolicy behind; its backing policy is accepted by Cilium and still allows or denies nothing ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5133, backport #5246).
|
||||
* **Proxmox CCM and CSI images**: the Proxmox cloud-controller-manager runs v0.16.1 and the Proxmox CCM and CSI driver images are pinned by digest. The v0.15.0 tag was re-pushed upstream to a build that panics on start, which left every new Proxmox worker with the uninitialized taint ([**@themoriarti**](https://github.com/themoriarti) in #4917, backport #5246).
|
||||
* **Autoscaler capacity of KubernetesNodes pools**: on the kubevirt substrate the cluster-autoscaler capacity annotation is planned on the cores the VM actually gets, so a leading-zero `resources.cpu` such as `"010"` (decoded as YAML, eight cores) no longer yields a wrong capacity. No pool rolls ([**@themoriarti**](https://github.com/themoriarti) in #4974, backport #5246).
|
||||
* **Tenant quota admission counts extended resources**: admission now reads the parent pool's usage of extended resources and hugepages under the `requests.*` key the controller counts, so a child tenant can no longer get a share the parent has already used ([**@lexfrei**](https://github.com/lexfrei) in #5184, backport #5246).
|
||||
* **Postgres backup jobs fail as soon as the CloudNativePG Backup fails**: a Postgres `BackupJob` now fails with the recorded error instead of after 30 minutes, and reports a `WaitingForCNPGBackup` condition with the Backup's phase while it waits ([**@scooby87**](https://github.com/scooby87) in #5211, backport #5246).
|
||||
* **MariaDB backups and restores report real failures**: a `BackupJob` or `RestoreJob` whose operator Job failed now ends Failed instead of Succeeded, and a failed backup no longer creates a Ready Backup without an archive ([**@scooby87**](https://github.com/scooby87) in #5207, backport #5246).
|
||||
* **ClickHouse restores use the right archive**: a `RestoreJob` restores the archive its Backup was taken as, not the newest one. New Backups record the archive name; Backups taken before this release are restored best effort from the newest archive stamped no later than their `takenAt`, with a warning in the restore pod log. A ClickHouse `BackupJob` created under a name whose Backup still exists now fails instead of reporting that Backup as its result ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5208, backport #5246).
|
||||
* **Backup S3 endpoint changes are followed**: the backup controller re-renders the Velero BSL, the default backup strategies and the platform bucket's credentials Secret when their S3 endpoint is stale, so after the platform bucket's BucketAccess is re-granted for a new `publishing.host`, backups move to the new endpoint without forcing releases by hand ([**@scooby87**](https://github.com/scooby87) in #5230, backport #5246).
|
||||
* **Backup clients run on a dedicated image**: the ClickHouse (Altinity) and RabbitMQ backup strategy Pods now run a small `backup-http-client` image instead of the 349 MB `platform-migrations` image, so a platform migration change no longer re-digests the backup clients. Air-gapped mirrors need the new `ghcr.io/cozystack/cozystack/backup-http-client` repository ([**@androndo**](https://github.com/androndo) in #4756, backport #5246).
|
||||
* **PostgreSQL system databases**: `databases` refuses `postgres`, `template0` and `template1`, and the init job never runs `DROP DATABASE` on them, which used to fail the init job on every upgrade and leave the HelmRelease in `UpgradeFailed` ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5155, backport #5246).
|
||||
* **PostgreSQL init script is stable**: the declared databases render in a stable order, so an upgrade that leaves the databases unchanged no longer rewrites the init-script Secret ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5091, backport #5246).
|
||||
* **Redis with a single replica**: an instance with `replicas: 1` runs one Sentinel instead of three, without a Sentinel PodDisruptionBudget, so node drains are no longer blocked by it; instances with two replicas or more are unchanged ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5151, backport #5246).
|
||||
* **MariaDB operator on a custom cluster domain**: the operator uses the platform's `networking.clusterDomain` instead of a hard-coded `cozy.local`, so replication and User/Database/Grant reconciliation work on clusters with a non-default domain ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #5123, backport #5246).
|
||||
* **tcp-balancer no longer OOMs**: HAProxy takes its fd and thread caps from the replica's own memory and CPU limits instead of the node, so a `t1.nano` balancer is no longer OOMKilled on nodes with many CPUs ([**@yankawai**](https://github.com/yankawai) in #5005, backport #5246).
|
||||
* **etcd alert noise**: `etcdHighNumberOfFailedGRPCRequests` counts only the gRPC codes the etcd mixin counts, so the Canceled watch streams of a stopping Kubernetes API server no longer fire it on every member ([**@yankawai**](https://github.com/yankawai) in #4985, backport #5246).
|
||||
* **Cilium IPAM hook on larger clusters**: the hook passes the CiliumNode list to `jq` on stdin and no longer fails with "Argument list too long" on clusters with more than a few nodes, which blocked the cilium upgrade and every package depending on networking ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5212, backport #5246).
|
||||
* **OIDC bootstrap Job with many users**: the Job no longer fails to start for a tenant cluster with about 240 or more `oidc.users`; the bindings now come from a ConfigMap the Job mounts ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5213, backport #5246).
|
||||
* **Dashboard tenant list links to the tenant page**: the tenant name in the tenant list opens its detail page, where Delete lives, and top-level tenants get their Edit button back ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #4004, backport #5246).
|
||||
* **Dead `registries` platform value removed**: the platform chart no longer has the `registries` value or the `patch-containerd` Secret, which nothing read. Mirrors for tenant workers are the per-pool `talos.registryMirrors` of `KubernetesNodes` ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #5190, backport #5246).
|
||||
|
||||
## Security
|
||||
|
||||
* **Marketplace digest pinning and signature verification**: see the marketplace entry under Fixes (#5082).
|
||||
|
||||
## Development, Testing, and CI/CD
|
||||
|
||||
* **Chainsaw asserts survive transient API errors**: an e2e assert keeps polling through an error such as an etcd leader change instead of failing the suite early ([**@lexfrei**](https://github.com/lexfrei) in #4867, backport #5246).
|
||||
* **Proxmox e2e hygiene**: the Proxmox tenant kubeconfig is kept per run and removed afterwards, the HelmRelease guard reads the `KubernetesNodes` pool release, and the leak allowlist is validated against every suite layout ([**@themoriarti**](https://github.com/themoriarti) in #4971, #4970 and #4972, backports #5246).
|
||||
* **Frozen-line builds start with `GITHUB_TOKEN`**: the initial packages build dispatched when a release line is frozen now runs ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #4888, backport #5246).
|
||||
* **Unit tests after the release image restamp**: two unit tests that read fields the release build rewrites stay green on `release-1.7`; this is a release-branch-only fix ([**@themoriarti**](https://github.com/themoriarti) in #5179).
|
||||
|
||||
## Documentation
|
||||
|
||||
* **[website] Documentation**: the v1.7 docs gain the guides that lived only in the cozystack repository, the pending v1.7 docs PRs, a DRBD note for the replicated storage class, a refreshed Chainsaw e2e guide, restored legacy URLs and a grouping of older doc versions ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#763, cozystack/website#782, cozystack/website#675, cozystack/website#611, cozystack/website#775 and cozystack/website#767).
|
||||
* **[website] Blog and demo**: a full-stack observability post for v1.7, the Cozystack 1.7 announcement, a three-board Raspberry Pi 5 post, and a collapsible navbar with a closable hint on the live demo page ([**@tym83**](https://github.com/tym83) in cozystack/website#790, cozystack/website#801 and cozystack/website#802; [**@kvaps**](https://github.com/kvaps) in cozystack/website#789).
|
||||
* **[website] Pages deploy**: the deploy workflow drops the Dart Sass install ([**@kvaps**](https://github.com/kvaps) in cozystack/website#798).
|
||||
* **[ansible-cozystack] Installer v1.7.0**: tracks the Cozystack installer image at v1.7.0, released in ansible-cozystack v1.7.0 ([**@app/renovate**](https://github.com/apps/renovate) in cozystack/ansible-cozystack#89).
|
||||
|
||||
## Migration and Upgrades
|
||||
|
||||
Platform migrations do not change in this release: `migrations.targetVersion` stays at 61.
|
||||
|
||||
* **Remove system databases from PostgreSQL `databases` before upgrading (#5155).** A release that declares `postgres`, `template0` or `template1` under `databases` is refused at render; remove the entry first. A system database a previous version took over is handed back to the superuser on the next upgrade: the event trigger is removed, ownership and the `public` schema are restored, and its `<db>_admin` and `<db>_readonly` roles are dropped.
|
||||
* **Tenant quota updates can be rejected after upgrading (#5184).** Admission now counts the parent's usage of extended resources and hugepages, so an update to an existing tenant can be rejected when its quota for these resources exceeds the parent's remaining budget, including an update that leaves the quota unchanged.
|
||||
* **Air-gapped mirrors need `backup-http-client` (#4756).** Mirror `ghcr.io/cozystack/cozystack/backup-http-client` before upgrading, or ClickHouse and RabbitMQ backup Pods cannot pull their image.
|
||||
* **tcp-balancer pods roll once (#5005).** The new HAProxy `global` section changes the rendered config; `maxconn` follows the memory limit at about 16 connections per MiB (2035 on `t1.nano`).
|
||||
* **Redis scaled to one replica loses its Sentinel PDB (#5151).** A post-upgrade hook deletes the Sentinel PodDisruptionBudget the operator created earlier for an instance with `replicas: 1`; scaling an instance between 1 and 2 replicas scales its Sentinel Deployment between 1 and 3.
|
||||
* **Backups created by failed MariaDB runs are not cleaned up (#5207).** Backups made by failed runs before this fix are not removed; restoring from one can fail or select another dump from the shared prefix.
|
||||
* **Keys lost to the COSI grant bug stay lost (#5284).** Re-add the affected user to the Bucket as described in the Fixes entry.
|
||||
* **`registries` in the platform Package is ignored (#5190).** A Package that still sets it keeps installing; the `patch-containerd` Secret in `cozy-system` is deleted by the release.
|
||||
* **SecurityGroups now reach Kubernetes workers (#5124).** A SecurityGroup attached to a `Kubernetes` application starts applying to its worker nodes after the upgrade, so review attached groups before upgrading.
|
||||
|
||||
## Contributors
|
||||
|
||||
We'd like to thank all contributors who made this release possible:
|
||||
|
||||
* [**@androndo**](https://github.com/androndo)
|
||||
* [**@IvanHunters**](https://github.com/IvanHunters)
|
||||
* [**@kvaps**](https://github.com/kvaps)
|
||||
* [**@lexfrei**](https://github.com/lexfrei)
|
||||
* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri)
|
||||
* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil)
|
||||
* [**@scooby87**](https://github.com/scooby87)
|
||||
* [**@themoriarti**](https://github.com/themoriarti)
|
||||
* [**@tym83**](https://github.com/tym83)
|
||||
* [**@yankawai**](https://github.com/yankawai)
|
||||
|
||||
**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.7.0...v1.7.1
|
||||
@@ -60,7 +60,7 @@ git diff --name-only $PREV..$NEW -- 'packages/*/charts/*/Chart.yaml' 'packages/*
|
||||
done
|
||||
|
||||
# B. Out-of-Chart.yaml pins (Talos, tenant K8s, vendored images)
|
||||
for path in packages/core/talos/images/talos/profiles/installer.yaml packages/core/installer/values.yaml packages/apps/kubernetes/values.yaml; do
|
||||
for path in packages/core/talos/images/talos/profiles/installer.yaml packages/core/installer/values.yaml packages/apps/kubernetes/values.yaml packages/apps/kubernetes-nodes/values.yaml; do
|
||||
git diff $PREV..$NEW -- "$path" | head -40
|
||||
done
|
||||
git diff --name-only $PREV..$NEW -- 'images/*/Dockerfile'
|
||||
|
||||
@@ -55,7 +55,7 @@ Actors are decomposed by **trust boundary**: two components are distinct actors
|
||||
| GitOps / Flux delivery layer | T1 — platform control plane | Reconciles HelmReleases/Kustomizations into arbitrary privileged manifests; effectively cluster-admin. | Tenants cannot create HelmReleases (RBAC); not tenant-network-reachable. |
|
||||
| `cozystack-api` aggregated API server | T1 — platform control plane | Converts tenant CRs into HelmReleases; can CRUD HelmReleases, Secrets, and Cilium policies cluster-wide. Acts as the policy-enforcing gateway. | Constrains tenants via fixed chart reference, name/quota/host validation, and admission. |
|
||||
| System operators / controllers | T1 — platform control plane | Domain-scoped cluster privileges (provision control planes, mint certs/secrets, program CNI). | Distinct ServiceAccounts; isolated from tenants by RBAC and network policy. |
|
||||
| Tenant administrator (OIDC `<tenant>-admin` / `<tenant>-super-admin`) | T2 — tenant boundary | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` in its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) `super-admin` holds the `apps.cozystack.io/*` wildcard (and can create child `Tenant`s); `admin` has an enumerated allowlist excluding some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`). | Namespace-scoped RBAC + admission policies + quota; no HelmRelease or cross-tenant access. |
|
||||
| Tenant administrator (OIDC `<tenant>-admin` / `<tenant>-super-admin`) | T2 — tenant boundary | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` in its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); `get` on persistentvolumeclaims and `create` on `upload.cdi.kubevirt.io/uploadtokenrequests`, the pair `virtctl image-upload` needs to upload into an existing VMDisk; no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) `super-admin` holds the `apps.cozystack.io/*` wildcard (and can create child `Tenant`s); `admin` has an enumerated allowlist excluding some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`). | Namespace-scoped RBAC + admission policies + quota; no HelmRelease or cross-tenant access. |
|
||||
| Tenant user (OIDC `<tenant>-view` / `<tenant>-use`) | T2 — tenant boundary | Read-only view plus operational verbs (VM console/vnc/portforward, start/stop/restart). Cannot create Application CRs. | Strictly dominated by tenant-admin; cannot escalate. |
|
||||
| Parent vs child (nested) tenant | T2 — tenant boundary | Parent ServiceAccounts bind into descendant namespaces; children do not bind upward. Quota hierarchically capped. | Ancestor→descendant trust is one-way. |
|
||||
| Tenant workloads / managed services | T3 — workload boundary | Run tenant-controlled code; no cluster-API rights beyond their ServiceAccount. | Namespace-scoped RBAC/identity isolate API objects; on the network, Cilium egress confines each tenant pod to its own subtree + platform system services + `world`, so unrelated tenants cannot reach each other over the pod network (enforced on egress; ingress is open to `cluster` — see Non-goals). |
|
||||
|
||||
@@ -39,7 +39,7 @@ The actors below are decomposed **by trust boundary**, not by component: two com
|
||||
| GitOps / Flux delivery layer | T1 | Reconciles HelmReleases/Kustomizations into arbitrary privileged manifests; effectively cluster-admin. | Tenants cannot create HelmReleases/Kustomizations (RBAC); not tenant-network-reachable. |
|
||||
| `cozystack-api` aggregated API server (ServiceAccount) | T1 | CRUD HelmReleases, Secrets, and CiliumNetworkPolicies cluster-wide; reads Namespaces/RBAC/quotas/nodes. Acts as the policy-enforcing gateway that converts tenant CRs to HelmReleases. | A distinct, network-exposed attack surface from Flux; constrains tenants via fixed chart ref + name/quota/host validation + admission. |
|
||||
| System operators / controllers | T1 | Domain-scoped cluster privileges (provision control planes, mint certs/secrets, program CNI). | Distinct ServiceAccounts, but all cluster-admin-adjacent; isolated from tenants only by RBAC and network policy. |
|
||||
| Tenant administrator (OIDC groups `<tenant>-admin`, `<tenant>-super-admin`) | T2 | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` within its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) The `<tenant>-super-admin` group holds the `apps.cozystack.io/*` wildcard (and so can create child `Tenant`s); the `<tenant>-admin` group has an enumerated allowlist that excludes some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`, `external-dns`, `bootbox`). | Namespace-scoped RoleBindings + admission policies + quota; no access to HelmReleases, arbitrary core writes, or other tenants. |
|
||||
| Tenant administrator (OIDC groups `<tenant>-admin`, `<tenant>-super-admin`) | T2 | CRUD on `apps.cozystack.io` application kinds and `sdn.cozystack.io/securitygroups` within its namespace subtree; read on pods/services/ingress plus `delete` on a fixed set (pods/services/endpoints/events); `get` on persistentvolumeclaims and `create` on `upload.cdi.kubevirt.io/uploadtokenrequests`, the pair `virtctl image-upload` needs to upload into an existing VMDisk; no create/update of arbitrary core resources. (The ClusterRole also lists `persistentvolumes`, but that grant is inert under the namespace-scoped RoleBinding, since PVs are cluster-scoped.) The `<tenant>-super-admin` group holds the `apps.cozystack.io/*` wildcard (and so can create child `Tenant`s); the `<tenant>-admin` group has an enumerated allowlist that excludes some kinds (e.g. `tenants`, `monitoring`, `etcd`, `ingress`, `external-dns`, `bootbox`). | Namespace-scoped RoleBindings + admission policies + quota; no access to HelmReleases, arbitrary core writes, or other tenants. |
|
||||
| Tenant user (OIDC groups `<tenant>-view`, `<tenant>-use`) | T2 | Read-only view of Application CRs, pods, and services, plus operational verbs only (VM console/vnc/portforward, start/stop/restart). Cannot create Application CRs. | Strictly dominated by tenant-admin; cannot escalate to create/delete, so it is a distinct actor. |
|
||||
| Parent tenant vs child (nested) tenant | T2 | Parent ServiceAccounts are bound into descendant namespaces; child ServiceAccounts are not bound upward. Quota is hierarchically capped. | Ancestor→descendant only; compromising a parent compromises its children, not the reverse. |
|
||||
| Tenant workloads / managed services | T3 | Run tenant-controlled code; no cluster-API rights beyond their ServiceAccount. | Namespace-scoped RBAC/identity isolate their API objects. On the network, Cilium egress confines each tenant pod to its own tenant subtree, platform system services, and `world` (out-of-cluster), so unrelated tenants cannot reach each other over the pod network — enforced on the egress side (tenant-pod ingress is open to `cluster`; see Non-goals for residual caveats). |
|
||||
@@ -92,7 +92,7 @@ The following are explicitly out of scope for the platform's security guarantees
|
||||
|
||||
The single most load-bearing isolation control is the combination of two facts:
|
||||
|
||||
1. The tenant ClusterRoles (`packages/system/cozystack-basics/templates/clusterroles.yaml`) grant **no** `helm.toolkit.fluxcd.io` access and **no** create/update of arbitrary core resources — only CRUD on `apps.cozystack.io` application kinds (super-admin holds the `*` wildcard; admin an enumerated allowlist) and `sdn.cozystack.io/securitygroups`, read access to pods/services/ingresses, and (for tenant-admins) `delete` on a fixed set of core objects (pods/services/endpoints/events; the `persistentvolumes` entry in the ClusterRole is inert under the namespace-scoped RoleBinding). Tenant isolation is enforced by the *absence* of privileged grants in RBAC, not by an admission policy that whitelists `apps.cozystack.io/*` writes.
|
||||
1. The tenant ClusterRoles (`packages/system/cozystack-basics/templates/clusterroles.yaml`) grant **no** `helm.toolkit.fluxcd.io` access and **no** create/update of arbitrary core resources — only CRUD on `apps.cozystack.io` application kinds (super-admin holds the `*` wildcard; admin an enumerated allowlist) and `sdn.cozystack.io/securitygroups`, read access to pods/services/ingresses, and (for tenant-admins) `delete` on a fixed set of core objects (pods/services/endpoints/events; the `persistentvolumes` entry in the ClusterRole is inert under the namespace-scoped RoleBinding), `get` on persistentvolumeclaims and `create` on `upload.cdi.kubevirt.io/uploadtokenrequests` (for uploading an image into an existing VMDisk with `virtctl image-upload`). Tenant isolation is enforced by the *absence* of privileged grants in RBAC, not by an admission policy that whitelists `apps.cozystack.io/*` writes.
|
||||
2. The Application-to-HelmRelease conversion (`pkg/registry/apps/application/rest.go`) takes the chart reference, release-name prefix, and `valuesFrom` from server-side configuration; the tenant controls only the Helm values (`app.Spec`).
|
||||
|
||||
If a future `ApplicationDefinition`, RBAC change, or API change ever let a tenant influence the chart reference, `valuesFrom`, or write HelmReleases directly, the T2→T1 isolation would collapse. Changes touching tenant RBAC or the Application REST layer should be reviewed against this invariant.
|
||||
|
||||
@@ -9,6 +9,10 @@ load test_helper
|
||||
# scope: they are re-vendored by `make update`, so a bad severity there is
|
||||
# fixed upstream or in patches/, not by editing the label.
|
||||
ALERT_SEVERITY_ALLOWED=" security critical major minor warning indeterminate informational normal ok cleared debug trace unknown "
|
||||
# The alerts that exist only to be dropped, the one place `none` is right.
|
||||
# InfoInhibitor is the source side of an Alertmanager inhibit rule, which the
|
||||
# platform config does not have, so delivering it at any severity is noise.
|
||||
ALERT_SEVERITY_BLACKHOLED=" InfoInhibitor "
|
||||
|
||||
helm_actions_stripped() {
|
||||
awk '
|
||||
@@ -104,6 +108,14 @@ alert_severity_contract() {
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ "$severity" = "none" ]; then
|
||||
for blackholed in $ALERT_SEVERITY_BLACKHOLED; do
|
||||
if [ "$blackholed" = "$alert" ]; then
|
||||
accepted=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ "$accepted" -eq 0 ]; then
|
||||
echo "$file: $alert -> '$severity'" >&2
|
||||
bad=$((bad + 1))
|
||||
@@ -128,6 +140,83 @@ alert_severity_contract() {
|
||||
alert_severity_contract
|
||||
}
|
||||
|
||||
@test "InfoInhibitor is labelled for the blackhole route" {
|
||||
rows=$(mktemp) || return 1
|
||||
if ! alert_severity_rows > "$rows"; then
|
||||
rm -f "$rows"
|
||||
return 1
|
||||
fi
|
||||
tab=$(printf '\t')
|
||||
found=$(grep -c "${tab}InfoInhibitor${tab}" "$rows" || true)
|
||||
delivered=$(grep "${tab}InfoInhibitor${tab}" "$rows" | grep -vc "${tab}none\$" || true)
|
||||
rm -f "$rows"
|
||||
[ "$found" -gt 0 ]
|
||||
[ "$delivered" -eq 0 ]
|
||||
}
|
||||
|
||||
@test "severity none passes only on an alert that exists to be dropped" {
|
||||
fixture=$(mktemp -d)
|
||||
mkdir -p "$fixture/packages/system/example/alerts"
|
||||
printf '%s\n' \
|
||||
'apiVersion: operator.victoriametrics.com/v1beta1' \
|
||||
'kind: VMRule' \
|
||||
'spec:' \
|
||||
' groups:' \
|
||||
' - name: general' \
|
||||
' rules:' \
|
||||
' - alert: InfoInhibitor' \
|
||||
' labels:' \
|
||||
' severity: none' \
|
||||
> "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
|
||||
git -C "$fixture" init -q
|
||||
git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
|
||||
|
||||
inhibitor_succeeded=0
|
||||
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || inhibitor_succeeded=$?
|
||||
|
||||
printf '%s\n' \
|
||||
'apiVersion: operator.victoriametrics.com/v1beta1' \
|
||||
'kind: VMRule' \
|
||||
'spec:' \
|
||||
' groups:' \
|
||||
' - name: silent' \
|
||||
' rules:' \
|
||||
' - alert: SilentAlert' \
|
||||
' labels:' \
|
||||
' severity: none' \
|
||||
> "$fixture/packages/system/example/alerts/01-silent.yaml"
|
||||
git -C "$fixture" add packages/system/example/alerts/01-silent.yaml
|
||||
|
||||
silent_succeeded=0
|
||||
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || silent_succeeded=$?
|
||||
rm -rf "$fixture"
|
||||
[ "$inhibitor_succeeded" -eq 0 ]
|
||||
[ "$silent_succeeded" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "an alert that exists to be dropped still fails on a refused severity" {
|
||||
fixture=$(mktemp -d)
|
||||
mkdir -p "$fixture/packages/system/example/alerts"
|
||||
printf '%s\n' \
|
||||
'apiVersion: operator.victoriametrics.com/v1beta1' \
|
||||
'kind: VMRule' \
|
||||
'spec:' \
|
||||
' groups:' \
|
||||
' - name: general' \
|
||||
' rules:' \
|
||||
' - alert: InfoInhibitor' \
|
||||
' labels:' \
|
||||
' severity: info' \
|
||||
> "$fixture/packages/system/example/alerts/00-inhibitor.yaml"
|
||||
git -C "$fixture" init -q
|
||||
git -C "$fixture" add packages/system/example/alerts/00-inhibitor.yaml
|
||||
|
||||
contract_succeeded=0
|
||||
ALERT_SEVERITY_ROOT="$fixture" alert_severity_contract >/dev/null 2>&1 || contract_succeeded=$?
|
||||
rm -rf "$fixture"
|
||||
[ "$contract_succeeded" -ne 0 ]
|
||||
}
|
||||
|
||||
@test "a parser failure cannot leave the contract green" {
|
||||
fixture=$(mktemp -d)
|
||||
mkdir -p "$fixture/packages/system/example/alerts"
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bats
|
||||
# Unit tests for hack/backport-batch-tests-needed.sh, which decides whether a
|
||||
# pull request runs `make test-backport-batch`, and for its wiring into the
|
||||
# checks job of pull-requests.yaml.
|
||||
#
|
||||
# Test-level EXIT traps replace Bats' own handler and hide failing TAP results.
|
||||
# Cleanup follows aborting assertions; see docs/agents/e2e-testing.md.
|
||||
|
||||
load test_helper
|
||||
|
||||
CLASSIFIER="hack/backport-batch-tests-needed.sh"
|
||||
WF=".github/workflows/pull-requests.yaml"
|
||||
|
||||
classify() {
|
||||
list="$1"
|
||||
shift
|
||||
printf '%s\n' "$@" > "$list"
|
||||
"$CLASSIFIER" "$list"
|
||||
}
|
||||
|
||||
@test "a change to the tool, an imported package, a page its tests read or the module runs the tests" {
|
||||
tmp=$(mktemp)
|
||||
[ "$(classify "$tmp" cmd/backport-batch/report_test.go)" = true ]
|
||||
[ "$(classify "$tmp" internal/apigate/surface.go)" = true ]
|
||||
[ "$(classify "$tmp" pkg/config/config.go)" = true ]
|
||||
[ "$(classify "$tmp" docs/release/backports.md)" = true ]
|
||||
[ "$(classify "$tmp" docs/release/backport-tasks/conflict.md)" = true ]
|
||||
[ "$(classify "$tmp" go.sum)" = true ]
|
||||
# One input among unrelated paths is enough.
|
||||
[ "$(classify "$tmp" packages/apps/mariadb/values.yaml pkg/config/config.go)" = true ]
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
@test "a change that reaches none of its inputs does not run the tests" {
|
||||
tmp=$(mktemp)
|
||||
[ "$(classify "$tmp" packages/apps/mariadb/values.yaml)" = false ]
|
||||
[ "$(classify "$tmp" cmd/api-gate/main.go)" = false ]
|
||||
[ "$(classify "$tmp" hack/backport-batch-scheduled.sh)" = false ]
|
||||
[ "$(classify "$tmp" docs/release.md docs/release/other.md)" = false ]
|
||||
# A file entry is not a prefix: backports.md does not cover backports.md.bak,
|
||||
# and a directory entry does not cover a sibling sharing its name as prefix.
|
||||
[ "$(classify "$tmp" docs/release/backports.md.bak)" = false ]
|
||||
[ "$(classify "$tmp" pkg/configx/x.go)" = false ]
|
||||
[ "$(classify "$tmp" internal/apigate-old/x.go)" = false ]
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
@test "entries are trimmed as the Go test trims them" {
|
||||
tmp=$(mktemp -d)
|
||||
printf '%s\n' cmd/backport-batch/main.go > "$tmp/diff"
|
||||
printf ' cmd/backport-batch/ \r\n' > "$tmp/inputs"
|
||||
[ "$("$CLASSIFIER" "$tmp/diff" "$tmp/inputs")" = true ]
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
@test "an inputs file with no entry is refused rather than read as never" {
|
||||
tmp=$(mktemp -d)
|
||||
printf '%s\n' cmd/backport-batch/main.go > "$tmp/diff"
|
||||
printf '# only a comment\n\n' > "$tmp/inputs"
|
||||
if "$CLASSIFIER" "$tmp/diff" "$tmp/inputs" >"$tmp/out" 2>"$tmp/err"; then
|
||||
echo "an empty inputs file must fail, it printed: $(cat "$tmp/out")" >&2
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'lists no path' "$tmp/err"
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
@test "the checks job runs the target on the classifier's answer alone" {
|
||||
plan_run=$(yq -r '.jobs.plan.steps[] | select(.id == "p") | .run' "$WF")
|
||||
printf '%s\n' "$plan_run" | grep -qF 'backport_batch="$(hack/backport-batch-tests-needed.sh /tmp/changed_norenames.txt)"'
|
||||
[ "$(yq -r '.jobs.plan.outputs.backport_batch' "$WF")" = '${{ steps.p.outputs.backport_batch }}' ]
|
||||
|
||||
step_if=$(yq -r '.jobs.checks.steps[] | select(.name == "Run backport-batch tests") | .if' "$WF")
|
||||
step_run=$(yq -r '.jobs.checks.steps[] | select(.name == "Run backport-batch tests") | .run' "$WF")
|
||||
# !cancelled(): a failed unit lane above must not skip this suite too.
|
||||
[ "$step_if" = "\${{ !cancelled() && needs.plan.outputs.backport_batch == 'true' }}" ]
|
||||
[ "$step_run" = 'make test-backport-batch' ]
|
||||
|
||||
# And nowhere else: the unit lane every code PR runs must not carry it.
|
||||
prereqs=$(sed -n 's/^unit-tests:[[:space:]]*//p' Makefile)
|
||||
[ -n "$prereqs" ]
|
||||
case " $prereqs " in
|
||||
*" test-backport-batch "*)
|
||||
echo "make unit-tests still runs test-backport-batch on every code PR: $prereqs" >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
}
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/bin/sh
|
||||
# Print whether a changed-path list touches an input of
|
||||
# `go test ./cmd/backport-batch/...`, as listed in
|
||||
# cmd/backport-batch/test-inputs.txt. An entry ending in / matches every path
|
||||
# below it, any other entry matches that file alone. The caller supplies the
|
||||
# rename-blind list so both sides of a rename are classified.
|
||||
#
|
||||
# Usage: hack/backport-batch-tests-needed.sh <changed-files> [<inputs-file>]
|
||||
set -eu
|
||||
|
||||
inputs="${2:-cmd/backport-batch/test-inputs.txt}"
|
||||
if [ "$#" -lt 1 ] || [ "$#" -gt 2 ] || [ ! -r "$1" ] || [ ! -r "$inputs" ]; then
|
||||
echo "usage: backport-batch-tests-needed.sh <readable-changed-files> [<readable-inputs-file>]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# An inputs file with no entry would answer false for every diff and switch the
|
||||
# tests off without a word; refuse instead.
|
||||
if ! grep -qvE '^[[:space:]]*(#|$)' "$inputs"; then
|
||||
echo "backport-batch-tests-needed: $inputs lists no path" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Entries are trimmed the way TestInputsListed trims them, so a trailing space
|
||||
# or a CR cannot make the Go test accept an entry this never matches.
|
||||
awk '
|
||||
NR == FNR {
|
||||
e = $0
|
||||
sub(/^[[:space:]]+/, "", e)
|
||||
sub(/[[:space:]]+$/, "", e)
|
||||
if (e != "" && e !~ /^#/) entries[++n] = e
|
||||
next
|
||||
}
|
||||
{
|
||||
for (i = 1; i <= n; i++) {
|
||||
e = entries[i]
|
||||
if ((substr(e, length(e)) == "/" && index($0, e) == 1) || $0 == e) {
|
||||
found = 1
|
||||
exit
|
||||
}
|
||||
}
|
||||
}
|
||||
END { print (found ? "true" : "false") }
|
||||
' "$inputs" "$1"
|
||||
@@ -67,7 +67,13 @@ if [ -z "$CHANGED" ] || [ "$CHANGED" = "FULL" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if grep -qE "$full_rebuild_pattern" "$CHANGED"; then
|
||||
# CI-only Go tools that no image builds. Dropped before the full-rebuild check,
|
||||
# whose cmd/ would otherwise rebuild everything for an edit no image contains.
|
||||
# hack/select-e2e.sh skips the same two, and a diff made only of them and docs/
|
||||
# never reaches this script's build jobs (the plan job's `code` is false).
|
||||
tooling_pattern='^cmd/(backport-batch|api-gate)/'
|
||||
|
||||
if grep -vE "$tooling_pattern" "$CHANGED" | grep -qE "$full_rebuild_pattern"; then
|
||||
emit_json "$units"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -97,6 +97,23 @@ load test_helper
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
@test "the CI-only tools do not force the full matrix" {
|
||||
tmp=$(mktemp)
|
||||
# No image builds cmd/backport-batch or cmd/api-gate, so they select nothing
|
||||
# and leave the rest of the diff to its own rules.
|
||||
printf '%s\n' cmd/backport-batch/report_test.go cmd/api-gate/main.go > "$tmp"
|
||||
out=$(hack/build-matrix.sh "$tmp")
|
||||
[ "$out" = '[]' ]
|
||||
printf '%s\n' cmd/backport-batch/main.go packages/apps/mariadb/Makefile > "$tmp"
|
||||
out=$(hack/build-matrix.sh "$tmp")
|
||||
[ "$out" = '["packages/apps/mariadb"]' ]
|
||||
# Every other cmd/ directory is a shipped binary and still rebuilds all.
|
||||
printf '%s\n' cmd/backport-batch/main.go cmd/cozystack-api/main.go > "$tmp"
|
||||
out=$(hack/build-matrix.sh "$tmp")
|
||||
[ "$out" = "$(hack/build-matrix.sh)" ]
|
||||
rm -f "$tmp"
|
||||
}
|
||||
|
||||
@test "a package with no image target selects nothing" {
|
||||
tmp=$(mktemp)
|
||||
# postgres ships no in-repo image build, so it is not a build unit.
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env bats
|
||||
# -----------------------------------------------------------------------------
|
||||
# The tenant cluster's pod and service CIDRs are written by two charts:
|
||||
#
|
||||
# * packages/apps/kubernetes renders them into the CAPI Cluster's
|
||||
# spec.clusterNetwork, which the control plane is built from;
|
||||
# * packages/apps/kubernetes-nodes renders them into every worker's Talos
|
||||
# machine config (cluster.network.podSubnets / serviceSubnets), inside the
|
||||
# TalosConfigTemplate the talos-reconcile Job applies.
|
||||
#
|
||||
# Each chart carries its own literal, and neither render fails when the two
|
||||
# disagree: the workers boot, join, and route pod and Service traffic over
|
||||
# ranges the control plane does not use. This guard renders both charts and
|
||||
# compares the lists, so a range moved on one side only fails here.
|
||||
#
|
||||
# The worker side is read the way the Job produces it: the heredoc is cut out
|
||||
# of the Job's command, run through sh, and the TalosConfigTemplate it prints is
|
||||
# parsed, then the machine config inside it. A text grep of the Job would also
|
||||
# match a comment.
|
||||
#
|
||||
# Requires: helm, yq (mikefarah v4+). Runs under both `bats` and the in-repo
|
||||
# cozytest.sh runner, so TMP is provisioned inline per test and cleaned without
|
||||
# an EXIT trap.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
load test_helper
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
|
||||
|
||||
# $1: output directory. Writes pods and services, one CIDR per line.
|
||||
cluster_cidrs() {
|
||||
helm template kubernetes-myk8s "$REPO_ROOT/packages/apps/kubernetes" \
|
||||
--namespace tenant-test \
|
||||
-f "$REPO_ROOT/packages/apps/kubernetes/tests/values-ci.yaml" \
|
||||
> "$1/cluster.yaml" 2> "$1/cluster.err" \
|
||||
|| { cat "$1/cluster.err" >&2; return 1; }
|
||||
yq 'select(.kind == "Cluster" and .apiVersion == "cluster.x-k8s.io/v1beta1") | .spec.clusterNetwork.pods.cidrBlocks[]' \
|
||||
"$1/cluster.yaml" > "$1/cluster.pods"
|
||||
yq 'select(.kind == "Cluster" and .apiVersion == "cluster.x-k8s.io/v1beta1") | .spec.clusterNetwork.services.cidrBlocks[]' \
|
||||
"$1/cluster.yaml" > "$1/cluster.services"
|
||||
}
|
||||
|
||||
# $1: output directory. Writes pool.pods and pool.services, one CIDR per line.
|
||||
pool_cidrs() {
|
||||
helm template kubernetes-nodes-myk8s-md0 "$REPO_ROOT/packages/apps/kubernetes-nodes" \
|
||||
--namespace tenant-test \
|
||||
--set cluster=myk8s \
|
||||
--set _cluster.cluster-domain=cozy.local \
|
||||
--set instanceType="" \
|
||||
--set resources.cpu=2 --set resources.memory=4Gi \
|
||||
--show-only templates/talos-reconcile-job.yaml \
|
||||
> "$1/pool.yaml" 2> "$1/pool.err" \
|
||||
|| { cat "$1/pool.err" >&2; return 1; }
|
||||
yq 'select(.kind == "Job") | .spec.template.spec.containers[0].command[2]' "$1/pool.yaml" > "$1/cmd.sh"
|
||||
awk '
|
||||
/(^|="?\$\()cat <<EOF/ { print "cat <<EOF"; inblock=1; next }
|
||||
inblock && /^EOF$/ { print "EOF"; inblock=0; next }
|
||||
inblock { print }
|
||||
' "$1/cmd.sh" > "$1/heredoc.sh"
|
||||
sh "$1/heredoc.sh" > "$1/tct.yaml" 2> "$1/heredoc.err" \
|
||||
|| { cat "$1/heredoc.err" >&2; return 1; }
|
||||
yq 'select(.kind == "TalosConfigTemplate") | .spec.template.spec.data' "$1/tct.yaml" > "$1/machineconfig.yaml"
|
||||
yq '.cluster.network.podSubnets[]' "$1/machineconfig.yaml" > "$1/pool.pods"
|
||||
yq '.cluster.network.serviceSubnets[]' "$1/machineconfig.yaml" > "$1/pool.services"
|
||||
}
|
||||
|
||||
# $1: directory, $2: pods or services, $3: what to call it in a message.
|
||||
assert_agree() {
|
||||
cluster=$(cat "$1/cluster.$2")
|
||||
pool=$(cat "$1/pool.$2")
|
||||
# An empty read means a field moved or a render came out short; comparing
|
||||
# "" with "" would pass on nothing.
|
||||
if [ -z "$cluster" ] || [ -z "$pool" ]; then
|
||||
echo "FAIL: could not read the $3 CIDRs from both renders."
|
||||
echo " kubernetes Cluster.spec.clusterNetwork: '${cluster:-<missing>}'"
|
||||
echo " kubernetes-nodes machine config cluster.network: '${pool:-<missing>}'"
|
||||
return 1
|
||||
fi
|
||||
if [ "$cluster" != "$pool" ]; then
|
||||
echo "FAIL: the tenant $3 CIDRs differ between the two charts."
|
||||
echo " packages/apps/kubernetes (CAPI Cluster): $cluster"
|
||||
echo " packages/apps/kubernetes-nodes (worker machine config): $pool"
|
||||
echo "Workers would route $3 traffic over a range the control plane does not"
|
||||
echo "use. Change both charts together. Moving the worker side also renames"
|
||||
echo "the content-hashed TalosConfigTemplate and rolls every worker."
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
@test "kubernetes and kubernetes-nodes render the same pod CIDRs" {
|
||||
TMP=$(mktemp -d)
|
||||
cluster_cidrs "$TMP"
|
||||
pool_cidrs "$TMP"
|
||||
assert_agree "$TMP" pods pod
|
||||
rm -rf "$TMP"
|
||||
}
|
||||
|
||||
@test "kubernetes and kubernetes-nodes render the same service CIDRs" {
|
||||
TMP=$(mktemp -d)
|
||||
cluster_cidrs "$TMP"
|
||||
pool_cidrs "$TMP"
|
||||
assert_agree "$TMP" services service
|
||||
rm -rf "$TMP"
|
||||
}
|
||||
@@ -78,19 +78,16 @@ CATALOG_VALUES="$REPO_ROOT/packages/system/kubernetes-worker-image/values.yaml"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Second family of default-drift guards on the same chart: the Talos <-> Kubernetes
|
||||
# support matrix. The render guard keyed only on `talos.version`, so a pool could
|
||||
# reach a different Talos release through `osImage.builtin.version` /
|
||||
# `osImage.factory.version` and boot a kubelet outside its support window with no
|
||||
# render failure. The table now lives once in templates/_helpers.tpl and both
|
||||
# paths consult it.
|
||||
# support matrix. A pool reaches a Talos release through `talos.version` or through
|
||||
# `osImage.builtin.version` / `osImage.factory.version`, and the table in
|
||||
# templates/_helpers.tpl has to be consulted for whichever one the pool boots.
|
||||
#
|
||||
# The symmetry is structural, not observable from a render: the matrix ships one
|
||||
# row (`v1.13`) whose Kubernetes list is a superset of `values.schema.json`'s
|
||||
# `version` enum, so no schema-valid pairing can make either call site fail today.
|
||||
# A helm-unittest case would have to put an out-of-enum `version` in, and
|
||||
# helm-unittest validates the schema, so it cannot. These two guards cover what a
|
||||
# render cannot: that both call sites exist, and that the table still means
|
||||
# something for the defaults the chart ships.
|
||||
# The matrix ships one row (`v1.13`) whose Kubernetes list is a superset of
|
||||
# `values.schema.json`'s `version` enum, so no schema-valid pairing makes either
|
||||
# call site fail today. hack/kubernetes-nodes-talos-matrix.bats renders the
|
||||
# refusals with the schema skipped; these two guards cover what it does not: that
|
||||
# both call sites stay wired, and that the table still means something for the
|
||||
# defaults the chart ships.
|
||||
|
||||
NODES_TEMPLATE="$REPO_ROOT/packages/apps/kubernetes-nodes/templates/nodegroup.yaml"
|
||||
NODES_SCHEMA="$REPO_ROOT/packages/apps/kubernetes-nodes/values.schema.json"
|
||||
@@ -148,30 +145,3 @@ NODES_SCHEMA="$REPO_ROOT/packages/apps/kubernetes-nodes/values.schema.json"
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Third guard, and the premise the first one rests on. The catalog is keyed off
|
||||
# the kubernetes-nodes chart's `talos` defaults, but the parent kubernetes chart
|
||||
# carries its own copy of the same block since the Phase 2 split, and its own
|
||||
# values.yaml says to keep the two in sync. Nothing enforced that: the pool
|
||||
# chart's `make update` copies files/versions.yaml and images/kubectl.tag from
|
||||
# the parent and not the talos block, so a one-sided bump leaves the parent's
|
||||
# support-matrix guard reasoning about one Talos release while every worker boots
|
||||
# another — and the catalog check above would go on passing, because it only ever
|
||||
# reads one of the two files.
|
||||
|
||||
PARENT_VALUES="$REPO_ROOT/packages/apps/kubernetes/values.yaml"
|
||||
|
||||
@test "the parent kubernetes chart and kubernetes-nodes agree on their Talos defaults" {
|
||||
for key in version schematicID imageFactoryURL installerRepository registryMirrors; do
|
||||
parent=$(yq ".talos.${key}" "$PARENT_VALUES")
|
||||
pool=$(yq ".talos.${key}" "$APP_VALUES")
|
||||
if [ "$parent" != "$pool" ]; then
|
||||
echo "talos.${key} differs between the two charts:" >&2
|
||||
echo " packages/apps/kubernetes/values.yaml: $parent" >&2
|
||||
echo " packages/apps/kubernetes-nodes/values.yaml: $pool" >&2
|
||||
echo "both are declared 'keep in sync' and a worker pool follows the second while the parent's support-matrix guard reads the first" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
@@ -35,8 +35,9 @@
|
||||
# a human's job, and reviewers should not assume CI did it for them.
|
||||
#
|
||||
# A docs-only PR skips builds and E2E, but the `checks` job still runs the Bats
|
||||
# lane. Otherwise editing this map, or another document audited by a unit
|
||||
# contract, would exempt the changed input from its own guard.
|
||||
# lane whenever a docs/ path changed. Otherwise editing this map, or another
|
||||
# document audited by a unit contract, would exempt the changed input from its
|
||||
# own guard.
|
||||
|
||||
load test_helper
|
||||
|
||||
@@ -117,9 +118,18 @@ map_section() {
|
||||
|
||||
[ "$job_if" = "!contains(github.event.pull_request.labels.*.name, 'release')" ]
|
||||
[ "$code_if" = "needs.plan.outputs.code == 'true'" ]
|
||||
[ "$docs_if" = "needs.plan.outputs.code == 'false'" ]
|
||||
[ "$docs_if" = "needs.plan.outputs.code == 'false' && needs.plan.outputs.docs == 'true'" ]
|
||||
[ "$docs_run" = 'make bats-unit-tests' ]
|
||||
[ "$code_run" = 'make unit-tests test-controllers test-controllers-envtest -j4 -k --output-sync=target' ]
|
||||
|
||||
# `code` is false for the CI-only tools too, so the docs lane is keyed on a
|
||||
# docs/ path being present, not on `code` alone. Pin that the output means
|
||||
# exactly that and that Bats gets installed for it.
|
||||
plan_run=$(yq -r '.jobs.plan.steps[] | select(.id == "p") | .run' "$WORKFLOW")
|
||||
printf '%s\n' "$plan_run" | grep -qF "if grep -qE '^docs/' /tmp/changed_norenames.txt; then"
|
||||
[ "$(yq -r '.jobs.plan.outputs.docs' "$WORKFLOW")" = '${{ steps.p.outputs.docs }}' ]
|
||||
toolchain_if=$(yq -r '.jobs.checks.steps[] | select(.name == "Set up test toolchain") | .if' "$WORKFLOW")
|
||||
[ "$toolchain_if" = "needs.plan.outputs.code == 'true' || needs.plan.outputs.docs == 'true'" ]
|
||||
}
|
||||
|
||||
@test "the trigger map and the PR-template checklist list the same repositories" {
|
||||
|
||||
@@ -18,7 +18,7 @@ cozy_e2e_storage_class() {
|
||||
esac
|
||||
}
|
||||
|
||||
# talos_spec_block: emit the tenant Kubernetes CR `spec.talos` block with the Talos
|
||||
# talos_spec_block: emit the tenant KubernetesNodes CR `spec.talos` block with the Talos
|
||||
# OS image cache (imageFactoryURL) when the in-sandbox cache is up. Prints nothing
|
||||
# when it falls back to the public default, so the chart default applies.
|
||||
# Indented for insertion directly under `spec:` in the heredoc below.
|
||||
@@ -5702,8 +5702,9 @@ YAML
|
||||
|
||||
# Point worker DataVolume imports at the in-sandbox Talos OS image cache when it
|
||||
# is up, falling back to the public factory otherwise. Worker image pulls use
|
||||
# the chart's public-registry default. Emitted right under spec: as
|
||||
# `talos: { imageFactoryURL: ... }`, or nothing when the default applies.
|
||||
# the chart's public-registry default. Emitted right under spec: of the
|
||||
# KubernetesNodes pool below as `talos: { imageFactoryURL: ... }`, or nothing
|
||||
# when the default applies. The Kubernetes CR takes no worker image field.
|
||||
local talos_block
|
||||
talos_block=$(talos_spec_block)
|
||||
|
||||
@@ -5714,7 +5715,6 @@ metadata:
|
||||
name: "${test_name}"
|
||||
namespace: tenant-test
|
||||
spec:
|
||||
${talos_block}
|
||||
${oidc_block}
|
||||
addons:
|
||||
certManager:
|
||||
|
||||
@@ -49,10 +49,10 @@
|
||||
# Best-effort: never fail the suite on the band-aid. Remove once tenant workers
|
||||
# no longer bulk-pull the OS image from the public internet in CI.
|
||||
local sid ver
|
||||
sid=$(yq '.talos.schematicID' packages/apps/kubernetes/values.yaml 2>/dev/null)
|
||||
ver=$(yq '.talos.version' packages/apps/kubernetes/values.yaml 2>/dev/null)
|
||||
sid=$(yq '.talos.schematicID' packages/apps/kubernetes-nodes/values.yaml 2>/dev/null)
|
||||
ver=$(yq '.talos.version' packages/apps/kubernetes-nodes/values.yaml 2>/dev/null)
|
||||
if [ -z "$sid" ] || [ "$sid" = "null" ] || [ -z "$ver" ] || [ "$ver" = "null" ]; then
|
||||
echo "WARNING: could not read talos.schematicID/version from values.yaml — skipping mirror (fallback to public factory)"
|
||||
echo "WARNING: could not read talos.schematicID/version from packages/apps/kubernetes-nodes/values.yaml — skipping mirror (fallback to public factory)"
|
||||
return 0
|
||||
fi
|
||||
# The CiliumClusterwideNetworkPolicy in the manifest is intentionally NOT applied
|
||||
@@ -561,3 +561,40 @@ data:
|
||||
version: "${version}"
|
||||
EOF
|
||||
}
|
||||
|
||||
@test "Tenant namespace delete policy refuses a direct delete and keeps the Tenant uninstall" {
|
||||
if ! kubectl api-resources --api-group=admissionregistration.k8s.io \
|
||||
2>/dev/null | grep -qw validatingadmissionpolicies; then
|
||||
skip "ValidatingAdmissionPolicy API not available on this cluster"
|
||||
fi
|
||||
kubectl get validatingadmissionpolicy cozystack-tenant-namespace-delete-policy
|
||||
kubectl get validatingadmissionpolicybinding cozystack-tenant-namespace-delete-policy
|
||||
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: apps.cozystack.io/v1alpha1
|
||||
kind: Tenant
|
||||
metadata:
|
||||
name: deleteprobe
|
||||
namespace: tenant-root
|
||||
spec: {}
|
||||
EOF
|
||||
timeout 60 sh -ec 'until kubectl get hr/tenant-deleteprobe -n tenant-root >/dev/null 2>&1; do sleep 2; done'
|
||||
kubectl wait hr/tenant-deleteprobe -n tenant-root --timeout=1m --for=condition=ready
|
||||
|
||||
local selected output rc still_there
|
||||
selected=$(kubectl get namespaces --selector tenant.cozystack.io/tenant-root --field-selector metadata.name=tenant-deleteprobe -o name)
|
||||
[ "$selected" = "namespace/tenant-deleteprobe" ]
|
||||
|
||||
output=$(kubectl delete namespace tenant-deleteprobe --dry-run=server 2>&1) && rc=0 || rc=$?
|
||||
echo "kubectl delete exit=$rc, output=$output"
|
||||
[ "$rc" -ne 0 ]
|
||||
echo "$output" | grep -qF "ValidatingAdmissionPolicy 'cozystack-tenant-namespace-delete-policy'"
|
||||
echo "$output" | grep -qF 'kubectl delete tenants.apps.cozystack.io deleteprobe --namespace tenant-root.'
|
||||
|
||||
kubectl delete tenants.apps.cozystack.io deleteprobe -n tenant-root --timeout=5m
|
||||
kubectl wait namespace/tenant-deleteprobe --for=delete --timeout=5m
|
||||
if ! still_there=$(kubectl get namespace tenant-deleteprobe --ignore-not-found -o name); then
|
||||
echo "FAIL: reading the tenant-deleteprobe namespace did not answer"; false
|
||||
fi
|
||||
if [ -n "$still_there" ]; then echo "FAIL: deleting the Tenant must remove its namespace"; false; fi
|
||||
}
|
||||
@@ -14,7 +14,7 @@
|
||||
# This Deployment fetches the image ONCE at e2e-install time with an aggressive
|
||||
# curl retry loop (decoupling the one flaky external fetch from the many
|
||||
# per-worker imports), then serves it locally over plain HTTP with range support.
|
||||
# Tenant `Kubernetes` CRs point `spec.talos.imageFactoryURL` at this Service
|
||||
# Tenant `KubernetesNodes` CRs point `spec.talos.imageFactoryURL` at this Service
|
||||
# (see hack/e2e-chainsaw/_lib/run-kubernetes.sh) so every worker imports from the
|
||||
# warm local copy instead of the public internet. Tenant namespaces run under a
|
||||
# default-deny Cilium egress installed by the tenant chart, whose allow rules cover
|
||||
@@ -28,7 +28,7 @@
|
||||
# mirror can only help, never make CI worse.
|
||||
#
|
||||
# The __SCHEMATIC_ID__ / __TALOS_VERSION__ placeholders are substituted at apply
|
||||
# time by hack/e2e-install-cozystack.bats from packages/apps/kubernetes/values.yaml.
|
||||
# time by hack/e2e-install-cozystack.bats from packages/apps/kubernetes-nodes/values.yaml.
|
||||
#
|
||||
# CI band-aid, not a product artifact. The durable fix is the overridable
|
||||
# `talos.imageFactoryURL` chart value this manifest exercises; a self-hosted
|
||||
|
||||
@@ -266,24 +266,56 @@ resolve_silent_conclusions() {
|
||||
plan_block="$(job_block plan "$PULL_REQUESTS" | code_lines)"
|
||||
[ -n "$plan_block" ]
|
||||
|
||||
count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^docs/' /tmp/changed_norenames.txt" || true)"
|
||||
count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^(.*)' /tmp/changed_norenames.txt" || true)"
|
||||
[ "${count:-0}" -eq 1 ]
|
||||
|
||||
# The rename-detected list may still feed the build matrix, but never this.
|
||||
count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^docs/' /tmp/changed.txt" || true)"
|
||||
count="$(printf '%s\n' "$plan_block" | grep -c "grep -qvE '\^(.*)' /tmp/changed.txt" || true)"
|
||||
[ "${count:-0}" -eq 0 ]
|
||||
|
||||
# The fork half of the same agreement: both sides of a rename are inspected.
|
||||
# Pinned on the docs-only expression itself — `previous_filename` also appears
|
||||
# Pinned on the skip expression itself — `previous_filename` also appears
|
||||
# in the TIA file list a few lines down, so counting the identifier would stay
|
||||
# satisfied by that one with this check deleted.
|
||||
block="$(job_block resolve "$FORK" | code_lines)"
|
||||
[ -n "$block" ]
|
||||
count="$(printf '%s\n' "$block" \
|
||||
| grep -c 'f\.previous_filename && !isDocs(f\.previous_filename)' || true)"
|
||||
| grep -c 'f\.previous_filename && !isNoRuntime(f\.previous_filename)' || true)"
|
||||
[ "${count:-0}" -eq 1 ]
|
||||
}
|
||||
|
||||
@test "both lanes skip e2e for the same path prefixes" {
|
||||
# The rename test above pins HOW each lane reads its list; this one pins
|
||||
# WHAT it skips. A prefix added to one lane only is a fork PR that one lane
|
||||
# reads as nothing-to-build and the other as needing the pr-patch artifact,
|
||||
# which nothing will ever produce.
|
||||
plan_prefixes="$(job_block plan "$PULL_REQUESTS" | code_lines \
|
||||
| sed -nE "s,.*grep -qvE '\^\(([^)]*)\)' /tmp/changed_norenames\.txt.*,\1,p" \
|
||||
| tr '|' '\n' | sort)"
|
||||
fork_prefixes="$(job_block resolve "$FORK" | code_lines \
|
||||
| sed -nE "s,.*const noRuntimePrefixes = \[([^]]*)\];.*,\1,p" \
|
||||
| tr ',' '\n' | tr -d " '" | sort)"
|
||||
[ -n "$plan_prefixes" ]
|
||||
printf '%s\n' "$plan_prefixes" | grep -qx 'docs/'
|
||||
if [ "$plan_prefixes" != "$fork_prefixes" ]; then
|
||||
echo "plan skips:" >&2; printf ' %s\n' $plan_prefixes >&2
|
||||
echo "fork skips:" >&2; printf ' %s\n' $fork_prefixes >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The tools in that list must also be the ones the build matrix and the suite
|
||||
# selector leave alone in a mixed diff; otherwise a tool skipped on its own
|
||||
# still rebuilds everything beside a one-line package change.
|
||||
tools="$(printf '%s\n' "$plan_prefixes" | grep -vx 'docs/')"
|
||||
[ -n "$tools" ]
|
||||
matrix_tools="$(sed -nE "s,^tooling_pattern='\^cmd/\(([^)]*)\)/'$,\1,p" "$REPO_ROOT/hack/build-matrix.sh" \
|
||||
| tr '|' '\n' | sed 's,^,cmd/,; s,$,/,' | sort)"
|
||||
select_tools="$(sed -nE 's,^ (cmd/[^)]*)\)$,\1,p' "$REPO_ROOT/hack/select-e2e.sh" \
|
||||
| tr '|' '\n' | sed 's,\*$,,' | sort)"
|
||||
[ "$tools" = "$matrix_tools" ]
|
||||
[ "$tools" = "$select_tools" ]
|
||||
}
|
||||
|
||||
@test "same-repo gate: every publishing guard names the same label set" {
|
||||
# `plan`, `verify-release-candidate`, `e2e-report` and the concurrency key
|
||||
# each decide from the label name whether this run owns the status. Renaming
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bats
|
||||
# Unit test: the refusals of the Talos to Kubernetes support-matrix guard in
|
||||
# packages/apps/kubernetes-nodes (kubernetes-nodes.assertTalosSupportsKubernetes,
|
||||
# called from templates/nodegroup.yaml for talos.version and again for the
|
||||
# release osImage resolves to).
|
||||
#
|
||||
# On the shipped chart a refusal is unreachable: the values schema offers only
|
||||
# Kubernetes versions the v1.13 row lists, and files/versions.yaml, which another
|
||||
# template fails on first, holds the same set. The guard is there for the day a
|
||||
# Talos row stops listing a version the chart still offers. So this renders a
|
||||
# copy of the chart with --skip-schema-validation and one extra versions.yaml row
|
||||
# for a version the matrix does not list. The pairings the guard lets through
|
||||
# are in the chart's own tests/talos_k8s_support_matrix_test.yaml.
|
||||
|
||||
load test_helper
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
|
||||
CHART="$REPO_ROOT/packages/apps/kubernetes-nodes"
|
||||
|
||||
# render_pool <args...> renders the pool chart copy with the given --set flags and
|
||||
# prints helm's output, errors included. The copy lives in the test's own tmpdir.
|
||||
render_pool() {
|
||||
copy="$BATS_TEST_TMPDIR/chart"
|
||||
if [ ! -d "$copy" ]; then
|
||||
cp -RL "$CHART" "$copy"
|
||||
printf '"v1.30": "v1.30.14"\n' >> "$copy/files/versions.yaml"
|
||||
fi
|
||||
helm template kubernetes-nodes-myk8s-md0 "$copy" --namespace tenant-test \
|
||||
--skip-schema-validation \
|
||||
--set cluster=myk8s \
|
||||
--set _cluster.cluster-domain=cozy.local \
|
||||
--set minReplicas=1 --set maxReplicas=3 \
|
||||
--set instanceType= --set resources.cpu=2 --set resources.memory=4Gi \
|
||||
--set diskSize=20Gi --set storageClass=replicated \
|
||||
"$@" 2>&1
|
||||
}
|
||||
|
||||
@test "an in-window pairing renders on the same copy, so the refusals below are the guard's" {
|
||||
out=$(render_pool --set version=v1.31) || { echo "$out" >&2; return 1; }
|
||||
printf '%s\n' "$out" | grep -q '^kind: KubevirtMachineTemplate$'
|
||||
}
|
||||
|
||||
@test "a Kubernetes version outside the row of talos.version is refused" {
|
||||
if out=$(render_pool --set version=v1.30); then
|
||||
echo "render succeeded with Kubernetes v1.30 on the default Talos" >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$out" | grep -qF "Kubernetes v1.30 is not supported by Talos v1.13.6. Supported versions: [v1.31 v1.32 v1.33 v1.34 v1.35 v1.36]. Bump 'version' or set talos.version explicitly." \
|
||||
|| { echo "$out" >&2; return 1; }
|
||||
}
|
||||
|
||||
@test "an older patch release of a listed Talos minor is checked against the same row" {
|
||||
if out=$(render_pool --set version=v1.30 --set talos.version=v1.13.0); then
|
||||
echo "render succeeded with Kubernetes v1.30 on Talos v1.13.0: the guard is not keyed on the minor" >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$out" | grep -qF 'Kubernetes v1.30 is not supported by Talos v1.13.0.' \
|
||||
|| { echo "$out" >&2; return 1; }
|
||||
}
|
||||
|
||||
# talos.version names a minor the matrix does not list, so the first call lets the
|
||||
# pool through and only the call fed the osImage release can refuse it.
|
||||
@test "a Kubernetes version outside the row of osImage.factory.version is refused" {
|
||||
if out=$(render_pool --set version=v1.30 --set talos.version=v1.12.0 --set osImage.factory.version=v1.13.0); then
|
||||
echo "render succeeded with Kubernetes v1.30 on Talos v1.13.0 reached through osImage.factory" >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$out" | grep -qF 'Kubernetes v1.30 is not supported by Talos v1.13.0.' \
|
||||
&& printf '%s\n' "$out" | grep -qF 'nodeGroup md0 reaches this Talos release through its osImage override' \
|
||||
|| { echo "$out" >&2; return 1; }
|
||||
}
|
||||
|
||||
@test "a Kubernetes version outside the row of osImage.builtin.version is refused" {
|
||||
if out=$(render_pool --set version=v1.30 --set talos.version=v1.12.0 --set osImage.builtin.version=v1.13.0); then
|
||||
echo "render succeeded with Kubernetes v1.30 on Talos v1.13.0 reached through osImage.builtin" >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$out" | grep -qF 'Kubernetes v1.30 is not supported by Talos v1.13.0.' \
|
||||
&& printf '%s\n' "$out" | grep -qF 'nodeGroup md0 reaches this Talos release through its osImage override' \
|
||||
|| { echo "$out" >&2; return 1; }
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bats
|
||||
# Replays the LINSTOR error-report alerts from
|
||||
# packages/system/piraeus-operator/alerts/piraeus-datastore.yaml through
|
||||
# vmalert-tool, pinned to the VictoriaMetrics release that the
|
||||
# victoria-metrics-operator deploys by default. Its increase() counts the first
|
||||
# samples of a series as new when they are small, and the report counts are
|
||||
# small, so the cases pin that a series which starts over after a controller
|
||||
# move, a label change or a disconnect does not report old error reports again.
|
||||
# Only pulling the image touches the network.
|
||||
|
||||
load test_helper
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
|
||||
VMALERT_TOOL_IMAGE="victoriametrics/vmalert-tool:v1.136.0@sha256:417346d7b6fdb66d8e3984887c2ad114535ec7d983eeef53381b56c776f333f1"
|
||||
|
||||
@test "LINSTOR error-report alerts pass their vmalert-tool cases" {
|
||||
docker info >/dev/null 2>&1 || {
|
||||
echo "docker is required to run the cases in $VMALERT_TOOL_IMAGE" >&2
|
||||
return 1
|
||||
}
|
||||
work=$(mktemp -d)
|
||||
yq '.spec' "$REPO_ROOT/packages/system/piraeus-operator/alerts/piraeus-datastore.yaml" > "$work/rules.yaml"
|
||||
cp "$REPO_ROOT/hack/testdata/linstor-error-report-alerts/unittest.yaml" "$work/unittest.yaml"
|
||||
rc=0
|
||||
docker run --rm --network none -v "$work:/w:ro" "$VMALERT_TOOL_IMAGE" \
|
||||
unittest --files /w/unittest.yaml > "$work/out" 2>&1 || rc=$?
|
||||
cat "$work/out"
|
||||
passed=$(grep -cx 'SUCCESS' "$work/out" || true)
|
||||
rm -rf "$work"
|
||||
[ "$rc" -eq 0 ]
|
||||
[ "$passed" -eq 1 ]
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
# -----------------------------------------------------------------------------
|
||||
# Unit tests for talos_spec_block in hack/e2e-chainsaw/_lib/run-kubernetes.sh
|
||||
#
|
||||
# It is the composition point for the tenant Kubernetes CR's `spec.talos`: the
|
||||
# It is the composition point for the tenant KubernetesNodes CR's `spec.talos`: the
|
||||
# Talos OS image cache contributes imageFactoryURL when available. The result is
|
||||
# spliced into an unquoted heredoc directly under `spec:`, so an indentation slip
|
||||
# does not fail this function -- it produces a CR the API rejects, and every
|
||||
@@ -32,7 +32,7 @@ load test_helper
|
||||
# Splice a talos_spec_block result under `spec:` of a minimal CR and print the
|
||||
# document, so yq can be asked what the tenant apiserver would see.
|
||||
spec_doc() {
|
||||
printf 'apiVersion: apps.cozystack.io/v1alpha1\nkind: Kubernetes\nmetadata:\n name: t\nspec:\n%s\n host: ""\n' "$1"
|
||||
printf 'apiVersion: apps.cozystack.io/v1alpha1\nkind: KubernetesNodes\nmetadata:\n name: t\nspec:\n%s\n cluster: t\n' "$1"
|
||||
}
|
||||
|
||||
@test "no Talos image cache renders nothing, so the chart default applies" {
|
||||
@@ -59,6 +59,6 @@ spec_doc() {
|
||||
# talos_spec_block writes imageFactoryURL by hand. If the chart ever renames
|
||||
# it, the CR would be rejected only at e2e time, and this file is the cheapest
|
||||
# place to notice.
|
||||
yq -e '.talos | has("imageFactoryURL")' packages/apps/kubernetes/values.yaml >/dev/null \
|
||||
|| { echo "packages/apps/kubernetes/values.yaml has no talos.imageFactoryURL; talos_spec_block would emit a key the chart rejects" >&2; exit 1; }
|
||||
yq -e '.talos | has("imageFactoryURL")' packages/apps/kubernetes-nodes/values.yaml >/dev/null \
|
||||
|| { echo "packages/apps/kubernetes-nodes/values.yaml has no talos.imageFactoryURL; talos_spec_block would emit a key the pool chart ignores" >&2; exit 1; }
|
||||
}
|
||||
+11
-2
@@ -8,8 +8,9 @@
|
||||
# Output:
|
||||
# - empty no E2E impact, decided by one of the rules that select
|
||||
# nothing: step 1 for docs/, dashboards/ and *.md, step 2 for
|
||||
# a *.disabled Chainsaw suite and for an
|
||||
# examples/backups/<app>/ with no suite, and
|
||||
# a *.disabled Chainsaw suite, for an
|
||||
# examples/backups/<app>/ with no suite and for the CI-only
|
||||
# tools cmd/backport-batch/ and cmd/api-gate/, and
|
||||
# inert_config_pattern for repo meta and agent config
|
||||
# - <suite names> selected per the PackageSource dependency graph
|
||||
# - full list any path that affects all tests, OR an unrecognised
|
||||
@@ -497,6 +498,14 @@ while IFS= read -r file || [ -n "$file" ]; do
|
||||
echo "select-e2e: '$file' belongs to no single backup walkthrough — escalating to the full suite" >&2
|
||||
trigger_full=1
|
||||
continue ;;
|
||||
cmd/backport-batch/*|cmd/api-gate/*)
|
||||
# CI-only Go tools. No image builds them and nothing in the e2e sandbox
|
||||
# runs them, so an edit cannot regress a suite. Matched here because
|
||||
# full_suite_pattern takes all of cmd/ and is checked before the inert
|
||||
# list. The plan job's `code` output and e2e-fork.yaml skip e2e for a
|
||||
# diff made only of these and docs/; this arm keeps them from escalating
|
||||
# a diff that also touches something else.
|
||||
continue ;;
|
||||
esac
|
||||
|
||||
# 3. Full-suite trigger
|
||||
|
||||
@@ -202,6 +202,23 @@ assert_full_suite() {
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
@test "the CI-only tools select nothing and do not escalate the rest" {
|
||||
tmp=$(mktemp -d)
|
||||
cp -r packages/core/platform/sources "$tmp/sources"
|
||||
printf '%s\n' cmd/backport-batch/report_test.go cmd/api-gate/main.go > "$tmp/diff"
|
||||
output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources")
|
||||
[ -z "$output" ]
|
||||
# Beside a package, the package alone decides the selection.
|
||||
printf '%s\n' cmd/backport-batch/main.go packages/apps/postgres/values.yaml > "$tmp/diff"
|
||||
output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources")
|
||||
assert_selection "a CI-only tool must not widen a package selection" "$output" "postgres"
|
||||
# Any other cmd/ directory is a shipped binary and still escalates.
|
||||
echo "cmd/cozystack-api/main.go" > "$tmp/diff"
|
||||
output=$(hack/select-e2e.sh "$tmp/diff" "$tmp/sources" 2>/dev/null)
|
||||
assert_full_suite "$output"
|
||||
rm -rf "$tmp"
|
||||
}
|
||||
|
||||
@test "kubernetes-application maps to the two kubernetes suites" {
|
||||
tmp=$(mktemp -d)
|
||||
cp -r packages/core/platform/sources "$tmp/sources"
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
rule_files:
|
||||
- rules.yaml
|
||||
|
||||
evaluation_interval: 15s
|
||||
|
||||
tests:
|
||||
- name: controller pod moves to another node
|
||||
interval: 30s
|
||||
input_series:
|
||||
- series: 'up{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a"}'
|
||||
values: '1x38 0 stale'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a", hostname="node-a", module="SATELLITE"}'
|
||||
values: '1x38 stale'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", controller_node="node-a", hostname="node-b", module="SATELLITE"}'
|
||||
values: '2x38 stale'
|
||||
- series: 'up{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b"}'
|
||||
values: '_x41 1x79'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b", hostname="node-a", module="SATELLITE"}'
|
||||
values: '_x41 1x58 2x20'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.2:3370", controller_node="node-b", hostname="node-b", module="SATELLITE"}'
|
||||
values: '_x41 2x79'
|
||||
alert_rule_test:
|
||||
- eval_time: 21m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
- eval_time: 28m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
- eval_time: 35m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
- eval_time: 45m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
- eval_time: 51m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
hostname: node-a
|
||||
severity: warning
|
||||
exp_annotations:
|
||||
description: |
|
||||
LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes.
|
||||
Use "linstor error-reports list --nodes node-a" to inspect the reports.
|
||||
|
||||
- name: an external label of the scrape changes
|
||||
interval: 30s
|
||||
input_series:
|
||||
- series: 'up{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack"}'
|
||||
values: '1x39 stale'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack", hostname="node-a", module="SATELLITE"}'
|
||||
values: '1x39 stale'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cozystack", hostname="linstor-controller-abc", module="CONTROLLER"}'
|
||||
values: '1x39 stale'
|
||||
- series: 'up{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a"}'
|
||||
values: '_x41 1x79'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a", hostname="node-a", module="SATELLITE"}'
|
||||
values: '_x41 1x79'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", cluster="cluster-a", hostname="linstor-controller-abc", module="CONTROLLER"}'
|
||||
values: '_x41 1x79'
|
||||
alert_rule_test:
|
||||
- eval_time: 21m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
- eval_time: 30m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
- eval_time: 21m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorControllerErrorRate
|
||||
- eval_time: 30m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorControllerErrorRate
|
||||
|
||||
- name: a satellite drops out of the metric while it is disconnected
|
||||
interval: 30s
|
||||
input_series:
|
||||
- series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}'
|
||||
values: '1x220'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}'
|
||||
values: '1x38 stale _x80 1x100'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-b", module="SATELLITE"}'
|
||||
values: '2x220'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-c", module="SATELLITE"}'
|
||||
values: '1x38 stale _x80 2x100'
|
||||
alert_rule_test:
|
||||
- eval_time: 61m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
hostname: node-c
|
||||
severity: warning
|
||||
exp_annotations:
|
||||
description: |
|
||||
LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes.
|
||||
Use "linstor error-reports list --nodes node-c" to inspect the reports.
|
||||
- eval_time: 70m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
hostname: node-c
|
||||
severity: warning
|
||||
exp_annotations:
|
||||
description: |
|
||||
LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes.
|
||||
Use "linstor error-reports list --nodes node-c" to inspect the reports.
|
||||
- eval_time: 76m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
|
||||
- name: a new report fires for the 15 minute window
|
||||
interval: 30s
|
||||
input_series:
|
||||
- series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}'
|
||||
values: '1x120'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}'
|
||||
values: '1x39 2x80'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-b", module="SATELLITE"}'
|
||||
values: '2x120'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-c", module="SATELLITE"}'
|
||||
values: '0x39 1x80'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="linstor-controller-abc", module="CONTROLLER"}'
|
||||
values: '0x39 1x80'
|
||||
alert_rule_test:
|
||||
- eval_time: 21m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
hostname: node-a
|
||||
severity: warning
|
||||
exp_annotations:
|
||||
description: |
|
||||
LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes.
|
||||
Use "linstor error-reports list --nodes node-a" to inspect the reports.
|
||||
- exp_labels:
|
||||
hostname: node-c
|
||||
severity: warning
|
||||
exp_annotations:
|
||||
description: |
|
||||
LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes.
|
||||
Use "linstor error-reports list --nodes node-c" to inspect the reports.
|
||||
- eval_time: 34m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
hostname: node-a
|
||||
severity: warning
|
||||
exp_annotations:
|
||||
description: |
|
||||
LINSTOR Satellite "node-a" reports 1 errors in the last 15 minutes.
|
||||
Use "linstor error-reports list --nodes node-a" to inspect the reports.
|
||||
- exp_labels:
|
||||
hostname: node-c
|
||||
severity: warning
|
||||
exp_annotations:
|
||||
description: |
|
||||
LINSTOR Satellite "node-c" reports 1 errors in the last 15 minutes.
|
||||
Use "linstor error-reports list --nodes node-c" to inspect the reports.
|
||||
- eval_time: 36m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
- eval_time: 21m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorControllerErrorRate
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
hostname: linstor-controller-abc
|
||||
severity: warning
|
||||
exp_annotations:
|
||||
description: |
|
||||
LINSTOR Controller reports 1 errors in the last 15 minutes.
|
||||
Use "linstor error-reports list --since 15minutes" to see them.
|
||||
- eval_time: 36m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorControllerErrorRate
|
||||
|
||||
- name: a failed controller scrape still holds the alert back
|
||||
interval: 30s
|
||||
input_series:
|
||||
- series: 'up{job="linstor-controller", instance="10.0.0.1:3370"}'
|
||||
values: '1x19 0x140 1x60'
|
||||
- series: 'linstor_error_reports_count{job="linstor-controller", instance="10.0.0.1:3370", hostname="node-a", module="SATELLITE"}'
|
||||
values: '1x19 stale _x140 1x60'
|
||||
alert_rule_test:
|
||||
- eval_time: 85m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
- eval_time: 93m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
- eval_time: 100m
|
||||
groupname: linstor.rules
|
||||
alertname: linstorSatelliteErrorRate
|
||||
@@ -0,0 +1,572 @@
|
||||
rule_files:
|
||||
- rules.yaml
|
||||
|
||||
evaluation_interval: 15s
|
||||
|
||||
tests:
|
||||
- name: two of three StatefulSet pods on one node
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", node="node-b", created_by_kind="StatefulSet", created_by_name="db"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
alert_rule_test:
|
||||
- eval_time: 59m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
- eval_time: 61m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
cluster: cozystack
|
||||
namespace: tenant-a
|
||||
owner_kind: StatefulSet
|
||||
owner_name: db
|
||||
node: node-a
|
||||
severity: warning
|
||||
exported_instance: tenant-a/db
|
||||
service: workload-placement
|
||||
exp_annotations:
|
||||
summary: Pods of one workload share a node.
|
||||
description: >-
|
||||
2 running pods of StatefulSet tenant-a/db are on node node-a in cluster cozystack,
|
||||
although the cluster has a schedulable node for each of its pods. Losing this node
|
||||
stops these pods together, which can cost a quorum-based workload its quorum. The
|
||||
scheduler does not move running pods, so they stay together until they are recreated.
|
||||
|
||||
- name: StatefulSet pods spread over three nodes
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", node="node-a", created_by_kind="StatefulSet", created_by_name="db"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", node="node-b", created_by_kind="StatefulSet", created_by_name="db"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", node="node-c", created_by_kind="StatefulSet", created_by_name="db"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-0", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-1", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="db-2", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
alert_rule_test:
|
||||
- eval_time: 61m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
- eval_time: 120m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
|
||||
- name: five ReplicaSet pods on three nodes
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-bbbbb", node="node-a", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ddddd", node="node-b", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-eeeee", node="node-c", created_by_kind="ReplicaSet", created_by_name="worker-7c8d9"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-aaaaa", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-bbbbb", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ccccc", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-ddddd", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="worker-7c8d9-eeeee", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
alert_rule_test:
|
||||
- eval_time: 61m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
- eval_time: 120m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
|
||||
- name: DaemonSet surge pod next to the old one
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-x7k2p", node="node-a", created_by_kind="DaemonSet", created_by_name="node-agent"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-q9z4m", node="node-a", created_by_kind="DaemonSet", created_by_name="node-agent"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-b5n8w", node="node-b", created_by_kind="DaemonSet", created_by_name="node-agent"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-x7k2p", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-q9z4m", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="cozy-system", pod="node-agent-b5n8w", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
alert_rule_test:
|
||||
- eval_time: 61m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
- eval_time: 120m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
|
||||
- name: two pods of one Job on one node
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-h2k4m", node="node-a", created_by_kind="Job", created_by_name="backup-29345678"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-p8r6t", node="node-a", created_by_kind="Job", created_by_name="backup-29345678"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-h2k4m", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="backup-29345678-p8r6t", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
alert_rule_test:
|
||||
- eval_time: 61m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
|
||||
- name: etcd members of one EtcdCluster on one node
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-x7k2p", node="node-a", created_by_kind="EtcdMember", created_by_name="etcd-x7k2p"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-q9z4m", node="node-a", created_by_kind="EtcdMember", created_by_name="etcd-q9z4m"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-b5n8w", node="node-b", created_by_kind="EtcdMember", created_by_name="etcd-b5n8w"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-0", node="node-b", created_by_kind="EtcdMember", created_by_name="etcd-0"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-1", node="node-c", created_by_kind="EtcdMember", created_by_name="etcd-1"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-2", node="node-c", created_by_kind="EtcdMember", created_by_name="etcd-2"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-x7k2p", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-q9z4m", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="etcd-b5n8w", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-0", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-1", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-b", pod="etcd-2", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
alert_rule_test:
|
||||
- eval_time: 61m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
cluster: cozystack
|
||||
namespace: tenant-a
|
||||
owner_kind: EtcdCluster
|
||||
owner_name: etcd
|
||||
node: node-a
|
||||
severity: warning
|
||||
exported_instance: tenant-a/etcd
|
||||
service: workload-placement
|
||||
exp_annotations:
|
||||
summary: Pods of one workload share a node.
|
||||
description: >-
|
||||
2 running pods of EtcdCluster tenant-a/etcd are on node node-a in cluster cozystack,
|
||||
although the cluster has a schedulable node for each of its pods. Losing this node
|
||||
stops these pods together, which can cost a quorum-based workload its quorum. The
|
||||
scheduler does not move running pods, so they stay together until they are recreated.
|
||||
- exp_labels:
|
||||
cluster: cozystack
|
||||
namespace: tenant-b
|
||||
owner_kind: EtcdCluster
|
||||
owner_name: etcd
|
||||
node: node-c
|
||||
severity: warning
|
||||
exported_instance: tenant-b/etcd
|
||||
service: workload-placement
|
||||
exp_annotations:
|
||||
summary: Pods of one workload share a node.
|
||||
description: >-
|
||||
2 running pods of EtcdCluster tenant-b/etcd are on node node-c in cluster cozystack,
|
||||
although the cluster has a schedulable node for each of its pods. Losing this node
|
||||
stops these pods together, which can cost a quorum-based workload its quorum. The
|
||||
scheduler does not move running pods, so they stay together until they are recreated.
|
||||
|
||||
- name: replacement pod next to its terminating predecessor for twenty minutes
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}'
|
||||
values: '1x30 stale'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}'
|
||||
values: '_x10 1x140'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", phase="Running", node="node-c"}'
|
||||
values: '1x30 stale'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", phase="Running", node="node-c"}'
|
||||
values: '_x10 1x140'
|
||||
alert_rule_test:
|
||||
- eval_time: 29m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
- eval_time: 75m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
- eval_time: 120m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
|
||||
- name: the same overlap held past the hour
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}'
|
||||
values: '1x100 stale'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="web-6b8f4"}'
|
||||
values: '_x10 1x140'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-aaaaa", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-bbbbb", phase="Running", node="node-c"}'
|
||||
values: '1x100 stale'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="web-6b8f4-ccccc", phase="Running", node="node-c"}'
|
||||
values: '_x10 1x140'
|
||||
alert_rule_test:
|
||||
- eval_time: 69m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
- eval_time: 71m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
cluster: cozystack
|
||||
namespace: tenant-a
|
||||
owner_kind: ReplicaSet
|
||||
owner_name: web-6b8f4
|
||||
node: node-b
|
||||
severity: warning
|
||||
exported_instance: tenant-a/web-6b8f4
|
||||
service: workload-placement
|
||||
exp_annotations:
|
||||
summary: Pods of one workload share a node.
|
||||
description: >-
|
||||
2 running pods of ReplicaSet tenant-a/web-6b8f4 are on node node-b in cluster cozystack,
|
||||
although the cluster has a schedulable node for each of its pods. Losing this node
|
||||
stops these pods together, which can cost a quorum-based workload its quorum. The
|
||||
scheduler does not move running pods, so they stay together until they are recreated.
|
||||
- eval_time: 105m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
|
||||
- name: a cordoned node does not count until it is uncordoned
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '0x150'
|
||||
- series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '0x150'
|
||||
- series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x30 0x120'
|
||||
- series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unschedulable", value="", effect="NoSchedule"}'
|
||||
values: '1x30 stale'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", node="node-b", created_by_kind="StatefulSet", created_by_name="kv"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", phase="Running", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", phase="Running", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", phase="Running", node="node-a"}'
|
||||
values: '1x150'
|
||||
alert_rule_test:
|
||||
- eval_time: 61m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
- eval_time: 90m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
- eval_time: 92m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
cluster: cozystack
|
||||
namespace: tenant-a
|
||||
owner_kind: StatefulSet
|
||||
owner_name: kv
|
||||
node: node-a
|
||||
severity: warning
|
||||
exported_instance: tenant-a/kv
|
||||
service: workload-placement
|
||||
exp_annotations:
|
||||
summary: Pods of one workload share a node.
|
||||
description: >-
|
||||
2 running pods of StatefulSet tenant-a/kv are on node node-a in cluster cozystack,
|
||||
although the cluster has a schedulable node for each of its pods. Losing this node
|
||||
stops these pods together, which can cost a quorum-based workload its quorum. The
|
||||
scheduler does not move running pods, so they stay together until they are recreated.
|
||||
|
||||
- name: a node that is down does not count
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '0x150'
|
||||
- series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '0x150'
|
||||
- series: 'kube_node_spec_unschedulable{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '0x150'
|
||||
- series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unreachable", value="", effect="NoSchedule"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_spec_taint{job="kube-state-metrics", cluster="cozystack", node="node-c", key="node.kubernetes.io/unreachable", value="", effect="NoExecute"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", node="node-a", created_by_kind="StatefulSet", created_by_name="kv"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", node="node-b", created_by_kind="StatefulSet", created_by_name="kv"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-0", phase="Running", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-1", phase="Running", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kv-2", phase="Running", node="node-a"}'
|
||||
values: '1x150'
|
||||
alert_rule_test:
|
||||
- eval_time: 61m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
- eval_time: 120m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
|
||||
- name: pods that are not running do not count
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-aaaaa", node="node-a", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", node="node-a", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-ccccc", node="node-b", created_by_kind="ReplicaSet", created_by_name="api-5c6d7"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-aaaaa", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", phase="Running", node="node-c"}'
|
||||
values: '0x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-bbbbb", phase="Failed", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="api-5c6d7-ccccc", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
alert_rule_test:
|
||||
- eval_time: 61m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
|
||||
- name: one pod each in two clusters that share workload and node names
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="kubernetes-example", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="kube-system", pod="coredns-5d78c9869d-x7k2p", node="node-a", created_by_kind="ReplicaSet", created_by_name="coredns-5d78c9869d"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="kubernetes-example", namespace="kube-system", pod="coredns-5d78c9869d-q9z4m", node="node-a", created_by_kind="ReplicaSet", created_by_name="coredns-5d78c9869d"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="kube-system", pod="coredns-5d78c9869d-x7k2p", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="kubernetes-example", namespace="kube-system", pod="coredns-5d78c9869d-q9z4m", phase="Running", node="node-b"}'
|
||||
values: '1x150'
|
||||
alert_rule_test:
|
||||
- eval_time: 61m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
|
||||
- name: ReplicaSet, CNPG Cluster and StrimziPodSet pods on one node
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-a"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-b"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_node_info{job="kube-state-metrics", cluster="cozystack", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-h2k4m", node="node-a", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-p8r6t", node="node-a", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-w3x5z", node="node-b", created_by_kind="ReplicaSet", created_by_name="rfs-cache-6f7d8b9c5d"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-1", node="node-a", created_by_kind="Cluster", created_by_name="postgres-db"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-2", node="node-b", created_by_kind="Cluster", created_by_name="postgres-db"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-3", node="node-b", created_by_kind="Cluster", created_by_name="postgres-db"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-0", node="node-c", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-1", node="node-c", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_info{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-2", node="node-a", created_by_kind="StrimziPodSet", created_by_name="kafka-kafka"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-h2k4m", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-p8r6t", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="rfs-cache-6f7d8b9c5d-w3x5z", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-1", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-2", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="postgres-db-3", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-0", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-1", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
- series: 'kube_pod_status_phase{job="kube-state-metrics", cluster="cozystack", namespace="tenant-a", pod="kafka-kafka-2", phase="Running", node="node-c"}'
|
||||
values: '1x150'
|
||||
alert_rule_test:
|
||||
- eval_time: 61m
|
||||
groupname: workload-placement
|
||||
alertname: PodReplicasColocated
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
cluster: cozystack
|
||||
namespace: tenant-a
|
||||
owner_kind: ReplicaSet
|
||||
owner_name: rfs-cache-6f7d8b9c5d
|
||||
node: node-a
|
||||
severity: warning
|
||||
exported_instance: tenant-a/rfs-cache-6f7d8b9c5d
|
||||
service: workload-placement
|
||||
exp_annotations:
|
||||
summary: Pods of one workload share a node.
|
||||
description: >-
|
||||
2 running pods of ReplicaSet tenant-a/rfs-cache-6f7d8b9c5d are on node node-a in cluster
|
||||
cozystack, although the cluster has a schedulable node for each of its pods. Losing this
|
||||
node stops these pods together, which can cost a quorum-based workload its quorum. The
|
||||
scheduler does not move running pods, so they stay together until they are recreated.
|
||||
- exp_labels:
|
||||
cluster: cozystack
|
||||
namespace: tenant-a
|
||||
owner_kind: Cluster
|
||||
owner_name: postgres-db
|
||||
node: node-b
|
||||
severity: warning
|
||||
exported_instance: tenant-a/postgres-db
|
||||
service: workload-placement
|
||||
exp_annotations:
|
||||
summary: Pods of one workload share a node.
|
||||
description: >-
|
||||
2 running pods of Cluster tenant-a/postgres-db are on node node-b in cluster cozystack,
|
||||
although the cluster has a schedulable node for each of its pods. Losing this node
|
||||
stops these pods together, which can cost a quorum-based workload its quorum. The
|
||||
scheduler does not move running pods, so they stay together until they are recreated.
|
||||
- exp_labels:
|
||||
cluster: cozystack
|
||||
namespace: tenant-a
|
||||
owner_kind: StrimziPodSet
|
||||
owner_name: kafka-kafka
|
||||
node: node-c
|
||||
severity: warning
|
||||
exported_instance: tenant-a/kafka-kafka
|
||||
service: workload-placement
|
||||
exp_annotations:
|
||||
summary: Pods of one workload share a node.
|
||||
description: >-
|
||||
2 running pods of StrimziPodSet tenant-a/kafka-kafka are on node node-c in cluster
|
||||
cozystack, although the cluster has a schedulable node for each of its pods. Losing
|
||||
this node stops these pods together, which can cost a quorum-based workload its quorum.
|
||||
The scheduler does not move running pods, so they stay together until they are
|
||||
recreated.
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
load test_helper
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)"
|
||||
VMALERT_TOOL_IMAGE="victoriametrics/vmalert-tool:v1.136.0@sha256:417346d7b6fdb66d8e3984887c2ad114535ec7d983eeef53381b56c776f333f1"
|
||||
|
||||
@test "workload placement alerts pass their vmalert-tool cases" {
|
||||
docker info >/dev/null 2>&1 || {
|
||||
echo "docker is required to run the cases in $VMALERT_TOOL_IMAGE" >&2
|
||||
return 1
|
||||
}
|
||||
work=$(mktemp -d)
|
||||
rc=0
|
||||
yq '.spec' "$REPO_ROOT/packages/system/monitoring-agents/alerts/workload-placement.yaml" > "$work/rules.yaml" || rc=$?
|
||||
cp "$REPO_ROOT/hack/testdata/workload-placement-alerts/unittest.yaml" "$work/unittest.yaml" || rc=$?
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
docker run --rm --network none -v "$work:/w:ro" "$VMALERT_TOOL_IMAGE" \
|
||||
unittest --files /w/unittest.yaml > "$work/out" 2>&1 || rc=$?
|
||||
cat "$work/out"
|
||||
fi
|
||||
passed=$(grep -cx 'SUCCESS' "$work/out" 2>/dev/null || true)
|
||||
rm -rf "$work"
|
||||
[ "$rc" -eq 0 ]
|
||||
[ "$passed" -eq 1 ]
|
||||
}
|
||||
@@ -631,7 +631,7 @@ func (r *BackupJobReconciler) createAltinityBackupArtifact(
|
||||
// used name meets the earlier run's Backup here. Only one recording
|
||||
// this run's archive was created by an earlier reconcile of this run.
|
||||
if got := existing.Spec.DriverMetadata[altinityArchiveMetaKey]; got != archive {
|
||||
return nil, fmt.Errorf("Backup %s/%s already exists and records archive %q, not %q that this run wrote",
|
||||
return nil, fmt.Errorf("existing Backup %s/%s records archive %q, not %q that this run wrote",
|
||||
backup.Namespace, backup.Name, got, archive)
|
||||
}
|
||||
return existing, nil
|
||||
|
||||
@@ -39,6 +39,13 @@ type s3ObjectLister interface {
|
||||
StatObject(ctx context.Context, bucket, key string) (int64, error)
|
||||
}
|
||||
|
||||
type s3ObjectRemover interface {
|
||||
// RemoveObject deletes key. S3 answers 204 for a key that does not
|
||||
// exist and minio-go takes 204 as success; a backend answering a
|
||||
// missing key with any other status makes it an error.
|
||||
RemoveObject(ctx context.Context, bucket, key string) error
|
||||
}
|
||||
|
||||
// s3Connection is what a driver resolves from its rendered strategy to
|
||||
// read the bucket with the same identity the backup was written with.
|
||||
type s3Connection struct {
|
||||
@@ -67,8 +74,10 @@ type objectStore struct {
|
||||
// reader reads Secrets, which the manager's client fetches uncached,
|
||||
// and lists operator Backups, which it serves from the cache.
|
||||
reader client.Reader
|
||||
// newLister overrides the S3 client; nil means a real minio client.
|
||||
newLister func(conn s3Connection) (s3ObjectLister, error)
|
||||
// newLister and newRemover override the S3 client; nil means a real
|
||||
// minio client.
|
||||
newLister func(conn s3Connection) (s3ObjectLister, error)
|
||||
newRemover func(conn s3Connection) (s3ObjectRemover, error)
|
||||
}
|
||||
|
||||
func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) {
|
||||
@@ -78,6 +87,17 @@ func (s objectStore) lister(conn s3Connection) (s3ObjectLister, error) {
|
||||
return newMinioObjectLister(conn)
|
||||
}
|
||||
|
||||
func (s objectStore) remover(conn s3Connection) (s3ObjectRemover, error) {
|
||||
if s.newRemover != nil {
|
||||
return s.newRemover(conn)
|
||||
}
|
||||
c, err := newMinioObjectClient(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (s objectStore) secretValue(ctx context.Context, namespace, name, key string) (string, error) {
|
||||
secret := &corev1.Secret{}
|
||||
if err := s.reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil {
|
||||
@@ -94,6 +114,14 @@ func (r *BackupJobReconciler) objectStore() objectStore {
|
||||
return objectStore{reader: r.Client, newLister: r.newS3ObjectLister}
|
||||
}
|
||||
|
||||
func (r *RestoreJobReconciler) objectStore() objectStore {
|
||||
return objectStore{reader: r.Client, newLister: r.newS3ObjectLister}
|
||||
}
|
||||
|
||||
func (r *BackupReconciler) objectStore() objectStore {
|
||||
return objectStore{reader: r.Client, newLister: r.newS3ObjectLister, newRemover: r.newS3ObjectRemover}
|
||||
}
|
||||
|
||||
func (r *BackupJobReconciler) s3Lister(conn s3Connection) (s3ObjectLister, error) {
|
||||
return r.objectStore().lister(conn)
|
||||
}
|
||||
@@ -103,6 +131,14 @@ type minioObjectLister struct {
|
||||
}
|
||||
|
||||
func newMinioObjectLister(conn s3Connection) (s3ObjectLister, error) {
|
||||
c, err := newMinioObjectClient(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func newMinioObjectClient(conn s3Connection) (*minioObjectLister, error) {
|
||||
opts := &minio.Options{
|
||||
Creds: credentials.NewStaticV4(conn.AccessKey, conn.SecretKey, conn.SessionToken),
|
||||
Region: conn.Region,
|
||||
@@ -168,6 +204,10 @@ func (l *minioObjectLister) StatObject(ctx context.Context, bucket, key string)
|
||||
return info.Size, nil
|
||||
}
|
||||
|
||||
func (l *minioObjectLister) RemoveObject(ctx context.Context, bucket, key string) error {
|
||||
return l.client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
// collectObjects drains a minio listing. minio-go ends a listing whose
|
||||
// context is cancelled between pages by closing the channel without an
|
||||
// error item, so a cancelled context is checked once the channel closes:
|
||||
|
||||
@@ -39,9 +39,22 @@ type BackupReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
Recorder record.EventRecorder
|
||||
// CredentialsConfig lets the Rabbitmq cleanup path project cozy-backups-creds
|
||||
// into the Backup's namespace before spawning its best-effort delete Job.
|
||||
// CredentialsConfig locates the platform backup credentials: the Job-based
|
||||
// cleanups project them into the Backup's namespace, and the MariaDB
|
||||
// cleanup reads them to reach the platform bucket.
|
||||
CredentialsConfig BackupCredentialsConfig
|
||||
// APIReader is the manager's uncached reader. The MariaDB cleanup lists
|
||||
// the RestoreJobs and operator Restores of a Backup, and the operator
|
||||
// Backups that may own its dump, through it: a restore that started
|
||||
// before the delete, or a run that has already uploaded its dump, may
|
||||
// not be in the cache yet. Wired in SetupWithManager.
|
||||
APIReader client.Reader
|
||||
|
||||
// newS3ObjectLister and newS3ObjectRemover override the S3 client the
|
||||
// MariaDB cleanup deletes a dump with; nil means a real minio client.
|
||||
// Set by unit tests.
|
||||
newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error)
|
||||
newS3ObjectRemover func(conn s3Connection) (s3ObjectRemover, error)
|
||||
}
|
||||
|
||||
func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
@@ -126,22 +139,20 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv
|
||||
// orchestration" for tenant-facing guidance.
|
||||
return ctrl.Result{}, nil
|
||||
case strategyv1alpha1.MariaDBStrategyKind:
|
||||
// Cozystack Backup deletion does NOT delete the operator-side
|
||||
// k8s.mariadb.com/Backup CR or the backing S3/PVC archive.
|
||||
// Lifecycle of that CR is owned by mariadb-operator's
|
||||
// MaxRetention setting and by tenants / explicit teardown flows;
|
||||
// the controller's RBAC has no delete verb on
|
||||
// k8s.mariadb.com/backups for that reason. Surfacing this as an
|
||||
// explicit branch (rather than falling through to the Velero
|
||||
// default) keeps the contract honest: the MariaDB driver does
|
||||
// not own the archive, so it does not delete it on CR removal.
|
||||
// Mirrors the Altinity branch's "we do not own S3" stance.
|
||||
return ctrl.Result{}, nil
|
||||
// mariadb-operator's maxRetention prunes only from inside the next
|
||||
// backup run of the same prefix and a Plan never deletes a Backup, so
|
||||
// a dump on the platform bucket is deleted here or never. A dump in
|
||||
// a tenant's own bucket is deleted too: the Backup proves it owns
|
||||
// it, and tenants cannot delete Backups, so the delete comes from an
|
||||
// administrator or from namespace teardown. Teardown can remove the
|
||||
// operator Backup first, and the dump is then left behind with a
|
||||
// Warning, since its storage is read from it (see cleanupMariaDBBackup).
|
||||
return r.cleanupMariaDBBackup(ctx, backup)
|
||||
case strategyv1alpha1.MongoDBStrategyKind:
|
||||
// Split by flow (see cleanupMongoDBBackup). A legacy backup writes to the
|
||||
// tenant's own bucket and its lifecycle stays the tenant's / psmdb
|
||||
// operator's — that branch is a no-op, the same "we do not own the
|
||||
// archive" contract as Altinity / MariaDB / FDB. A useSystemBucket backup
|
||||
// archive" contract as Altinity / FDB. A useSystemBucket backup
|
||||
// lands in the shared cozy-backups bucket with no psmdb task retention to
|
||||
// prune it, so the driver OWNS it: this deletes the operator
|
||||
// PerconaServerMongoDBBackup CR and waits for its delete-backup finalizer
|
||||
@@ -154,7 +165,7 @@ func (r *BackupReconciler) cleanupOnDelete(ctx context.Context, backup *backupsv
|
||||
// is governed by the next BackupJob's stopOtherFoundationDBBackups
|
||||
// call, by manual operator action, or by
|
||||
// examples/backups/foundationdb/cleanup.sh. Same "we do not own
|
||||
// the archive" contract as Altinity / MariaDB — the explicit
|
||||
// the archive" contract as Altinity — the explicit
|
||||
// branch guards the seam against a future driver refactor that
|
||||
// might incidentally stamp velero.io/backup-name onto FDB
|
||||
// driverMetadata via a shared helper.
|
||||
@@ -294,7 +305,18 @@ func (r *BackupReconciler) cleanupCNPGBackup(ctx context.Context, backup *backup
|
||||
|
||||
// SetupWithManager registers the BackupReconciler with the Manager.
|
||||
func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
r.APIReader = mgr.GetAPIReader()
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&backupsv1alpha1.Backup{}).
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
// apiReader returns the uncached reader. Reconcilers built by hand in unit
|
||||
// tests have none and fall back to their fake Client, which has no cache to
|
||||
// bypass.
|
||||
func (r *BackupReconciler) apiReader() client.Reader {
|
||||
if r.APIReader != nil {
|
||||
return r.APIReader
|
||||
}
|
||||
return r.Client
|
||||
}
|
||||
@@ -176,11 +176,8 @@ func TestBackupCleanup_Altinity_DoesNotFallThroughToVelero(t *testing.T) {
|
||||
|
||||
// TestBackupCleanup_MariaDB_DoesNotFallThroughToVelero locks in the same
|
||||
// invariant as the Altinity test above for the MariaDB strategy. The
|
||||
// dispatcher must explicitly route MariaDB-strategy Backups to a no-op
|
||||
// cleanup rather than fall through to the Velero default - the MariaDB
|
||||
// driver does not own the operator-side k8s.mariadb.com/Backup CR or the
|
||||
// S3/PVC archive behind it (rbac.yaml grants no delete verb on
|
||||
// k8s.mariadb.com/backups for that reason). The test seeds the Velero
|
||||
// dispatcher must route MariaDB-strategy Backups to their own cleanup
|
||||
// rather than fall through to the Velero default. The test seeds the Velero
|
||||
// metadata keys plus a matching velero.io/Backup; with the explicit branch
|
||||
// no DeleteBackupRequest is synthesised, and the Velero Backup survives.
|
||||
// Without the branch the default falls through to cleanupVeleroBackup,
|
||||
|
||||
@@ -222,13 +222,13 @@ func cnpgListStub(prefix string, seen func(*http.Request)) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
if _, ok := req.URL.Query()["location"]; ok {
|
||||
fmt.Fprint(w, `<?xml version="1.0" encoding="UTF-8"?><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></LocationConstraint>`)
|
||||
_, _ = fmt.Fprint(w, `<?xml version="1.0" encoding="UTF-8"?><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/"></LocationConstraint>`)
|
||||
return
|
||||
}
|
||||
if seen != nil {
|
||||
seen(req)
|
||||
}
|
||||
fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
|
||||
_, _ = fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>bkt</Name><Prefix>%[1]s</Prefix><KeyCount>2</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>
|
||||
<Contents><Key>%[1]sbackup.info</Key><Size>7</Size><ETag>"a"</ETag><LastModified>2026-10-21T10:01:00.000Z</LastModified></Contents>
|
||||
|
||||
@@ -2718,9 +2718,12 @@ func buildBarmanPlugin(objectStoreName, serverName string) cnpgtypes.PluginConfi
|
||||
// high-water mark on a 38 MB database is 254 MiB, so 128Mi cannot hold it. A
|
||||
// tenant that leaves resourceQuotas empty has no LimitRange, and there the
|
||||
// sidecar had no requests and no limit at all. 256Mi holds the measured working
|
||||
// set and 1Gi is four times the measurement. No CPU limit is set, matching the
|
||||
// entityOperator precedent in packages/apps/kafka: a throttled sidecar stalls
|
||||
// WAL archiving instead of failing it.
|
||||
// set and 1Gi is four times the measurement. The CPU limit is explicit for the
|
||||
// same reason: left unset, that LimitRange gives the sidecar its 250m default,
|
||||
// so gzip compression of full backups and WAL segments runs on a quarter of a
|
||||
// core (a 1.6 GB database took 30 minutes to back up). One core keeps the
|
||||
// sidecar from starving the instance while removing that ceiling; it counts
|
||||
// against the tenant's limits.cpu budget, one core per instance pod.
|
||||
//
|
||||
// Checksum: since botocore ~1.36 the default (when_supported) attaches a
|
||||
// flexible checksum to every PutObject, which non-AWS S3-compatible backends
|
||||
@@ -2740,6 +2743,7 @@ func barmanSidecarConfiguration() *cnpgtypes.InstanceSidecarConfiguration {
|
||||
corev1.ResourceMemory: resource.MustParse("256Mi"),
|
||||
},
|
||||
Limits: corev1.ResourceList{
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
||||
},
|
||||
},
|
||||
|
||||
@@ -2156,8 +2156,11 @@ func TestApplyClusterPluginBackup_PatchesExistingCluster(t *testing.T) {
|
||||
sc.Resources.Requests.Cpu().MilliValue() != 100 {
|
||||
t.Errorf("ObjectStore leaves Barman subject to undersized tenant resource defaults: %+v", sc.Resources)
|
||||
}
|
||||
if _, ok := sc.Resources.Limits[corev1.ResourceCPU]; ok {
|
||||
t.Errorf("ObjectStore caps sidecar CPU, which throttles WAL archiving instead of failing it: %+v", sc.Resources.Limits)
|
||||
// Without an explicit CPU limit the tenant LimitRange caps the sidecar at
|
||||
// its 250m default, and every backup and WAL upload compresses on a
|
||||
// quarter of a core.
|
||||
if sc.Resources.Limits.Cpu().MilliValue() != 1000 {
|
||||
t.Errorf("ObjectStore leaves the sidecar CPU limit to the tenant LimitRange default: %+v", sc.Resources.Limits)
|
||||
}
|
||||
// The ObjectStore must be owner-referenced to the Cluster so Kubernetes GC
|
||||
// removes it when the Cluster is deleted (no orphan in the platform flow,
|
||||
@@ -2830,7 +2833,7 @@ func TestReconcileCNPG_ObjectStoreApplyRecovers(t *testing.T) {
|
||||
}
|
||||
|
||||
failing = false
|
||||
j, _ = f.pass()
|
||||
_, _ = f.pass()
|
||||
f.settleDestination()
|
||||
j, backups := f.pass()
|
||||
if len(backups) != 1 {
|
||||
|
||||
@@ -134,14 +134,14 @@ var etcdClusterGVR = schema.GroupVersionResource{
|
||||
// the wrong CRD via the typed client.
|
||||
func validateEtcdApplicationRef(ref corev1.TypedLocalObjectReference) error {
|
||||
if ref.Kind != etcdAppKind {
|
||||
return fmt.Errorf("Etcd strategy supports applicationRef.kind=%q, got %q", etcdAppKind, ref.Kind)
|
||||
return fmt.Errorf("the Etcd strategy supports applicationRef.kind=%q, got %q", etcdAppKind, ref.Kind)
|
||||
}
|
||||
apiGroup := ""
|
||||
if ref.APIGroup != nil {
|
||||
apiGroup = *ref.APIGroup
|
||||
}
|
||||
if apiGroup != "" && apiGroup != etcdapp.GroupName {
|
||||
return fmt.Errorf("Etcd strategy supports applicationRef.apiGroup=%q, got %q", etcdapp.GroupName, apiGroup)
|
||||
return fmt.Errorf("the Etcd strategy supports applicationRef.apiGroup=%q, got %q", etcdapp.GroupName, apiGroup)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -104,10 +104,10 @@ func kafkaClusterName(appName string) string {
|
||||
// apps.cozystack.io/Kafka application.
|
||||
func validateKafkaApplicationRef(ref corev1.TypedLocalObjectReference) error {
|
||||
if ref.Kind != kafkaApplicationKind {
|
||||
return fmt.Errorf("Kafka strategy only supports applicationRef.kind=%q, got %q", kafkaApplicationKind, ref.Kind)
|
||||
return fmt.Errorf("the Kafka strategy only supports applicationRef.kind=%q, got %q", kafkaApplicationKind, ref.Kind)
|
||||
}
|
||||
if ref.APIGroup != nil && *ref.APIGroup != "" && *ref.APIGroup != backupsv1alpha1.DefaultApplicationAPIGroup {
|
||||
return fmt.Errorf("Kafka strategy only supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup)
|
||||
return fmt.Errorf("the Kafka strategy only supports applicationRef.apiGroup=%q, got %q", backupsv1alpha1.DefaultApplicationAPIGroup, *ref.APIGroup)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -528,7 +528,7 @@ func (r *BackupJobReconciler) createKafkaBackupArtifact(
|
||||
// RabbitMQ driver's guard).
|
||||
if existing.Spec.ApplicationRef.Kind != backup.Spec.ApplicationRef.Kind ||
|
||||
existing.Spec.ApplicationRef.Name != backup.Spec.ApplicationRef.Name {
|
||||
return nil, fmt.Errorf("Backup %s/%s already exists for a different application (%s/%s, not %s/%s)",
|
||||
return nil, fmt.Errorf("existing Backup %s/%s belongs to a different application (%s/%s, not %s/%s)",
|
||||
backup.Namespace, backup.Name,
|
||||
existing.Spec.ApplicationRef.Kind, existing.Spec.ApplicationRef.Name,
|
||||
backup.Spec.ApplicationRef.Kind, backup.Spec.ApplicationRef.Name)
|
||||
|
||||
@@ -39,19 +39,19 @@ type mariadbDump struct {
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
// mariadbArtifactSizeBytes measures the dump mariadb-operator uploaded for
|
||||
// mdbBackup. Returns 0 - leaving Artifact.SizeBytes absent - when the dump
|
||||
// cannot be found or told apart: an unknown size must not be reported as a
|
||||
// measured one.
|
||||
func (r *BackupJobReconciler) mariadbArtifactSizeBytes(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) int64 {
|
||||
// attributedMariaDBDump returns the dump mariadb-operator uploaded for
|
||||
// mdbBackup. ok is false when the dump cannot be found or told apart, and
|
||||
// the Backup then reports neither a size nor a dump: an unknown dump must
|
||||
// not be reported as a measured one.
|
||||
func (r *BackupJobReconciler) attributedMariaDBDump(ctx context.Context, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (dump mariadbDump, ok bool) {
|
||||
ctx, cancel := context.WithTimeout(ctx, artifactSizeProbeTimeout)
|
||||
defer cancel()
|
||||
dump, err := findMariaDBDump(ctx, r.objectStore(), mdbBackup, completedAt)
|
||||
if err != nil {
|
||||
getLogger(ctx).Info("MariaDB backup size not measured; leaving it unset", "backup", mdbBackup.Name, "error", err)
|
||||
return 0
|
||||
getLogger(ctx).Info("MariaDB dump not attributed; leaving its size and name unset", "backup", mdbBackup.Name, "error", err)
|
||||
return mariadbDump{}, false
|
||||
}
|
||||
return dump.Size
|
||||
return dump, true
|
||||
}
|
||||
|
||||
// findMariaDBDump lists the bucket mdbBackup wrote to and returns the one
|
||||
@@ -60,22 +60,26 @@ func (r *BackupJobReconciler) mariadbArtifactSizeBytes(ctx context.Context, mdbB
|
||||
// from can have been edited since, and pointing at another bucket now says
|
||||
// nothing about where this dump went.
|
||||
func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, completedAt time.Time) (mariadbDump, error) {
|
||||
lister, _, err := mariadbStorage(ctx, store, mdbBackup)
|
||||
if err != nil {
|
||||
return mariadbDump{}, err
|
||||
}
|
||||
return findMariaDBDumpWith(ctx, lister, store.reader, mdbBackup, completedAt, false)
|
||||
}
|
||||
|
||||
// findMariaDBDumpWith is findMariaDBDump reading the bucket through lister,
|
||||
// for a caller that reaches the bucket by another endpoint than the one
|
||||
// mdbBackup names. withUnfinished lets runs that have not completed contest
|
||||
// the dump too, as neighbourMariaDBRuns describes.
|
||||
func findMariaDBDumpWith(ctx context.Context, lister s3ObjectLister, reader client.Reader, mdbBackup *mariadbtypes.Backup, completedAt time.Time, withUnfinished bool) (mariadbDump, error) {
|
||||
s3 := mdbBackup.Spec.Storage.S3
|
||||
if s3 == nil {
|
||||
return mariadbDump{}, errors.New("the operator Backup does not store to S3")
|
||||
}
|
||||
others, err := finishedMariaDBRuns(ctx, store.reader, mdbBackup)
|
||||
others, err := neighbourMariaDBRuns(ctx, reader, mdbBackup, withUnfinished)
|
||||
if err != nil {
|
||||
return mariadbDump{}, fmt.Errorf("list operator Backups: %w", err)
|
||||
}
|
||||
conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3)
|
||||
if err != nil {
|
||||
return mariadbDump{}, fmt.Errorf("resolve S3 credentials: %w", err)
|
||||
}
|
||||
lister, err := store.lister(conn)
|
||||
if err != nil {
|
||||
return mariadbDump{}, fmt.Errorf("build S3 client: %w", err)
|
||||
}
|
||||
// Non-recursive, as mariadb-operator lists it: the default prefix is also
|
||||
// the default CNPG destinationPath, so an app of each kind with the same
|
||||
// name shares it and a recursive listing would walk the WAL archive.
|
||||
@@ -88,6 +92,44 @@ func findMariaDBDump(ctx context.Context, store objectStore, mdbBackup *mariadbt
|
||||
return matchMariaDBDump(objects, prefix, own, others)
|
||||
}
|
||||
|
||||
// mariadbDumpExists reports whether the dump file name is still in the
|
||||
// storage mdbBackup wrote to.
|
||||
func mariadbDumpExists(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup, name string) (bool, error) {
|
||||
lister, s3, err := mariadbStorage(ctx, store, mdbBackup)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
key := mariadbOperatorS3Prefix(s3.Prefix) + name
|
||||
objects, err := lister.ListObjects(ctx, s3.Bucket, key, false)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err)
|
||||
}
|
||||
for _, obj := range objects {
|
||||
if obj.Key == key {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// mariadbStorage opens the bucket mdbBackup wrote to, with the credentials
|
||||
// its own spec names.
|
||||
func mariadbStorage(ctx context.Context, store objectStore, mdbBackup *mariadbtypes.Backup) (s3ObjectLister, *mariadbtypes.S3Storage, error) {
|
||||
s3 := mdbBackup.Spec.Storage.S3
|
||||
if s3 == nil {
|
||||
return nil, nil, errors.New("the operator Backup does not store to S3")
|
||||
}
|
||||
conn, err := mariadbS3Connection(ctx, store, mdbBackup.Namespace, s3)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("resolve S3 credentials: %w", err)
|
||||
}
|
||||
lister, err := store.lister(conn)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("build S3 client: %w", err)
|
||||
}
|
||||
return lister, s3, nil
|
||||
}
|
||||
|
||||
// mariadbRun is the span in which an operator Backup can have written its
|
||||
// dump: its lifetime, widened by mariadbSizeProbeClockSkew on each side.
|
||||
type mariadbRun struct {
|
||||
@@ -95,6 +137,10 @@ type mariadbRun struct {
|
||||
lower, upper time.Time
|
||||
}
|
||||
|
||||
// mariadbUnfinishedRunEnd closes the window of a run that has not completed,
|
||||
// which can still write its dump at any time.
|
||||
var mariadbUnfinishedRunEnd = time.Date(9999, time.December, 31, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
func newMariaDBRun(b *mariadbtypes.Backup, created, completed time.Time) mariadbRun {
|
||||
return mariadbRun{
|
||||
name: b.Namespace + "/" + b.Name,
|
||||
@@ -120,8 +166,8 @@ func (r mariadbRun) placesOnEitherClock(d mariadbDump) bool {
|
||||
// dump in the window on one clock only may be this run's own, misplaced by
|
||||
// a skewed clock, which leaves any other candidate unproven. A neighbour's
|
||||
// dump is still taken if this run's own is missing from the listing, or
|
||||
// misplaced on both clocks, and the neighbour's operator Backup is gone or
|
||||
// has not completed.
|
||||
// misplaced on both clocks, and the neighbour's operator Backup is gone or,
|
||||
// unless others includes unfinished runs, has not completed.
|
||||
func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, others []mariadbRun) (mariadbDump, error) {
|
||||
var near []mariadbDump
|
||||
for _, obj := range objects {
|
||||
@@ -153,14 +199,17 @@ func matchMariaDBDump(objects []s3ObjectInfo, prefix string, own mariadbRun, oth
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// finishedMariaDBRuns returns the runs of the other operator Backups that
|
||||
// neighbourMariaDBRuns returns the runs of the other operator Backups that
|
||||
// write under the same bucket and prefix as b and have completed, whether
|
||||
// their Job succeeded or failed. One still running is left out: its window
|
||||
// has no end yet, and an operator Backup stuck short of Complete, as one
|
||||
// whose Job pod never starts is, would otherwise contest every later run of
|
||||
// the application for good. The endpoint is not compared, since one has
|
||||
// several spellings, and a false neighbour only leaves a size unset.
|
||||
func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup) ([]mariadbRun, error) {
|
||||
// their Job succeeded or failed. One still running is left out unless
|
||||
// withUnfinished is set: its window has no end yet, and an operator Backup
|
||||
// stuck short of Complete, as one whose Job pod never starts is, would
|
||||
// otherwise contest every later run of the application for good. That is
|
||||
// the price of deleting a dump, since a run can upload its dump and prune
|
||||
// older ones through maxRetention before it completes. The endpoint is not
|
||||
// compared, since one has several spellings, and a false neighbour only
|
||||
// keeps a dump unattributed.
|
||||
func neighbourMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbtypes.Backup, withUnfinished bool) ([]mariadbRun, error) {
|
||||
list := &mariadbtypes.BackupList{}
|
||||
if err := reader.List(ctx, list); err != nil {
|
||||
return nil, err
|
||||
@@ -177,6 +226,9 @@ func finishedMariaDBRuns(ctx context.Context, reader client.Reader, b *mariadbty
|
||||
}
|
||||
cond := apimeta.FindStatusCondition(other.Status.Conditions, mariadbtypes.ConditionTypeComplete)
|
||||
if cond == nil || cond.Status != metav1.ConditionTrue {
|
||||
if withUnfinished {
|
||||
runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, mariadbUnfinishedRunEnd))
|
||||
}
|
||||
continue
|
||||
}
|
||||
runs = append(runs, newMariaDBRun(other, other.CreationTimestamp.Time, cond.LastTransitionTime.Time))
|
||||
|
||||
@@ -29,6 +29,14 @@ func dumpKey(prefix string, ts time.Time, compression string) string {
|
||||
return fmt.Sprintf("%sbackup.%s.%s.sql", prefix, ts.UTC().Format(time.RFC3339), compression)
|
||||
}
|
||||
|
||||
// probedSize is the size a Backup reports for a probe result.
|
||||
func probedSize(dump mariadbDump, attributed bool) int64 {
|
||||
if !attributed {
|
||||
return 0
|
||||
}
|
||||
return dump.Size
|
||||
}
|
||||
|
||||
// mariadbBackupSucceeded is the Complete condition mariadb-operator sets on a
|
||||
// Backup whose Job succeeded. A failed Job also gets Status=True, with
|
||||
// Reason=JobFailed, so a fixture without the reason is not a success.
|
||||
@@ -105,7 +113,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) {
|
||||
{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4242, LastModified: created.Add(time.Minute)},
|
||||
{Key: prefix + "0-backup-target.txt", Size: 7},
|
||||
}}
|
||||
got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed)
|
||||
got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed))
|
||||
if got != 4242 {
|
||||
t.Errorf("size = %d, want 4242", got)
|
||||
}
|
||||
@@ -130,7 +138,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) {
|
||||
}
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 3, LastModified: created.Add(time.Minute)}}}
|
||||
r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), ca)
|
||||
if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed); got != 3 {
|
||||
if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, tlsS3, created), completed)); got != 3 {
|
||||
t.Errorf("size = %d, want 3", got)
|
||||
}
|
||||
if !l.conn.Secure || string(l.conn.CAPEM) != "PEM" {
|
||||
@@ -148,7 +156,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) {
|
||||
}
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 4, LastModified: created.Add(time.Minute)}}}
|
||||
r := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds"), sts)
|
||||
if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed); got != 4 {
|
||||
if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, stsS3, created), completed)); got != 4 {
|
||||
t.Errorf("size = %d, want 4", got)
|
||||
}
|
||||
if l.conn.SessionToken != "TOKEN" {
|
||||
@@ -160,7 +168,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey(prefix, created.Add(-30*time.Second), ""), Size: 9, LastModified: created.Add(time.Minute)},
|
||||
}}
|
||||
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 9 {
|
||||
if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 9 {
|
||||
t.Errorf("size = %d, want 9", got)
|
||||
}
|
||||
})
|
||||
@@ -173,7 +181,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) {
|
||||
{Key: dumpKey(prefix, created.Add(-4*time.Minute), "gzip"), Size: 7, LastModified: created.Add(time.Minute)},
|
||||
{Key: dumpKey(prefix, created.Add(30*time.Second), "gzip"), Size: 666, LastModified: created.Add(-4 * time.Minute)},
|
||||
}}
|
||||
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
|
||||
if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 {
|
||||
t.Errorf("size = %d, want 0: no dump is inside the window on both clocks", got)
|
||||
}
|
||||
})
|
||||
@@ -182,7 +190,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey(prefix, completed.Add(30*time.Second), "gzip"), Size: 5, LastModified: completed.Add(30 * time.Second)},
|
||||
}}
|
||||
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 5 {
|
||||
if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 5 {
|
||||
t.Errorf("size = %d, want 5", got)
|
||||
}
|
||||
})
|
||||
@@ -191,7 +199,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) {
|
||||
r := reconcilerWithLister(t, &fakeS3Lister{}, s3CredsSecret("tenant", "creds"))
|
||||
r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil }
|
||||
withShortProbeTimeout(t, func() {
|
||||
if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
|
||||
if got := probedSize(r.attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 {
|
||||
t.Errorf("size = %d, want 0", got)
|
||||
}
|
||||
})
|
||||
@@ -202,7 +210,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) {
|
||||
{Key: dumpKey(prefix, created.Add(time.Minute), "gzip"), Size: 1, LastModified: created.Add(time.Minute)},
|
||||
{Key: dumpKey(prefix, created.Add(2*time.Minute), "gzip"), Size: 2, LastModified: created.Add(2 * time.Minute)},
|
||||
}}
|
||||
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
|
||||
if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 {
|
||||
t.Errorf("size = %d, want 0 (ambiguous)", got)
|
||||
}
|
||||
})
|
||||
@@ -211,14 +219,14 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: dumpKey(prefix, completed.Add(time.Hour), "gzip"), Size: 1, LastModified: completed.Add(time.Hour)},
|
||||
}}
|
||||
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
|
||||
if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 {
|
||||
t.Errorf("size = %d, want 0", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("list error leaves size unset", func(t *testing.T) {
|
||||
l := &fakeS3Lister{err: errors.New("AccessDenied")}
|
||||
if got := reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
|
||||
if got := probedSize(reconcilerWithLister(t, l, s3CredsSecret("tenant", "creds")).attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 {
|
||||
t.Errorf("size = %d, want 0", got)
|
||||
}
|
||||
})
|
||||
@@ -226,7 +234,7 @@ func TestMariaDBArtifactSizeBytes(t *testing.T) {
|
||||
t.Run("missing credentials Secret leaves size unset", func(t *testing.T) {
|
||||
c := newMariaDBStrategyTestClient(t)
|
||||
r := &BackupJobReconciler{Client: c, Scheme: c.Scheme()}
|
||||
if got := r.mariadbArtifactSizeBytes(context.Background(), mdbBackup, completed); got != 0 {
|
||||
if got := probedSize(r.attributedMariaDBDump(context.Background(), mdbBackup, completed)); got != 0 {
|
||||
t.Errorf("size = %d, want 0", got)
|
||||
}
|
||||
})
|
||||
@@ -464,7 +472,7 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) {
|
||||
gotPath, gotPrefix, gotAuth = req.URL.Path, req.URL.Query().Get("prefix"), req.Header.Get("Authorization")
|
||||
gotDelimiter = req.URL.Query().Get("delimiter")
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
|
||||
_, _ = fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>bkt</Name><Prefix>tenant/db/</Prefix><KeyCount>1</KeyCount><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>
|
||||
<Contents><Key>%s</Key><Size>31337</Size><ETag>"x"</ETag><LastModified>2026-10-21T10:01:00.000Z</LastModified></Contents>
|
||||
@@ -481,7 +489,7 @@ func TestMariaDBArtifactSizeBytes_MinioClient(t *testing.T) {
|
||||
s3.Prefix = "tenant/db/"
|
||||
s3.Region = "us-east-1"
|
||||
|
||||
if got := r.mariadbArtifactSizeBytes(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute)); got != 31337 {
|
||||
if got := probedSize(r.attributedMariaDBDump(context.Background(), operatorBackupStoringTo(t, s3, created), created.Add(3*time.Minute))); got != 31337 {
|
||||
t.Fatalf("size = %d, want 31337", got)
|
||||
}
|
||||
if gotPath != "/bkt/" || gotPrefix != "tenant/db/" || gotDelimiter != "/" {
|
||||
@@ -523,6 +531,69 @@ func TestCreateMariaDBBackupArtifact_SizeBytes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestCreateMariaDBBackupArtifact_RecordsDump pins that the Backup names
|
||||
// its dump only when the probe attributed one to the operator Backup.
|
||||
func TestCreateMariaDBBackupArtifact_RecordsDump(t *testing.T) {
|
||||
apps := mariadbapp.GroupName
|
||||
created := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
|
||||
completed := metav1.NewTime(created.Add(3 * time.Minute))
|
||||
rendered := newRenderedMariaDBTemplate()
|
||||
rendered.Storage.S3.Prefix = "src/"
|
||||
resolved := &ResolvedBackupConfig{
|
||||
StrategyRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "mariadb-strategy-default"},
|
||||
Parameters: map[string]string{},
|
||||
}
|
||||
own := s3ObjectInfo{Key: "src/backup.2026-10-21T10:01:00Z.gzip.sql", Size: 2048, LastModified: created.Add(2 * time.Minute)}
|
||||
second := s3ObjectInfo{Key: "src/backup.2026-10-21T10:02:00Z.gzip.sql", Size: 1, LastModified: created.Add(2 * time.Minute)}
|
||||
offset := s3ObjectInfo{Key: "src/backup.2026-10-21T14:01:00+04:00.sql", Size: 7, LastModified: created.Add(2 * time.Minute)}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
lister *fakeS3Lister
|
||||
wantFile, wantTime string
|
||||
}{
|
||||
{name: "an attributed dump is recorded", lister: &fakeS3Lister{objects: []s3ObjectInfo{own}},
|
||||
wantFile: "backup.2026-10-21T10:01:00Z.gzip.sql", wantTime: "2026-10-21T10:01:00Z"},
|
||||
{name: "a dump named with an offset is timed in UTC", lister: &fakeS3Lister{objects: []s3ObjectInfo{offset}},
|
||||
wantFile: "backup.2026-10-21T14:01:00+04:00.sql", wantTime: "2026-10-21T10:01:00Z"},
|
||||
{name: "two dumps in the lifetime record none", lister: &fakeS3Lister{objects: []s3ObjectInfo{own, second}}},
|
||||
{name: "no dump in the lifetime records none", lister: &fakeS3Lister{}},
|
||||
{name: "a failed listing records none", lister: &fakeS3Lister{err: errors.New("AccessDenied")}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
mdbBackup := operatorBackupStoringTo(t, rendered.Storage.S3, created)
|
||||
mdbBackup.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(completed)}
|
||||
job := &backupsv1alpha1.BackupJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "bj-dump"},
|
||||
Spec: backupsv1alpha1.BackupJobSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
|
||||
},
|
||||
}
|
||||
r := reconcilerWithLister(t, tc.lister, job, s3CredsSecret("tenant", "creds"))
|
||||
if _, err := r.createMariaDBBackupArtifact(context.Background(), job, resolved, mdbBackup, rendered); err != nil {
|
||||
t.Fatalf("createMariaDBBackupArtifact: %v", err)
|
||||
}
|
||||
persisted := &backupsv1alpha1.Backup{}
|
||||
if err := r.Get(context.Background(), client.ObjectKey{Namespace: "tenant", Name: "bj-dump"}, persisted); err != nil {
|
||||
t.Fatalf("get Backup: %v", err)
|
||||
}
|
||||
md := persisted.Spec.DriverMetadata
|
||||
file, hasFile := md[mariadbDumpFileKey]
|
||||
at, hasTime := md[mariadbDumpTimeKey]
|
||||
if tc.wantFile == "" {
|
||||
if hasFile || hasTime {
|
||||
t.Errorf("driverMetadata = %v; want no dump recorded", md)
|
||||
}
|
||||
return
|
||||
}
|
||||
if file != tc.wantFile || at != tc.wantTime {
|
||||
t.Errorf("recorded dump %q at %q; want %q at %q", file, at, tc.wantFile, tc.wantTime)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCreateMariaDBBackupArtifact_StrategyEditedMidBackup pins that the
|
||||
// probe reads where the operator Backup wrote its dump, not where the
|
||||
// strategy, edited while the backup ran, now points.
|
||||
|
||||
@@ -0,0 +1,531 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/tools/record"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
|
||||
|
||||
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
||||
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbapp"
|
||||
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes"
|
||||
)
|
||||
|
||||
// The source run of the restore fixture: operator Backup op-bk of
|
||||
// application src, alive from mdbRestoreT0 to mdbRestoreT0+40s, writing
|
||||
// under tenant/src/ with mdbRestoreDump as its dump.
|
||||
var mdbRestoreT0 = time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
const (
|
||||
mdbRestorePrefix = "tenant/src/"
|
||||
mdbRestoreDump = "backup.2026-10-21T10:00:05Z.gzip.sql"
|
||||
mdbRestoreTime = "2026-10-21T10:00:05Z"
|
||||
mdbNewerDump = "backup.2026-10-21T11:00:05Z.gzip.sql"
|
||||
)
|
||||
|
||||
type mariadbRestoreFixture struct {
|
||||
c client.Client
|
||||
r *RestoreJobReconciler
|
||||
rj *backupsv1alpha1.RestoreJob
|
||||
backup *backupsv1alpha1.Backup
|
||||
src *mariadbtypes.Backup
|
||||
lister *fakeS3Lister
|
||||
events *record.FakeRecorder
|
||||
}
|
||||
|
||||
// newMariaDBRestoreFixture seeds a RestoreJob of Backup src-bk, whose
|
||||
// driverMetadata is driverMD plus the operator Backup's name, into the
|
||||
// existing application target. extra objects are seeded too.
|
||||
func newMariaDBRestoreFixture(t *testing.T, driverMD map[string]string, lister *fakeS3Lister, extra ...client.Object) *mariadbRestoreFixture {
|
||||
t.Helper()
|
||||
apps := mariadbapp.GroupName
|
||||
s3 := newRenderedMariaDBTemplate().Storage.S3
|
||||
s3.Prefix = "tenant/src"
|
||||
src := operatorBackupStoringTo(t, s3, mdbRestoreT0)
|
||||
src.Status.Conditions = []metav1.Condition{mariadbBackupSucceeded(metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)))}
|
||||
|
||||
md := map[string]string{mariadbBackupNameKey: src.Name}
|
||||
for k, v := range driverMD {
|
||||
md[k] = v
|
||||
}
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "src-bk"},
|
||||
Spec: backupsv1alpha1.BackupSpec{
|
||||
ApplicationRef: corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "src", APIGroup: &apps},
|
||||
TakenAt: metav1.NewTime(mdbRestoreT0.Add(40 * time.Second)),
|
||||
DriverMetadata: md,
|
||||
},
|
||||
}
|
||||
started := metav1.Now()
|
||||
rj := &backupsv1alpha1.RestoreJob{
|
||||
ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "rj"},
|
||||
Spec: backupsv1alpha1.RestoreJobSpec{
|
||||
BackupRef: corev1.LocalObjectReference{Name: backup.Name},
|
||||
TargetApplicationRef: &corev1.TypedLocalObjectReference{Kind: "MariaDB", Name: "target", APIGroup: &apps},
|
||||
},
|
||||
Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &started, Phase: backupsv1alpha1.RestoreJobPhaseRunning},
|
||||
}
|
||||
target := &mariadbtypes.MariaDB{ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: mariadbNameForApp("target")}}
|
||||
objs := append([]client.Object{src, backup, rj, target}, extra...)
|
||||
c := newMariaDBStrategyTestClient(t, objs...)
|
||||
fetched := &backupsv1alpha1.RestoreJob{}
|
||||
if err := c.Get(context.Background(), client.ObjectKeyFromObject(rj), fetched); err != nil {
|
||||
t.Fatalf("get RestoreJob: %v", err)
|
||||
}
|
||||
events := record.NewFakeRecorder(10)
|
||||
return &mariadbRestoreFixture{
|
||||
c: c,
|
||||
rj: fetched,
|
||||
backup: backup,
|
||||
src: src,
|
||||
lister: lister,
|
||||
events: events,
|
||||
r: &RestoreJobReconciler{
|
||||
Client: c,
|
||||
Recorder: events,
|
||||
newS3ObjectLister: func(conn s3Connection) (s3ObjectLister, error) {
|
||||
lister.conn = conn
|
||||
return lister, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *mariadbRestoreFixture) reconcile(t *testing.T) ctrl.Result {
|
||||
t.Helper()
|
||||
res, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup)
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileMariaDBRestore: %v", err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (f *mariadbRestoreFixture) restores(t *testing.T) []mariadbtypes.Restore {
|
||||
t.Helper()
|
||||
list := &mariadbtypes.RestoreList{}
|
||||
if err := f.c.List(context.Background(), list, client.InNamespace("tenant")); err != nil {
|
||||
t.Fatalf("list operator Restores: %v", err)
|
||||
}
|
||||
return list.Items
|
||||
}
|
||||
|
||||
// wantFailed asserts the RestoreJob failed with a message holding every
|
||||
// fragment, and that no operator Restore was created.
|
||||
func (f *mariadbRestoreFixture) wantFailed(t *testing.T, fragments ...string) {
|
||||
t.Helper()
|
||||
if f.rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
||||
t.Fatalf("phase = %q, want Failed", f.rj.Status.Phase)
|
||||
}
|
||||
for _, s := range fragments {
|
||||
if !strings.Contains(f.rj.Status.Message, s) {
|
||||
t.Errorf("message %q does not contain %q", f.rj.Status.Message, s)
|
||||
}
|
||||
}
|
||||
if got := f.restores(t); len(got) != 0 {
|
||||
t.Errorf("created %d operator Restores; want none", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// wantWaiting asserts the RestoreJob is kept waiting on the dump check
|
||||
// with Ready=False saying why, and that no operator Restore was created.
|
||||
func (f *mariadbRestoreFixture) wantWaiting(t *testing.T, res ctrl.Result, fragments ...string) {
|
||||
t.Helper()
|
||||
if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
|
||||
t.Fatalf("RestoreJob failed: %s", f.rj.Status.Message)
|
||||
}
|
||||
if res.RequeueAfter != mariadbDumpCheckRetryInterval {
|
||||
t.Errorf("RequeueAfter = %s, want %s", res.RequeueAfter, mariadbDumpCheckRetryInterval)
|
||||
}
|
||||
persisted := &backupsv1alpha1.RestoreJob{}
|
||||
if err := f.c.Get(context.Background(), client.ObjectKeyFromObject(f.rj), persisted); err != nil {
|
||||
t.Fatalf("get RestoreJob: %v", err)
|
||||
}
|
||||
cond := apimeta.FindStatusCondition(persisted.Status.Conditions, "Ready")
|
||||
if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != "DumpCheckPending" {
|
||||
t.Fatalf("Ready = %+v, want False/DumpCheckPending", cond)
|
||||
}
|
||||
for _, s := range fragments {
|
||||
if !strings.Contains(cond.Message, s) {
|
||||
t.Errorf("Ready message %q does not contain %q", cond.Message, s)
|
||||
}
|
||||
}
|
||||
if got := f.restores(t); len(got) != 0 {
|
||||
t.Errorf("created %d operator Restores; want none", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
// wantRestore asserts exactly one operator Restore of op-bk into target,
|
||||
// asking for targetTime.
|
||||
func (f *mariadbRestoreFixture) wantRestore(t *testing.T, targetTime string) {
|
||||
t.Helper()
|
||||
got := f.restores(t)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("created %d operator Restores; want one", len(got))
|
||||
}
|
||||
spec := got[0].Spec
|
||||
if spec.BackupRef == nil || spec.BackupRef.Name != f.src.Name || spec.MariaDBRef.Name != mariadbNameForApp("target") {
|
||||
t.Errorf("Restore spec = %+v, want backupRef %s into %s", spec, f.src.Name, mariadbNameForApp("target"))
|
||||
}
|
||||
if spec.TargetRecoveryTime != targetTime {
|
||||
t.Errorf("targetRecoveryTime = %q, want %q", spec.TargetRecoveryTime, targetTime)
|
||||
}
|
||||
}
|
||||
|
||||
func recordedDump(name string) map[string]string {
|
||||
return map[string]string{mariadbDumpFileKey: name, mariadbDumpTimeKey: mdbRestoreTime}
|
||||
}
|
||||
|
||||
// A Backup being deleted is deleting its dump, so a restore that starts
|
||||
// now can find it gone once the operator reads it, and the operator then
|
||||
// restores the nearest other dump. The Backup is read from the API server,
|
||||
// since the copy the reconcile was handed can predate the delete.
|
||||
func TestReconcileMariaDBRestore_RefusesABackupBeingDeleted(t *testing.T) {
|
||||
creds := s3CredsSecret("tenant", "creds")
|
||||
present := func() *fakeS3Lister {
|
||||
return &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}}}
|
||||
}
|
||||
cases := map[string]map[string]string{"recorded dump": recordedDump(mdbRestoreDump), "legacy Backup": nil}
|
||||
for name, md := range cases {
|
||||
t.Run(name+", being deleted", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, md, present(), creds)
|
||||
ctx := context.Background()
|
||||
live := &backupsv1alpha1.Backup{}
|
||||
if err := f.c.Get(ctx, client.ObjectKeyFromObject(f.backup), live); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
live.Finalizers = []string{backupFinalizer}
|
||||
if err := f.c.Update(ctx, live); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.c.Delete(ctx, live); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.reconcile(t)
|
||||
f.wantFailed(t, "src-bk", "being deleted")
|
||||
})
|
||||
t.Run(name+", already gone", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, md, present(), creds)
|
||||
if err := f.c.Delete(context.Background(), f.backup.DeepCopy()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.reconcile(t)
|
||||
f.wantFailed(t, "src-bk", "is gone")
|
||||
})
|
||||
}
|
||||
// The refusal applies only before the operator Restore exists. One
|
||||
// created by an earlier reconcile but not yet in the cache must be
|
||||
// found and followed, or the RestoreJob fails under a running restore
|
||||
// and the cleanup deletes the dump that restore reads.
|
||||
t.Run("an operator Restore the cache has not seen yet is followed", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds)
|
||||
ctx := context.Background()
|
||||
live := &backupsv1alpha1.Backup{}
|
||||
if err := f.c.Get(ctx, client.ObjectKeyFromObject(f.backup), live); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
live.Finalizers = []string{backupFinalizer}
|
||||
if err := f.c.Update(ctx, live); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.c.Delete(ctx, live); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.c.Get(ctx, client.ObjectKeyFromObject(live), live); err != nil || live.DeletionTimestamp.IsZero() {
|
||||
t.Fatalf("Backup not left being deleted: %v", err)
|
||||
}
|
||||
live.ResourceVersion = ""
|
||||
apiServer := newMariaDBStrategyTestClient(t, live, &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: "tenant", Name: "rj-abcde",
|
||||
Labels: map[string]string{
|
||||
backupsv1alpha1.OwningJobNameLabel: "rj",
|
||||
backupsv1alpha1.OwningJobNamespaceLabel: "tenant",
|
||||
},
|
||||
}})
|
||||
f.r.APIReader = apiServer
|
||||
f.reconcile(t)
|
||||
if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
|
||||
t.Fatalf("RestoreJob failed under a running restore: %s", f.rj.Status.Message)
|
||||
}
|
||||
if got := f.restores(t); len(got) != 0 {
|
||||
t.Errorf("created %d operator Restores; want the existing one followed", len(got))
|
||||
}
|
||||
})
|
||||
// A failed lookup says nothing about whether the operator Restore
|
||||
// exists, so the RestoreJob must not be failed under one that runs.
|
||||
t.Run("an operator Restore lookup that fails is retried", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds)
|
||||
f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
|
||||
List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error {
|
||||
if _, ok := list.(*mariadbtypes.RestoreList); ok {
|
||||
return errors.New("etcdserver: request timed out")
|
||||
}
|
||||
return c.List(ctx, list, opts...)
|
||||
},
|
||||
})
|
||||
if _, err := f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup); err == nil {
|
||||
t.Error("err = nil, want the failed lookup returned for a retry")
|
||||
}
|
||||
if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
|
||||
t.Fatalf("RestoreJob failed on a lookup error: %s", f.rj.Status.Message)
|
||||
}
|
||||
if got := f.restores(t); len(got) != 0 {
|
||||
t.Errorf("created %d operator Restores without knowing whether one exists", len(got))
|
||||
}
|
||||
})
|
||||
t.Run("a Backup that cannot be read keeps the RestoreJob waiting", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), present(), creds)
|
||||
f.r.APIReader = interceptor.NewClient(f.c.(client.WithWatch), interceptor.Funcs{
|
||||
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
|
||||
if _, ok := obj.(*backupsv1alpha1.Backup); ok {
|
||||
return errors.New("apiserver unavailable")
|
||||
}
|
||||
return c.Get(ctx, key, obj, opts...)
|
||||
},
|
||||
})
|
||||
f.wantWaiting(t, f.reconcile(t), "src-bk", "apiserver unavailable")
|
||||
})
|
||||
}
|
||||
|
||||
func TestReconcileMariaDBRestore_RecordedDump(t *testing.T) {
|
||||
creds := s3CredsSecret("tenant", "creds")
|
||||
|
||||
t.Run("restores the recorded dump by its own timestamp", func(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{
|
||||
{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)},
|
||||
{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)},
|
||||
}}
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds)
|
||||
f.reconcile(t)
|
||||
f.wantRestore(t, mdbRestoreTime)
|
||||
if l.bucket != "bkt" || l.prefix != mdbRestorePrefix+mdbRestoreDump || l.conn.AccessKey != "AKID" {
|
||||
t.Errorf("checked %s/%s as %q, want bkt/%s%s with the operator Backup's credentials", l.bucket, l.prefix, l.conn.AccessKey, mdbRestorePrefix, mdbRestoreDump)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a recorded dump gone from storage fails the RestoreJob", func(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: []s3ObjectInfo{{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}}}
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds)
|
||||
f.reconcile(t)
|
||||
f.wantFailed(t, mdbRestoreDump, "no longer exists in storage", "s3://bkt/"+mdbRestorePrefix)
|
||||
})
|
||||
|
||||
t.Run("a storage that cannot be read keeps the RestoreJob waiting", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds)
|
||||
f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "Access Denied.")
|
||||
})
|
||||
|
||||
t.Run("missing credentials keep the RestoreJob waiting", func(t *testing.T) {
|
||||
l := &fakeS3Lister{}
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l)
|
||||
f.wantWaiting(t, f.reconcile(t), mdbRestoreDump, "resolve S3 credentials")
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 listed %d times without credentials", l.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("a hanging endpoint is given up at the check timeout", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{}, creds)
|
||||
f.r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil }
|
||||
var res ctrl.Result
|
||||
var err error
|
||||
withShortDumpCheckTimeout(t, func() { res, err = f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) })
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileMariaDBRestore: %v", err)
|
||||
}
|
||||
f.wantWaiting(t, res, mdbRestoreDump, context.DeadlineExceeded.Error())
|
||||
})
|
||||
|
||||
t.Run("a storage still unreadable at the deadline fails the RestoreJob with the cause", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), &fakeS3Lister{err: errors.New("Access Denied.")}, creds)
|
||||
long := metav1.NewTime(time.Now().Add(-mariadbDefaultRestoreDeadline - time.Minute))
|
||||
f.rj.Status.StartedAt = &long
|
||||
f.reconcile(t)
|
||||
f.wantFailed(t, mdbRestoreDump, "Access Denied.", "gave up after")
|
||||
})
|
||||
|
||||
t.Run("a recorded name that is not a dump fails the RestoreJob unchecked", func(t *testing.T) {
|
||||
l := &fakeS3Lister{}
|
||||
f := newMariaDBRestoreFixture(t, recordedDump("dump.sql"), l, creds)
|
||||
f.reconcile(t)
|
||||
f.wantFailed(t, `"dump.sql"`)
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 listed %d times for a name that is not a dump", l.calls)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an operator Restore created earlier is followed without a check", func(t *testing.T) {
|
||||
l := &fakeS3Lister{err: errors.New("Access Denied.")}
|
||||
existing := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: "tenant", Name: "rj-abcde",
|
||||
Labels: map[string]string{
|
||||
backupsv1alpha1.OwningJobNameLabel: "rj",
|
||||
backupsv1alpha1.OwningJobNamespaceLabel: "tenant",
|
||||
},
|
||||
}}
|
||||
f := newMariaDBRestoreFixture(t, recordedDump(mdbRestoreDump), l, creds, existing)
|
||||
res := f.reconcile(t)
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 listed %d times for a RestoreJob whose operator Restore exists", l.calls)
|
||||
}
|
||||
if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed || res.RequeueAfter != mariadbPollInterval {
|
||||
t.Errorf("phase %q, requeue %s; want the existing Restore polled", f.rj.Status.Phase, res.RequeueAfter)
|
||||
}
|
||||
if got := f.restores(t); len(got) != 1 {
|
||||
t.Errorf("%d operator Restores; want only the existing one", len(got))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestReconcileMariaDBRestore_LegacyBackup drives Backups that record no
|
||||
// dump: one proven to belong to the source run is restored by its
|
||||
// timestamp, and anything else restores the newest dump with a Warning.
|
||||
func TestReconcileMariaDBRestore_LegacyBackup(t *testing.T) {
|
||||
creds := s3CredsSecret("tenant", "creds")
|
||||
own := s3ObjectInfo{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}
|
||||
newer := s3ObjectInfo{Key: mdbRestorePrefix + mdbNewerDump, LastModified: mdbRestoreT0.Add(time.Hour)}
|
||||
second := s3ObjectInfo{Key: mdbRestorePrefix + "backup.2026-10-21T10:00:30Z.gzip.sql", LastModified: mdbRestoreT0.Add(35 * time.Second)}
|
||||
|
||||
t.Run("a dump proven to be the run's own is restored by its timestamp", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{objects: []s3ObjectInfo{own, newer}}, creds)
|
||||
f.reconcile(t)
|
||||
f.wantRestore(t, mdbRestoreTime)
|
||||
if ev := drainEvents(f.events); len(ev) != 0 {
|
||||
t.Errorf("events %q; want none for a proven dump", ev)
|
||||
}
|
||||
})
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
lister *fakeS3Lister
|
||||
cause string
|
||||
}{
|
||||
{name: "two dumps in the run's lifetime", lister: &fakeS3Lister{objects: []s3ObjectInfo{own, second, newer}}, cause: "cannot be told apart"},
|
||||
{name: "no dump in the run's lifetime", lister: &fakeS3Lister{objects: []s3ObjectInfo{newer}}, cause: "no dump in the bucket belongs"},
|
||||
{name: "a storage that cannot be read", lister: &fakeS3Lister{err: errors.New("Access Denied.")}, cause: "Access Denied."},
|
||||
}
|
||||
wantBestEffort := func(t *testing.T, f *mariadbRestoreFixture, cause string) {
|
||||
t.Helper()
|
||||
f.wantRestore(t, "")
|
||||
ev := drainEvents(f.events)
|
||||
if len(ev) != 1 || !strings.HasPrefix(ev[0], "Warning BestEffortDump ") || !strings.Contains(ev[0], cause) || !strings.Contains(ev[0], "src-bk records no dump") {
|
||||
t.Errorf("events %q; want one BestEffortDump warning naming the Backup and %q", ev, cause)
|
||||
}
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name+" restores the newest dump with a warning", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, nil, tc.lister, creds)
|
||||
f.reconcile(t)
|
||||
wantBestEffort(t, f, tc.cause)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("a hanging endpoint is given up at the check timeout", func(t *testing.T) {
|
||||
f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{}, creds)
|
||||
f.r.newS3ObjectLister = func(s3Connection) (s3ObjectLister, error) { return blockingLister{}, nil }
|
||||
var err error
|
||||
withShortDumpCheckTimeout(t, func() { _, err = f.r.reconcileMariaDBRestore(context.Background(), f.rj, f.backup) })
|
||||
if err != nil {
|
||||
t.Fatalf("reconcileMariaDBRestore: %v", err)
|
||||
}
|
||||
wantBestEffort(t, f, context.DeadlineExceeded.Error())
|
||||
})
|
||||
}
|
||||
|
||||
// TestReconcileMariaDBRestore_FailedBackupRun drives a Backup made of an
|
||||
// operator Backup whose Job failed: there is no dump of its own to
|
||||
// restore, whatever the Backup records.
|
||||
func TestReconcileMariaDBRestore_FailedBackupRun(t *testing.T) {
|
||||
creds := s3CredsSecret("tenant", "creds")
|
||||
failed := func(status metav1.ConditionStatus) []metav1.Condition {
|
||||
return []metav1.Condition{{
|
||||
Type: mariadbtypes.ConditionTypeComplete, Status: status, Reason: mariadbtypes.ConditionReasonJobFailed,
|
||||
Message: "Failed", LastTransitionTime: metav1.NewTime(mdbRestoreT0.Add(3 * time.Minute)),
|
||||
}}
|
||||
}
|
||||
listing := []s3ObjectInfo{{Key: mdbRestorePrefix + mdbRestoreDump, LastModified: mdbRestoreT0.Add(20 * time.Second)}}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
driverMD map[string]string
|
||||
status metav1.ConditionStatus
|
||||
}{
|
||||
{name: "Complete=True/JobFailed with no recorded dump", status: metav1.ConditionTrue},
|
||||
{name: "Complete=True/JobFailed with a recorded dump", driverMD: recordedDump(mdbRestoreDump), status: metav1.ConditionTrue},
|
||||
{name: "Complete=False/JobFailed", status: metav1.ConditionFalse},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name+" fails the RestoreJob", func(t *testing.T) {
|
||||
l := &fakeS3Lister{objects: listing}
|
||||
f := newMariaDBRestoreFixture(t, tc.driverMD, l, creds)
|
||||
f.src.Status.Conditions = failed(tc.status)
|
||||
if err := f.c.Update(context.Background(), f.src); err != nil {
|
||||
t.Fatalf("update operator Backup: %v", err)
|
||||
}
|
||||
f.reconcile(t)
|
||||
f.wantFailed(t, "k8s.mariadb.com/Backup tenant/op-bk failed", "left no dump to restore")
|
||||
if l.calls != 0 {
|
||||
t.Errorf("S3 listed %d times for a failed backup run", l.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("an operator Restore created earlier is followed", func(t *testing.T) {
|
||||
existing := &mariadbtypes.Restore{ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: "tenant", Name: "rj-abcde",
|
||||
Labels: map[string]string{
|
||||
backupsv1alpha1.OwningJobNameLabel: "rj",
|
||||
backupsv1alpha1.OwningJobNamespaceLabel: "tenant",
|
||||
},
|
||||
}}
|
||||
f := newMariaDBRestoreFixture(t, nil, &fakeS3Lister{objects: listing}, creds, existing)
|
||||
f.src.Status.Conditions = failed(metav1.ConditionTrue)
|
||||
if err := f.c.Update(context.Background(), f.src); err != nil {
|
||||
t.Fatalf("update operator Backup: %v", err)
|
||||
}
|
||||
f.reconcile(t)
|
||||
if f.rj.Status.Phase == backupsv1alpha1.RestoreJobPhaseFailed {
|
||||
t.Errorf("RestoreJob failed although its operator Restore was already created: %s", f.rj.Status.Message)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func drainEvents(r *record.FakeRecorder) []string {
|
||||
var out []string
|
||||
for {
|
||||
select {
|
||||
case e := <-r.Events:
|
||||
out = append(out, e)
|
||||
default:
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// withShortDumpCheckTimeout shortens the restore's dump check timeout and
|
||||
// fails the test if fn is not back well before the default would expire.
|
||||
func withShortDumpCheckTimeout(t *testing.T, fn func()) {
|
||||
t.Helper()
|
||||
saved := mariadbRestoreDumpCheckTimeout
|
||||
mariadbRestoreDumpCheckTimeout = 50 * time.Millisecond
|
||||
defer func() { mariadbRestoreDumpCheckTimeout = saved }()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
fn()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the dump check did not give up at its timeout")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
package backupcontroller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1"
|
||||
"github.com/cozystack/cozystack/internal/backupcontroller/mariadbtypes"
|
||||
)
|
||||
|
||||
const (
|
||||
mariadbSkipArtifactCleanupAnnotation = "backups.cozystack.io/skip-artifact-cleanup"
|
||||
|
||||
// Retry cadence while a dump cannot be deleted or a restore from the
|
||||
// Backup is still running.
|
||||
mariadbCleanupRetryInterval = 30 * time.Second
|
||||
)
|
||||
|
||||
// mariadbDumpCleanupTimeout bounds the object-store round trips of one
|
||||
// cleanup attempt. The Backup controller has a single worker, so an
|
||||
// unreachable endpoint stalls every other Backup for this long. A variable
|
||||
// so tests can shorten it.
|
||||
var mariadbDumpCleanupTimeout = 10 * time.Second
|
||||
|
||||
// cleanupMariaDBBackup deletes the dump a MariaDB Backup was taken as, then
|
||||
// the operator Backup that wrote it. Every run of an application writes
|
||||
// under one prefix, so the only key deleted is one proven to be this
|
||||
// Backup's: the dump it records, unless contestedMariaDBDump finds another
|
||||
// run that could have written it, or, for a Backup taken before dumps were
|
||||
// recorded, the one findMariaDBDump attributes to its operator Backup. When
|
||||
// no dump is proven the Backup is released with a Warning and nothing is
|
||||
// deleted, the operator Backup included, since it is what any later
|
||||
// attribution would start from. A failed delete holds the Backup and
|
||||
// retries, as the Redis and Rabbitmq cleanups do.
|
||||
func (r *BackupReconciler) cleanupMariaDBBackup(ctx context.Context, backup *backupsv1alpha1.Backup) (ctrl.Result, error) {
|
||||
logger := getLogger(ctx)
|
||||
if backup.Annotations[mariadbSkipArtifactCleanupAnnotation] == "true" {
|
||||
logger.Debug("skipping MariaDB artifact cleanup per annotation", "backup", backup.Name, "annotation", mariadbSkipArtifactCleanupAnnotation)
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
opName := backup.Spec.DriverMetadata[mariadbBackupNameKey]
|
||||
if opName == "" {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, "the dump", "the Backup records no k8s.mariadb.com/Backup"), nil
|
||||
}
|
||||
|
||||
// The operator Restore reads the dump through the operator Backup, so
|
||||
// neither may go while a restore from this Backup runs.
|
||||
restore, err := r.activeRestoreOf(ctx, backup, opName)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if restore != "" {
|
||||
logger.Debug("holding MariaDB Backup cleanup while a restore from it runs", "backup", backup.Name, "restore", restore)
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactCleanupWaiting",
|
||||
"%s is restoring from this Backup; its dump is deleted once the restore finishes. Annotate the Backup %s=true to release it now and leave the dump",
|
||||
restore, mariadbSkipArtifactCleanupAnnotation)
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil
|
||||
}
|
||||
|
||||
op := &mariadbtypes.Backup{}
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: backup.Namespace, Name: opName}, op); err != nil {
|
||||
if apierrors.IsNotFound(err) || apimeta.IsNoMatchError(err) {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, "the dump",
|
||||
fmt.Sprintf("k8s.mariadb.com/Backup %s is gone, so the storage and credentials its dump was written with cannot be read", opName)), nil
|
||||
}
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
s3 := op.Spec.Storage.S3
|
||||
if s3 == nil {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, fmt.Sprintf("the dump on the storage volume of k8s.mariadb.com/Backup %s", opName),
|
||||
"PVC and Volume storage is not reachable from the controller"), nil
|
||||
}
|
||||
prefix := mariadbOperatorS3Prefix(s3.Prefix)
|
||||
location := fmt.Sprintf("s3://%s/%s", s3.Bucket, prefix)
|
||||
|
||||
// A failed Job is Complete=True too, with reason JobFailed, and wrote no
|
||||
// dump: whatever its window holds is another run's.
|
||||
if cond := apimeta.FindStatusCondition(op.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond == nil ||
|
||||
cond.Status != metav1.ConditionTrue || cond.Reason != mariadbtypes.ConditionReasonJobComplete {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, location,
|
||||
fmt.Sprintf("k8s.mariadb.com/Backup %s did not complete successfully, so no dump is its own", opName)), nil
|
||||
}
|
||||
|
||||
name := backup.Spec.DriverMetadata[mariadbDumpFileKey]
|
||||
if name != "" {
|
||||
if _, ok := parseMariaDBLogicalBackupFile(name); !ok {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, location,
|
||||
fmt.Sprintf("driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name)), nil
|
||||
}
|
||||
}
|
||||
|
||||
conn, err := r.mariadbCleanupConnection(ctx, op)
|
||||
if err != nil {
|
||||
return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("resolve S3 credentials: %w", err))
|
||||
}
|
||||
|
||||
s3ctx, cancel := context.WithTimeout(ctx, mariadbDumpCleanupTimeout)
|
||||
defer cancel()
|
||||
store := r.objectStore()
|
||||
lister, err := store.lister(conn)
|
||||
if err != nil {
|
||||
return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err))
|
||||
}
|
||||
absent := false
|
||||
if name != "" {
|
||||
other, found, err := contestedMariaDBDump(s3ctx, lister, r.apiReader(), op, name)
|
||||
absent = !found
|
||||
if err != nil {
|
||||
return r.retryMariaDBCleanup(ctx, backup, location+name, err)
|
||||
}
|
||||
if other != "" {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, location+name,
|
||||
fmt.Sprintf("the recorded dump also falls in the lifetime of k8s.mariadb.com/Backup %s, which may have written it; the operator Backup is kept", other)), nil
|
||||
}
|
||||
} else {
|
||||
dump, err := findMariaDBDumpWith(s3ctx, lister, r.apiReader(), op, backup.Spec.TakenAt.Time, true)
|
||||
if errors.Is(err, errMariaDBDumpNotFound) || errors.Is(err, errMariaDBDumpAmbiguous) {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, location,
|
||||
fmt.Sprintf("the Backup records no dump and none is proven to be the one k8s.mariadb.com/Backup %s wrote (%v); the operator Backup is kept", opName, err)), nil
|
||||
}
|
||||
if err != nil {
|
||||
return r.retryMariaDBCleanup(ctx, backup, location, err)
|
||||
}
|
||||
name = dump.Name
|
||||
// Recorded first: once the dump is gone a retry could not match it
|
||||
// again, and could match a neighbour's instead.
|
||||
base := backup.DeepCopy()
|
||||
backup.Spec.DriverMetadata[mariadbDumpFileKey] = name
|
||||
backup.Spec.DriverMetadata[mariadbDumpTimeKey] = dump.Timestamp.UTC().Format(time.RFC3339)
|
||||
if err := r.Patch(ctx, backup, client.MergeFrom(base)); err != nil {
|
||||
return ctrl.Result{}, fmt.Errorf("record dump %s on Backup %s: %w", name, backup.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
remover, err := store.remover(conn)
|
||||
if err != nil {
|
||||
return r.retryMariaDBCleanup(ctx, backup, location, fmt.Errorf("build S3 client: %w", err))
|
||||
}
|
||||
if err := remover.RemoveObject(s3ctx, s3.Bucket, prefix+name); err != nil {
|
||||
return r.retryMariaDBCleanup(ctx, backup, location+name, err)
|
||||
}
|
||||
policy := metav1.DeletePropagationBackground
|
||||
if err := r.Delete(ctx, op, &client.DeleteOptions{PropagationPolicy: &policy}); err != nil && !apierrors.IsNotFound(err) {
|
||||
return ctrl.Result{}, fmt.Errorf("delete k8s.mariadb.com/Backup %s/%s: %w", op.Namespace, op.Name, err)
|
||||
}
|
||||
if absent {
|
||||
logger.Info("MariaDB dump already absent; deleted its operator Backup", "backup", backup.Name, "dump", location+name, "endpoint", conn.Endpoint, "operatorBackup", opName)
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(backup, corev1.EventTypeNormal, "ArtifactAlreadyAbsent",
|
||||
"%s was not found when listed through %s, so it was deleted earlier or never reached that storage; k8s.mariadb.com/Backup %s is deleted",
|
||||
location+name, conn.Endpoint, opName)
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
logger.Info("deleted MariaDB dump and its operator Backup", "backup", backup.Name, "dump", location+name, "operatorBackup", opName)
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
// contestedMariaDBDump returns the name of another operator Backup on op's
|
||||
// bucket and prefix whose run, finished or not, could have written the dump
|
||||
// name, or "" when none could, and whether the listing found the dump. The
|
||||
// size probe that recorded name let only runs finished by then contest it,
|
||||
// and a run can upload its dump and prune op's through maxRetention before
|
||||
// it completes. A dump no longer in the bucket is contested by nobody:
|
||||
// deleting it again removes nothing.
|
||||
func contestedMariaDBDump(ctx context.Context, lister s3ObjectLister, reader client.Reader, op *mariadbtypes.Backup, name string) (string, bool, error) {
|
||||
ts, ok := parseMariaDBLogicalBackupFile(name)
|
||||
if !ok {
|
||||
return "", false, fmt.Errorf("%q is not the name of a mariadb-operator dump", name)
|
||||
}
|
||||
s3 := op.Spec.Storage.S3
|
||||
key := mariadbOperatorS3Prefix(s3.Prefix) + name
|
||||
objects, err := lister.ListObjects(ctx, s3.Bucket, key, false)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("list s3://%s/%s: %w", s3.Bucket, key, err)
|
||||
}
|
||||
i := slices.IndexFunc(objects, func(obj s3ObjectInfo) bool { return obj.Key == key })
|
||||
if i < 0 {
|
||||
return "", false, nil
|
||||
}
|
||||
others, err := neighbourMariaDBRuns(ctx, reader, op, true)
|
||||
if err != nil {
|
||||
return "", true, fmt.Errorf("list operator Backups: %w", err)
|
||||
}
|
||||
d := mariadbDump{Key: key, Name: name, Timestamp: ts, LastModified: objects[i].LastModified}
|
||||
for _, other := range others {
|
||||
if other.placesOnEitherClock(d) {
|
||||
return other.name, true, nil
|
||||
}
|
||||
}
|
||||
return "", true, nil
|
||||
}
|
||||
|
||||
// activeRestoreOf names a restore from backup that has not finished: a
|
||||
// RestoreJob of it, or an operator Restore reading the dump of opName, the
|
||||
// operator Backup it was made of; "" when there is none. The operator
|
||||
// Restore is checked on its own because a RestoreJob can fail, on its
|
||||
// deadline, while the Restore it created keeps reading the dump.
|
||||
// mariadb-operator marks a Restore Complete=True once its Job has
|
||||
// completed or failed.
|
||||
func (r *BackupReconciler) activeRestoreOf(ctx context.Context, backup *backupsv1alpha1.Backup, opName string) (string, error) {
|
||||
list := &backupsv1alpha1.RestoreJobList{}
|
||||
if err := r.apiReader().List(ctx, list, client.InNamespace(backup.Namespace)); err != nil {
|
||||
return "", fmt.Errorf("list RestoreJobs: %w", err)
|
||||
}
|
||||
for i := range list.Items {
|
||||
rj := &list.Items[i]
|
||||
if rj.Spec.BackupRef.Name != backup.Name {
|
||||
continue
|
||||
}
|
||||
if rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseSucceeded && rj.Status.Phase != backupsv1alpha1.RestoreJobPhaseFailed {
|
||||
return "RestoreJob " + rj.Name, nil
|
||||
}
|
||||
}
|
||||
restores := &mariadbtypes.RestoreList{}
|
||||
if err := r.apiReader().List(ctx, restores, client.InNamespace(backup.Namespace)); err != nil {
|
||||
// Without the operator's CRDs no Restore can run; the operator
|
||||
// Backup read that follows releases the Backup.
|
||||
if apimeta.IsNoMatchError(err) {
|
||||
return "", nil
|
||||
}
|
||||
return "", fmt.Errorf("list k8s.mariadb.com/Restores: %w", err)
|
||||
}
|
||||
for i := range restores.Items {
|
||||
restore := &restores.Items[i]
|
||||
if restore.Spec.BackupRef == nil || restore.Spec.BackupRef.Name != opName {
|
||||
continue
|
||||
}
|
||||
if !apimeta.IsStatusConditionTrue(restore.Status.Conditions, mariadbtypes.ConditionTypeComplete) {
|
||||
return "k8s.mariadb.com/Restore " + restore.Name, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// mariadbCleanupConnection returns how to reach the bucket op wrote to. A
|
||||
// dump on the platform bucket, which op names by its bucket and by the
|
||||
// Secret the platform credentials are projected to, is reached as the
|
||||
// platform reaches it now, so a Backup taken before the endpoint moved is
|
||||
// still deleted. Any other bucket, or the platform bucket while the
|
||||
// platform credentials are missing or malformed, is reached as op recorded
|
||||
// it; a delete that fails there is handled by retryMariaDBCleanup like any
|
||||
// other. A failed read of the platform credentials is retried instead: the
|
||||
// recorded endpoint may be the one the platform moved off, where the dump
|
||||
// would be found missing and reported deleted.
|
||||
func (r *BackupReconciler) mariadbCleanupConnection(ctx context.Context, op *mariadbtypes.Backup) (s3Connection, error) {
|
||||
if r.CredentialsConfig.IsEnabled() {
|
||||
conn, bucket, err := r.platformS3Connection(ctx)
|
||||
var projErr *ProjectionError
|
||||
switch {
|
||||
case errors.As(err, &projErr) && projErr.Reason == ReasonAPIError:
|
||||
return s3Connection{}, err
|
||||
case err != nil:
|
||||
getLogger(ctx).Info("cannot read the platform backup credentials; using the operator Backup's own", "operatorBackup", op.Name, "error", err)
|
||||
case bucket == op.Spec.Storage.S3.Bucket && op.Spec.Storage.S3.AccessKeyIdSecretKeyRef.Name == r.CredentialsConfig.TargetSecretName:
|
||||
// A strategy can pin a private CA for the platform bucket; the
|
||||
// platform credentials carry none.
|
||||
if tls := op.Spec.Storage.S3.TLS; conn.Secure && tls != nil && tls.CASecretKeyRef != nil {
|
||||
ca, err := r.objectStore().secretValue(ctx, op.Namespace, tls.CASecretKeyRef.Name, tls.CASecretKeyRef.Key)
|
||||
if err != nil {
|
||||
return s3Connection{}, err
|
||||
}
|
||||
conn.CAPEM = []byte(ca)
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
}
|
||||
return mariadbS3Connection(ctx, r.objectStore(), op.Namespace, op.Spec.Storage.S3)
|
||||
}
|
||||
|
||||
// platformS3Connection reads the platform's backup bucket from the Secret
|
||||
// cozy-backups-creds is projected from, and its TLS from the scheme of
|
||||
// BACKUP_STORAGE_ENDPOINT, which is how the default strategy sets
|
||||
// tls.enabled. It reads the source rather than projecting it: projecting
|
||||
// writes a Secret, which a terminating namespace refuses.
|
||||
func (r *BackupReconciler) platformS3Connection(ctx context.Context) (s3Connection, string, error) {
|
||||
cfg := r.CredentialsConfig
|
||||
src := &corev1.Secret{}
|
||||
if err := r.Get(ctx, types.NamespacedName{Namespace: cfg.SourceNamespace, Name: cfg.SourceSecretName}, src); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMissing,
|
||||
Message: fmt.Sprintf("source credentials Secret %s/%s not found", cfg.SourceNamespace, cfg.SourceSecretName)}
|
||||
}
|
||||
return s3Connection{}, "", &ProjectionError{Reason: ReasonAPIError, Message: fmt.Sprintf("read source credentials Secret: %v", err)}
|
||||
}
|
||||
creds, err := parseSourceSecret(src)
|
||||
if err != nil {
|
||||
return s3Connection{}, "", err
|
||||
}
|
||||
if creds.endpoint == "" {
|
||||
creds.endpoint = stripScheme(cfg.Endpoint)
|
||||
}
|
||||
if creds.endpoint == "" {
|
||||
return s3Connection{}, "", &ProjectionError{Reason: ReasonSourceMalformed,
|
||||
Message: fmt.Sprintf("endpoint is empty in both source Secret %s/%s and BACKUP_STORAGE_ENDPOINT", cfg.SourceNamespace, cfg.SourceSecretName)}
|
||||
}
|
||||
if creds.region == "" {
|
||||
creds.region = cfg.Region
|
||||
}
|
||||
return s3Connection{
|
||||
Endpoint: creds.endpoint,
|
||||
Secure: strings.HasPrefix(cfg.Endpoint, "https://"),
|
||||
Region: creds.region,
|
||||
AccessKey: creds.accessKey,
|
||||
SecretKey: creds.secretKey,
|
||||
}, creds.bucket, nil
|
||||
}
|
||||
|
||||
// retryMariaDBCleanup holds the Backup after a failed attempt and says so,
|
||||
// unless its namespace is going away, where nothing it needs can be read
|
||||
// any longer and holding it would hold the namespace.
|
||||
func (r *BackupReconciler) retryMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location string, cause error) (ctrl.Result, error) {
|
||||
terminating, err := r.namespaceTerminating(ctx, backup.Namespace)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
if terminating {
|
||||
return r.releaseMariaDBCleanup(ctx, backup, location, fmt.Sprintf("namespace is terminating and the dump cannot be deleted: %v", cause)), nil
|
||||
}
|
||||
getLogger(ctx).Info("cannot delete MariaDB dump; retrying", "backup", backup.Name, "location", location, "error", cause)
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactCleanupFailed",
|
||||
"cannot delete the dump under %s: %v; retrying. Annotate the Backup %s=true to release it and leave the dump",
|
||||
location, cause, mariadbSkipArtifactCleanupAnnotation)
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: mariadbCleanupRetryInterval}, nil
|
||||
}
|
||||
|
||||
// releaseMariaDBCleanup lets the Backup go without deleting anything further
|
||||
// and records a Warning naming what was left, so the give-up is visible.
|
||||
func (r *BackupReconciler) releaseMariaDBCleanup(ctx context.Context, backup *backupsv1alpha1.Backup, location, reason string) ctrl.Result {
|
||||
getLogger(ctx).Info("releasing Backup without deleting its dump", "backup", backup.Name, "location", location, "reason", reason)
|
||||
if r.Recorder != nil {
|
||||
r.Recorder.Eventf(backup, corev1.EventTypeWarning, "ArtifactNotDeleted", "left %s behind: %s", location, reason)
|
||||
}
|
||||
return ctrl.Result{}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -4,6 +4,7 @@ package backupcontroller
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
@@ -30,8 +31,6 @@ import (
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const (
|
||||
mariadbFieldManager = "cozystack-mariadb-backup-driver"
|
||||
|
||||
// mariadb-operator's MariaDB ApplicationDefinition (release.prefix). The
|
||||
// cozystack mariadb-rd renders the HelmRelease with releaseName =
|
||||
// "mariadb-" + appName (see packages/system/mariadb-rd/cozyrds/mariadb.yaml,
|
||||
@@ -48,6 +47,15 @@ const (
|
||||
mariadbBackupNamespaceKey = "k8s.mariadb.com/backup-namespace"
|
||||
mariadbStorageURIKey = "k8s.mariadb.com/storage-uri"
|
||||
|
||||
// The dump the operator Backup wrote: its file name under the storage
|
||||
// prefix, and the RFC3339 timestamp in that name. Every run of an
|
||||
// application writes under the same prefix, so only this name tells its
|
||||
// dump from the others, and a restore asks for exactly that dump.
|
||||
// Recorded only when findMariaDBDump proved the dump belongs to the
|
||||
// operator Backup.
|
||||
mariadbDumpFileKey = "k8s.mariadb.com/dump-file"
|
||||
mariadbDumpTimeKey = "k8s.mariadb.com/dump-time"
|
||||
|
||||
// Polling cadence for the operator Backup/Restore lifecycle.
|
||||
mariadbPollInterval = 5 * time.Second
|
||||
|
||||
@@ -61,6 +69,12 @@ const (
|
||||
// to terminate. Tenants override via spec.options.restoreTimeoutSeconds.
|
||||
mariadbDefaultRestoreDeadline = 30 * time.Minute
|
||||
|
||||
// Retry cadence while the dump a restore asks for cannot be checked.
|
||||
// Longer than mariadbPollInterval because each attempt can hold the
|
||||
// RestoreJob controller's single worker for
|
||||
// mariadbRestoreDumpCheckTimeout.
|
||||
mariadbDumpCheckRetryInterval = 30 * time.Second
|
||||
|
||||
// mariadbBackupSnapshotKind is the Kind stamped onto the snapshot
|
||||
// persisted in Backup.status.underlyingResources. The same shape carries
|
||||
// the storage descriptor and the rendered strategy parameters used at
|
||||
@@ -74,6 +88,12 @@ const (
|
||||
// existing API surface.
|
||||
var mariadbBackupSnapshotAPIVersion = backupsv1alpha1.GroupVersion.String()
|
||||
|
||||
// mariadbRestoreDumpCheckTimeout bounds the object-store round trips a
|
||||
// restore makes before it creates the operator Restore. The RestoreJob
|
||||
// controller has a single worker, so an unreachable endpoint stalls every
|
||||
// other RestoreJob for this long. A variable so tests can shorten it.
|
||||
var mariadbRestoreDumpCheckTimeout = 10 * time.Second
|
||||
|
||||
// mariadbNameForApp returns the k8s.mariadb.com/MariaDB CR name for a
|
||||
// cozystack MariaDB application instance. The mapping mirrors the mariadb
|
||||
// ApplicationDefinition (release.prefix=mariadb-).
|
||||
@@ -422,7 +442,11 @@ func (r *BackupJobReconciler) createMariaDBBackupArtifact(
|
||||
if uri := driverMD[mariadbStorageURIKey]; uri != "" {
|
||||
status.Artifact = &backupsv1alpha1.BackupArtifact{URI: uri}
|
||||
if probe {
|
||||
status.Artifact.SizeBytes = r.mariadbArtifactSizeBytes(ctx, mdbBackup, takenAt.Time)
|
||||
if dump, ok := r.attributedMariaDBDump(ctx, mdbBackup, takenAt.Time); ok {
|
||||
status.Artifact.SizeBytes = dump.Size
|
||||
driverMD[mariadbDumpFileKey] = dump.Name
|
||||
driverMD[mariadbDumpTimeKey] = dump.Timestamp.UTC().Format(time.RFC3339)
|
||||
}
|
||||
}
|
||||
}
|
||||
backup := &backupsv1alpha1.Backup{
|
||||
@@ -594,9 +618,26 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
mdbRestore, err := r.ensureMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName)
|
||||
mdbRestore, err := r.findMariaDBRestoreForJob(ctx, restoreJob)
|
||||
if err != nil {
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err))
|
||||
// Retried: failing the RestoreJob here could fail it under an
|
||||
// operator Restore that is running.
|
||||
return ctrl.Result{}, fmt.Errorf("find k8s.mariadb.com/Restore of RestoreJob %s/%s: %w", restoreJob.Namespace, restoreJob.Name, err)
|
||||
}
|
||||
// The dump is chosen once, as the operator Restore is created; one that
|
||||
// already exists has made its choice.
|
||||
if mdbRestore == nil {
|
||||
targetTime, err := r.mariadbRestoreTargetTime(ctx, restoreJob, backup, srcBackup)
|
||||
var refused mariadbRestoreRefused
|
||||
if errors.As(err, &refused) {
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, refused.Error())
|
||||
}
|
||||
if err != nil {
|
||||
return r.awaitMariaDBDumpCheck(ctx, restoreJob, options, err)
|
||||
}
|
||||
if mdbRestore, err = r.createMariaDBRestore(ctx, restoreJob, sourceBackupName, targetMDBName, targetTime); err != nil {
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure k8s.mariadb.com/Restore: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
cond := apimeta.FindStatusCondition(mdbRestore.Status.Conditions, mariadbtypes.ConditionTypeComplete)
|
||||
@@ -630,12 +671,114 @@ func (r *RestoreJobReconciler) reconcileMariaDBRestore(ctx context.Context, rest
|
||||
}
|
||||
}
|
||||
|
||||
// ensureMariaDBRestore creates a k8s.mariadb.com/Restore CR labelled with
|
||||
// the RestoreJob, or returns the existing one if a previous reconcile
|
||||
// already created it.
|
||||
func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName string) (*mariadbtypes.Restore, error) {
|
||||
// mariadbRestoreRefused is why the dump a RestoreJob asks for cannot be
|
||||
// restored. It fails the RestoreJob, where any other error from choosing
|
||||
// the dump is retried.
|
||||
type mariadbRestoreRefused string
|
||||
|
||||
func (e mariadbRestoreRefused) Error() string { return string(e) }
|
||||
|
||||
// mariadbRestoreTargetTime returns the targetRecoveryTime of the operator
|
||||
// Restore for backup: the timestamp in the name of the dump backup
|
||||
// records. mariadb-operator restores the dump whose timestamp equals the
|
||||
// target and, when there is none, the nearest one, so the dump is first
|
||||
// checked to still exist. A Backup that records no dump is resolved by
|
||||
// legacyMariaDBRestoreTargetTime.
|
||||
func (r *RestoreJobReconciler) mariadbRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) (string, error) {
|
||||
// Deleting a Backup deletes its dump, and the operator restores the
|
||||
// nearest other dump once its own is gone, so a restore must not start
|
||||
// from a Backup being deleted. The copy in hand can predate the delete;
|
||||
// the cleanup holds the Backup only for restores that exist when it
|
||||
// lists them.
|
||||
live := &backupsv1alpha1.Backup{}
|
||||
if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(backup), live); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is gone, and its dump may be gone with it", backup.Namespace, backup.Name))
|
||||
}
|
||||
return "", fmt.Errorf("read Backup %s/%s: %w", backup.Namespace, backup.Name, err)
|
||||
}
|
||||
if !live.DeletionTimestamp.IsZero() {
|
||||
return "", mariadbRestoreRefused(fmt.Sprintf("Backup %s/%s is being deleted, and its dump with it", backup.Namespace, backup.Name))
|
||||
}
|
||||
// A failed Job is Complete=True too, with reason JobFailed, and a
|
||||
// BackupJob that took Complete=True for success made a Backup of it.
|
||||
if cond := apimeta.FindStatusCondition(src.Status.Conditions, mariadbtypes.ConditionTypeComplete); cond != nil && cond.Reason == mariadbtypes.ConditionReasonJobFailed {
|
||||
return "", mariadbRestoreRefused(fmt.Sprintf(
|
||||
"the backup run of k8s.mariadb.com/Backup %s/%s failed (Complete=%s/%s) and left no dump to restore; restoring would bring back another run's dump",
|
||||
src.Namespace, src.Name, cond.Status, cond.Reason))
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, mariadbRestoreDumpCheckTimeout)
|
||||
defer cancel()
|
||||
name := backup.Spec.DriverMetadata[mariadbDumpFileKey]
|
||||
if name == "" {
|
||||
return r.legacyMariaDBRestoreTargetTime(ctx, rj, backup, src), nil
|
||||
}
|
||||
ts, ok := parseMariaDBLogicalBackupFile(name)
|
||||
if !ok {
|
||||
return "", mariadbRestoreRefused(fmt.Sprintf("Backup driverMetadata %s=%q is not the name of a mariadb-operator dump", mariadbDumpFileKey, name))
|
||||
}
|
||||
exists, err := mariadbDumpExists(ctx, r.objectStore(), src, name)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot check that dump %s of k8s.mariadb.com/Backup %s/%s still exists in storage: %w", name, src.Namespace, src.Name, err)
|
||||
}
|
||||
if !exists {
|
||||
s3 := src.Spec.Storage.S3
|
||||
return "", mariadbRestoreRefused(fmt.Sprintf(
|
||||
"dump %s of k8s.mariadb.com/Backup %s/%s no longer exists in storage (s3://%s/%s); restoring now would bring back a different dump",
|
||||
name, src.Namespace, src.Name, s3.Bucket, mariadbOperatorS3Prefix(s3.Prefix)))
|
||||
}
|
||||
return ts.UTC().Format(time.RFC3339), nil
|
||||
}
|
||||
|
||||
// legacyMariaDBRestoreTargetTime chooses the dump of a Backup taken before
|
||||
// Backups recorded theirs: the one findMariaDBDump proves belongs to src,
|
||||
// with the completion time the size probe was given, which takenAt keeps.
|
||||
// When none is proven, for any reason, it returns empty, and the operator
|
||||
// restores the newest dump under the prefix as such a restore always did:
|
||||
// refusing would leave these Backups unrestorable, and the controller may
|
||||
// never have been able to reach their storage. A Warning event says the
|
||||
// dump is a best effort.
|
||||
func (r *RestoreJobReconciler) legacyMariaDBRestoreTargetTime(ctx context.Context, rj *backupsv1alpha1.RestoreJob, backup *backupsv1alpha1.Backup, src *mariadbtypes.Backup) string {
|
||||
dump, err := findMariaDBDump(ctx, r.objectStore(), src, backup.Spec.TakenAt.Time)
|
||||
if err != nil {
|
||||
getLogger(ctx).Info("no dump proven for a MariaDB Backup that records none; restoring the newest", "restorejob", rj.Name, "backup", backup.Name, "error", err)
|
||||
r.Recorder.Eventf(rj, corev1.EventTypeWarning, "BestEffortDump",
|
||||
"Backup %s records no dump and none in storage could be proven to belong to k8s.mariadb.com/Backup %s/%s (%v); mariadb-operator restores the newest dump under its prefix as a best effort, which may be another backup run's",
|
||||
backup.Name, src.Namespace, src.Name, err)
|
||||
return ""
|
||||
}
|
||||
return dump.Timestamp.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// awaitMariaDBDumpCheck keeps the RestoreJob waiting, with Ready=False
|
||||
// saying why, while the dump it asks for cannot be checked, and fails it
|
||||
// with the cause once the restore deadline has passed.
|
||||
func (r *RestoreJobReconciler) awaitMariaDBDumpCheck(ctx context.Context, rj *backupsv1alpha1.RestoreJob, options MariaDBRestoreOptions, cause error) (ctrl.Result, error) {
|
||||
deadline := options.effectiveRestoreDeadline()
|
||||
if rj.Status.StartedAt != nil && time.Since(rj.Status.StartedAt.Time) > deadline {
|
||||
return r.markRestoreJobFailed(ctx, rj, fmt.Sprintf(
|
||||
"%v; gave up after %s (override via spec.options.restoreTimeoutSeconds)", cause, deadline))
|
||||
}
|
||||
getLogger(ctx).Info("cannot check the MariaDB dump to restore; retrying", "restorejob", rj.Name, "error", cause)
|
||||
apimeta.SetStatusCondition(&rj.Status.Conditions, metav1.Condition{
|
||||
Type: "Ready",
|
||||
Status: metav1.ConditionFalse,
|
||||
Reason: "DumpCheckPending",
|
||||
Message: cause.Error(),
|
||||
})
|
||||
if err := r.Status().Update(ctx, rj); err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
return ctrl.Result{RequeueAfter: mariadbDumpCheckRetryInterval}, nil
|
||||
}
|
||||
|
||||
// findMariaDBRestoreForJob returns the k8s.mariadb.com/Restore labelled
|
||||
// with the RestoreJob, if a previous reconcile already created one. It is
|
||||
// read uncached: one the cache has not caught up with would be created
|
||||
// again, or, once its Backup is being deleted, refused while it runs.
|
||||
func (r *RestoreJobReconciler) findMariaDBRestoreForJob(ctx context.Context, rj *backupsv1alpha1.RestoreJob) (*mariadbtypes.Restore, error) {
|
||||
list := &mariadbtypes.RestoreList{}
|
||||
if err := r.List(ctx, list,
|
||||
if err := r.apiReader().List(ctx, list,
|
||||
client.InNamespace(rj.Namespace),
|
||||
client.MatchingLabels{
|
||||
backupsv1alpha1.OwningJobNameLabel: rj.Name,
|
||||
@@ -644,10 +787,15 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(list.Items) > 0 {
|
||||
return &list.Items[0], nil
|
||||
if len(list.Items) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return &list.Items[0], nil
|
||||
}
|
||||
|
||||
// createMariaDBRestore creates the k8s.mariadb.com/Restore of the
|
||||
// RestoreJob, labelled with it. targetRecoveryTime is RFC3339, or empty.
|
||||
func (r *RestoreJobReconciler) createMariaDBRestore(ctx context.Context, rj *backupsv1alpha1.RestoreJob, sourceBackupName, targetMariaDBName, targetRecoveryTime string) (*mariadbtypes.Restore, error) {
|
||||
obj := &mariadbtypes.Restore{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: rj.Namespace,
|
||||
@@ -663,6 +811,7 @@ func (r *RestoreJobReconciler) ensureMariaDBRestore(ctx context.Context, rj *bac
|
||||
Kind: mariadbtypes.BackupKind,
|
||||
Name: sourceBackupName,
|
||||
},
|
||||
TargetRecoveryTime: targetRecoveryTime,
|
||||
},
|
||||
}
|
||||
if err := r.Create(ctx, obj); err != nil {
|
||||
|
||||
@@ -317,7 +317,7 @@ func TestEnsureMariaDBBackup_IdempotentByLabel(t *testing.T) {
|
||||
// Restore-side ensure idempotency + target rewrite
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) {
|
||||
func TestFindMariaDBRestoreForJob_FindsTheCreatedRestore(t *testing.T) {
|
||||
c := newMariaDBStrategyTestClient(t)
|
||||
r := &RestoreJobReconciler{Client: c, Scheme: c.Scheme()}
|
||||
rj := newMariaDBRestoreJob("rj-1", "tenant")
|
||||
@@ -325,16 +325,16 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) {
|
||||
t.Fatalf("seed RestoreJob: %v", err)
|
||||
}
|
||||
|
||||
first, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target")
|
||||
if err != nil {
|
||||
t.Fatalf("first ensureMariaDBRestore: %v", err)
|
||||
if none, err := r.findMariaDBRestoreForJob(context.Background(), rj); err != nil || none != nil {
|
||||
t.Fatalf("findMariaDBRestoreForJob before create = %v, %v; want none", none, err)
|
||||
}
|
||||
second, err := r.ensureMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target")
|
||||
created, err := r.createMariaDBRestore(context.Background(), rj, "src-backup", "mariadb-target", "2026-10-21T10:00:05Z")
|
||||
if err != nil {
|
||||
t.Fatalf("second ensureMariaDBRestore: %v", err)
|
||||
t.Fatalf("createMariaDBRestore: %v", err)
|
||||
}
|
||||
if first.Name != second.Name {
|
||||
t.Errorf("expected idempotent reuse: first=%q second=%q", first.Name, second.Name)
|
||||
found, err := r.findMariaDBRestoreForJob(context.Background(), rj)
|
||||
if err != nil || found == nil || found.Name != created.Name {
|
||||
t.Fatalf("findMariaDBRestoreForJob = %v, %v; want the Restore just created", found, err)
|
||||
}
|
||||
list := &mariadbtypes.RestoreList{}
|
||||
if err := c.List(context.Background(), list, client.InNamespace("tenant")); err != nil {
|
||||
@@ -350,6 +350,9 @@ func TestEnsureMariaDBRestore_IdempotentByLabel(t *testing.T) {
|
||||
if got.Spec.BackupRef == nil || got.Spec.BackupRef.Name != "src-backup" || got.Spec.BackupRef.Kind != mariadbtypes.BackupKind {
|
||||
t.Errorf("BackupRef mismatch: %#v", got.Spec.BackupRef)
|
||||
}
|
||||
if got.Spec.TargetRecoveryTime != "2026-10-21T10:00:05Z" {
|
||||
t.Errorf("TargetRecoveryTime: got %q want 2026-10-21T10:00:05Z", got.Spec.TargetRecoveryTime)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -528,8 +528,8 @@ func (r *BackupJobReconciler) reconcileMongoDB(ctx context.Context, j *backupsv1
|
||||
}
|
||||
}
|
||||
|
||||
switch state := mdbBackup.Status.State; {
|
||||
case state == psmdbtypes.StateReady:
|
||||
switch state := mdbBackup.Status.State; state {
|
||||
case psmdbtypes.StateReady:
|
||||
// Backup completed successfully. Materialise the Cozystack artifact and
|
||||
// finalise the BackupJob. Idempotent: reuse the existing artifact if a
|
||||
// previous reconcile created it and then raced on the status update.
|
||||
@@ -570,7 +570,7 @@ func (r *BackupJobReconciler) reconcileMongoDB(ctx context.Context, j *backupsv1
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
|
||||
case state == psmdbtypes.StateError || state == psmdbtypes.StateRejected:
|
||||
case psmdbtypes.StateError, psmdbtypes.StateRejected:
|
||||
message := mdbBackup.Status.Error
|
||||
// The residual half of the injection race: on the useSystemBucket flow the
|
||||
// operator can latch state=error because it resolved the injected storage
|
||||
@@ -1740,8 +1740,8 @@ func (r *RestoreJobReconciler) reconcileMongoDBRestore(ctx context.Context, rest
|
||||
return r.markRestoreJobFailed(ctx, restoreJob, fmt.Sprintf("failed to ensure psmdb.percona.com/PerconaServerMongoDBRestore: %v", err))
|
||||
}
|
||||
|
||||
switch state := mdbRestore.Status.State; {
|
||||
case state == psmdbtypes.StateReady:
|
||||
switch state := mdbRestore.Status.State; state {
|
||||
case psmdbtypes.StateReady:
|
||||
now := metav1.Now()
|
||||
restoreJob.Status.CompletedAt = &now
|
||||
restoreJob.Status.Phase = backupsv1alpha1.RestoreJobPhaseSucceeded
|
||||
@@ -1756,7 +1756,7 @@ func (r *RestoreJobReconciler) reconcileMongoDBRestore(ctx context.Context, rest
|
||||
}
|
||||
return ctrl.Result{}, nil
|
||||
|
||||
case state == psmdbtypes.StateError || state == psmdbtypes.StateRejected:
|
||||
case psmdbtypes.StateError, psmdbtypes.StateRejected:
|
||||
message := mdbRestore.Status.Error
|
||||
if message == "" {
|
||||
message = fmt.Sprintf("psmdb.percona.com/PerconaServerMongoDBRestore %s/%s reported state=%s", mdbRestore.Namespace, mdbRestore.Name, state)
|
||||
@@ -1855,7 +1855,7 @@ func (r *RestoreJobReconciler) resolveMongoDBBackupSource(ctx context.Context, b
|
||||
}
|
||||
if destination == "" {
|
||||
return nil, fmt.Errorf(
|
||||
"Backup has no restorable destination (the operator PerconaServerMongoDBBackup was reaped and no snapshot destination is persisted); re-run the BackupJob")
|
||||
"no restorable destination for the Backup (the operator PerconaServerMongoDBBackup was reaped and no snapshot destination is persisted); re-run the BackupJob")
|
||||
}
|
||||
src := &psmdbtypes.BackupSource{
|
||||
Type: psmdbtypes.BackupTypeLogical,
|
||||
|
||||
@@ -513,7 +513,7 @@ func (r *BackupJobReconciler) createRabbitmqBackupArtifact(
|
||||
// this run's success while describing something else.
|
||||
if existing.Spec.ApplicationRef.Kind != backup.Spec.ApplicationRef.Kind ||
|
||||
existing.Spec.ApplicationRef.Name != backup.Spec.ApplicationRef.Name {
|
||||
return nil, fmt.Errorf("Backup %s/%s already exists for a different application (%s/%s, not %s/%s)",
|
||||
return nil, fmt.Errorf("existing Backup %s/%s belongs to a different application (%s/%s, not %s/%s)",
|
||||
backup.Namespace, backup.Name,
|
||||
existing.Spec.ApplicationRef.Kind, existing.Spec.ApplicationRef.Name,
|
||||
backup.Spec.ApplicationRef.Kind, backup.Spec.ApplicationRef.Name)
|
||||
|
||||
@@ -57,6 +57,11 @@ type RestoreJobReconciler struct {
|
||||
// see a peer's StartedAt before the cache does. Wired in
|
||||
// SetupWithManager.
|
||||
APIReader client.Reader
|
||||
|
||||
// newS3ObjectLister overrides the S3 client used to check that a
|
||||
// backup's object still exists; nil means a real minio client. Set by
|
||||
// unit tests.
|
||||
newS3ObjectLister func(conn s3Connection) (s3ObjectLister, error)
|
||||
}
|
||||
|
||||
func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
||||
|
||||
@@ -65,12 +65,14 @@ func (s captureSink) WithName(string) logr.LogSink { return s }
|
||||
func TestRunThenDrain_FencesLeftBehindAreAnErrorNamingThem(t *testing.T) {
|
||||
logger, entries := newCaptureLogger()
|
||||
ctx := log.IntoContext(context.Background(), logger)
|
||||
RunThenDrain(ctx,
|
||||
if err := RunThenDrain(ctx,
|
||||
func(context.Context) error { return nil },
|
||||
func(context.Context) (int, error) {
|
||||
return 1, errors.Join(context.DeadlineExceeded, errors.New("still fenced: tenant-root/vm-disk-0"))
|
||||
},
|
||||
time.Minute)
|
||||
time.Minute); err != nil {
|
||||
t.Fatalf("RunThenDrain = %v, want the run's nil error", err)
|
||||
}
|
||||
for _, e := range *entries {
|
||||
if e.err != nil && strings.Contains(e.err.Error(), "tenant-root/vm-disk-0") {
|
||||
return
|
||||
@@ -84,10 +86,12 @@ func TestRunThenDrain_FencesLeftBehindAreAnErrorNamingThem(t *testing.T) {
|
||||
func TestRunThenDrain_AnUnansweredListingIsNotACountOfMinusOne(t *testing.T) {
|
||||
logger, entries := newCaptureLogger()
|
||||
ctx := log.IntoContext(context.Background(), logger)
|
||||
RunThenDrain(ctx,
|
||||
if err := RunThenDrain(ctx,
|
||||
func(context.Context) error { return nil },
|
||||
func(context.Context) (int, error) { return -1, context.DeadlineExceeded },
|
||||
time.Minute)
|
||||
time.Minute); err != nil {
|
||||
t.Fatalf("RunThenDrain = %v, want the run's nil error", err)
|
||||
}
|
||||
found := false
|
||||
for _, e := range *entries {
|
||||
for i := 0; i+1 < len(e.kv); i += 2 {
|
||||
@@ -108,10 +112,12 @@ func TestRunThenDrain_AnUnansweredListingIsNotACountOfMinusOne(t *testing.T) {
|
||||
func TestRunThenDrain_ACompleteDrainSaysSo(t *testing.T) {
|
||||
logger, entries := newCaptureLogger()
|
||||
ctx := log.IntoContext(context.Background(), logger)
|
||||
RunThenDrain(ctx,
|
||||
if err := RunThenDrain(ctx,
|
||||
func(context.Context) error { return nil },
|
||||
func(context.Context) (int, error) { return 0, nil },
|
||||
time.Minute)
|
||||
time.Minute); err != nil {
|
||||
t.Fatalf("RunThenDrain = %v, want the run's nil error", err)
|
||||
}
|
||||
for _, e := range *entries {
|
||||
if e.err == nil && strings.Contains(e.msg, "Released every uninstall fence") {
|
||||
return
|
||||
|
||||
@@ -486,11 +486,12 @@ func ownsSuspensionManagedFields(hr *helmv2.HelmRelease) bool {
|
||||
// fieldsManageSuspend reports whether a managed-fields set includes spec.suspend,
|
||||
// encoded by the apiserver as {"f:spec":{"f:suspend":{}}}.
|
||||
func fieldsManageSuspend(fs *metav1.FieldsV1) bool {
|
||||
if fs == nil || len(fs.Raw) == 0 {
|
||||
raw := fs.GetRawBytes()
|
||||
if len(raw) == 0 {
|
||||
return false
|
||||
}
|
||||
var top map[string]json.RawMessage
|
||||
if err := json.Unmarshal(fs.Raw, &top); err != nil {
|
||||
if err := json.Unmarshal(raw, &top); err != nil {
|
||||
return false
|
||||
}
|
||||
specRaw, ok := top["f:spec"]
|
||||
|
||||
@@ -83,7 +83,7 @@ func suspendManagedFields(manager string, op metav1.ManagedFieldsOperationType)
|
||||
Operation: op,
|
||||
APIVersion: helmv2.GroupVersion.String(),
|
||||
FieldsType: "FieldsV1",
|
||||
FieldsV1: &metav1.FieldsV1{Raw: []byte(`{"f:spec":{"f:suspend":{}}}`)},
|
||||
FieldsV1: metav1.NewFieldsV1(`{"f:spec":{"f:suspend":{}}}`),
|
||||
}}
|
||||
}
|
||||
|
||||
@@ -286,7 +286,7 @@ func TestOwnsSuspensionManagedFields(t *testing.T) {
|
||||
{"no managed fields", nil, false},
|
||||
{"plunger apply of other fields", []metav1.ManagedFieldsEntry{{
|
||||
Manager: suspendFieldManager, Operation: metav1.ManagedFieldsOperationApply,
|
||||
FieldsType: "FieldsV1", FieldsV1: &metav1.FieldsV1{Raw: []byte(`{"f:spec":{"f:chart":{}}}`)},
|
||||
FieldsType: "FieldsV1", FieldsV1: metav1.NewFieldsV1(`{"f:spec":{"f:chart":{}}}`),
|
||||
}}, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
|
||||
@@ -3,8 +3,10 @@ package kubeovnplunger
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -153,12 +155,9 @@ func (r *KubeOVNPlunger) SetupWithManager(mgr ctrl.Manager, kubeOVNNamespace, ap
|
||||
Complete(r); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
|
||||
go srv.ListenAndServe()
|
||||
<-ctx.Done()
|
||||
_ = srv.Shutdown(context.Background())
|
||||
return nil
|
||||
}))
|
||||
if err := mgr.Add(serveStatus(srv)); err != nil {
|
||||
return err
|
||||
}
|
||||
return mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
|
||||
ticker := time.NewTicker(rescanInterval)
|
||||
defer ticker.Stop()
|
||||
@@ -180,6 +179,25 @@ func (r *KubeOVNPlunger) SetupWithManager(mgr ctrl.Manager, kubeOVNNamespace, ap
|
||||
}))
|
||||
}
|
||||
|
||||
// serveStatus runs the status stream for as long as the manager does. A server
|
||||
// that cannot start fails the manager, as the metrics server does.
|
||||
func serveStatus(s *sse.Server) manager.RunnableFunc {
|
||||
return func(ctx context.Context) error {
|
||||
served := make(chan error, 1)
|
||||
go func() { served <- s.ListenAndServe() }()
|
||||
select {
|
||||
case err := <-served:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
}
|
||||
_ = s.Shutdown(context.Background())
|
||||
if err := <-served; !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
type ExecOptions struct {
|
||||
Namespace string
|
||||
Pod string
|
||||
|
||||
@@ -219,7 +219,7 @@ func (r *KubeOVNPlunger) WriteClusterMetrics(db string, snaps []ovnstatus.Health
|
||||
r.metrics.allAgree.WithLabelValues(db, cid).Set(b2f(ecv.AllAgree))
|
||||
r.metrics.membersExpected.WithLabelValues(db, cid).Set(float64(expectedReplicas))
|
||||
r.metrics.membersObserved.WithLabelValues(db, cid).Set(float64(ecv.MembersCount))
|
||||
r.metrics.ipsExpected.WithLabelValues(db, cid).Set(float64(len(ecv.ConsensusResult.TruthView.Members))) // optional; or len(hints.ExpectedIPs)
|
||||
r.metrics.ipsExpected.WithLabelValues(db, cid).Set(float64(len(ecv.TruthView.Members))) // optional; or len(hints.ExpectedIPs)
|
||||
r.metrics.ipsObserved.WithLabelValues(db, cid).Set(float64(ecv.DistinctIPCount))
|
||||
r.metrics.excessMembers.WithLabelValues(db, cid).Set(float64(ecv.ExpectedExcess))
|
||||
r.metrics.missingMembers.WithLabelValues(db, cid).Set(float64(ecv.ExpectedShortfall))
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package kubeovnplunger
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cozystack/cozystack/internal/sse"
|
||||
)
|
||||
|
||||
func runStatusServer(ctx context.Context, t *testing.T, addr string) (bool, error) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
t.Cleanup(cancel)
|
||||
s := sse.New(sse.Options{Addr: addr, Logger: log.New(io.Discard, "", 0)})
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- serveStatus(s)(ctx) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
return true, err
|
||||
case <-time.After(2 * time.Second):
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// A status server that cannot listen fails its runnable, and with it the
|
||||
// manager, rather than leaving the plunger running without it.
|
||||
func TestStatusServerReportsListenFailure(t *testing.T) {
|
||||
taken, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = taken.Close() })
|
||||
returned, err := runStatusServer(context.Background(), t, taken.Addr().String())
|
||||
if !returned || err == nil {
|
||||
t.Fatalf("serveStatus on an occupied address = %v (returned %v), want an error", err, returned)
|
||||
}
|
||||
}
|
||||
|
||||
// Stopping the manager stops the server, and that is not an error.
|
||||
func TestStatusServerStopsWithoutError(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
time.AfterFunc(100*time.Millisecond, cancel)
|
||||
returned, err := runStatusServer(ctx, t, "127.0.0.1:0")
|
||||
if !returned || err != nil {
|
||||
t.Fatalf("serveStatus after cancel = %v (returned %v), want nil", err, returned)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package controller
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
@@ -15,10 +14,6 @@ import (
|
||||
cozyv1alpha1 "github.com/cozystack/cozystack/api/v1alpha1"
|
||||
)
|
||||
|
||||
const (
|
||||
deletionRequeueDelay = 30 * time.Second
|
||||
)
|
||||
|
||||
// WorkloadMonitorReconciler reconciles a WorkloadMonitor object
|
||||
type WorkloadReconciler struct {
|
||||
client.Client
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"k8s.io/client-go/tools/record"
|
||||
"k8s.io/client-go/util/retry"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
"k8s.io/utils/pointer"
|
||||
"k8s.io/utils/ptr"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/cache"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
@@ -374,6 +374,25 @@ func (r *WorkloadMonitorReconciler) isBucketClaimReady(bc *cosiv1alpha1.BucketCl
|
||||
return bc.Status.BucketReady
|
||||
}
|
||||
|
||||
// bucketClaimsMessage names every BucketClaim that has not been provisioned,
|
||||
// with its bucket class, and is empty when there is none.
|
||||
func (r *WorkloadMonitorReconciler) bucketClaimsMessage(claims []cosiv1alpha1.BucketClaim) string {
|
||||
var pending []string
|
||||
for i := range claims {
|
||||
bc := &claims[i]
|
||||
if r.isBucketClaimReady(bc) {
|
||||
continue
|
||||
}
|
||||
entry := fmt.Sprintf("BucketClaim %s is not provisioned", bc.Name)
|
||||
if bc.Spec.BucketClassName != "" {
|
||||
entry = fmt.Sprintf("BucketClaim %s of bucket class %s is not provisioned", bc.Name, bc.Spec.BucketClassName)
|
||||
}
|
||||
pending = append(pending, entry)
|
||||
}
|
||||
sort.Strings(pending)
|
||||
return strings.Join(pending, "; ")
|
||||
}
|
||||
|
||||
// isServiceReady checks if the service has an external IP bound
|
||||
func (r *WorkloadMonitorReconciler) isServiceReady(svc *corev1.Service) bool {
|
||||
return len(svc.Status.LoadBalancer.Ingress) > 0
|
||||
@@ -423,8 +442,8 @@ func updateOwnerReferences(obj metav1.Object, monitor client.Object) {
|
||||
Name: monitor.GetName(),
|
||||
UID: monitor.GetUID(),
|
||||
// Set Controller to false to avoid conflict as multiple controllers are not allowed
|
||||
Controller: pointer.BoolPtr(false),
|
||||
BlockOwnerDeletion: pointer.BoolPtr(true),
|
||||
Controller: ptr.To(false),
|
||||
BlockOwnerDeletion: ptr.To(true),
|
||||
}
|
||||
owners = append(owners, newOwnerRef)
|
||||
}
|
||||
@@ -525,10 +544,10 @@ func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, p
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("querying Prometheus: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("Prometheus returned status %d", resp.StatusCode)
|
||||
return nil, fmt.Errorf("unexpected status %d from Prometheus", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
@@ -549,7 +568,7 @@ func (r *WorkloadMonitorReconciler) queryAllBucketMetrics(ctx context.Context, p
|
||||
return nil, fmt.Errorf("parsing Prometheus response: %w", err)
|
||||
}
|
||||
if promResp.Status != "success" {
|
||||
return nil, fmt.Errorf("Prometheus query finished with status %q", promResp.Status)
|
||||
return nil, fmt.Errorf("the Prometheus query finished with status %q", promResp.Status)
|
||||
}
|
||||
|
||||
for _, r := range promResp.Data.Result {
|
||||
@@ -1157,6 +1176,8 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
|
||||
}
|
||||
}
|
||||
|
||||
bucketsMessage := r.bucketClaimsMessage(bucketClaimList.Items)
|
||||
|
||||
// Update WorkloadMonitor status based on observed pods
|
||||
monitor.Status.ObservedReplicas = observedReplicas
|
||||
monitor.Status.AvailableReplicas = availableReplicas
|
||||
@@ -1174,22 +1195,30 @@ func (r *WorkloadMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Requ
|
||||
// Default to operational = true, but check MinReplicas if set.
|
||||
// Use fresh.Spec to avoid making decisions based on a stale cached copy
|
||||
// when the spec was updated between the initial read and this retry.
|
||||
fresh.Status.Operational = pointer.Bool(true)
|
||||
fresh.Status.Operational = ptr.To(true)
|
||||
if fresh.Spec.MinReplicas != nil && availableReplicas < *fresh.Spec.MinReplicas {
|
||||
fresh.Status.Operational = pointer.Bool(false)
|
||||
fresh.Status.Operational = ptr.To(false)
|
||||
}
|
||||
// Only the DataVolume check writes Message and Reason, so the stored ones
|
||||
// are the last DataVolume verdict: kept while no reader is installed yet, so a
|
||||
// controller restart does not report a stuck disk as ready.
|
||||
// A stored DataVolume verdict is the last one: kept while no reader is
|
||||
// installed yet, so a controller restart does not report a stuck disk as
|
||||
// ready. BucketClaims are listed on every pass, so their verdict is
|
||||
// recomputed and only fills fields no DataVolume verdict holds.
|
||||
if dataVolumesRead {
|
||||
fresh.Status.Message = dataVolumesMessage
|
||||
fresh.Status.Reason = ""
|
||||
if dataVolumesMessage != "" {
|
||||
fresh.Status.Reason = cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady
|
||||
}
|
||||
} else if fresh.Status.Reason == cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady {
|
||||
fresh.Status.Message = ""
|
||||
fresh.Status.Reason = ""
|
||||
}
|
||||
if fresh.Status.Message == "" && bucketsMessage != "" {
|
||||
fresh.Status.Message = bucketsMessage
|
||||
fresh.Status.Reason = cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady
|
||||
}
|
||||
if fresh.Status.Message != "" {
|
||||
fresh.Status.Operational = pointer.Bool(false)
|
||||
fresh.Status.Operational = ptr.To(false)
|
||||
}
|
||||
if equality.Semantic.DeepEqual(stored, &fresh.Status) {
|
||||
return nil
|
||||
|
||||
@@ -870,7 +870,7 @@ func TestReconcileNoBucketClaimSkips(t *testing.T) {
|
||||
|
||||
func TestQueryAllBucketMetrics(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[
|
||||
_, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[
|
||||
{"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"bucket-aaa"},"value":[1713000000,"10485864"]},
|
||||
{"metric":{"__name__":"SeaweedFS_s3_bucket_physical_size_bytes","bucket":"bucket-aaa"},"value":[1713000000,"20971728"]},
|
||||
{"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"bucket-bbb"},"value":[1713000000,"0"]}
|
||||
@@ -909,7 +909,7 @@ func TestQueryAllBucketMetrics(t *testing.T) {
|
||||
|
||||
func TestQueryAllBucketMetricsEmpty(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
|
||||
_, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
@@ -953,7 +953,7 @@ func TestQueryAllBucketMetricsPathPrefixAndUserinfo(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
gotUser, gotPass, gotOK = r.BasicAuth()
|
||||
fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
|
||||
_, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
@@ -1219,7 +1219,7 @@ func TestReconcileBucketMetricsFromEndpointOverride(t *testing.T) {
|
||||
if r.URL.Path != "/vm/select/0/prometheus/api/v1/query" {
|
||||
t.Errorf("unexpected query path %q", r.URL.Path)
|
||||
}
|
||||
fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[
|
||||
_, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[
|
||||
{"metric":{"__name__":"SeaweedFS_s3_bucket_size_bytes","bucket":"cosi-abc123"},"value":[1713000000,"4096"]}
|
||||
]}}`)
|
||||
}))
|
||||
@@ -1331,7 +1331,7 @@ func TestReconcileRetainsLastKnownSizesOnQueryFailure(t *testing.T) {
|
||||
|
||||
func TestReconcileRetainsLastKnownSizesWhenBucketMissingFromResult(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
|
||||
_, _ = fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
@@ -2975,3 +2975,135 @@ func TestReconcileBucketApp_LegacySizeWinsOverOwnerlessNewWorkload(t *testing.T)
|
||||
t.Errorf("size = %s, want the legacy 689Gi over the stale 10Gi", got.String())
|
||||
}
|
||||
}
|
||||
|
||||
func newBucketClaim(name string, labels map[string]string, class string, ready bool) *cosiv1alpha1.BucketClaim {
|
||||
return &cosiv1alpha1.BucketClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default", Labels: labels},
|
||||
Spec: cosiv1alpha1.BucketClaimSpec{
|
||||
BucketClassName: class,
|
||||
Protocols: []cosiv1alpha1.Protocol{cosiv1alpha1.ProtocolS3},
|
||||
},
|
||||
Status: cosiv1alpha1.BucketClaimStatus{BucketReady: ready},
|
||||
}
|
||||
}
|
||||
|
||||
func reconcileBucketMonitor(t *testing.T, c client.Client, s *runtime.Scheme, withReader bool) *cozyv1alpha1.WorkloadMonitor {
|
||||
t.Helper()
|
||||
reconciler := &WorkloadMonitorReconciler{Client: c, Scheme: s}
|
||||
if withReader {
|
||||
reconciler.DataVolumeReader = c
|
||||
}
|
||||
req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "my-bucket", Namespace: "default"}}
|
||||
if _, err := reconciler.Reconcile(context.TODO(), req); err != nil {
|
||||
t.Fatalf("Reconcile returned error: %v", err)
|
||||
}
|
||||
updated := &cozyv1alpha1.WorkloadMonitor{}
|
||||
if err := c.Get(context.TODO(), req.NamespacedName, updated); err != nil {
|
||||
t.Fatalf("Failed to get updated WorkloadMonitor: %v", err)
|
||||
}
|
||||
if updated.Status.Operational == nil {
|
||||
t.Fatal("Expected Operational to be set, got nil")
|
||||
}
|
||||
return updated
|
||||
}
|
||||
|
||||
func newBucketMonitorClient(s *runtime.Scheme, objs ...client.Object) client.Client {
|
||||
monitor := &cozyv1alpha1.WorkloadMonitor{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "my-bucket", Namespace: "default"},
|
||||
Spec: cozyv1alpha1.WorkloadMonitorSpec{
|
||||
Kind: "bucket",
|
||||
Type: "s3",
|
||||
Selector: map[string]string{"app.kubernetes.io/instance": "my-bucket"},
|
||||
Replicas: ptr.To[int32](0),
|
||||
MinReplicas: ptr.To[int32](0),
|
||||
},
|
||||
}
|
||||
return fake.NewClientBuilder().
|
||||
WithScheme(s).
|
||||
WithObjects(append([]client.Object{monitor}, objs...)...).
|
||||
WithStatusSubresource(monitor).
|
||||
Build()
|
||||
}
|
||||
|
||||
func TestReconcile_BucketClaimDecidesOperational(t *testing.T) {
|
||||
selected := map[string]string{"app.kubernetes.io/instance": "my-bucket"}
|
||||
cases := []struct {
|
||||
name string
|
||||
claim *cosiv1alpha1.BucketClaim
|
||||
operational bool
|
||||
message string
|
||||
}{
|
||||
{"provisioned", newBucketClaim("my-bucket", selected, "tenant-root", true), true, ""},
|
||||
{"not provisioned", newBucketClaim("my-bucket", selected, "tenant-root", false), false, "BucketClaim my-bucket of bucket class tenant-root is not provisioned"},
|
||||
{"not provisioned without a class", newBucketClaim("my-bucket", selected, "", false), false, "BucketClaim my-bucket is not provisioned"},
|
||||
{"outside the selector", newBucketClaim("other", map[string]string{"app.kubernetes.io/instance": "other"}, "tenant-root", false), true, ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := newTestScheme()
|
||||
got := reconcileBucketMonitor(t, newBucketMonitorClient(s, tc.claim), s, true)
|
||||
if *got.Status.Operational != tc.operational {
|
||||
t.Errorf("Operational = %v, want %v", *got.Status.Operational, tc.operational)
|
||||
}
|
||||
if got.Status.Message != tc.message {
|
||||
t.Errorf("Message = %q, want %q", got.Status.Message, tc.message)
|
||||
}
|
||||
wantReason := ""
|
||||
if tc.message != "" {
|
||||
wantReason = cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady
|
||||
}
|
||||
if got.Status.Reason != wantReason {
|
||||
t.Errorf("Reason = %q, want %q", got.Status.Reason, wantReason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcile_ProvisionedBucketClaimClearsTheVerdict(t *testing.T) {
|
||||
for _, withReader := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("dataVolumeReader=%v", withReader), func(t *testing.T) {
|
||||
s := newTestScheme()
|
||||
claim := newBucketClaim("my-bucket", map[string]string{"app.kubernetes.io/instance": "my-bucket"}, "tenant-root", false)
|
||||
c := newBucketMonitorClient(s, claim)
|
||||
|
||||
got := reconcileBucketMonitor(t, c, s, withReader)
|
||||
if *got.Status.Operational || got.Status.Reason != cozyv1alpha1.WorkloadMonitorReasonBucketClaimNotReady {
|
||||
t.Fatalf("before provisioning: Operational = %v, Reason = %q", *got.Status.Operational, got.Status.Reason)
|
||||
}
|
||||
|
||||
stored := &cosiv1alpha1.BucketClaim{}
|
||||
if err := c.Get(context.TODO(), types.NamespacedName{Name: "my-bucket", Namespace: "default"}, stored); err != nil {
|
||||
t.Fatalf("Failed to get BucketClaim: %v", err)
|
||||
}
|
||||
stored.Status.BucketReady = true
|
||||
if err := c.Update(context.TODO(), stored); err != nil {
|
||||
t.Fatalf("Failed to update BucketClaim: %v", err)
|
||||
}
|
||||
|
||||
got = reconcileBucketMonitor(t, c, s, withReader)
|
||||
if !*got.Status.Operational || got.Status.Message != "" || got.Status.Reason != "" {
|
||||
t.Errorf("after provisioning: Operational = %v, Message = %q, Reason = %q", *got.Status.Operational, got.Status.Message, got.Status.Reason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcile_DataVolumeVerdictOutranksBucketClaim(t *testing.T) {
|
||||
s := newTestScheme()
|
||||
selected := map[string]string{"app.kubernetes.io/instance": "my-bucket"}
|
||||
phase := "ImportInProgress"
|
||||
c := newBucketMonitorClient(s,
|
||||
newBucketClaim("my-bucket", selected, "tenant-root", false),
|
||||
newDataVolume("my-bucket", selected, &phase),
|
||||
)
|
||||
got := reconcileBucketMonitor(t, c, s, true)
|
||||
if *got.Status.Operational {
|
||||
t.Error("Operational = true, want false")
|
||||
}
|
||||
if got.Status.Reason != cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady {
|
||||
t.Errorf("Reason = %q, want %q", got.Status.Reason, cozyv1alpha1.WorkloadMonitorReasonDataVolumeNotReady)
|
||||
}
|
||||
if got.Status.Message != "DataVolume my-bucket is ImportInProgress" {
|
||||
t.Errorf("Message = %q", got.Status.Message)
|
||||
}
|
||||
}
|
||||
Loaded 100 of 375 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user